tcg/tcg-op-gvec.h

   1 /*
   2  * Generic vector operation expansion
   3  *
   4  * Copyright (c) 2018 Linaro
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 /*
  21  * "Generic" vectors.  All operands are given as offsets from ENV,
  22  * and therefore cannot also be allocated via tcg_global_mem_new_*.
  23  * OPRSZ is the byte size of the vector upon which the operation is performed.
  24  * MAXSZ is the byte size of the full vector; bytes beyond OPSZ are cleared.
  25  *
  26  * All sizes must be 8 or any multiple of 16.
  27  * When OPRSZ is 8, the alignment may be 8, otherwise must be 16.
  28  * Operands may completely, but not partially, overlap.
  29  */
  30
  31 /* Expand a call to a gvec-style helper, with pointers to two vector
  32    operands, and a descriptor (see tcg-gvec-desc.h).  */
  33 typedef void gen_helper_gvec_2(TCGv_ptr, TCGv_ptr, TCGv_i32);
  34 void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
  35                         uint32_t oprsz, uint32_t maxsz, int32_t data,
  36                         gen_helper_gvec_2 *fn);
  37
  38 /* Similarly, passing an extra data value.  */
  39 typedef void gen_helper_gvec_2i(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
  40 void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
  41                          uint32_t oprsz, uint32_t maxsz, int32_t data,
  42                          gen_helper_gvec_2i *fn);
  43
  44 /* Similarly, passing an extra pointer (e.g. env or float_status).  */
  45 typedef void gen_helper_gvec_2_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
  46 void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
  47                         TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
  48                         int32_t data, gen_helper_gvec_2_ptr *fn);
  49
  50 /* Similarly, with three vector operands.  */
  51 typedef void gen_helper_gvec_3(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
  52 void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
  53                         uint32_t oprsz, uint32_t maxsz, int32_t data,
  54                         gen_helper_gvec_3 *fn);
  55
  56 /* Similarly, with four vector operands.  */
  57 typedef void gen_helper_gvec_4(TCGv_ptr, TCGv_ptr, TCGv_ptr,
  58                                TCGv_ptr, TCGv_i32);
  59 void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
  60                         uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
  61                         int32_t data, gen_helper_gvec_4 *fn);
  62
  63 /* Similarly, with five vector operands.  */
  64 typedef void gen_helper_gvec_5(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr,
  65                                TCGv_ptr, TCGv_i32);
  66 void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
  67                         uint32_t cofs, uint32_t xofs, uint32_t oprsz,
  68                         uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn);
  69
  70 typedef void gen_helper_gvec_3_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr,
  71                                    TCGv_ptr, TCGv_i32);
  72 void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
  73                         TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
  74                         int32_t data, gen_helper_gvec_3_ptr *fn);
  75
  76 typedef void gen_helper_gvec_4_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr,
  77                                    TCGv_ptr, TCGv_ptr, TCGv_i32);
  78 void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
  79                         uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
  80                         uint32_t maxsz, int32_t data,
  81                         gen_helper_gvec_4_ptr *fn);
  82
  83 /* Expand a gvec operation.  Either inline or out-of-line depending on
  84    the actual vector size and the operations supported by the host.  */
  85 typedef struct {
  86     /* Expand inline as a 64-bit or 32-bit integer.
  87        Only one of these will be non-NULL.  */
  88     void (*fni8)(TCGv_i64, TCGv_i64);
  89     void (*fni4)(TCGv_i32, TCGv_i32);
  90     /* Expand inline with a host vector type.  */
  91     void (*fniv)(unsigned, TCGv_vec, TCGv_vec);
  92     /* Expand out-of-line helper w/descriptor.  */
  93     gen_helper_gvec_2 *fno;
  94     /* The optional opcodes, if any, utilized by .fniv.  */
  95     const TCGOpcode *opt_opc;
  96     /* The data argument to the out-of-line helper.  */
  97     int32_t data;
  98     /* The vector element size, if applicable.  */
  99     uint8_t vece;
 100     /* Prefer i64 to v64.  */
 101     bool prefer_i64;
 102 } GVecGen2;
 103
 104 typedef struct {
 105     /* Expand inline as a 64-bit or 32-bit integer.
 106        Only one of these will be non-NULL.  */
 107     void (*fni8)(TCGv_i64, TCGv_i64, int64_t);
 108     void (*fni4)(TCGv_i32, TCGv_i32, int32_t);
 109     /* Expand inline with a host vector type.  */
 110     void (*fniv)(unsigned, TCGv_vec, TCGv_vec, int64_t);
 111     /* Expand out-of-line helper w/descriptor, data in descriptor.  */
 112     gen_helper_gvec_2 *fno;
 113     /* Expand out-of-line helper w/descriptor, data as argument.  */
 114     gen_helper_gvec_2i *fnoi;
 115     /* The optional opcodes, if any, utilized by .fniv.  */
 116     const TCGOpcode *opt_opc;
 117     /* The vector element size, if applicable.  */
 118     uint8_t vece;
 119     /* Prefer i64 to v64.  */
 120     bool prefer_i64;
 121     /* Load dest as a 3rd source operand.  */
 122     bool load_dest;
 123 } GVecGen2i;
 124
 125 typedef struct {
 126     /* Expand inline as a 64-bit or 32-bit integer.
 127        Only one of these will be non-NULL.  */
 128     void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
 129     void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
 130     /* Expand inline with a host vector type.  */
 131     void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
 132     /* Expand out-of-line helper w/descriptor.  */
 133     gen_helper_gvec_2i *fno;
 134     /* The optional opcodes, if any, utilized by .fniv.  */
 135     const TCGOpcode *opt_opc;
 136     /* The data argument to the out-of-line helper.  */
 137     uint32_t data;
 138     /* The vector element size, if applicable.  */
 139     uint8_t vece;
 140     /* Prefer i64 to v64.  */
 141     bool prefer_i64;
 142     /* Load scalar as 1st source operand.  */
 143     bool scalar_first;
 144 } GVecGen2s;
 145
 146 typedef struct {
 147     /* Expand inline as a 64-bit or 32-bit integer.
 148        Only one of these will be non-NULL.  */
 149     void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
 150     void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
 151     /* Expand inline with a host vector type.  */
 152     void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
 153     /* Expand out-of-line helper w/descriptor.  */
 154     gen_helper_gvec_3 *fno;
 155     /* The optional opcodes, if any, utilized by .fniv.  */
 156     const TCGOpcode *opt_opc;
 157     /* The data argument to the out-of-line helper.  */
 158     int32_t data;
 159     /* The vector element size, if applicable.  */
 160     uint8_t vece;
 161     /* Prefer i64 to v64.  */
 162     bool prefer_i64;
 163     /* Load dest as a 3rd source operand.  */
 164     bool load_dest;
 165 } GVecGen3;
 166
 167 typedef struct {
 168     /*
 169      * Expand inline as a 64-bit or 32-bit integer. Only one of these will be
 170      * non-NULL.
 171      */
 172     void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t);
 173     void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t);
 174     /* Expand inline with a host vector type.  */
 175     void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, int64_t);
 176     /* Expand out-of-line helper w/descriptor, data in descriptor.  */
 177     gen_helper_gvec_3 *fno;
 178     /* The optional opcodes, if any, utilized by .fniv.  */
 179     const TCGOpcode *opt_opc;
 180     /* The vector element size, if applicable.  */
 181     uint8_t vece;
 182     /* Prefer i64 to v64.  */
 183     bool prefer_i64;
 184     /* Load dest as a 3rd source operand.  */
 185     bool load_dest;
 186 } GVecGen3i;
 187
 188 typedef struct {
 189     /* Expand inline as a 64-bit or 32-bit integer.
 190        Only one of these will be non-NULL.  */
 191     void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64);
 192     void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32);
 193     /* Expand inline with a host vector type.  */
 194     void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec);
 195     /* Expand out-of-line helper w/descriptor.  */
 196     gen_helper_gvec_4 *fno;
 197     /* The optional opcodes, if any, utilized by .fniv.  */
 198     const TCGOpcode *opt_opc;
 199     /* The data argument to the out-of-line helper.  */
 200     int32_t data;
 201     /* The vector element size, if applicable.  */
 202     uint8_t vece;
 203     /* Prefer i64 to v64.  */
 204     bool prefer_i64;
 205     /* Write aofs as a 2nd dest operand.  */
 206     bool write_aofs;
 207 } GVecGen4;
 208
 209 void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
 210                     uint32_t oprsz, uint32_t maxsz, const GVecGen2 *);
 211 void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
 212                      uint32_t maxsz, int64_t c, const GVecGen2i *);
 213 void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
 214                      uint32_t maxsz, TCGv_i64 c, const GVecGen2s *);
 215 void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
 216                     uint32_t oprsz, uint32_t maxsz, const GVecGen3 *);
 217 void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
 218                      uint32_t oprsz, uint32_t maxsz, int64_t c,
 219                      const GVecGen3i *);
 220 void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
 221                     uint32_t oprsz, uint32_t maxsz, const GVecGen4 *);
 222
 223 /* Expand a specific vector operation.  */
 224
 225 void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
 226                       uint32_t oprsz, uint32_t maxsz);
 227 void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
 228                       uint32_t oprsz, uint32_t maxsz);
 229 void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
 230                       uint32_t oprsz, uint32_t maxsz);
 231 void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
 232                       uint32_t oprsz, uint32_t maxsz);
 233
 234 void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
 235                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 236 void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
 237                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 238 void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
 239                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 240
 241 void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
 242                        int64_t c, uint32_t oprsz, uint32_t maxsz);
 243 void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
 244                        int64_t c, uint32_t oprsz, uint32_t maxsz);
 245
 246 void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
 247                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
 248 void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
 249                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
 250 void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
 251                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
 252
 253 /* Saturated arithmetic.  */
 254 void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
 255                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 256 void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
 257                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 258 void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
 259                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 260 void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
 261                         uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 262
 263 /* Min/max.  */
 264 void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
 265                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 266 void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
 267                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 268 void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
 269                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 270 void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
 271                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 272
 273 void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
 274                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 275 void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
 276                      uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 277 void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
 278                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 279 void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
 280                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 281 void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
 282                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 283 void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
 284                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 285 void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
 286                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 287 void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
 288                       uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 289
 290 void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
 291                        int64_t c, uint32_t oprsz, uint32_t maxsz);
 292 void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
 293                        int64_t c, uint32_t oprsz, uint32_t maxsz);
 294 void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
 295                       int64_t c, uint32_t oprsz, uint32_t maxsz);
 296
 297 void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
 298                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
 299 void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
 300                        TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
 301 void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
 302                       TCGv_i64 c, uint32_t oprsz, uint32_t maxsz);
 303
 304 void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
 305                           uint32_t s, uint32_t m);
 306 void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t s,
 307                           uint32_t m, TCGv_i32);
 308 void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t s,
 309                           uint32_t m, TCGv_i64);
 310
 311 void tcg_gen_gvec_dup8i(uint32_t dofs, uint32_t s, uint32_t m, uint8_t x);
 312 void tcg_gen_gvec_dup16i(uint32_t dofs, uint32_t s, uint32_t m, uint16_t x);
 313 void tcg_gen_gvec_dup32i(uint32_t dofs, uint32_t s, uint32_t m, uint32_t x);
 314 void tcg_gen_gvec_dup64i(uint32_t dofs, uint32_t s, uint32_t m, uint64_t x);
 315
 316 void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
 317                        int64_t shift, uint32_t oprsz, uint32_t maxsz);
 318 void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
 319                        int64_t shift, uint32_t oprsz, uint32_t maxsz);
 320 void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
 321                        int64_t shift, uint32_t oprsz, uint32_t maxsz);
 322
 323 void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
 324                        TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz);
 325 void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
 326                        TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz);
 327 void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
 328                        TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz);
 329
 330 /*
 331  * Perform vector shift by vector element, modulo the element size.
 332  * E.g.  D[i] = A[i] << (B[i] % (8 << vece)).
 333  */
 334 void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
 335                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 336 void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
 337                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 338 void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
 339                        uint32_t bofs, uint32_t oprsz, uint32_t maxsz);
 340
 341 void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
 342                       uint32_t aofs, uint32_t bofs,
 343                       uint32_t oprsz, uint32_t maxsz);
 344
 345 /*
 346  * 64-bit vector operations.  Use these when the register has been allocated
 347  * with tcg_global_mem_new_i64, and so we cannot also address it via pointer.
 348  * OPRSZ = MAXSZ = 8.
 349  */
 350
 351 void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 a);
 352 void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 a);
 353 void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 a);
 354
 355 void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
 356 void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
 357 void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
 358
 359 void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
 360 void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
 361 void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b);
 362
 363 void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
 364 void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
 365 void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
 366 void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
 367 void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);
 368 void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t);