tcg/tcg-op-vec.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2018 Linaro, Inc.
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu-common.h"
  22 #include "cpu.h"
  23 #include "exec/exec-all.h"
  24 #include "tcg.h"
  25 #include "tcg-op.h"
  26 #include "tcg-mo.h"
  27
  28 /* Reduce the number of ifdefs below.  This assumes that all uses of
  29    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  30    the compiler can eliminate.  */
  31 #if TCG_TARGET_REG_BITS == 64
  32 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  33 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  34 #define TCGV_LOW  TCGV_LOW_link_error
  35 #define TCGV_HIGH TCGV_HIGH_link_error
  36 #endif
  37
  38 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
  39 {
  40     TCGOp *op = tcg_emit_op(opc);
  41     TCGOP_VECL(op) = type - TCG_TYPE_V64;
  42     TCGOP_VECE(op) = vece;
  43     op->args[0] = r;
  44     op->args[1] = a;
  45 }
  46
  47 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
  48                TCGArg r, TCGArg a, TCGArg b)
  49 {
  50     TCGOp *op = tcg_emit_op(opc);
  51     TCGOP_VECL(op) = type - TCG_TYPE_V64;
  52     TCGOP_VECE(op) = vece;
  53     op->args[0] = r;
  54     op->args[1] = a;
  55     op->args[2] = b;
  56 }
  57
  58 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
  59                TCGArg r, TCGArg a, TCGArg b, TCGArg c)
  60 {
  61     TCGOp *op = tcg_emit_op(opc);
  62     TCGOP_VECL(op) = type - TCG_TYPE_V64;
  63     TCGOP_VECE(op) = vece;
  64     op->args[0] = r;
  65     op->args[1] = a;
  66     op->args[2] = b;
  67     op->args[3] = c;
  68 }
  69
  70 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
  71 {
  72     TCGTemp *rt = tcgv_vec_temp(r);
  73     TCGTemp *at = tcgv_vec_temp(a);
  74     TCGType type = rt->base_type;
  75
  76     /* Must enough inputs for the output.  */
  77     tcg_debug_assert(at->base_type >= type);
  78     vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
  79 }
  80
  81 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
  82                         TCGv_vec r, TCGv_vec a, TCGv_vec b)
  83 {
  84     TCGTemp *rt = tcgv_vec_temp(r);
  85     TCGTemp *at = tcgv_vec_temp(a);
  86     TCGTemp *bt = tcgv_vec_temp(b);
  87     TCGType type = rt->base_type;
  88
  89     /* Must enough inputs for the output.  */
  90     tcg_debug_assert(at->base_type >= type);
  91     tcg_debug_assert(bt->base_type >= type);
  92     vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
  93 }
  94
  95 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
  96 {
  97     if (r != a) {
  98         vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
  99     }
 100 }
 101
 102 #define MO_REG  (TCG_TARGET_REG_BITS == 64 ? MO_64 : MO_32)
 103
 104 static void do_dupi_vec(TCGv_vec r, unsigned vece, TCGArg a)
 105 {
 106     TCGTemp *rt = tcgv_vec_temp(r);
 107     vec_gen_2(INDEX_op_dupi_vec, rt->base_type, vece, temp_arg(rt), a);
 108 }
 109
 110 TCGv_vec tcg_const_zeros_vec(TCGType type)
 111 {
 112     TCGv_vec ret = tcg_temp_new_vec(type);
 113     do_dupi_vec(ret, MO_REG, 0);
 114     return ret;
 115 }
 116
 117 TCGv_vec tcg_const_ones_vec(TCGType type)
 118 {
 119     TCGv_vec ret = tcg_temp_new_vec(type);
 120     do_dupi_vec(ret, MO_REG, -1);
 121     return ret;
 122 }
 123
 124 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
 125 {
 126     TCGTemp *t = tcgv_vec_temp(m);
 127     return tcg_const_zeros_vec(t->base_type);
 128 }
 129
 130 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
 131 {
 132     TCGTemp *t = tcgv_vec_temp(m);
 133     return tcg_const_ones_vec(t->base_type);
 134 }
 135
 136 void tcg_gen_dup64i_vec(TCGv_vec r, uint64_t a)
 137 {
 138     if (TCG_TARGET_REG_BITS == 32 && a == deposit64(a, 32, 32, a)) {
 139         do_dupi_vec(r, MO_32, a);
 140     } else if (TCG_TARGET_REG_BITS == 64 || a == (uint64_t)(int32_t)a) {
 141         do_dupi_vec(r, MO_64, a);
 142     } else {
 143         TCGv_i64 c = tcg_const_i64(a);
 144         tcg_gen_dup_i64_vec(MO_64, r, c);
 145         tcg_temp_free_i64(c);
 146     }
 147 }
 148
 149 void tcg_gen_dup32i_vec(TCGv_vec r, uint32_t a)
 150 {
 151     do_dupi_vec(r, MO_REG, dup_const(MO_32, a));
 152 }
 153
 154 void tcg_gen_dup16i_vec(TCGv_vec r, uint32_t a)
 155 {
 156     do_dupi_vec(r, MO_REG, dup_const(MO_16, a));
 157 }
 158
 159 void tcg_gen_dup8i_vec(TCGv_vec r, uint32_t a)
 160 {
 161     do_dupi_vec(r, MO_REG, dup_const(MO_8, a));
 162 }
 163
 164 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
 165 {
 166     do_dupi_vec(r, MO_REG, dup_const(vece, a));
 167 }
 168
 169 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
 170 {
 171     TCGArg ri = tcgv_vec_arg(r);
 172     TCGTemp *rt = arg_temp(ri);
 173     TCGType type = rt->base_type;
 174
 175     if (TCG_TARGET_REG_BITS == 64) {
 176         TCGArg ai = tcgv_i64_arg(a);
 177         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 178     } else if (vece == MO_64) {
 179         TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
 180         TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
 181         vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
 182     } else {
 183         TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
 184         vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 185     }
 186 }
 187
 188 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
 189 {
 190     TCGArg ri = tcgv_vec_arg(r);
 191     TCGArg ai = tcgv_i32_arg(a);
 192     TCGTemp *rt = arg_temp(ri);
 193     TCGType type = rt->base_type;
 194
 195     vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
 196 }
 197
 198 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
 199 {
 200     TCGArg ri = tcgv_vec_arg(r);
 201     TCGArg bi = tcgv_ptr_arg(b);
 202     TCGTemp *rt = arg_temp(ri);
 203     TCGType type = rt->base_type;
 204
 205     vec_gen_3(opc, type, 0, ri, bi, o);
 206 }
 207
 208 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 209 {
 210     vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
 211 }
 212
 213 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
 214 {
 215     vec_gen_ldst(INDEX_op_st_vec, r, b, o);
 216 }
 217
 218 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
 219 {
 220     TCGArg ri = tcgv_vec_arg(r);
 221     TCGArg bi = tcgv_ptr_arg(b);
 222     TCGTemp *rt = arg_temp(ri);
 223     TCGType type = rt->base_type;
 224
 225     tcg_debug_assert(low_type >= TCG_TYPE_V64);
 226     tcg_debug_assert(low_type <= type);
 227     vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
 228 }
 229
 230 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 231 {
 232     vec_gen_op3(INDEX_op_add_vec, vece, r, a, b);
 233 }
 234
 235 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 236 {
 237     vec_gen_op3(INDEX_op_sub_vec, vece, r, a, b);
 238 }
 239
 240 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 241 {
 242     vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
 243 }
 244
 245 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 246 {
 247     vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
 248 }
 249
 250 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 251 {
 252     vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
 253 }
 254
 255 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 256 {
 257     if (TCG_TARGET_HAS_andc_vec) {
 258         vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
 259     } else {
 260         TCGv_vec t = tcg_temp_new_vec_matching(r);
 261         tcg_gen_not_vec(0, t, b);
 262         tcg_gen_and_vec(0, r, a, t);
 263         tcg_temp_free_vec(t);
 264     }
 265 }
 266
 267 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 268 {
 269     if (TCG_TARGET_HAS_orc_vec) {
 270         vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
 271     } else {
 272         TCGv_vec t = tcg_temp_new_vec_matching(r);
 273         tcg_gen_not_vec(0, t, b);
 274         tcg_gen_or_vec(0, r, a, t);
 275         tcg_temp_free_vec(t);
 276     }
 277 }
 278
 279 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 280 {
 281     if (TCG_TARGET_HAS_not_vec) {
 282         vec_gen_op2(INDEX_op_not_vec, 0, r, a);
 283     } else {
 284         TCGv_vec t = tcg_const_ones_vec_matching(r);
 285         tcg_gen_xor_vec(0, r, a, t);
 286         tcg_temp_free_vec(t);
 287     }
 288 }
 289
 290 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
 291 {
 292     if (TCG_TARGET_HAS_neg_vec) {
 293         vec_gen_op2(INDEX_op_neg_vec, vece, r, a);
 294     } else {
 295         TCGv_vec t = tcg_const_zeros_vec_matching(r);
 296         tcg_gen_sub_vec(vece, r, t, a);
 297         tcg_temp_free_vec(t);
 298     }
 299 }
 300
 301 static void do_shifti(TCGOpcode opc, unsigned vece,
 302                       TCGv_vec r, TCGv_vec a, int64_t i)
 303 {
 304     TCGTemp *rt = tcgv_vec_temp(r);
 305     TCGTemp *at = tcgv_vec_temp(a);
 306     TCGArg ri = temp_arg(rt);
 307     TCGArg ai = temp_arg(at);
 308     TCGType type = rt->base_type;
 309     int can;
 310
 311     tcg_debug_assert(at->base_type == type);
 312     tcg_debug_assert(i >= 0 && i < (8 << vece));
 313
 314     if (i == 0) {
 315         tcg_gen_mov_vec(r, a);
 316         return;
 317     }
 318
 319     can = tcg_can_emit_vec_op(opc, type, vece);
 320     if (can > 0) {
 321         vec_gen_3(opc, type, vece, ri, ai, i);
 322     } else {
 323         /* We leave the choice of expansion via scalar or vector shift
 324            to the target.  Often, but not always, dupi can feed a vector
 325            shift easier than a scalar.  */
 326         tcg_debug_assert(can < 0);
 327         tcg_expand_vec_op(opc, type, vece, ri, ai, i);
 328     }
 329 }
 330
 331 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 332 {
 333     do_shifti(INDEX_op_shli_vec, vece, r, a, i);
 334 }
 335
 336 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 337 {
 338     do_shifti(INDEX_op_shri_vec, vece, r, a, i);
 339 }
 340
 341 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
 342 {
 343     do_shifti(INDEX_op_sari_vec, vece, r, a, i);
 344 }
 345
 346 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
 347                      TCGv_vec r, TCGv_vec a, TCGv_vec b)
 348 {
 349     TCGTemp *rt = tcgv_vec_temp(r);
 350     TCGTemp *at = tcgv_vec_temp(a);
 351     TCGTemp *bt = tcgv_vec_temp(b);
 352     TCGArg ri = temp_arg(rt);
 353     TCGArg ai = temp_arg(at);
 354     TCGArg bi = temp_arg(bt);
 355     TCGType type = rt->base_type;
 356     int can;
 357
 358     tcg_debug_assert(at->base_type == type);
 359     tcg_debug_assert(bt->base_type == type);
 360     can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
 361     if (can > 0) {
 362         vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 363     } else {
 364         tcg_debug_assert(can < 0);
 365         tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
 366     }
 367 }
 368
 369 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
 370 {
 371     TCGTemp *rt = tcgv_vec_temp(r);
 372     TCGTemp *at = tcgv_vec_temp(a);
 373     TCGTemp *bt = tcgv_vec_temp(b);
 374     TCGArg ri = temp_arg(rt);
 375     TCGArg ai = temp_arg(at);
 376     TCGArg bi = temp_arg(bt);
 377     TCGType type = rt->base_type;
 378     int can;
 379
 380     tcg_debug_assert(at->base_type == type);
 381     tcg_debug_assert(bt->base_type == type);
 382     can = tcg_can_emit_vec_op(INDEX_op_mul_vec, type, vece);
 383     if (can > 0) {
 384         vec_gen_3(INDEX_op_mul_vec, type, vece, ri, ai, bi);
 385     } else {
 386         tcg_debug_assert(can < 0);
 387         tcg_expand_vec_op(INDEX_op_mul_vec, type, vece, ri, ai, bi);
 388     }
 389 }