gcc/optabs-query.cc

   1 /* IR-agnostic target query functions relating to optabs
   2    Copyright (C) 1987-2022 Free Software Foundation, Inc.
   3
   4 This file is part of GCC.
   5
   6 GCC is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 3, or (at your option) any later
   9 version.
  10
  11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with GCC; see the file COPYING3.  If not see
  18 <http://www.gnu.org/licenses/>.  */
  19
  20
  21 #include "config.h"
  22 #include "system.h"
  23 #include "coretypes.h"
  24 #include "target.h"
  25 #include "insn-codes.h"
  26 #include "optabs-query.h"
  27 #include "optabs-libfuncs.h"
  28 #include "insn-config.h"
  29 #include "rtl.h"
  30 #include "recog.h"
  31 #include "vec-perm-indices.h"
  32
  33 struct target_optabs default_target_optabs;
  34 struct target_optabs *this_fn_optabs = &default_target_optabs;
  35 #if SWITCHABLE_TARGET
  36 struct target_optabs *this_target_optabs = &default_target_optabs;
  37 #endif
  38
  39 /* Return the insn used to perform conversion OP from mode FROM_MODE
  40    to mode TO_MODE; return CODE_FOR_nothing if the target does not have
  41    such an insn, or if it is unsuitable for optimization type OPT_TYPE.  */
  42
  43 insn_code
  44 convert_optab_handler (convert_optab optab, machine_mode to_mode,
  45                        machine_mode from_mode, optimization_type opt_type)
  46 {
  47   insn_code icode = convert_optab_handler (optab, to_mode, from_mode);
  48   if (icode == CODE_FOR_nothing
  49       || !targetm.optab_supported_p (optab, to_mode, from_mode, opt_type))
  50     return CODE_FOR_nothing;
  51   return icode;
  52 }
  53
  54 /* Return the insn used to implement mode MODE of OP; return
  55    CODE_FOR_nothing if the target does not have such an insn,
  56    or if it is unsuitable for optimization type OPT_TYPE.  */
  57
  58 insn_code
  59 direct_optab_handler (convert_optab optab, machine_mode mode,
  60                       optimization_type opt_type)
  61 {
  62   insn_code icode = direct_optab_handler (optab, mode);
  63   if (icode == CODE_FOR_nothing
  64       || !targetm.optab_supported_p (optab, mode, mode, opt_type))
  65     return CODE_FOR_nothing;
  66   return icode;
  67 }
  68
  69 /* Enumerates the possible types of structure operand to an
  70    extraction_insn.  */
  71 enum extraction_type { ET_unaligned_mem, ET_reg };
  72
  73 /* Check whether insv, extv or extzv pattern ICODE can be used for an
  74    insertion or extraction of type TYPE on a structure of mode MODE.
  75    Return true if so and fill in *INSN accordingly.  STRUCT_OP is the
  76    operand number of the structure (the first sign_extract or zero_extract
  77    operand) and FIELD_OP is the operand number of the field (the other
  78    side of the set from the sign_extract or zero_extract).  */
  79
  80 static bool
  81 get_traditional_extraction_insn (extraction_insn *insn,
  82                                  enum extraction_type type,
  83                                  machine_mode mode,
  84                                  enum insn_code icode,
  85                                  int struct_op, int field_op)
  86 {
  87   const struct insn_data_d *data = &insn_data[icode];
  88
  89   machine_mode struct_mode = data->operand[struct_op].mode;
  90   if (struct_mode == VOIDmode)
  91     struct_mode = word_mode;
  92   if (mode != struct_mode)
  93     return false;
  94
  95   machine_mode field_mode = data->operand[field_op].mode;
  96   if (field_mode == VOIDmode)
  97     field_mode = word_mode;
  98
  99   machine_mode pos_mode = data->operand[struct_op + 2].mode;
 100   if (pos_mode == VOIDmode)
 101     pos_mode = word_mode;
 102
 103   insn->icode = icode;
 104   insn->field_mode = as_a <scalar_int_mode> (field_mode);
 105   if (type == ET_unaligned_mem)
 106     insn->struct_mode = byte_mode;
 107   else if (struct_mode == BLKmode)
 108     insn->struct_mode = opt_scalar_int_mode ();
 109   else
 110     insn->struct_mode = as_a <scalar_int_mode> (struct_mode);
 111   insn->pos_mode = as_a <scalar_int_mode> (pos_mode);
 112   return true;
 113 }
 114
 115 /* Return true if an optab exists to perform an insertion or extraction
 116    of type TYPE in mode MODE.  Describe the instruction in *INSN if so.
 117
 118    REG_OPTAB is the optab to use for register structures and
 119    MISALIGN_OPTAB is the optab to use for misaligned memory structures.
 120    POS_OP is the operand number of the bit position.  */
 121
 122 static bool
 123 get_optab_extraction_insn (class extraction_insn *insn,
 124                            enum extraction_type type,
 125                            machine_mode mode, direct_optab reg_optab,
 126                            direct_optab misalign_optab, int pos_op)
 127 {
 128   direct_optab optab = (type == ET_unaligned_mem ? misalign_optab : reg_optab);
 129   enum insn_code icode = direct_optab_handler (optab, mode);
 130   if (icode == CODE_FOR_nothing)
 131     return false;
 132
 133   const struct insn_data_d *data = &insn_data[icode];
 134
 135   machine_mode pos_mode = data->operand[pos_op].mode;
 136   if (pos_mode == VOIDmode)
 137     pos_mode = word_mode;
 138
 139   insn->icode = icode;
 140   insn->field_mode = as_a <scalar_int_mode> (mode);
 141   if (type == ET_unaligned_mem)
 142     insn->struct_mode = opt_scalar_int_mode ();
 143   else
 144     insn->struct_mode = insn->field_mode;
 145   insn->pos_mode = as_a <scalar_int_mode> (pos_mode);
 146   return true;
 147 }
 148
 149 /* Return true if an instruction exists to perform an insertion or
 150    extraction (PATTERN says which) of type TYPE in mode MODE.
 151    Describe the instruction in *INSN if so.  */
 152
 153 static bool
 154 get_extraction_insn (extraction_insn *insn,
 155                      enum extraction_pattern pattern,
 156                      enum extraction_type type,
 157                      machine_mode mode)
 158 {
 159   switch (pattern)
 160     {
 161     case EP_insv:
 162       if (targetm.have_insv ()
 163           && get_traditional_extraction_insn (insn, type, mode,
 164                                               targetm.code_for_insv, 0, 3))
 165         return true;
 166       return get_optab_extraction_insn (insn, type, mode, insv_optab,
 167                                         insvmisalign_optab, 2);
 168
 169     case EP_extv:
 170       if (targetm.have_extv ()
 171           && get_traditional_extraction_insn (insn, type, mode,
 172                                               targetm.code_for_extv, 1, 0))
 173         return true;
 174       return get_optab_extraction_insn (insn, type, mode, extv_optab,
 175                                         extvmisalign_optab, 3);
 176
 177     case EP_extzv:
 178       if (targetm.have_extzv ()
 179           && get_traditional_extraction_insn (insn, type, mode,
 180                                               targetm.code_for_extzv, 1, 0))
 181         return true;
 182       return get_optab_extraction_insn (insn, type, mode, extzv_optab,
 183                                         extzvmisalign_optab, 3);
 184
 185     default:
 186       gcc_unreachable ();
 187     }
 188 }
 189
 190 /* Return true if an instruction exists to access a field of mode
 191    FIELDMODE in a structure that has STRUCT_BITS significant bits.
 192    Describe the "best" such instruction in *INSN if so.  PATTERN and
 193    TYPE describe the type of insertion or extraction we want to perform.
 194
 195    For an insertion, the number of significant structure bits includes
 196    all bits of the target.  For an extraction, it need only include the
 197    most significant bit of the field.  Larger widths are acceptable
 198    in both cases.  */
 199
 200 static bool
 201 get_best_extraction_insn (extraction_insn *insn,
 202                           enum extraction_pattern pattern,
 203                           enum extraction_type type,
 204                           unsigned HOST_WIDE_INT struct_bits,
 205                           machine_mode field_mode)
 206 {
 207   opt_scalar_int_mode mode_iter;
 208   FOR_EACH_MODE_FROM (mode_iter, smallest_int_mode_for_size (struct_bits))
 209     {
 210       scalar_int_mode mode = mode_iter.require ();
 211       if (get_extraction_insn (insn, pattern, type, mode))
 212         {
 213           FOR_EACH_MODE_FROM (mode_iter, mode)
 214             {
 215               mode = mode_iter.require ();
 216               if (maybe_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (field_mode))
 217                   || TRULY_NOOP_TRUNCATION_MODES_P (insn->field_mode,
 218                                                     field_mode))
 219                 break;
 220               get_extraction_insn (insn, pattern, type, mode);
 221             }
 222           return true;
 223         }
 224     }
 225   return false;
 226 }
 227
 228 /* Return true if an instruction exists to access a field of mode
 229    FIELDMODE in a register structure that has STRUCT_BITS significant bits.
 230    Describe the "best" such instruction in *INSN if so.  PATTERN describes
 231    the type of insertion or extraction we want to perform.
 232
 233    For an insertion, the number of significant structure bits includes
 234    all bits of the target.  For an extraction, it need only include the
 235    most significant bit of the field.  Larger widths are acceptable
 236    in both cases.  */
 237
 238 bool
 239 get_best_reg_extraction_insn (extraction_insn *insn,
 240                               enum extraction_pattern pattern,
 241                               unsigned HOST_WIDE_INT struct_bits,
 242                               machine_mode field_mode)
 243 {
 244   return get_best_extraction_insn (insn, pattern, ET_reg, struct_bits,
 245                                    field_mode);
 246 }
 247
 248 /* Return true if an instruction exists to access a field of BITSIZE
 249    bits starting BITNUM bits into a memory structure.  Describe the
 250    "best" such instruction in *INSN if so.  PATTERN describes the type
 251    of insertion or extraction we want to perform and FIELDMODE is the
 252    natural mode of the extracted field.
 253
 254    The instructions considered here only access bytes that overlap
 255    the bitfield; they do not touch any surrounding bytes.  */
 256
 257 bool
 258 get_best_mem_extraction_insn (extraction_insn *insn,
 259                               enum extraction_pattern pattern,
 260                               HOST_WIDE_INT bitsize, HOST_WIDE_INT bitnum,
 261                               machine_mode field_mode)
 262 {
 263   unsigned HOST_WIDE_INT struct_bits = (bitnum % BITS_PER_UNIT
 264                                         + bitsize
 265                                         + BITS_PER_UNIT - 1);
 266   struct_bits -= struct_bits % BITS_PER_UNIT;
 267   return get_best_extraction_insn (insn, pattern, ET_unaligned_mem,
 268                                    struct_bits, field_mode);
 269 }
 270
 271 /* Return the insn code used to extend FROM_MODE to TO_MODE.
 272    UNSIGNEDP specifies zero-extension instead of sign-extension.  If
 273    no such operation exists, CODE_FOR_nothing will be returned.  */
 274
 275 enum insn_code
 276 can_extend_p (machine_mode to_mode, machine_mode from_mode,
 277               int unsignedp)
 278 {
 279   if (unsignedp < 0 && targetm.have_ptr_extend ())
 280     return targetm.code_for_ptr_extend;
 281
 282   convert_optab tab = unsignedp ? zext_optab : sext_optab;
 283   return convert_optab_handler (tab, to_mode, from_mode);
 284 }
 285
 286 /* Return the insn code to convert fixed-point mode FIXMODE to floating-point
 287    mode FLTMODE, or CODE_FOR_nothing if no such instruction exists.
 288    UNSIGNEDP specifies whether FIXMODE is unsigned.  */
 289
 290 enum insn_code
 291 can_float_p (machine_mode fltmode, machine_mode fixmode,
 292              int unsignedp)
 293 {
 294   convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
 295   return convert_optab_handler (tab, fltmode, fixmode);
 296 }
 297
 298 /* Return the insn code to convert floating-point mode FLTMODE to fixed-point
 299    mode FIXMODE, or CODE_FOR_nothing if no such instruction exists.
 300    UNSIGNEDP specifies whether FIXMODE is unsigned.
 301
 302    On a successful return, set *TRUNCP_PTR to true if it is necessary to
 303    output an explicit FTRUNC before the instruction.  */
 304
 305 enum insn_code
 306 can_fix_p (machine_mode fixmode, machine_mode fltmode,
 307            int unsignedp, bool *truncp_ptr)
 308 {
 309   convert_optab tab;
 310   enum insn_code icode;
 311
 312   tab = unsignedp ? ufixtrunc_optab : sfixtrunc_optab;
 313   icode = convert_optab_handler (tab, fixmode, fltmode);
 314   if (icode != CODE_FOR_nothing)
 315     {
 316       *truncp_ptr = false;
 317       return icode;
 318     }
 319
 320   /* FIXME: This requires a port to define both FIX and FTRUNC pattern
 321      for this to work.  We need to rework the fix* and ftrunc* patterns
 322      and documentation.  */
 323   tab = unsignedp ? ufix_optab : sfix_optab;
 324   icode = convert_optab_handler (tab, fixmode, fltmode);
 325   if (icode != CODE_FOR_nothing
 326       && optab_handler (ftrunc_optab, fltmode) != CODE_FOR_nothing)
 327     {
 328       *truncp_ptr = true;
 329       return icode;
 330     }
 331
 332   return CODE_FOR_nothing;
 333 }
 334
 335 /* Return nonzero if a conditional move of mode MODE is supported.
 336
 337    This function is for combine so it can tell whether an insn that looks
 338    like a conditional move is actually supported by the hardware.  If we
 339    guess wrong we lose a bit on optimization, but that's it.  */
 340 /* ??? sparc64 supports conditionally moving integers values based on fp
 341    comparisons, and vice versa.  How do we handle them?  */
 342
 343 bool
 344 can_conditionally_move_p (machine_mode mode)
 345 {
 346   return direct_optab_handler (movcc_optab, mode) != CODE_FOR_nothing;
 347 }
 348
 349 /* If a target doesn't implement a permute on a vector with multibyte
 350    elements, we can try to do the same permute on byte elements.
 351    If this makes sense for vector mode MODE then return the appropriate
 352    byte vector mode.  */
 353
 354 opt_machine_mode
 355 qimode_for_vec_perm (machine_mode mode)
 356 {
 357   if (GET_MODE_INNER (mode) != QImode)
 358     return related_vector_mode (mode, QImode, GET_MODE_SIZE (mode));
 359   return opt_machine_mode ();
 360 }
 361
 362 /* Return true if selector SEL can be represented in the integer
 363    equivalent of vector mode MODE.  */
 364
 365 bool
 366 selector_fits_mode_p (machine_mode mode, const vec_perm_indices &sel)
 367 {
 368   unsigned HOST_WIDE_INT mask = GET_MODE_MASK (GET_MODE_INNER (mode));
 369   return (mask == HOST_WIDE_INT_M1U
 370           || sel.all_in_range_p (0, mask + 1));
 371 }
 372
 373 /* Return true if VEC_PERM_EXPRs with variable selector operands can be
 374    expanded using SIMD extensions of the CPU.  MODE is the mode of the
 375    vectors being permuted.  */
 376
 377 bool
 378 can_vec_perm_var_p (machine_mode mode)
 379 {
 380   /* If the target doesn't implement a vector mode for the vector type,
 381      then no operations are supported.  */
 382   if (!VECTOR_MODE_P (mode))
 383     return false;
 384
 385   if (direct_optab_handler (vec_perm_optab, mode) != CODE_FOR_nothing)
 386     return true;
 387
 388   /* We allow fallback to a QI vector mode, and adjust the mask.  */
 389   machine_mode qimode;
 390   if (!qimode_for_vec_perm (mode).exists (&qimode)
 391       || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
 392     return false;
 393
 394   if (direct_optab_handler (vec_perm_optab, qimode) == CODE_FOR_nothing)
 395     return false;
 396
 397   /* In order to support the lowering of variable permutations,
 398      we need to support shifts and adds.  */
 399   if (GET_MODE_UNIT_SIZE (mode) > 2
 400       && optab_handler (ashl_optab, mode) == CODE_FOR_nothing
 401       && optab_handler (vashl_optab, mode) == CODE_FOR_nothing)
 402     return false;
 403   if (optab_handler (add_optab, qimode) == CODE_FOR_nothing)
 404     return false;
 405
 406   return true;
 407 }
 408
 409 /* Return true if the target directly supports VEC_PERM_EXPRs on vectors
 410    of mode MODE using the selector SEL.  ALLOW_VARIABLE_P is true if it
 411    is acceptable to force the selector into a register and use a variable
 412    permute (if the target supports that).
 413
 414    Note that additional permutations representing whole-vector shifts may
 415    also be handled via the vec_shr or vec_shl optab, but only where the
 416    second input vector is entirely constant zeroes; this case is not dealt
 417    with here.  */
 418
 419 bool
 420 can_vec_perm_const_p (machine_mode mode, const vec_perm_indices &sel,
 421                       bool allow_variable_p)
 422 {
 423   /* If the target doesn't implement a vector mode for the vector type,
 424      then no operations are supported.  */
 425   if (!VECTOR_MODE_P (mode))
 426     return false;
 427
 428   /* It's probably cheaper to test for the variable case first.  */
 429   if (allow_variable_p && selector_fits_mode_p (mode, sel))
 430     {
 431       if (direct_optab_handler (vec_perm_optab, mode) != CODE_FOR_nothing)
 432         return true;
 433
 434       /* Unlike can_vec_perm_var_p, we don't need to test for optabs
 435          related computing the QImode selector, since that happens at
 436          compile time.  */
 437       machine_mode qimode;
 438       if (qimode_for_vec_perm (mode).exists (&qimode))
 439         {
 440           vec_perm_indices qimode_indices;
 441           qimode_indices.new_expanded_vector (sel, GET_MODE_UNIT_SIZE (mode));
 442           if (selector_fits_mode_p (qimode, qimode_indices)
 443               && (direct_optab_handler (vec_perm_optab, qimode)
 444                   != CODE_FOR_nothing))
 445             return true;
 446         }
 447     }
 448
 449   if (targetm.vectorize.vec_perm_const != NULL)
 450     {
 451       if (targetm.vectorize.vec_perm_const (mode, NULL_RTX, NULL_RTX,
 452                                             NULL_RTX, sel))
 453         return true;
 454
 455       /* ??? For completeness, we ought to check the QImode version of
 456          vec_perm_const_optab.  But all users of this implicit lowering
 457          feature implement the variable vec_perm_optab, and the ia64
 458          port specifically doesn't want us to lower V2SF operations
 459          into integer operations.  */
 460     }
 461
 462   return false;
 463 }
 464
 465 /* Find a widening optab even if it doesn't widen as much as we want.
 466    E.g. if from_mode is HImode, and to_mode is DImode, and there is no
 467    direct HI->SI insn, then return SI->DI, if that exists.  */
 468
 469 enum insn_code
 470 find_widening_optab_handler_and_mode (optab op, machine_mode to_mode,
 471                                       machine_mode from_mode,
 472                                       machine_mode *found_mode)
 473 {
 474   machine_mode limit_mode = to_mode;
 475   if (is_a <scalar_int_mode> (from_mode))
 476     {
 477       gcc_checking_assert (is_a <scalar_int_mode> (to_mode)
 478                            && known_lt (GET_MODE_PRECISION (from_mode),
 479                                         GET_MODE_PRECISION (to_mode)));
 480       /* The modes after FROM_MODE are all MODE_INT, so the only
 481          MODE_PARTIAL_INT mode we consider is FROM_MODE itself.
 482          If LIMIT_MODE is MODE_PARTIAL_INT, stop at the containing
 483          MODE_INT.  */
 484       if (GET_MODE_CLASS (limit_mode) == MODE_PARTIAL_INT)
 485         limit_mode = GET_MODE_WIDER_MODE (limit_mode).require ();
 486     }
 487   else
 488     gcc_checking_assert (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
 489                          && from_mode < to_mode);
 490   FOR_EACH_MODE (from_mode, from_mode, limit_mode)
 491     {
 492       enum insn_code handler = convert_optab_handler (op, to_mode, from_mode);
 493
 494       if (handler != CODE_FOR_nothing)
 495         {
 496           if (found_mode)
 497             *found_mode = from_mode;
 498           return handler;
 499         }
 500     }
 501
 502   return CODE_FOR_nothing;
 503 }
 504
 505 /* Return non-zero if a highpart multiply is supported of can be synthisized.
 506    For the benefit of expand_mult_highpart, the return value is 1 for direct,
 507    2 for even/odd widening, and 3 for hi/lo widening.  */
 508
 509 int
 510 can_mult_highpart_p (machine_mode mode, bool uns_p)
 511 {
 512   optab op;
 513
 514   op = uns_p ? umul_highpart_optab : smul_highpart_optab;
 515   if (optab_handler (op, mode) != CODE_FOR_nothing)
 516     return 1;
 517
 518   /* If the mode is an integral vector, synth from widening operations.  */
 519   if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
 520     return 0;
 521
 522   poly_int64 nunits = GET_MODE_NUNITS (mode);
 523
 524   op = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
 525   if (optab_handler (op, mode) != CODE_FOR_nothing)
 526     {
 527       op = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
 528       if (optab_handler (op, mode) != CODE_FOR_nothing)
 529         {
 530           /* The encoding has 2 interleaved stepped patterns.  */
 531           vec_perm_builder sel (nunits, 2, 3);
 532           for (unsigned int i = 0; i < 6; ++i)
 533             sel.quick_push (!BYTES_BIG_ENDIAN
 534                             + (i & ~1)
 535                             + ((i & 1) ? nunits : 0));
 536           vec_perm_indices indices (sel, 2, nunits);
 537           if (can_vec_perm_const_p (mode, indices))
 538             return 2;
 539         }
 540     }
 541
 542   op = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
 543   if (optab_handler (op, mode) != CODE_FOR_nothing)
 544     {
 545       op = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
 546       if (optab_handler (op, mode) != CODE_FOR_nothing)
 547         {
 548           /* The encoding has a single stepped pattern.  */
 549           vec_perm_builder sel (nunits, 1, 3);
 550           for (unsigned int i = 0; i < 3; ++i)
 551             sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
 552           vec_perm_indices indices (sel, 2, nunits);
 553           if (can_vec_perm_const_p (mode, indices))
 554             return 3;
 555         }
 556     }
 557
 558   return 0;
 559 }
 560
 561 /* Return true if target supports vector masked load/store for mode.  */
 562
 563 bool
 564 can_vec_mask_load_store_p (machine_mode mode,
 565                            machine_mode mask_mode,
 566                            bool is_load)
 567 {
 568   optab op = is_load ? maskload_optab : maskstore_optab;
 569   machine_mode vmode;
 570
 571   /* If mode is vector mode, check it directly.  */
 572   if (VECTOR_MODE_P (mode))
 573     return convert_optab_handler (op, mode, mask_mode) != CODE_FOR_nothing;
 574
 575   /* Otherwise, return true if there is some vector mode with
 576      the mask load/store supported.  */
 577
 578   /* See if there is any chance the mask load or store might be
 579      vectorized.  If not, punt.  */
 580   scalar_mode smode;
 581   if (!is_a <scalar_mode> (mode, &smode))
 582     return false;
 583
 584   vmode = targetm.vectorize.preferred_simd_mode (smode);
 585   if (VECTOR_MODE_P (vmode)
 586       && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
 587       && convert_optab_handler (op, vmode, mask_mode) != CODE_FOR_nothing)
 588     return true;
 589
 590   auto_vector_modes vector_modes;
 591   targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
 592   for (machine_mode base_mode : vector_modes)
 593     if (related_vector_mode (base_mode, smode).exists (&vmode)
 594         && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
 595         && convert_optab_handler (op, vmode, mask_mode) != CODE_FOR_nothing)
 596       return true;
 597   return false;
 598 }
 599
 600 /* If target supports vector load/store with length for vector mode MODE,
 601    return the corresponding vector mode, otherwise return opt_machine_mode ().
 602    There are two flavors for vector load/store with length, one is to measure
 603    length with bytes, the other is to measure length with lanes.
 604    As len_{load,store} optabs point out, for the flavor with bytes, we use
 605    VnQI to wrap the other supportable same size vector modes.  */
 606
 607 opt_machine_mode
 608 get_len_load_store_mode (machine_mode mode, bool is_load)
 609 {
 610   optab op = is_load ? len_load_optab : len_store_optab;
 611   gcc_assert (VECTOR_MODE_P (mode));
 612
 613   /* Check if length in lanes supported for this mode directly.  */
 614   if (direct_optab_handler (op, mode))
 615     return mode;
 616
 617   /* Check if length in bytes supported for same vector size VnQI.  */
 618   machine_mode vmode;
 619   poly_uint64 nunits = GET_MODE_SIZE (mode);
 620   if (related_vector_mode (mode, QImode, nunits).exists (&vmode)
 621       && direct_optab_handler (op, vmode))
 622     return vmode;
 623
 624   return opt_machine_mode ();
 625 }
 626
 627 /* Return true if there is a compare_and_swap pattern.  */
 628
 629 bool
 630 can_compare_and_swap_p (machine_mode mode, bool allow_libcall)
 631 {
 632   enum insn_code icode;
 633
 634   /* Check for __atomic_compare_and_swap.  */
 635   icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
 636   if (icode != CODE_FOR_nothing)
 637     return true;
 638
 639   /* Check for __sync_compare_and_swap.  */
 640   icode = optab_handler (sync_compare_and_swap_optab, mode);
 641   if (icode != CODE_FOR_nothing)
 642     return true;
 643   if (allow_libcall && optab_libfunc (sync_compare_and_swap_optab, mode))
 644     return true;
 645
 646   /* No inline compare and swap.  */
 647   return false;
 648 }
 649
 650 /* Return true if an atomic exchange can be performed.  */
 651
 652 bool
 653 can_atomic_exchange_p (machine_mode mode, bool allow_libcall)
 654 {
 655   enum insn_code icode;
 656
 657   /* Check for __atomic_exchange.  */
 658   icode = direct_optab_handler (atomic_exchange_optab, mode);
 659   if (icode != CODE_FOR_nothing)
 660     return true;
 661
 662   /* Don't check __sync_test_and_set, as on some platforms that
 663      has reduced functionality.  Targets that really do support
 664      a proper exchange should simply be updated to the __atomics.  */
 665
 666   return can_compare_and_swap_p (mode, allow_libcall);
 667 }
 668
 669 /* Return true if an atomic load can be performed without falling back to
 670    a compare-and-swap.  */
 671
 672 bool
 673 can_atomic_load_p (machine_mode mode)
 674 {
 675   enum insn_code icode;
 676
 677   /* Does the target supports the load directly?  */
 678   icode = direct_optab_handler (atomic_load_optab, mode);
 679   if (icode != CODE_FOR_nothing)
 680     return true;
 681
 682   /* If the size of the object is greater than word size on this target,
 683      then we assume that a load will not be atomic.  Also see
 684      expand_atomic_load.  */
 685   return known_le (GET_MODE_PRECISION (mode), BITS_PER_WORD);
 686 }
 687
 688 /* Determine whether "1 << x" is relatively cheap in word_mode.  */
 689
 690 bool
 691 lshift_cheap_p (bool speed_p)
 692 {
 693   /* FIXME: This should be made target dependent via this "this_target"
 694      mechanism, similar to e.g. can_copy_init_p in gcse.cc.  */
 695   static bool init[2] = { false, false };
 696   static bool cheap[2] = { true, true };
 697
 698   /* If the targer has no lshift in word_mode, the operation will most
 699      probably not be cheap.  ??? Does GCC even work for such targets?  */
 700   if (optab_handler (ashl_optab, word_mode) == CODE_FOR_nothing)
 701     return false;
 702
 703   if (!init[speed_p])
 704     {
 705       rtx reg = gen_raw_REG (word_mode, 10000);
 706       int cost = set_src_cost (gen_rtx_ASHIFT (word_mode, const1_rtx, reg),
 707                                word_mode, speed_p);
 708       cheap[speed_p] = cost < COSTS_N_INSNS (3);
 709       init[speed_p] = true;
 710     }
 711
 712   return cheap[speed_p];
 713 }
 714
 715 /* If MODE is not VOIDmode, return true if vector conversion optab OP supports
 716    that mode, given that the second mode is always an integer vector.
 717    If MODE is VOIDmode, return true if OP supports any vector mode.  */
 718
 719 static bool
 720 supports_vec_convert_optab_p (optab op, machine_mode mode)
 721 {
 722   int start = mode == VOIDmode ? 0 : mode;
 723   int end = mode == VOIDmode ? MAX_MACHINE_MODE : mode;
 724   for (int i = start; i <= end; ++i)
 725     if (VECTOR_MODE_P ((machine_mode) i))
 726       for (int j = MIN_MODE_VECTOR_INT; j < MAX_MODE_VECTOR_INT; ++j)
 727         if (convert_optab_handler (op, (machine_mode) i,
 728                                    (machine_mode) j) != CODE_FOR_nothing)
 729           return true;
 730
 731   return false;
 732 }
 733
 734 /* If MODE is not VOIDmode, return true if vec_gather_load is available for
 735    that mode.  If MODE is VOIDmode, return true if gather_load is available
 736    for at least one vector mode.  */
 737
 738 bool
 739 supports_vec_gather_load_p (machine_mode mode)
 740 {
 741   if (!this_fn_optabs->supports_vec_gather_load[mode])
 742     this_fn_optabs->supports_vec_gather_load[mode]
 743       = (supports_vec_convert_optab_p (gather_load_optab, mode)
 744          || supports_vec_convert_optab_p (mask_gather_load_optab, mode)
 745          ? 1 : -1);
 746
 747   return this_fn_optabs->supports_vec_gather_load[mode] > 0;
 748 }
 749
 750 /* If MODE is not VOIDmode, return true if vec_scatter_store is available for
 751    that mode.  If MODE is VOIDmode, return true if scatter_store is available
 752    for at least one vector mode.  */
 753
 754 bool
 755 supports_vec_scatter_store_p (machine_mode mode)
 756 {
 757   if (!this_fn_optabs->supports_vec_scatter_store[mode])
 758     this_fn_optabs->supports_vec_scatter_store[mode]
 759       = (supports_vec_convert_optab_p (scatter_store_optab, mode)
 760          || supports_vec_convert_optab_p (mask_scatter_store_optab, mode)
 761          ? 1 : -1);
 762
 763   return this_fn_optabs->supports_vec_scatter_store[mode] > 0;
 764 }
 765
 766 /* Whether we can extract part of the vector mode MODE as
 767    (scalar or vector) mode EXTR_MODE.  */
 768
 769 bool
 770 can_vec_extract (machine_mode mode, machine_mode extr_mode)
 771 {
 772   unsigned m;
 773   if (!VECTOR_MODE_P (mode)
 774       || !constant_multiple_p (GET_MODE_SIZE (mode),
 775                                GET_MODE_SIZE (extr_mode), &m))
 776     return false;
 777
 778   if (convert_optab_handler (vec_extract_optab, mode, extr_mode)
 779       != CODE_FOR_nothing)
 780     return true;
 781
 782   /* Besides a direct vec_extract we can also use an element extract from
 783      an integer vector mode with elements of the size of the extr_mode.  */
 784   scalar_int_mode imode;
 785   machine_mode vmode;
 786   if (!int_mode_for_size (GET_MODE_BITSIZE (extr_mode), 0).exists (&imode)
 787       || !related_vector_mode (mode, imode, m).exists (&vmode)
 788       || (convert_optab_handler (vec_extract_optab, vmode, imode)
 789           == CODE_FOR_nothing))
 790     return false;
 791   /* We assume we can pun mode to vmode and imode to extr_mode.  */
 792   return true;
 793 }