gcc/internal-fn.c

   1 /* Internal functions.
   2    Copyright (C) 2011-2014 Free Software Foundation, Inc.
   3
   4 This file is part of GCC.
   5
   6 GCC is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 3, or (at your option) any later
   9 version.
  10
  11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with GCC; see the file COPYING3.  If not see
  18 <http://www.gnu.org/licenses/>.  */
  19
  20 #include "config.h"
  21 #include "system.h"
  22 #include "coretypes.h"
  23 #include "internal-fn.h"
  24 #include "tree.h"
  25 #include "stor-layout.h"
  26 #include "expr.h"
  27 #include "optabs.h"
  28 #include "basic-block.h"
  29 #include "tree-ssa-alias.h"
  30 #include "internal-fn.h"
  31 #include "gimple-expr.h"
  32 #include "is-a.h"
  33 #include "gimple.h"
  34 #include "ubsan.h"
  35 #include "target.h"
  36 #include "predict.h"
  37 #include "stringpool.h"
  38 #include "tree-ssanames.h"
  39 #include "diagnostic-core.h"
  40
  41 /* The names of each internal function, indexed by function number.  */
  42 const char *const internal_fn_name_array[] = {
  43 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
  44 #include "internal-fn.def"
  45 #undef DEF_INTERNAL_FN
  46   "<invalid-fn>"
  47 };
  48
  49 /* The ECF_* flags of each internal function, indexed by function number.  */
  50 const int internal_fn_flags_array[] = {
  51 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
  52 #include "internal-fn.def"
  53 #undef DEF_INTERNAL_FN
  54   0
  55 };
  56
  57 /* Fnspec of each internal function, indexed by function number.  */
  58 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
  59
  60 void
  61 init_internal_fns ()
  62 {
  63 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
  64   if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
  65     build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
  66 #include "internal-fn.def"
  67 #undef DEF_INTERNAL_FN
  68   internal_fn_fnspec_array[IFN_LAST] = 0;
  69 }
  70
  71 /* ARRAY_TYPE is an array of vector modes.  Return the associated insn
  72    for load-lanes-style optab OPTAB.  The insn must exist.  */
  73
  74 static enum insn_code
  75 get_multi_vector_move (tree array_type, convert_optab optab)
  76 {
  77   enum insn_code icode;
  78   enum machine_mode imode;
  79   enum machine_mode vmode;
  80
  81   gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
  82   imode = TYPE_MODE (array_type);
  83   vmode = TYPE_MODE (TREE_TYPE (array_type));
  84
  85   icode = convert_optab_handler (optab, imode, vmode);
  86   gcc_assert (icode != CODE_FOR_nothing);
  87   return icode;
  88 }
  89
  90 /* Expand LOAD_LANES call STMT.  */
  91
  92 static void
  93 expand_LOAD_LANES (gimple stmt)
  94 {
  95   struct expand_operand ops[2];
  96   tree type, lhs, rhs;
  97   rtx target, mem;
  98
  99   lhs = gimple_call_lhs (stmt);
 100   rhs = gimple_call_arg (stmt, 0);
 101   type = TREE_TYPE (lhs);
 102
 103   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 104   mem = expand_normal (rhs);
 105
 106   gcc_assert (MEM_P (mem));
 107   PUT_MODE (mem, TYPE_MODE (type));
 108
 109   create_output_operand (&ops[0], target, TYPE_MODE (type));
 110   create_fixed_operand (&ops[1], mem);
 111   expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops);
 112 }
 113
 114 /* Expand STORE_LANES call STMT.  */
 115
 116 static void
 117 expand_STORE_LANES (gimple stmt)
 118 {
 119   struct expand_operand ops[2];
 120   tree type, lhs, rhs;
 121   rtx target, reg;
 122
 123   lhs = gimple_call_lhs (stmt);
 124   rhs = gimple_call_arg (stmt, 0);
 125   type = TREE_TYPE (rhs);
 126
 127   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 128   reg = expand_normal (rhs);
 129
 130   gcc_assert (MEM_P (target));
 131   PUT_MODE (target, TYPE_MODE (type));
 132
 133   create_fixed_operand (&ops[0], target);
 134   create_input_operand (&ops[1], reg, TYPE_MODE (type));
 135   expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops);
 136 }
 137
 138 static void
 139 expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED)
 140 {
 141   gcc_unreachable ();
 142 }
 143
 144 /* This should get expanded in adjust_simduid_builtins.  */
 145
 146 static void
 147 expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED)
 148 {
 149   gcc_unreachable ();
 150 }
 151
 152 /* This should get expanded in adjust_simduid_builtins.  */
 153
 154 static void
 155 expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED)
 156 {
 157   gcc_unreachable ();
 158 }
 159
 160 /* This should get expanded in adjust_simduid_builtins.  */
 161
 162 static void
 163 expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED)
 164 {
 165   gcc_unreachable ();
 166 }
 167
 168 /* This should get expanded in the sanopt pass.  */
 169
 170 static void
 171 expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED)
 172 {
 173   gcc_unreachable ();
 174 }
 175
 176 /* This should get expanded in the sanopt pass.  */
 177
 178 static void
 179 expand_ASAN_CHECK (gimple stmt ATTRIBUTE_UNUSED)
 180 {
 181   gcc_unreachable ();
 182 }
 183
 184 /* Add sub/add overflow checking to the statement STMT.
 185    CODE says whether the operation is +, or -.  */
 186
 187 void
 188 ubsan_expand_si_overflow_addsub_check (tree_code code, gimple stmt)
 189 {
 190   rtx res, op0, op1;
 191   tree lhs, fn, arg0, arg1;
 192   rtx done_label, do_error, target = NULL_RTX;
 193
 194   lhs = gimple_call_lhs (stmt);
 195   arg0 = gimple_call_arg (stmt, 0);
 196   arg1 = gimple_call_arg (stmt, 1);
 197   done_label = gen_label_rtx ();
 198   do_error = gen_label_rtx ();
 199   do_pending_stack_adjust ();
 200   op0 = expand_normal (arg0);
 201   op1 = expand_normal (arg1);
 202
 203   enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
 204   if (lhs)
 205     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 206
 207   enum insn_code icode
 208     = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode);
 209   if (icode != CODE_FOR_nothing)
 210     {
 211       struct expand_operand ops[4];
 212       rtx last = get_last_insn ();
 213
 214       res = gen_reg_rtx (mode);
 215       create_output_operand (&ops[0], res, mode);
 216       create_input_operand (&ops[1], op0, mode);
 217       create_input_operand (&ops[2], op1, mode);
 218       create_fixed_operand (&ops[3], do_error);
 219       if (maybe_expand_insn (icode, 4, ops))
 220         {
 221           last = get_last_insn ();
 222           if (profile_status_for_fn (cfun) != PROFILE_ABSENT
 223               && JUMP_P (last)
 224               && any_condjump_p (last)
 225               && !find_reg_note (last, REG_BR_PROB, 0))
 226             add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
 227           emit_jump (done_label);
 228         }
 229       else
 230         {
 231           delete_insns_since (last);
 232           icode = CODE_FOR_nothing;
 233         }
 234     }
 235
 236   if (icode == CODE_FOR_nothing)
 237     {
 238       rtx sub_check = gen_label_rtx ();
 239       int pos_neg = 3;
 240
 241       /* Compute the operation.  On RTL level, the addition is always
 242          unsigned.  */
 243       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
 244                           op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
 245
 246       /* If we can prove one of the arguments (for MINUS_EXPR only
 247          the second operand, as subtraction is not commutative) is always
 248          non-negative or always negative, we can do just one comparison
 249          and conditional jump instead of 2 at runtime, 3 present in the
 250          emitted code.  If one of the arguments is CONST_INT, all we
 251          need is to make sure it is op1, then the first
 252          emit_cmp_and_jump_insns will be just folded.  Otherwise try
 253          to use range info if available.  */
 254       if (code == PLUS_EXPR && CONST_INT_P (op0))
 255         {
 256           rtx tem = op0;
 257           op0 = op1;
 258           op1 = tem;
 259         }
 260       else if (CONST_INT_P (op1))
 261         ;
 262       else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME)
 263         {
 264           double_int arg0_min, arg0_max;
 265           if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE)
 266             {
 267               if (!arg0_min.is_negative ())
 268                 pos_neg = 1;
 269               else if (arg0_max.is_negative ())
 270                 pos_neg = 2;
 271             }
 272           if (pos_neg != 3)
 273             {
 274               rtx tem = op0;
 275               op0 = op1;
 276               op1 = tem;
 277             }
 278         }
 279       if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME)
 280         {
 281           double_int arg1_min, arg1_max;
 282           if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE)
 283             {
 284               if (!arg1_min.is_negative ())
 285                 pos_neg = 1;
 286               else if (arg1_max.is_negative ())
 287                 pos_neg = 2;
 288             }
 289         }
 290
 291       /* If the op1 is negative, we have to use a different check.  */
 292       if (pos_neg == 3)
 293         emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode,
 294                                  false, sub_check, PROB_EVEN);
 295
 296       /* Compare the result of the operation with one of the operands.  */
 297       if (pos_neg & 1)
 298         emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE,
 299                                  NULL_RTX, mode, false, done_label,
 300                                  PROB_VERY_LIKELY);
 301
 302       /* If we get here, we have to print the error.  */
 303       if (pos_neg == 3)
 304         {
 305           emit_jump (do_error);
 306
 307           emit_label (sub_check);
 308         }
 309
 310       /* We have k = a + b for b < 0 here.  k <= a must hold.  */
 311       if (pos_neg & 2)
 312         emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE,
 313                                  NULL_RTX, mode, false, done_label,
 314                                  PROB_VERY_LIKELY);
 315     }
 316
 317   emit_label (do_error);
 318   /* Expand the ubsan builtin call.  */
 319   push_temp_slots ();
 320   fn = ubsan_build_overflow_builtin (code, gimple_location (stmt),
 321                                      TREE_TYPE (arg0), arg0, arg1);
 322   expand_normal (fn);
 323   pop_temp_slots ();
 324   do_pending_stack_adjust ();
 325
 326   /* We're done.  */
 327   emit_label (done_label);
 328
 329   if (lhs)
 330     emit_move_insn (target, res);
 331 }
 332
 333 /* Add negate overflow checking to the statement STMT.  */
 334
 335 void
 336 ubsan_expand_si_overflow_neg_check (gimple stmt)
 337 {
 338   rtx res, op1;
 339   tree lhs, fn, arg1;
 340   rtx done_label, do_error, target = NULL_RTX;
 341
 342   lhs = gimple_call_lhs (stmt);
 343   arg1 = gimple_call_arg (stmt, 1);
 344   done_label = gen_label_rtx ();
 345   do_error = gen_label_rtx ();
 346
 347   do_pending_stack_adjust ();
 348   op1 = expand_normal (arg1);
 349
 350   enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
 351   if (lhs)
 352     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 353
 354   enum insn_code icode = optab_handler (negv3_optab, mode);
 355   if (icode != CODE_FOR_nothing)
 356     {
 357       struct expand_operand ops[3];
 358       rtx last = get_last_insn ();
 359
 360       res = gen_reg_rtx (mode);
 361       create_output_operand (&ops[0], res, mode);
 362       create_input_operand (&ops[1], op1, mode);
 363       create_fixed_operand (&ops[2], do_error);
 364       if (maybe_expand_insn (icode, 3, ops))
 365         {
 366           last = get_last_insn ();
 367           if (profile_status_for_fn (cfun) != PROFILE_ABSENT
 368               && JUMP_P (last)
 369               && any_condjump_p (last)
 370               && !find_reg_note (last, REG_BR_PROB, 0))
 371             add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
 372           emit_jump (done_label);
 373         }
 374       else
 375         {
 376           delete_insns_since (last);
 377           icode = CODE_FOR_nothing;
 378         }
 379     }
 380
 381   if (icode == CODE_FOR_nothing)
 382     {
 383       /* Compute the operation.  On RTL level, the addition is always
 384          unsigned.  */
 385       res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
 386
 387       /* Compare the operand with the most negative value.  */
 388       rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
 389       emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false,
 390                                done_label, PROB_VERY_LIKELY);
 391     }
 392
 393   emit_label (do_error);
 394   /* Expand the ubsan builtin call.  */
 395   push_temp_slots ();
 396   fn = ubsan_build_overflow_builtin (NEGATE_EXPR, gimple_location (stmt),
 397                                      TREE_TYPE (arg1), arg1, NULL_TREE);
 398   expand_normal (fn);
 399   pop_temp_slots ();
 400   do_pending_stack_adjust ();
 401
 402   /* We're done.  */
 403   emit_label (done_label);
 404
 405   if (lhs)
 406     emit_move_insn (target, res);
 407 }
 408
 409 /* Add mul overflow checking to the statement STMT.  */
 410
 411 void
 412 ubsan_expand_si_overflow_mul_check (gimple stmt)
 413 {
 414   rtx res, op0, op1;
 415   tree lhs, fn, arg0, arg1;
 416   rtx done_label, do_error, target = NULL_RTX;
 417
 418   lhs = gimple_call_lhs (stmt);
 419   arg0 = gimple_call_arg (stmt, 0);
 420   arg1 = gimple_call_arg (stmt, 1);
 421   done_label = gen_label_rtx ();
 422   do_error = gen_label_rtx ();
 423
 424   do_pending_stack_adjust ();
 425   op0 = expand_normal (arg0);
 426   op1 = expand_normal (arg1);
 427
 428   enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
 429   if (lhs)
 430     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 431
 432   enum insn_code icode = optab_handler (mulv4_optab, mode);
 433   if (icode != CODE_FOR_nothing)
 434     {
 435       struct expand_operand ops[4];
 436       rtx last = get_last_insn ();
 437
 438       res = gen_reg_rtx (mode);
 439       create_output_operand (&ops[0], res, mode);
 440       create_input_operand (&ops[1], op0, mode);
 441       create_input_operand (&ops[2], op1, mode);
 442       create_fixed_operand (&ops[3], do_error);
 443       if (maybe_expand_insn (icode, 4, ops))
 444         {
 445           last = get_last_insn ();
 446           if (profile_status_for_fn (cfun) != PROFILE_ABSENT
 447               && JUMP_P (last)
 448               && any_condjump_p (last)
 449               && !find_reg_note (last, REG_BR_PROB, 0))
 450             add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
 451           emit_jump (done_label);
 452         }
 453       else
 454         {
 455           delete_insns_since (last);
 456           icode = CODE_FOR_nothing;
 457         }
 458     }
 459
 460   if (icode == CODE_FOR_nothing)
 461     {
 462       struct separate_ops ops;
 463       enum machine_mode hmode
 464         = mode_for_size (GET_MODE_PRECISION (mode) / 2, MODE_INT, 1);
 465       ops.op0 = arg0;
 466       ops.op1 = arg1;
 467       ops.op2 = NULL_TREE;
 468       ops.location = gimple_location (stmt);
 469       if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
 470           && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
 471         {
 472           enum machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
 473           ops.code = WIDEN_MULT_EXPR;
 474           ops.type
 475             = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0);
 476
 477           res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
 478           rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res,
 479                                      GET_MODE_PRECISION (mode), NULL_RTX, 0);
 480           hipart = gen_lowpart (mode, hipart);
 481           res = gen_lowpart (mode, res);
 482           rtx signbit = expand_shift (RSHIFT_EXPR, mode, res,
 483                                       GET_MODE_PRECISION (mode) - 1,
 484                                       NULL_RTX, 0);
 485           /* RES is low half of the double width result, HIPART
 486              the high half.  There was overflow if
 487              HIPART is different from RES < 0 ? -1 : 0.  */
 488           emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode,
 489                                    false, done_label, PROB_VERY_LIKELY);
 490         }
 491       else if (hmode != BLKmode
 492                && 2 * GET_MODE_PRECISION (hmode) == GET_MODE_PRECISION (mode))
 493         {
 494           rtx large_op0 = gen_label_rtx ();
 495           rtx small_op0_large_op1 = gen_label_rtx ();
 496           rtx one_small_one_large = gen_label_rtx ();
 497           rtx both_ops_large = gen_label_rtx ();
 498           rtx after_hipart_neg = gen_label_rtx ();
 499           rtx after_lopart_neg = gen_label_rtx ();
 500           rtx do_overflow = gen_label_rtx ();
 501           rtx hipart_different = gen_label_rtx ();
 502
 503           int hprec = GET_MODE_PRECISION (hmode);
 504           rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
 505                                       NULL_RTX, 0);
 506           hipart0 = gen_lowpart (hmode, hipart0);
 507           rtx lopart0 = gen_lowpart (hmode, op0);
 508           rtx signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
 509                                        NULL_RTX, 0);
 510           rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
 511                                       NULL_RTX, 0);
 512           hipart1 = gen_lowpart (hmode, hipart1);
 513           rtx lopart1 = gen_lowpart (hmode, op1);
 514           rtx signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
 515                                        NULL_RTX, 0);
 516
 517           res = gen_reg_rtx (mode);
 518
 519           /* True if op0 resp. op1 are known to be in the range of
 520              halfstype.  */
 521           bool op0_small_p = false;
 522           bool op1_small_p = false;
 523           /* True if op0 resp. op1 are known to have all zeros or all ones
 524              in the upper half of bits, but are not known to be
 525              op{0,1}_small_p.  */
 526           bool op0_medium_p = false;
 527           bool op1_medium_p = false;
 528           /* -1 if op{0,1} is known to be negative, 0 if it is known to be
 529              nonnegative, 1 if unknown.  */
 530           int op0_sign = 1;
 531           int op1_sign = 1;
 532
 533           if (TREE_CODE (arg0) == SSA_NAME)
 534             {
 535               double_int arg0_min, arg0_max;
 536               if (get_range_info (arg0, &arg0_min, &arg0_max) == VR_RANGE)
 537                 {
 538                   if (arg0_max.sle (double_int::max_value (hprec, false))
 539                       && double_int::min_value (hprec, false).sle (arg0_min))
 540                     op0_small_p = true;
 541                   else if (arg0_max.sle (double_int::max_value (hprec, true))
 542                            && (~double_int::max_value (hprec,
 543                                                        true)).sle (arg0_min))
 544                     op0_medium_p = true;
 545                   if (!arg0_min.is_negative ())
 546                     op0_sign = 0;
 547                   else if (arg0_max.is_negative ())
 548                     op0_sign = -1;
 549                 }
 550             }
 551           if (TREE_CODE (arg1) == SSA_NAME)
 552             {
 553               double_int arg1_min, arg1_max;
 554               if (get_range_info (arg1, &arg1_min, &arg1_max) == VR_RANGE)
 555                 {
 556                   if (arg1_max.sle (double_int::max_value (hprec, false))
 557                       && double_int::min_value (hprec, false).sle (arg1_min))
 558                     op1_small_p = true;
 559                   else if (arg1_max.sle (double_int::max_value (hprec, true))
 560                            && (~double_int::max_value (hprec,
 561                                                        true)).sle (arg1_min))
 562                     op1_medium_p = true;
 563                   if (!arg1_min.is_negative ())
 564                     op1_sign = 0;
 565                   else if (arg1_max.is_negative ())
 566                     op1_sign = -1;
 567                 }
 568             }
 569
 570           int smaller_sign = 1;
 571           int larger_sign = 1;
 572           if (op0_small_p)
 573             {
 574               smaller_sign = op0_sign;
 575               larger_sign = op1_sign;
 576             }
 577           else if (op1_small_p)
 578             {
 579               smaller_sign = op1_sign;
 580               larger_sign = op0_sign;
 581             }
 582           else if (op0_sign == op1_sign)
 583             {
 584               smaller_sign = op0_sign;
 585               larger_sign = op0_sign;
 586             }
 587
 588           if (!op0_small_p)
 589             emit_cmp_and_jump_insns (signbit0, hipart0, NE, NULL_RTX, hmode,
 590                                      false, large_op0, PROB_UNLIKELY);
 591
 592           if (!op1_small_p)
 593             emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
 594                                      false, small_op0_large_op1,
 595                                      PROB_UNLIKELY);
 596
 597           /* If both op0 and op1 are sign extended from hmode to mode,
 598              the multiplication will never overflow.  We can do just one
 599              hmode x hmode => mode widening multiplication.  */
 600           if (GET_CODE (lopart0) == SUBREG)
 601             {
 602               SUBREG_PROMOTED_VAR_P (lopart0) = 1;
 603               SUBREG_PROMOTED_UNSIGNED_SET (lopart0, 0);
 604             }
 605           if (GET_CODE (lopart1) == SUBREG)
 606             {
 607               SUBREG_PROMOTED_VAR_P (lopart1) = 1;
 608               SUBREG_PROMOTED_UNSIGNED_SET (lopart1, 0);
 609             }
 610           tree halfstype = build_nonstandard_integer_type (hprec, 0);
 611           ops.op0 = make_tree (halfstype, lopart0);
 612           ops.op1 = make_tree (halfstype, lopart1);
 613           ops.code = WIDEN_MULT_EXPR;
 614           ops.type = TREE_TYPE (arg0);
 615           rtx thisres
 616             = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
 617           emit_move_insn (res, thisres);
 618           emit_jump (done_label);
 619
 620           emit_label (small_op0_large_op1);
 621
 622           /* If op0 is sign extended from hmode to mode, but op1 is not,
 623              just swap the arguments and handle it as op1 sign extended,
 624              op0 not.  */
 625           rtx larger = gen_reg_rtx (mode);
 626           rtx hipart = gen_reg_rtx (hmode);
 627           rtx lopart = gen_reg_rtx (hmode);
 628           emit_move_insn (larger, op1);
 629           emit_move_insn (hipart, hipart1);
 630           emit_move_insn (lopart, lopart0);
 631           emit_jump (one_small_one_large);
 632
 633           emit_label (large_op0);
 634
 635           if (!op1_small_p)
 636             emit_cmp_and_jump_insns (signbit1, hipart1, NE, NULL_RTX, hmode,
 637                                      false, both_ops_large, PROB_UNLIKELY);
 638
 639           /* If op1 is sign extended from hmode to mode, but op0 is not,
 640              prepare larger, hipart and lopart pseudos and handle it together
 641              with small_op0_large_op1.  */
 642           emit_move_insn (larger, op0);
 643           emit_move_insn (hipart, hipart0);
 644           emit_move_insn (lopart, lopart1);
 645
 646           emit_label (one_small_one_large);
 647
 648           /* lopart is the low part of the operand that is sign extended
 649              to mode, larger is the the other operand, hipart is the
 650              high part of larger and lopart0 and lopart1 are the low parts
 651              of both operands.
 652              We perform lopart0 * lopart1 and lopart * hipart widening
 653              multiplications.  */
 654           tree halfutype = build_nonstandard_integer_type (hprec, 1);
 655           ops.op0 = make_tree (halfutype, lopart0);
 656           ops.op1 = make_tree (halfutype, lopart1);
 657           rtx lo0xlo1
 658             = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
 659
 660           ops.op0 = make_tree (halfutype, lopart);
 661           ops.op1 = make_tree (halfutype, hipart);
 662           rtx loxhi = gen_reg_rtx (mode);
 663           rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
 664           emit_move_insn (loxhi, tem);
 665
 666           /* if (hipart < 0) loxhi -= lopart << (bitsize / 2);  */
 667           if (larger_sign == 0)
 668             emit_jump (after_hipart_neg);
 669           else if (larger_sign != -1)
 670             emit_cmp_and_jump_insns (hipart, const0_rtx, GE, NULL_RTX, hmode,
 671                                      false, after_hipart_neg, PROB_EVEN);
 672
 673           tem = convert_modes (mode, hmode, lopart, 1);
 674           tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
 675           tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
 676                                      1, OPTAB_DIRECT);
 677           emit_move_insn (loxhi, tem);
 678
 679           emit_label (after_hipart_neg);
 680
 681           /* if (lopart < 0) loxhi -= larger;  */
 682           if (smaller_sign == 0)
 683             emit_jump (after_lopart_neg);
 684           else if (smaller_sign != -1)
 685             emit_cmp_and_jump_insns (lopart, const0_rtx, GE, NULL_RTX, hmode,
 686                                      false, after_lopart_neg, PROB_EVEN);
 687
 688           tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
 689                                      1, OPTAB_DIRECT);
 690           emit_move_insn (loxhi, tem);
 691
 692           emit_label (after_lopart_neg);
 693
 694           /* loxhi += (uns) lo0xlo1 >> (bitsize / 2);  */
 695           tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
 696           tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
 697                                      1, OPTAB_DIRECT);
 698           emit_move_insn (loxhi, tem);
 699
 700           /* if (loxhi >> (bitsize / 2)
 701                  == (hmode) loxhi >> (bitsize / 2 - 1))  */
 702           rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
 703                                           NULL_RTX, 0);
 704           hipartloxhi = gen_lowpart (hmode, hipartloxhi);
 705           rtx lopartloxhi = gen_lowpart (hmode, loxhi);
 706           rtx signbitloxhi = expand_shift (RSHIFT_EXPR, hmode, lopartloxhi,
 707                                            hprec - 1, NULL_RTX, 0);
 708
 709           emit_cmp_and_jump_insns (signbitloxhi, hipartloxhi, NE, NULL_RTX,
 710                                    hmode, false, do_overflow,
 711                                    PROB_VERY_UNLIKELY);
 712
 713           /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1;  */
 714           rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
 715                                            NULL_RTX, 1);
 716           tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1);
 717
 718           tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
 719                                      1, OPTAB_DIRECT);
 720           if (tem != res)
 721             emit_move_insn (res, tem);
 722           emit_jump (done_label);
 723
 724           emit_label (both_ops_large);
 725
 726           /* If both operands are large (not sign extended from hmode),
 727              then perform the full multiplication which will be the result
 728              of the operation.  The only cases which don't overflow are
 729              some cases where both hipart0 and highpart1 are 0 or -1.  */
 730           ops.code = MULT_EXPR;
 731           ops.op0 = make_tree (TREE_TYPE (arg0), op0);
 732           ops.op1 = make_tree (TREE_TYPE (arg0), op1);
 733           tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
 734           emit_move_insn (res, tem);
 735
 736           if (!op0_medium_p)
 737             {
 738               tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
 739                                          NULL_RTX, 1, OPTAB_DIRECT);
 740               emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode,
 741                                        true, do_error, PROB_VERY_UNLIKELY);
 742             }
 743
 744           if (!op1_medium_p)
 745             {
 746               tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
 747                                          NULL_RTX, 1, OPTAB_DIRECT);
 748               emit_cmp_and_jump_insns (tem, const1_rtx, GTU, NULL_RTX, hmode,
 749                                        true, do_error, PROB_VERY_UNLIKELY);
 750             }
 751
 752           /* At this point hipart{0,1} are both in [-1, 0].  If they are the
 753              same, overflow happened if res is negative, if they are different,
 754              overflow happened if res is positive.  */
 755           if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
 756             emit_jump (hipart_different);
 757           else if (op0_sign == 1 || op1_sign == 1)
 758             emit_cmp_and_jump_insns (hipart0, hipart1, NE, NULL_RTX, hmode,
 759                                      true, hipart_different, PROB_EVEN);
 760
 761           emit_cmp_and_jump_insns (res, const0_rtx, LT, NULL_RTX, mode, false,
 762                                    do_error, PROB_VERY_UNLIKELY);
 763           emit_jump (done_label);
 764
 765           emit_label (hipart_different);
 766
 767           emit_cmp_and_jump_insns (res, const0_rtx, GE, NULL_RTX, mode, false,
 768                                    do_error, PROB_VERY_UNLIKELY);
 769           emit_jump (done_label);
 770
 771           emit_label (do_overflow);
 772
 773           /* Overflow, do full multiplication and fallthru into do_error.  */
 774           ops.op0 = make_tree (TREE_TYPE (arg0), op0);
 775           ops.op1 = make_tree (TREE_TYPE (arg0), op1);
 776           tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
 777           emit_move_insn (res, tem);
 778         }
 779       else
 780         {
 781           ops.code = MULT_EXPR;
 782           ops.type = TREE_TYPE (arg0);
 783           res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
 784           emit_jump (done_label);
 785         }
 786     }
 787
 788   emit_label (do_error);
 789   /* Expand the ubsan builtin call.  */
 790   push_temp_slots ();
 791   fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt),
 792                                      TREE_TYPE (arg0), arg0, arg1);
 793   expand_normal (fn);
 794   pop_temp_slots ();
 795   do_pending_stack_adjust ();
 796
 797   /* We're done.  */
 798   emit_label (done_label);
 799
 800   if (lhs)
 801     emit_move_insn (target, res);
 802 }
 803
 804 /* Expand UBSAN_CHECK_ADD call STMT.  */
 805
 806 static void
 807 expand_UBSAN_CHECK_ADD (gimple stmt)
 808 {
 809   ubsan_expand_si_overflow_addsub_check (PLUS_EXPR, stmt);
 810 }
 811
 812 /* Expand UBSAN_CHECK_SUB call STMT.  */
 813
 814 static void
 815 expand_UBSAN_CHECK_SUB (gimple stmt)
 816 {
 817   if (integer_zerop (gimple_call_arg (stmt, 0)))
 818     ubsan_expand_si_overflow_neg_check (stmt);
 819   else
 820     ubsan_expand_si_overflow_addsub_check (MINUS_EXPR, stmt);
 821 }
 822
 823 /* Expand UBSAN_CHECK_MUL call STMT.  */
 824
 825 static void
 826 expand_UBSAN_CHECK_MUL (gimple stmt)
 827 {
 828   ubsan_expand_si_overflow_mul_check (stmt);
 829 }
 830
 831 /* This should get folded in tree-vectorizer.c.  */
 832
 833 static void
 834 expand_LOOP_VECTORIZED (gimple stmt ATTRIBUTE_UNUSED)
 835 {
 836   gcc_unreachable ();
 837 }
 838
 839 static void
 840 expand_MASK_LOAD (gimple stmt)
 841 {
 842   struct expand_operand ops[3];
 843   tree type, lhs, rhs, maskt;
 844   rtx mem, target, mask;
 845
 846   maskt = gimple_call_arg (stmt, 2);
 847   lhs = gimple_call_lhs (stmt);
 848   if (lhs == NULL_TREE)
 849     return;
 850   type = TREE_TYPE (lhs);
 851   rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
 852                      gimple_call_arg (stmt, 1));
 853
 854   mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 855   gcc_assert (MEM_P (mem));
 856   mask = expand_normal (maskt);
 857   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 858   create_output_operand (&ops[0], target, TYPE_MODE (type));
 859   create_fixed_operand (&ops[1], mem);
 860   create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
 861   expand_insn (optab_handler (maskload_optab, TYPE_MODE (type)), 3, ops);
 862 }
 863
 864 static void
 865 expand_MASK_STORE (gimple stmt)
 866 {
 867   struct expand_operand ops[3];
 868   tree type, lhs, rhs, maskt;
 869   rtx mem, reg, mask;
 870
 871   maskt = gimple_call_arg (stmt, 2);
 872   rhs = gimple_call_arg (stmt, 3);
 873   type = TREE_TYPE (rhs);
 874   lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
 875                      gimple_call_arg (stmt, 1));
 876
 877   mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 878   gcc_assert (MEM_P (mem));
 879   mask = expand_normal (maskt);
 880   reg = expand_normal (rhs);
 881   create_fixed_operand (&ops[0], mem);
 882   create_input_operand (&ops[1], reg, TYPE_MODE (type));
 883   create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
 884   expand_insn (optab_handler (maskstore_optab, TYPE_MODE (type)), 3, ops);
 885 }
 886
 887 static void
 888 expand_ABNORMAL_DISPATCHER (gimple)
 889 {
 890 }
 891
 892 static void
 893 expand_BUILTIN_EXPECT (gimple stmt)
 894 {
 895   /* When guessing was done, the hints should be already stripped away.  */
 896   gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
 897
 898   rtx target;
 899   tree lhs = gimple_call_lhs (stmt);
 900   if (lhs)
 901     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
 902   else
 903     target = const0_rtx;
 904   rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
 905   if (lhs && val != target)
 906     emit_move_insn (target, val);
 907 }
 908
 909 /* Routines to expand each internal function, indexed by function number.
 910    Each routine has the prototype:
 911
 912        expand_<NAME> (gimple stmt)
 913
 914    where STMT is the statement that performs the call. */
 915 static void (*const internal_fn_expanders[]) (gimple) = {
 916 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
 917 #include "internal-fn.def"
 918 #undef DEF_INTERNAL_FN
 919   0
 920 };
 921
 922 /* Expand STMT, which is a call to internal function FN.  */
 923
 924 void
 925 expand_internal_call (gimple stmt)
 926 {
 927   internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt);
 928 }