gcc/tree-vectorizer.c

   1 /* Vectorizer
   2    Copyright (C) 2003-2018 Free Software Foundation, Inc.
   3    Contributed by Dorit Naishlos <dorit@il.ibm.com>
   4
   5 This file is part of GCC.
   6
   7 GCC is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 3, or (at your option) any later
  10 version.
  11
  12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with GCC; see the file COPYING3.  If not see
  19 <http://www.gnu.org/licenses/>.  */
  20
  21 /* Loop and basic block vectorizer.
  22
  23   This file contains drivers for the three vectorizers:
  24   (1) loop vectorizer (inter-iteration parallelism),
  25   (2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop
  26       vectorizer)
  27   (3) BB vectorizer (out-of-loops), aka SLP
  28
  29   The rest of the vectorizer's code is organized as follows:
  30   - tree-vect-loop.c - loop specific parts such as reductions, etc. These are
  31     used by drivers (1) and (2).
  32   - tree-vect-loop-manip.c - vectorizer's loop control-flow utilities, used by
  33     drivers (1) and (2).
  34   - tree-vect-slp.c - BB vectorization specific analysis and transformation,
  35     used by drivers (2) and (3).
  36   - tree-vect-stmts.c - statements analysis and transformation (used by all).
  37   - tree-vect-data-refs.c - vectorizer specific data-refs analysis and
  38     manipulations (used by all).
  39   - tree-vect-patterns.c - vectorizable code patterns detector (used by all)
  40
  41   Here's a poor attempt at illustrating that:
  42
  43      tree-vectorizer.c:
  44      loop_vect()  loop_aware_slp()  slp_vect()
  45           |        /           \          /
  46           |       /             \        /
  47           tree-vect-loop.c  tree-vect-slp.c
  48                 | \      \  /      /   |
  49                 |  \      \/      /    |
  50                 |   \     /\     /     |
  51                 |    \   /  \   /      |
  52          tree-vect-stmts.c  tree-vect-data-refs.c
  53                        \      /
  54                     tree-vect-patterns.c
  55 */
  56
  57 #include "config.h"
  58 #include "system.h"
  59 #include "coretypes.h"
  60 #include "backend.h"
  61 #include "tree.h"
  62 #include "gimple.h"
  63 #include "predict.h"
  64 #include "tree-pass.h"
  65 #include "ssa.h"
  66 #include "cgraph.h"
  67 #include "fold-const.h"
  68 #include "stor-layout.h"
  69 #include "gimple-iterator.h"
  70 #include "gimple-walk.h"
  71 #include "tree-ssa-loop-manip.h"
  72 #include "tree-ssa-loop-niter.h"
  73 #include "tree-cfg.h"
  74 #include "cfgloop.h"
  75 #include "tree-vectorizer.h"
  76 #include "tree-ssa-propagate.h"
  77 #include "dbgcnt.h"
  78 #include "tree-scalar-evolution.h"
  79 #include "stringpool.h"
  80 #include "attribs.h"
  81 #include "gimple-pretty-print.h"
  82
  83
  84 /* Loop or bb location, with hotness information.  */
  85 dump_user_location_t vect_location;
  86
  87 /* Vector mapping GIMPLE stmt to stmt_vec_info. */
  88 vec<stmt_vec_info> *stmt_vec_info_vec;
  89
  90 /* Dump a cost entry according to args to F.  */
  91
  92 void
  93 dump_stmt_cost (FILE *f, void *data, int count, enum vect_cost_for_stmt kind,
  94                 stmt_vec_info stmt_info, int misalign,
  95                 enum vect_cost_model_location where)
  96 {
  97   fprintf (f, "%p ", data);
  98   if (stmt_info)
  99     {
 100       print_gimple_expr (f, STMT_VINFO_STMT (stmt_info), 0, TDF_SLIM);
 101       fprintf (f, " ");
 102     }
 103   else
 104     fprintf (f, "<unknown> ");
 105   fprintf (f, "%d times ", count);
 106   const char *ks = "unknown";
 107   switch (kind)
 108     {
 109     case scalar_stmt:
 110       ks = "scalar_stmt";
 111       break;
 112     case scalar_load:
 113       ks = "scalar_load";
 114       break;
 115     case scalar_store:
 116       ks = "scalar_store";
 117       break;
 118     case vector_stmt:
 119       ks = "vector_stmt";
 120       break;
 121     case vector_load:
 122       ks = "vector_load";
 123       break;
 124     case vector_gather_load:
 125       ks = "vector_gather_load";
 126       break;
 127     case unaligned_load:
 128       ks = "unaligned_load";
 129       break;
 130     case unaligned_store:
 131       ks = "unaligned_store";
 132       break;
 133     case vector_store:
 134       ks = "unaligned_store";
 135       break;
 136     case vector_scatter_store:
 137       ks = "unaligned_store";
 138       break;
 139     case vec_to_scalar:
 140       ks = "unaligned_store";
 141       break;
 142     case scalar_to_vec:
 143       ks = "unaligned_store";
 144       break;
 145     case cond_branch_not_taken:
 146       ks = "unaligned_store";
 147       break;
 148     case cond_branch_taken:
 149       ks = "unaligned_store";
 150       break;
 151     case vec_perm:
 152       ks = "unaligned_store";
 153       break;
 154     case vec_promote_demote:
 155       ks = "unaligned_store";
 156       break;
 157     case vec_construct:
 158       ks = "unaligned_store";
 159       break;
 160     }
 161   fprintf (f, "%s ", ks);
 162   if (kind == unaligned_load || kind == unaligned_store)
 163     fprintf (f, "(misalign %d) ", misalign);
 164   const char *ws = "unknown";
 165   switch (where)
 166     {
 167     case vect_prologue:
 168       ws = "prologue";
 169       break;
 170     case vect_body:
 171       ws = "body";
 172       break;
 173     case vect_epilogue:
 174       ws = "epilogue";
 175       break;
 176     }
 177   fprintf (f, "in %s\n", ws);
 178 }
 179 \f
 180 /* For mapping simduid to vectorization factor.  */
 181
 182 struct simduid_to_vf : free_ptr_hash<simduid_to_vf>
 183 {
 184   unsigned int simduid;
 185   poly_uint64 vf;
 186
 187   /* hash_table support.  */
 188   static inline hashval_t hash (const simduid_to_vf *);
 189   static inline int equal (const simduid_to_vf *, const simduid_to_vf *);
 190 };
 191
 192 inline hashval_t
 193 simduid_to_vf::hash (const simduid_to_vf *p)
 194 {
 195   return p->simduid;
 196 }
 197
 198 inline int
 199 simduid_to_vf::equal (const simduid_to_vf *p1, const simduid_to_vf *p2)
 200 {
 201   return p1->simduid == p2->simduid;
 202 }
 203
 204 /* This hash maps the OMP simd array to the corresponding simduid used
 205    to index into it.  Like thus,
 206
 207         _7 = GOMP_SIMD_LANE (simduid.0)
 208         ...
 209         ...
 210         D.1737[_7] = stuff;
 211
 212
 213    This hash maps from the OMP simd array (D.1737[]) to DECL_UID of
 214    simduid.0.  */
 215
 216 struct simd_array_to_simduid : free_ptr_hash<simd_array_to_simduid>
 217 {
 218   tree decl;
 219   unsigned int simduid;
 220
 221   /* hash_table support.  */
 222   static inline hashval_t hash (const simd_array_to_simduid *);
 223   static inline int equal (const simd_array_to_simduid *,
 224                            const simd_array_to_simduid *);
 225 };
 226
 227 inline hashval_t
 228 simd_array_to_simduid::hash (const simd_array_to_simduid *p)
 229 {
 230   return DECL_UID (p->decl);
 231 }
 232
 233 inline int
 234 simd_array_to_simduid::equal (const simd_array_to_simduid *p1,
 235                               const simd_array_to_simduid *p2)
 236 {
 237   return p1->decl == p2->decl;
 238 }
 239
 240 /* Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF, IFN_GOMP_SIMD_LAST_LANE,
 241    into their corresponding constants and remove
 242    IFN_GOMP_SIMD_ORDERED_{START,END}.  */
 243
 244 static void
 245 adjust_simduid_builtins (hash_table<simduid_to_vf> *htab)
 246 {
 247   basic_block bb;
 248
 249   FOR_EACH_BB_FN (bb, cfun)
 250     {
 251       gimple_stmt_iterator i;
 252
 253       for (i = gsi_start_bb (bb); !gsi_end_p (i); )
 254         {
 255           poly_uint64 vf = 1;
 256           enum internal_fn ifn;
 257           gimple *stmt = gsi_stmt (i);
 258           tree t;
 259           if (!is_gimple_call (stmt)
 260               || !gimple_call_internal_p (stmt))
 261             {
 262               gsi_next (&i);
 263               continue;
 264             }
 265           ifn = gimple_call_internal_fn (stmt);
 266           switch (ifn)
 267             {
 268             case IFN_GOMP_SIMD_LANE:
 269             case IFN_GOMP_SIMD_VF:
 270             case IFN_GOMP_SIMD_LAST_LANE:
 271               break;
 272             case IFN_GOMP_SIMD_ORDERED_START:
 273             case IFN_GOMP_SIMD_ORDERED_END:
 274               if (integer_onep (gimple_call_arg (stmt, 0)))
 275                 {
 276                   enum built_in_function bcode
 277                     = (ifn == IFN_GOMP_SIMD_ORDERED_START
 278                        ? BUILT_IN_GOMP_ORDERED_START
 279                        : BUILT_IN_GOMP_ORDERED_END);
 280                   gimple *g
 281                     = gimple_build_call (builtin_decl_explicit (bcode), 0);
 282                   tree vdef = gimple_vdef (stmt);
 283                   gimple_set_vdef (g, vdef);
 284                   SSA_NAME_DEF_STMT (vdef) = g;
 285                   gimple_set_vuse (g, gimple_vuse (stmt));
 286                   gsi_replace (&i, g, true);
 287                   continue;
 288                 }
 289               gsi_remove (&i, true);
 290               unlink_stmt_vdef (stmt);
 291               continue;
 292             default:
 293               gsi_next (&i);
 294               continue;
 295             }
 296           tree arg = gimple_call_arg (stmt, 0);
 297           gcc_assert (arg != NULL_TREE);
 298           gcc_assert (TREE_CODE (arg) == SSA_NAME);
 299           simduid_to_vf *p = NULL, data;
 300           data.simduid = DECL_UID (SSA_NAME_VAR (arg));
 301           /* Need to nullify loop safelen field since it's value is not
 302              valid after transformation.  */
 303           if (bb->loop_father && bb->loop_father->safelen > 0)
 304             bb->loop_father->safelen = 0;
 305           if (htab)
 306             {
 307               p = htab->find (&data);
 308               if (p)
 309                 vf = p->vf;
 310             }
 311           switch (ifn)
 312             {
 313             case IFN_GOMP_SIMD_VF:
 314               t = build_int_cst (unsigned_type_node, vf);
 315               break;
 316             case IFN_GOMP_SIMD_LANE:
 317               t = build_int_cst (unsigned_type_node, 0);
 318               break;
 319             case IFN_GOMP_SIMD_LAST_LANE:
 320               t = gimple_call_arg (stmt, 1);
 321               break;
 322             default:
 323               gcc_unreachable ();
 324             }
 325           tree lhs = gimple_call_lhs (stmt);
 326           if (lhs)
 327             replace_uses_by (lhs, t);
 328           release_defs (stmt);
 329           gsi_remove (&i, true);
 330         }
 331     }
 332 }
 333
 334 /* Helper structure for note_simd_array_uses.  */
 335
 336 struct note_simd_array_uses_struct
 337 {
 338   hash_table<simd_array_to_simduid> **htab;
 339   unsigned int simduid;
 340 };
 341
 342 /* Callback for note_simd_array_uses, called through walk_gimple_op.  */
 343
 344 static tree
 345 note_simd_array_uses_cb (tree *tp, int *walk_subtrees, void *data)
 346 {
 347   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
 348   struct note_simd_array_uses_struct *ns
 349     = (struct note_simd_array_uses_struct *) wi->info;
 350
 351   if (TYPE_P (*tp))
 352     *walk_subtrees = 0;
 353   else if (VAR_P (*tp)
 354            && lookup_attribute ("omp simd array", DECL_ATTRIBUTES (*tp))
 355            && DECL_CONTEXT (*tp) == current_function_decl)
 356     {
 357       simd_array_to_simduid data;
 358       if (!*ns->htab)
 359         *ns->htab = new hash_table<simd_array_to_simduid> (15);
 360       data.decl = *tp;
 361       data.simduid = ns->simduid;
 362       simd_array_to_simduid **slot = (*ns->htab)->find_slot (&data, INSERT);
 363       if (*slot == NULL)
 364         {
 365           simd_array_to_simduid *p = XNEW (simd_array_to_simduid);
 366           *p = data;
 367           *slot = p;
 368         }
 369       else if ((*slot)->simduid != ns->simduid)
 370         (*slot)->simduid = -1U;
 371       *walk_subtrees = 0;
 372     }
 373   return NULL_TREE;
 374 }
 375
 376 /* Find "omp simd array" temporaries and map them to corresponding
 377    simduid.  */
 378
 379 static void
 380 note_simd_array_uses (hash_table<simd_array_to_simduid> **htab)
 381 {
 382   basic_block bb;
 383   gimple_stmt_iterator gsi;
 384   struct walk_stmt_info wi;
 385   struct note_simd_array_uses_struct ns;
 386
 387   memset (&wi, 0, sizeof (wi));
 388   wi.info = &ns;
 389   ns.htab = htab;
 390
 391   FOR_EACH_BB_FN (bb, cfun)
 392     for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 393       {
 394         gimple *stmt = gsi_stmt (gsi);
 395         if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
 396           continue;
 397         switch (gimple_call_internal_fn (stmt))
 398           {
 399           case IFN_GOMP_SIMD_LANE:
 400           case IFN_GOMP_SIMD_VF:
 401           case IFN_GOMP_SIMD_LAST_LANE:
 402             break;
 403           default:
 404             continue;
 405           }
 406         tree lhs = gimple_call_lhs (stmt);
 407         if (lhs == NULL_TREE)
 408           continue;
 409         imm_use_iterator use_iter;
 410         gimple *use_stmt;
 411         ns.simduid = DECL_UID (SSA_NAME_VAR (gimple_call_arg (stmt, 0)));
 412         FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, lhs)
 413           if (!is_gimple_debug (use_stmt))
 414             walk_gimple_op (use_stmt, note_simd_array_uses_cb, &wi);
 415       }
 416 }
 417
 418 /* Shrink arrays with "omp simd array" attribute to the corresponding
 419    vectorization factor.  */
 420
 421 static void
 422 shrink_simd_arrays
 423   (hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab,
 424    hash_table<simduid_to_vf> *simduid_to_vf_htab)
 425 {
 426   for (hash_table<simd_array_to_simduid>::iterator iter
 427          = simd_array_to_simduid_htab->begin ();
 428        iter != simd_array_to_simduid_htab->end (); ++iter)
 429     if ((*iter)->simduid != -1U)
 430       {
 431         tree decl = (*iter)->decl;
 432         poly_uint64 vf = 1;
 433         if (simduid_to_vf_htab)
 434           {
 435             simduid_to_vf *p = NULL, data;
 436             data.simduid = (*iter)->simduid;
 437             p = simduid_to_vf_htab->find (&data);
 438             if (p)
 439               vf = p->vf;
 440           }
 441         tree atype
 442           = build_array_type_nelts (TREE_TYPE (TREE_TYPE (decl)), vf);
 443         TREE_TYPE (decl) = atype;
 444         relayout_decl (decl);
 445       }
 446
 447   delete simd_array_to_simduid_htab;
 448 }
 449 \f
 450 /* Initialize the vec_info with kind KIND_IN and target cost data
 451    TARGET_COST_DATA_IN.  */
 452
 453 vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in,
 454                     vec_info_shared *shared_)
 455   : kind (kind_in),
 456     shared (shared_),
 457     target_cost_data (target_cost_data_in)
 458 {
 459   stmt_vec_infos.create (50);
 460   set_stmt_vec_info_vec (&stmt_vec_infos);
 461 }
 462
 463 vec_info::~vec_info ()
 464 {
 465   slp_instance instance;
 466   unsigned int i;
 467
 468   FOR_EACH_VEC_ELT (slp_instances, i, instance)
 469     vect_free_slp_instance (instance, true);
 470
 471   destroy_cost_data (target_cost_data);
 472   free_stmt_vec_infos (&stmt_vec_infos);
 473 }
 474
 475 vec_info_shared::vec_info_shared ()
 476   : datarefs (vNULL),
 477     datarefs_copy (vNULL),
 478     ddrs (vNULL)
 479 {
 480 }
 481
 482 vec_info_shared::~vec_info_shared ()
 483 {
 484   free_data_refs (datarefs);
 485   free_dependence_relations (ddrs);
 486   datarefs_copy.release ();
 487 }
 488
 489 void
 490 vec_info_shared::save_datarefs ()
 491 {
 492   if (!flag_checking)
 493     return;
 494   datarefs_copy.reserve_exact (datarefs.length ());
 495   for (unsigned i = 0; i < datarefs.length (); ++i)
 496     datarefs_copy.quick_push (*datarefs[i]);
 497 }
 498
 499 void
 500 vec_info_shared::check_datarefs ()
 501 {
 502   if (!flag_checking)
 503     return;
 504   gcc_assert (datarefs.length () == datarefs_copy.length ());
 505   for (unsigned i = 0; i < datarefs.length (); ++i)
 506     if (memcmp (&datarefs_copy[i], datarefs[i], sizeof (data_reference)) != 0)
 507       gcc_unreachable ();
 508 }
 509
 510 /* Record that STMT belongs to the vectorizable region.  Create and return
 511    an associated stmt_vec_info.  */
 512
 513 stmt_vec_info
 514 vec_info::add_stmt (gimple *stmt)
 515 {
 516   stmt_vec_info res = new_stmt_vec_info (stmt, this);
 517   set_vinfo_for_stmt (stmt, res);
 518   return res;
 519 }
 520
 521 /* If STMT has an associated stmt_vec_info, return that vec_info, otherwise
 522    return null.  It is safe to call this function on any statement, even if
 523    it might not be part of the vectorizable region.  */
 524
 525 stmt_vec_info
 526 vec_info::lookup_stmt (gimple *stmt)
 527 {
 528   unsigned int uid = gimple_uid (stmt);
 529   if (uid > 0 && uid - 1 < stmt_vec_infos.length ())
 530     {
 531       stmt_vec_info res = stmt_vec_infos[uid - 1];
 532       if (res && res->stmt == stmt)
 533         return res;
 534     }
 535   return NULL;
 536 }
 537
 538 /* If NAME is an SSA_NAME and its definition has an associated stmt_vec_info,
 539    return that stmt_vec_info, otherwise return null.  It is safe to call
 540    this on arbitrary operands.  */
 541
 542 stmt_vec_info
 543 vec_info::lookup_def (tree name)
 544 {
 545   if (TREE_CODE (name) == SSA_NAME
 546       && !SSA_NAME_IS_DEFAULT_DEF (name))
 547     return lookup_stmt (SSA_NAME_DEF_STMT (name));
 548   return NULL;
 549 }
 550
 551 /* See whether there is a single non-debug statement that uses LHS and
 552    whether that statement has an associated stmt_vec_info.  Return the
 553    stmt_vec_info if so, otherwise return null.  */
 554
 555 stmt_vec_info
 556 vec_info::lookup_single_use (tree lhs)
 557 {
 558   use_operand_p dummy;
 559   gimple *use_stmt;
 560   if (single_imm_use (lhs, &dummy, &use_stmt))
 561     return lookup_stmt (use_stmt);
 562   return NULL;
 563 }
 564
 565 /* Record that NEW_STMT_INFO now implements the same data reference
 566    as OLD_STMT_INFO.  */
 567
 568 void
 569 vec_info::move_dr (stmt_vec_info new_stmt_info, stmt_vec_info old_stmt_info)
 570 {
 571   gcc_assert (!is_pattern_stmt_p (old_stmt_info));
 572   STMT_VINFO_DR_INFO (old_stmt_info)->stmt = new_stmt_info;
 573   new_stmt_info->dr_aux = old_stmt_info->dr_aux;
 574   STMT_VINFO_DR_WRT_VEC_LOOP (new_stmt_info)
 575     = STMT_VINFO_DR_WRT_VEC_LOOP (old_stmt_info);
 576   STMT_VINFO_GATHER_SCATTER_P (new_stmt_info)
 577     = STMT_VINFO_GATHER_SCATTER_P (old_stmt_info);
 578 }
 579
 580 /* A helper function to free scev and LOOP niter information, as well as
 581    clear loop constraint LOOP_C_FINITE.  */
 582
 583 void
 584 vect_free_loop_info_assumptions (struct loop *loop)
 585 {
 586   scev_reset_htab ();
 587   /* We need to explicitly reset upper bound information since they are
 588      used even after free_numbers_of_iterations_estimates.  */
 589   loop->any_upper_bound = false;
 590   loop->any_likely_upper_bound = false;
 591   free_numbers_of_iterations_estimates (loop);
 592   loop_constraint_clear (loop, LOOP_C_FINITE);
 593 }
 594
 595 /* Return whether STMT is inside the region we try to vectorize.  */
 596
 597 bool
 598 vect_stmt_in_region_p (vec_info *vinfo, gimple *stmt)
 599 {
 600   if (!gimple_bb (stmt))
 601     return false;
 602
 603   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
 604     {
 605       struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
 606       if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
 607         return false;
 608     }
 609   else
 610     {
 611       bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
 612       if (gimple_bb (stmt) != BB_VINFO_BB (bb_vinfo)
 613           || gimple_uid (stmt) == -1U
 614           || gimple_code (stmt) == GIMPLE_PHI)
 615         return false;
 616     }
 617
 618   return true;
 619 }
 620
 621
 622 /* If LOOP has been versioned during ifcvt, return the internal call
 623    guarding it.  */
 624
 625 static gimple *
 626 vect_loop_vectorized_call (struct loop *loop)
 627 {
 628   basic_block bb = loop_preheader_edge (loop)->src;
 629   gimple *g;
 630   do
 631     {
 632       g = last_stmt (bb);
 633       if (g)
 634         break;
 635       if (!single_pred_p (bb))
 636         break;
 637       bb = single_pred (bb);
 638     }
 639   while (1);
 640   if (g && gimple_code (g) == GIMPLE_COND)
 641     {
 642       gimple_stmt_iterator gsi = gsi_for_stmt (g);
 643       gsi_prev (&gsi);
 644       if (!gsi_end_p (gsi))
 645         {
 646           g = gsi_stmt (gsi);
 647           if (gimple_call_internal_p (g, IFN_LOOP_VECTORIZED)
 648               && (tree_to_shwi (gimple_call_arg (g, 0)) == loop->num
 649                   || tree_to_shwi (gimple_call_arg (g, 1)) == loop->num))
 650             return g;
 651         }
 652     }
 653   return NULL;
 654 }
 655
 656 /* If LOOP has been versioned during loop distribution, return the gurading
 657    internal call.  */
 658
 659 static gimple *
 660 vect_loop_dist_alias_call (struct loop *loop)
 661 {
 662   basic_block bb;
 663   basic_block entry;
 664   struct loop *outer, *orig;
 665   gimple_stmt_iterator gsi;
 666   gimple *g;
 667
 668   if (loop->orig_loop_num == 0)
 669     return NULL;
 670
 671   orig = get_loop (cfun, loop->orig_loop_num);
 672   if (orig == NULL)
 673     {
 674       /* The original loop is somehow destroyed.  Clear the information.  */
 675       loop->orig_loop_num = 0;
 676       return NULL;
 677     }
 678
 679   if (loop != orig)
 680     bb = nearest_common_dominator (CDI_DOMINATORS, loop->header, orig->header);
 681   else
 682     bb = loop_preheader_edge (loop)->src;
 683
 684   outer = bb->loop_father;
 685   entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 686
 687   /* Look upward in dominance tree.  */
 688   for (; bb != entry && flow_bb_inside_loop_p (outer, bb);
 689        bb = get_immediate_dominator (CDI_DOMINATORS, bb))
 690     {
 691       g = last_stmt (bb);
 692       if (g == NULL || gimple_code (g) != GIMPLE_COND)
 693         continue;
 694
 695       gsi = gsi_for_stmt (g);
 696       gsi_prev (&gsi);
 697       if (gsi_end_p (gsi))
 698         continue;
 699
 700       g = gsi_stmt (gsi);
 701       /* The guarding internal function call must have the same distribution
 702          alias id.  */
 703       if (gimple_call_internal_p (g, IFN_LOOP_DIST_ALIAS)
 704           && (tree_to_shwi (gimple_call_arg (g, 0)) == loop->orig_loop_num))
 705         return g;
 706     }
 707   return NULL;
 708 }
 709
 710 /* Set the uids of all the statements in basic blocks inside loop
 711    represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
 712    call guarding the loop which has been if converted.  */
 713 static void
 714 set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
 715 {
 716   tree arg = gimple_call_arg (loop_vectorized_call, 1);
 717   basic_block *bbs;
 718   unsigned int i;
 719   struct loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
 720
 721   LOOP_VINFO_SCALAR_LOOP (loop_vinfo) = scalar_loop;
 722   gcc_checking_assert (vect_loop_vectorized_call (scalar_loop)
 723                        == loop_vectorized_call);
 724   /* If we are going to vectorize outer loop, prevent vectorization
 725      of the inner loop in the scalar loop - either the scalar loop is
 726      thrown away, so it is a wasted work, or is used only for
 727      a few iterations.  */
 728   if (scalar_loop->inner)
 729     {
 730       gimple *g = vect_loop_vectorized_call (scalar_loop->inner);
 731       if (g)
 732         {
 733           arg = gimple_call_arg (g, 0);
 734           get_loop (cfun, tree_to_shwi (arg))->dont_vectorize = true;
 735           fold_loop_internal_call (g, boolean_false_node);
 736         }
 737     }
 738   bbs = get_loop_body (scalar_loop);
 739   for (i = 0; i < scalar_loop->num_nodes; i++)
 740     {
 741       basic_block bb = bbs[i];
 742       gimple_stmt_iterator gsi;
 743       for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 744         {
 745           gimple *phi = gsi_stmt (gsi);
 746           gimple_set_uid (phi, 0);
 747         }
 748       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 749         {
 750           gimple *stmt = gsi_stmt (gsi);
 751           gimple_set_uid (stmt, 0);
 752         }
 753     }
 754   free (bbs);
 755 }
 756
 757 /* Try to vectorize LOOP.  */
 758
 759 static unsigned
 760 try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 761                       unsigned *num_vectorized_loops,
 762                       loop_p loop, loop_vec_info orig_loop_vinfo,
 763                       gimple *loop_vectorized_call,
 764                       gimple *loop_dist_alias_call)
 765 {
 766   unsigned ret = 0;
 767   vec_info_shared shared;
 768   vect_location = find_loop_location (loop);
 769   if (LOCATION_LOCUS (vect_location.get_location_t ()) != UNKNOWN_LOCATION
 770       && dump_enabled_p ())
 771     dump_printf (MSG_NOTE, "\nAnalyzing loop at %s:%d\n",
 772                  LOCATION_FILE (vect_location.get_location_t ()),
 773                  LOCATION_LINE (vect_location.get_location_t ()));
 774
 775   loop_vec_info loop_vinfo = vect_analyze_loop (loop, orig_loop_vinfo, &shared);
 776   loop->aux = loop_vinfo;
 777
 778   if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
 779     {
 780       /* Free existing information if loop is analyzed with some
 781          assumptions.  */
 782       if (loop_constraint_set_p (loop, LOOP_C_FINITE))
 783         vect_free_loop_info_assumptions (loop);
 784
 785       /* If we applied if-conversion then try to vectorize the
 786          BB of innermost loops.
 787          ???  Ideally BB vectorization would learn to vectorize
 788          control flow by applying if-conversion on-the-fly, the
 789          following retains the if-converted loop body even when
 790          only non-if-converted parts took part in BB vectorization.  */
 791       if (flag_tree_slp_vectorize != 0
 792           && loop_vectorized_call
 793           && ! loop->inner)
 794         {
 795           basic_block bb = loop->header;
 796           bool has_mask_load_store = false;
 797           for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
 798                !gsi_end_p (gsi); gsi_next (&gsi))
 799             {
 800               gimple *stmt = gsi_stmt (gsi);
 801               if (is_gimple_call (stmt)
 802                   && gimple_call_internal_p (stmt)
 803                   && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD
 804                       || gimple_call_internal_fn (stmt) == IFN_MASK_STORE))
 805                 {
 806                   has_mask_load_store = true;
 807                   break;
 808                 }
 809               gimple_set_uid (stmt, -1);
 810               gimple_set_visited (stmt, false);
 811             }
 812           if (! has_mask_load_store && vect_slp_bb (bb))
 813             {
 814               dump_printf_loc (MSG_NOTE, vect_location,
 815                                "basic block vectorized\n");
 816               fold_loop_internal_call (loop_vectorized_call,
 817                                        boolean_true_node);
 818               loop_vectorized_call = NULL;
 819               ret |= TODO_cleanup_cfg;
 820             }
 821         }
 822       /* If outer loop vectorization fails for LOOP_VECTORIZED guarded
 823          loop, don't vectorize its inner loop; we'll attempt to
 824          vectorize LOOP_VECTORIZED guarded inner loop of the scalar
 825          loop version.  */
 826       if (loop_vectorized_call && loop->inner)
 827         loop->inner->dont_vectorize = true;
 828       return ret;
 829     }
 830
 831   if (!dbg_cnt (vect_loop))
 832     {
 833       /* Free existing information if loop is analyzed with some
 834          assumptions.  */
 835       if (loop_constraint_set_p (loop, LOOP_C_FINITE))
 836         vect_free_loop_info_assumptions (loop);
 837       return ret;
 838     }
 839
 840   if (loop_vectorized_call)
 841     set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
 842
 843   unsigned HOST_WIDE_INT bytes;
 844   if (current_vector_size.is_constant (&bytes))
 845     dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
 846                      "loop vectorized vectorized using "
 847                      HOST_WIDE_INT_PRINT_UNSIGNED " byte "
 848                      "vectors\n", bytes);
 849   else
 850     dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
 851                      "loop vectorized using variable length vectors\n");
 852
 853   loop_p new_loop = vect_transform_loop (loop_vinfo);
 854   (*num_vectorized_loops)++;
 855   /* Now that the loop has been vectorized, allow it to be unrolled
 856      etc.  */
 857   loop->force_vectorize = false;
 858
 859   if (loop->simduid)
 860     {
 861       simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
 862       if (!simduid_to_vf_htab)
 863         simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
 864       simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
 865       simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
 866       *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
 867           = simduid_to_vf_data;
 868     }
 869
 870   if (loop_vectorized_call)
 871     {
 872       fold_loop_internal_call (loop_vectorized_call, boolean_true_node);
 873       loop_vectorized_call = NULL;
 874       ret |= TODO_cleanup_cfg;
 875     }
 876   if (loop_dist_alias_call)
 877     {
 878       tree value = gimple_call_arg (loop_dist_alias_call, 1);
 879       fold_loop_internal_call (loop_dist_alias_call, value);
 880       loop_dist_alias_call = NULL;
 881       ret |= TODO_cleanup_cfg;
 882     }
 883
 884   /* Epilogue of vectorized loop must be vectorized too.  */
 885   if (new_loop)
 886     ret |= try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
 887                                  new_loop, loop_vinfo, NULL, NULL);
 888
 889   return ret;
 890 }
 891
 892 /* Try to vectorize LOOP.  */
 893
 894 static unsigned
 895 try_vectorize_loop (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 896                     unsigned *num_vectorized_loops, loop_p loop)
 897 {
 898   if (!((flag_tree_loop_vectorize
 899          && optimize_loop_nest_for_speed_p (loop))
 900         || loop->force_vectorize))
 901     return 0;
 902
 903   return try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
 904                                loop, NULL,
 905                                vect_loop_vectorized_call (loop),
 906                                vect_loop_dist_alias_call (loop));
 907 }
 908
 909
 910 /* Function vectorize_loops.
 911
 912    Entry point to loop vectorization phase.  */
 913
 914 unsigned
 915 vectorize_loops (void)
 916 {
 917   unsigned int i;
 918   unsigned int num_vectorized_loops = 0;
 919   unsigned int vect_loops_num;
 920   struct loop *loop;
 921   hash_table<simduid_to_vf> *simduid_to_vf_htab = NULL;
 922   hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab = NULL;
 923   bool any_ifcvt_loops = false;
 924   unsigned ret = 0;
 925
 926   vect_loops_num = number_of_loops (cfun);
 927
 928   /* Bail out if there are no loops.  */
 929   if (vect_loops_num <= 1)
 930     return 0;
 931
 932   if (cfun->has_simduid_loops)
 933     note_simd_array_uses (&simd_array_to_simduid_htab);
 934
 935   set_stmt_vec_info_vec (NULL);
 936
 937   /*  ----------- Analyze loops. -----------  */
 938
 939   /* If some loop was duplicated, it gets bigger number
 940      than all previously defined loops.  This fact allows us to run
 941      only over initial loops skipping newly generated ones.  */
 942   FOR_EACH_LOOP (loop, 0)
 943     if (loop->dont_vectorize)
 944       {
 945         any_ifcvt_loops = true;
 946         /* If-conversion sometimes versions both the outer loop
 947            (for the case when outer loop vectorization might be
 948            desirable) as well as the inner loop in the scalar version
 949            of the loop.  So we have:
 950             if (LOOP_VECTORIZED (1, 3))
 951               {
 952                 loop1
 953                   loop2
 954               }
 955             else
 956               loop3 (copy of loop1)
 957                 if (LOOP_VECTORIZED (4, 5))
 958                   loop4 (copy of loop2)
 959                 else
 960                   loop5 (copy of loop4)
 961            If FOR_EACH_LOOP gives us loop3 first (which has
 962            dont_vectorize set), make sure to process loop1 before loop4;
 963            so that we can prevent vectorization of loop4 if loop1
 964            is successfully vectorized.  */
 965         if (loop->inner)
 966           {
 967             gimple *loop_vectorized_call
 968               = vect_loop_vectorized_call (loop);
 969             if (loop_vectorized_call
 970                 && vect_loop_vectorized_call (loop->inner))
 971               {
 972                 tree arg = gimple_call_arg (loop_vectorized_call, 0);
 973                 struct loop *vector_loop
 974                   = get_loop (cfun, tree_to_shwi (arg));
 975                 if (vector_loop && vector_loop != loop)
 976                   {
 977                     /* Make sure we don't vectorize it twice.  */
 978                     vector_loop->dont_vectorize = true;
 979                     ret |= try_vectorize_loop (simduid_to_vf_htab,
 980                                                &num_vectorized_loops,
 981                                                vector_loop);
 982                   }
 983               }
 984           }
 985       }
 986     else
 987       ret |= try_vectorize_loop (simduid_to_vf_htab, &num_vectorized_loops,
 988                                  loop);
 989
 990   vect_location = dump_user_location_t ();
 991
 992   statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops);
 993   if (dump_enabled_p ()
 994       || (num_vectorized_loops > 0 && dump_enabled_p ()))
 995     dump_printf_loc (MSG_NOTE, vect_location,
 996                      "vectorized %u loops in function.\n",
 997                      num_vectorized_loops);
 998
 999   /*  ----------- Finalize. -----------  */
1000
1001   if (any_ifcvt_loops)
1002     for (i = 1; i < number_of_loops (cfun); i++)
1003       {
1004         loop = get_loop (cfun, i);
1005         if (loop && loop->dont_vectorize)
1006           {
1007             gimple *g = vect_loop_vectorized_call (loop);
1008             if (g)
1009               {
1010                 fold_loop_internal_call (g, boolean_false_node);
1011                 ret |= TODO_cleanup_cfg;
1012                 g = NULL;
1013               }
1014             else
1015               g = vect_loop_dist_alias_call (loop);
1016
1017             if (g)
1018               {
1019                 fold_loop_internal_call (g, boolean_false_node);
1020                 ret |= TODO_cleanup_cfg;
1021               }
1022           }
1023       }
1024
1025   for (i = 1; i < number_of_loops (cfun); i++)
1026     {
1027       loop_vec_info loop_vinfo;
1028       bool has_mask_store;
1029
1030       loop = get_loop (cfun, i);
1031       if (!loop || !loop->aux)
1032         continue;
1033       loop_vinfo = (loop_vec_info) loop->aux;
1034       has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo);
1035       delete loop_vinfo;
1036       if (has_mask_store
1037           && targetm.vectorize.empty_mask_is_expensive (IFN_MASK_STORE))
1038         optimize_mask_stores (loop);
1039       loop->aux = NULL;
1040     }
1041
1042   /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins.  */
1043   if (cfun->has_simduid_loops)
1044     adjust_simduid_builtins (simduid_to_vf_htab);
1045
1046   /* Shrink any "omp array simd" temporary arrays to the
1047      actual vectorization factors.  */
1048   if (simd_array_to_simduid_htab)
1049     shrink_simd_arrays (simd_array_to_simduid_htab, simduid_to_vf_htab);
1050   delete simduid_to_vf_htab;
1051   cfun->has_simduid_loops = false;
1052
1053   if (num_vectorized_loops > 0)
1054     {
1055       /* If we vectorized any loop only virtual SSA form needs to be updated.
1056          ???  Also while we try hard to update loop-closed SSA form we fail
1057          to properly do this in some corner-cases (see PR56286).  */
1058       rewrite_into_loop_closed_ssa (NULL, TODO_update_ssa_only_virtuals);
1059       return TODO_cleanup_cfg;
1060     }
1061
1062   return ret;
1063 }
1064
1065
1066 /* Entry point to the simduid cleanup pass.  */
1067
1068 namespace {
1069
1070 const pass_data pass_data_simduid_cleanup =
1071 {
1072   GIMPLE_PASS, /* type */
1073   "simduid", /* name */
1074   OPTGROUP_NONE, /* optinfo_flags */
1075   TV_NONE, /* tv_id */
1076   ( PROP_ssa | PROP_cfg ), /* properties_required */
1077   0, /* properties_provided */
1078   0, /* properties_destroyed */
1079   0, /* todo_flags_start */
1080   0, /* todo_flags_finish */
1081 };
1082
1083 class pass_simduid_cleanup : public gimple_opt_pass
1084 {
1085 public:
1086   pass_simduid_cleanup (gcc::context *ctxt)
1087     : gimple_opt_pass (pass_data_simduid_cleanup, ctxt)
1088   {}
1089
1090   /* opt_pass methods: */
1091   opt_pass * clone () { return new pass_simduid_cleanup (m_ctxt); }
1092   virtual bool gate (function *fun) { return fun->has_simduid_loops; }
1093   virtual unsigned int execute (function *);
1094
1095 }; // class pass_simduid_cleanup
1096
1097 unsigned int
1098 pass_simduid_cleanup::execute (function *fun)
1099 {
1100   hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab = NULL;
1101
1102   note_simd_array_uses (&simd_array_to_simduid_htab);
1103
1104   /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins.  */
1105   adjust_simduid_builtins (NULL);
1106
1107   /* Shrink any "omp array simd" temporary arrays to the
1108      actual vectorization factors.  */
1109   if (simd_array_to_simduid_htab)
1110     shrink_simd_arrays (simd_array_to_simduid_htab, NULL);
1111   fun->has_simduid_loops = false;
1112   return 0;
1113 }
1114
1115 }  // anon namespace
1116
1117 gimple_opt_pass *
1118 make_pass_simduid_cleanup (gcc::context *ctxt)
1119 {
1120   return new pass_simduid_cleanup (ctxt);
1121 }
1122
1123
1124 /*  Entry point to basic block SLP phase.  */
1125
1126 namespace {
1127
1128 const pass_data pass_data_slp_vectorize =
1129 {
1130   GIMPLE_PASS, /* type */
1131   "slp", /* name */
1132   OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
1133   TV_TREE_SLP_VECTORIZATION, /* tv_id */
1134   ( PROP_ssa | PROP_cfg ), /* properties_required */
1135   0, /* properties_provided */
1136   0, /* properties_destroyed */
1137   0, /* todo_flags_start */
1138   TODO_update_ssa, /* todo_flags_finish */
1139 };
1140
1141 class pass_slp_vectorize : public gimple_opt_pass
1142 {
1143 public:
1144   pass_slp_vectorize (gcc::context *ctxt)
1145     : gimple_opt_pass (pass_data_slp_vectorize, ctxt)
1146   {}
1147
1148   /* opt_pass methods: */
1149   opt_pass * clone () { return new pass_slp_vectorize (m_ctxt); }
1150   virtual bool gate (function *) { return flag_tree_slp_vectorize != 0; }
1151   virtual unsigned int execute (function *);
1152
1153 }; // class pass_slp_vectorize
1154
1155 unsigned int
1156 pass_slp_vectorize::execute (function *fun)
1157 {
1158   basic_block bb;
1159
1160   bool in_loop_pipeline = scev_initialized_p ();
1161   if (!in_loop_pipeline)
1162     {
1163       loop_optimizer_init (LOOPS_NORMAL);
1164       scev_initialize ();
1165     }
1166
1167   /* Mark all stmts as not belonging to the current region and unvisited.  */
1168   FOR_EACH_BB_FN (bb, fun)
1169     {
1170       for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
1171            gsi_next (&gsi))
1172         {
1173           gimple *stmt = gsi_stmt (gsi);
1174           gimple_set_uid (stmt, -1);
1175           gimple_set_visited (stmt, false);
1176         }
1177     }
1178
1179   FOR_EACH_BB_FN (bb, fun)
1180     {
1181       if (vect_slp_bb (bb))
1182         dump_printf_loc (MSG_NOTE, vect_location, "basic block vectorized\n");
1183     }
1184
1185   if (!in_loop_pipeline)
1186     {
1187       scev_finalize ();
1188       loop_optimizer_finalize ();
1189     }
1190
1191   return 0;
1192 }
1193
1194 } // anon namespace
1195
1196 gimple_opt_pass *
1197 make_pass_slp_vectorize (gcc::context *ctxt)
1198 {
1199   return new pass_slp_vectorize (ctxt);
1200 }
1201
1202
1203 /* Increase alignment of global arrays to improve vectorization potential.
1204    TODO:
1205    - Consider also structs that have an array field.
1206    - Use ipa analysis to prune arrays that can't be vectorized?
1207      This should involve global alignment analysis and in the future also
1208      array padding.  */
1209
1210 static unsigned get_vec_alignment_for_type (tree);
1211 static hash_map<tree, unsigned> *type_align_map;
1212
1213 /* Return alignment of array's vector type corresponding to scalar type.
1214    0 if no vector type exists.  */
1215 static unsigned
1216 get_vec_alignment_for_array_type (tree type)
1217 {
1218   gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1219   poly_uint64 array_size, vector_size;
1220
1221   tree vectype = get_vectype_for_scalar_type (strip_array_types (type));
1222   if (!vectype
1223       || !poly_int_tree_p (TYPE_SIZE (type), &array_size)
1224       || !poly_int_tree_p (TYPE_SIZE (vectype), &vector_size)
1225       || maybe_lt (array_size, vector_size))
1226     return 0;
1227
1228   return TYPE_ALIGN (vectype);
1229 }
1230
1231 /* Return alignment of field having maximum alignment of vector type
1232    corresponding to it's scalar type. For now, we only consider fields whose
1233    offset is a multiple of it's vector alignment.
1234    0 if no suitable field is found.  */
1235 static unsigned
1236 get_vec_alignment_for_record_type (tree type)
1237 {
1238   gcc_assert (TREE_CODE (type) == RECORD_TYPE);
1239
1240   unsigned max_align = 0, alignment;
1241   HOST_WIDE_INT offset;
1242   tree offset_tree;
1243
1244   if (TYPE_PACKED (type))
1245     return 0;
1246
1247   unsigned *slot = type_align_map->get (type);
1248   if (slot)
1249     return *slot;
1250
1251   for (tree field = first_field (type);
1252        field != NULL_TREE;
1253        field = DECL_CHAIN (field))
1254     {
1255       /* Skip if not FIELD_DECL or if alignment is set by user.  */
1256       if (TREE_CODE (field) != FIELD_DECL
1257           || DECL_USER_ALIGN (field)
1258           || DECL_ARTIFICIAL (field))
1259         continue;
1260
1261       /* We don't need to process the type further if offset is variable,
1262          since the offsets of remaining members will also be variable.  */
1263       if (TREE_CODE (DECL_FIELD_OFFSET (field)) != INTEGER_CST
1264           || TREE_CODE (DECL_FIELD_BIT_OFFSET (field)) != INTEGER_CST)
1265         break;
1266
1267       /* Similarly stop processing the type if offset_tree
1268          does not fit in unsigned HOST_WIDE_INT.  */
1269       offset_tree = bit_position (field);
1270       if (!tree_fits_uhwi_p (offset_tree))
1271         break;
1272
1273       offset = tree_to_uhwi (offset_tree);
1274       alignment = get_vec_alignment_for_type (TREE_TYPE (field));
1275
1276       /* Get maximum alignment of vectorized field/array among those members
1277          whose offset is multiple of the vector alignment.  */
1278       if (alignment
1279           && (offset % alignment == 0)
1280           && (alignment > max_align))
1281         max_align = alignment;
1282     }
1283
1284   type_align_map->put (type, max_align);
1285   return max_align;
1286 }
1287
1288 /* Return alignment of vector type corresponding to decl's scalar type
1289    or 0 if it doesn't exist or the vector alignment is lesser than
1290    decl's alignment.  */
1291 static unsigned
1292 get_vec_alignment_for_type (tree type)
1293 {
1294   if (type == NULL_TREE)
1295     return 0;
1296
1297   gcc_assert (TYPE_P (type));
1298
1299   static unsigned alignment = 0;
1300   switch (TREE_CODE (type))
1301     {
1302       case ARRAY_TYPE:
1303         alignment = get_vec_alignment_for_array_type (type);
1304         break;
1305       case RECORD_TYPE:
1306         alignment = get_vec_alignment_for_record_type (type);
1307         break;
1308       default:
1309         alignment = 0;
1310         break;
1311     }
1312
1313   return (alignment > TYPE_ALIGN (type)) ? alignment : 0;
1314 }
1315
1316 /* Entry point to increase_alignment pass.  */
1317 static unsigned int
1318 increase_alignment (void)
1319 {
1320   varpool_node *vnode;
1321
1322   vect_location = dump_user_location_t ();
1323   type_align_map = new hash_map<tree, unsigned>;
1324
1325   /* Increase the alignment of all global arrays for vectorization.  */
1326   FOR_EACH_DEFINED_VARIABLE (vnode)
1327     {
1328       tree decl = vnode->decl;
1329       unsigned int alignment;
1330
1331       if ((decl_in_symtab_p (decl)
1332           && !symtab_node::get (decl)->can_increase_alignment_p ())
1333           || DECL_USER_ALIGN (decl) || DECL_ARTIFICIAL (decl))
1334         continue;
1335
1336       alignment = get_vec_alignment_for_type (TREE_TYPE (decl));
1337       if (alignment && vect_can_force_dr_alignment_p (decl, alignment))
1338         {
1339           vnode->increase_alignment (alignment);
1340           dump_printf (MSG_NOTE, "Increasing alignment of decl: ");
1341           dump_generic_expr (MSG_NOTE, TDF_SLIM, decl);
1342           dump_printf (MSG_NOTE, "\n");
1343         }
1344     }
1345
1346   delete type_align_map;
1347   return 0;
1348 }
1349
1350
1351 namespace {
1352
1353 const pass_data pass_data_ipa_increase_alignment =
1354 {
1355   SIMPLE_IPA_PASS, /* type */
1356   "increase_alignment", /* name */
1357   OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
1358   TV_IPA_OPT, /* tv_id */
1359   0, /* properties_required */
1360   0, /* properties_provided */
1361   0, /* properties_destroyed */
1362   0, /* todo_flags_start */
1363   0, /* todo_flags_finish */
1364 };
1365
1366 class pass_ipa_increase_alignment : public simple_ipa_opt_pass
1367 {
1368 public:
1369   pass_ipa_increase_alignment (gcc::context *ctxt)
1370     : simple_ipa_opt_pass (pass_data_ipa_increase_alignment, ctxt)
1371   {}
1372
1373   /* opt_pass methods: */
1374   virtual bool gate (function *)
1375     {
1376       return flag_section_anchors && flag_tree_loop_vectorize;
1377     }
1378
1379   virtual unsigned int execute (function *) { return increase_alignment (); }
1380
1381 }; // class pass_ipa_increase_alignment
1382
1383 } // anon namespace
1384
1385 simple_ipa_opt_pass *
1386 make_pass_ipa_increase_alignment (gcc::context *ctxt)
1387 {
1388   return new pass_ipa_increase_alignment (ctxt);
1389 }