| blob | 8cd02afdeab56de475266131eb6aa110bb0e1b21 |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2023 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
61 /* For lang_hooks.types.type_for_mode. */
64 /* Return the vectorized type for the given statement. */
66 tree
68 {
70 }
72 /* Return TRUE iff the given statement is in an inner loop relative to
73 the loop being vectorized. */
74 bool
76 {
88 }
90 /* Record the cost of a statement, either by directly informing the
91 target model or by saving it in a vector for later processing.
92 Return a preliminary estimate of the statement's cost. */
94 static unsigned
100 {
108 stmt_info_for_cost si
114 }
116 unsigned
121 {
124 }
126 unsigned
131 {
134 }
136 unsigned
140 {
145 }
147 /* Return a variable of type ELEM_TYPE[NELEMS]. */
149 static tree
151 {
154 }
156 /* ARRAY is an array of vectors created by create_vector_array.
157 Return an SSA_NAME for the vector in index N. The reference
158 is part of the vectorization of STMT_INFO and the vector is associated
159 with scalar destination SCALAR_DEST. */
161 static tree
165 {
182 }
184 /* ARRAY is an array of vectors created by create_vector_array.
185 Emit code to store SSA_NAME VECT in index N of the array.
186 The store is part of the vectorization of STMT_INFO. */
188 static void
192 {
193 tree array_ref;
202 }
204 /* PTR is a pointer to an array of type TYPE. Return a representation
205 of *PTR. The memory reference replaces those in FIRST_DR
206 (and its group). */
208 static tree
210 {
211 tree mem_ref;
214 /* Arrays have the same alignment as their type. */
217 }
219 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
220 Emit the clobber before *GSI. */
222 static void
225 {
229 }
231 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
233 /* Function vect_mark_relevant.
235 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
237 static void
240 {
249 /* If this stmt is an original stmt in a pattern, we might need to mark its
250 related pattern stmt instead of the original stmt. However, such stmts
251 may have their own uses that are not in any pattern, in such cases the
252 stmt itself should be marked. */
254 {
255 /* This is the last stmt in a sequence that was detected as a
256 pattern that can potentially be vectorized. Don't mark the stmt
257 as relevant/live because it's not going to be vectorized.
258 Instead mark the pattern-stmt that replaces it. */
262 "last stmt in pattern. don't mark"
272 {
275 "vec_stmt_relevant_p: forcing live pattern stmt "
278 }
284 }
292 {
297 }
300 }
303 /* Function is_simple_and_all_uses_invariant
305 Return true if STMT_INFO is simple and all uses of it are invariant. */
307 bool
309 loop_vec_info loop_vinfo)
310 {
311 tree op;
312 ssa_op_iter iter;
319 {
323 {
328 }
332 }
334 }
336 /* Function vect_stmt_relevant_p.
338 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
339 is "relevant for vectorization".
341 A stmt is considered "relevant for vectorization" if:
342 - it has uses outside the loop.
343 - it has vdefs (it alters memory).
344 - control stmts in the loop (except for the exit condition).
346 CHECKME: what other side effects would the vectorizer allow? */
348 static bool
351 {
353 ssa_op_iter op_iter;
354 imm_use_iterator imm_iter;
355 use_operand_p use_p;
356 def_operand_p def_p;
361 /* cond stmt other than loop exit cond. */
366 /* changing memory. */
370 {
375 }
377 /* uses outside the loop. */
379 {
381 {
384 {
392 /* We expect all such uses to be in the loop exit phis
393 (because of loop closed form) */
398 }
399 }
400 }
404 {
409 }
412 }
415 /* Function exist_non_indexing_operands_for_use_p
417 USE is one of the uses attached to STMT_INFO. Check if USE is
418 used in STMT_INFO for anything other than indexing an array. */
420 static bool
422 {
423 tree operand;
425 /* USE corresponds to some operand in STMT. If there is no data
426 reference in STMT, then any operand that corresponds to USE
427 is not indexing an array. */
431 /* STMT has a data_ref. FORNOW this means that its of one of
432 the following forms:
433 -1- ARRAY_REF = var
434 -2- var = ARRAY_REF
435 (This should have been verified in analyze_data_refs).
437 'var' in the second case corresponds to a def, not a use,
438 so USE cannot correspond to any operands that are not used
439 for array indexing.
441 Therefore, all we need to check is if STMT falls into the
442 first case, and whether var corresponds to USE. */
446 {
449 {
462 }
464 }
476 }
479 /*
480 Function process_use.
482 Inputs:
483 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
484 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
485 that defined USE. This is done by calling mark_relevant and passing it
486 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
487 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
488 be performed.
490 Outputs:
491 Generally, LIVE_P and RELEVANT are used to define the liveness and
492 relevance info of the DEF_STMT of this USE:
493 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
494 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
495 Exceptions:
496 - case 1: If USE is used only for address computations (e.g. array indexing),
497 which does not need to be directly vectorized, then the liveness/relevance
498 of the respective DEF_STMT is left unchanged.
499 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
500 we skip DEF_STMT cause it had already been processed.
501 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
502 "relevant" will be modified accordingly.
504 Return true if everything is as expected. Return false otherwise. */
506 static opt_result
510 {
511 stmt_vec_info dstmt_vinfo;
514 /* case 1: we are only interested in uses that need to be vectorized. Uses
515 that are used for address computation are not considered relevant. */
521 "not vectorized:"
530 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
531 We have to force the stmt live since the epilogue loop needs it to
532 continue computing the reduction. */
538 {
544 }
546 /* case 3a: outer-loop stmt defining an inner-loop stmt:
547 outer-loop-header-bb:
548 d = dstmt_vinfo
549 inner-loop:
550 stmt # use (d)
551 outer-loop-tail-bb:
552 ... */
554 {
560 {
581 }
582 }
584 /* case 3b: inner-loop stmt defining an outer-loop stmt:
585 outer-loop-header-bb:
586 ...
587 inner-loop:
588 d = dstmt_vinfo
589 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
590 stmt # use (d) */
592 {
598 {
616 }
617 }
618 /* We are also not interested in uses on loop PHI backedges that are
619 inductions. Otherwise we'll needlessly vectorize the IV increment
620 and cause hybrid SLP for SLP inductions. Unless the PHI is live
621 of course. */
628 {
633 }
638 }
641 /* Function vect_mark_stmts_to_be_vectorized.
643 Not all stmts in the loop need to be vectorized. For example:
645 for i...
646 for j...
647 1. T0 = i + j
648 2. T1 = a[T0]
650 3. j = j + 1
652 Stmt 1 and 3 do not need to be vectorized, because loop control and
653 addressing of vectorized data-refs are handled differently.
655 This pass detects such stmts. */
657 opt_result
659 {
663 gimple_stmt_iterator si;
665 basic_block bb;
673 /* 1. Init worklist. */
675 {
678 {
686 }
688 {
698 }
699 }
701 /* 2. Process_worklist */
703 {
704 use_operand_p use_p;
705 ssa_op_iter iter;
712 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
713 (DEF_STMT) as relevant/irrelevant according to the relevance property
714 of STMT. */
717 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
718 propagated as is to the DEF_STMTs of its USEs.
720 One exception is when STMT has been identified as defining a reduction
721 variable; in this case we set the relevance to vect_used_by_reduction.
722 This is because we distinguish between two kinds of relevant stmts -
723 those that are used by a reduction computation, and those that are
724 (also) used by a regular computation. This allows us later on to
725 identify stmts that are used solely by a reduction, and therefore the
726 order of the results that they produce does not have to be kept. */
729 {
758 }
761 {
762 /* Pattern statements are not inserted into the code, so
763 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
764 have to scan the RHS or function arguments instead. */
766 {
772 {
773 opt_result res
783 }
785 {
788 {
789 opt_result res
794 }
795 }
796 }
798 {
800 {
802 opt_result res
807 }
808 }
809 }
810 else
812 {
814 opt_result res
819 }
822 {
823 gather_scatter_info gs_info;
826 opt_result res
830 {
834 }
835 }
839 }
841 /* Function vect_model_simple_cost.
843 Models cost for simple operations, i.e. those that only emit ncopies of a
844 single op. Right now, this does not account for multiple insns that could
845 be generated for the single vector op. We will handle that shortly. */
847 static void
852 slp_tree node,
855 {
860 /* ??? Somehow we need to fix this at the callers. */
865 /* Cost the "broadcast" of a scalar operand in to a vector operand.
866 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
867 cost model. */
873 /* Pass the inside-of-loop statements to the target-specific cost model. */
879 "vect_model_simple_cost: inside_cost = %d, "
881 }
884 /* Model cost for type demotion and promotion operations. PWR is
885 normally zero for single-step promotions and demotions. It will be
886 one if two-step promotion/demotion is required, and so on. NCOPIES
887 is the number of vector results (and thus number of instructions)
888 for the narrowest end of the operation chain. Each additional
889 step doubles the number of instructions required. If WIDEN_ARITH
890 is true the stmt is doing widening arithmetic. */
892 static void
898 {
903 {
905 widen_arith
909 }
911 /* FORNOW: Assuming maximum 2 args per stmts. */
919 "vect_model_promotion_demotion_cost: inside_cost = %d, "
921 }
923 /* Returns true if the current function returns DECL. */
925 static bool
927 {
928 edge_iterator ei;
929 edge e;
931 {
937 /* We often end up with an aggregate copy to the result decl,
938 handle that case as well. First skip intermediate clobbers
939 though. */
941 do
942 {
944 }
950 }
952 }
954 /* Calculate cost of DR's memory access. */
955 void
957 dr_alignment_support alignment_support_scheme,
961 {
963 {
965 {
968 vect_body);
974 }
977 {
978 /* Here, we assign an additional cost for the unaligned store. */
984 "vect_model_store_cost: unaligned supported by "
987 }
990 {
997 }
1001 }
1002 }
1004 /* Calculate cost of DR's memory access. */
1005 void
1007 dr_alignment_support alignment_support_scheme,
1014 {
1016 {
1018 {
1027 }
1029 {
1030 /* Here, we assign an additional cost for the unaligned load. */
1037 "vect_model_load_cost: unaligned supported by "
1041 }
1043 {
1049 /* FIXME: If the misalignment remains fixed across the iterations of
1050 the containing loop, the following cost should be added to the
1051 prologue costs. */
1061 }
1063 {
1066 "vect_model_load_cost: unaligned software "
1069 /* Unaligned software pipeline has a load of an address, an initial
1070 load, and possibly a mask operation to "prime" the loop. However,
1071 if this is an access in a group of loads, which provide grouped
1072 access, then the above cost should only be considered for one
1073 access in the group. Inside the loop, there is a load op
1074 and a realignment op. */
1077 {
1085 }
1094 "vect_model_load_cost: explicit realign optimized"
1098 }
1101 {
1108 }
1112 }
1113 }
1115 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1116 the loop preheader for the vectorized stmt STMT_VINFO. */
1118 static void
1121 {
1124 else
1130 }
1132 /* Function vect_init_vector.
1134 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1135 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1136 vector type a vector with all elements equal to VAL is created first.
1137 Place the initialization at GSI if it is not NULL. Otherwise, place the
1138 initialization at the loop preheader.
1139 Return the DEF of INIT_STMT.
1140 It will be used in the vectorization of STMT_INFO. */
1142 tree
1145 {
1147 tree new_temp;
1149 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1151 {
1154 {
1155 /* Scalar boolean value should be transformed into
1156 all zeros or all ones value before building a vector. */
1158 {
1164 else
1165 {
1171 }
1172 }
1173 else
1174 {
1179 else
1180 /* ??? Condition vectorization expects us to do
1181 promotion of invariant/external defs. */
1185 {
1189 }
1190 }
1191 }
1193 }
1199 }
1202 /* Function vect_get_vec_defs_for_operand.
1204 OP is an operand in STMT_VINFO. This function returns a vector of
1205 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1207 In the case that OP is an SSA_NAME which is defined in the loop, then
1208 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1210 In case OP is an invariant or constant, a new stmt that creates a vector def
1211 needs to be introduced. VECTYPE may be used to specify a required type for
1212 vector invariant. */
1214 void
1218 {
1228 stmt_vec_info def_stmt_info;
1237 {
1239 tree vector_type;
1246 else
1253 }
1254 else
1255 {
1261 }
1262 }
1265 /* Get vectorized definitions for OP0 and OP1. */
1267 void
1274 {
1276 {
1285 }
1286 else
1287 {
1300 }
1301 }
1303 void
1310 {
1316 }
1318 /* Helper function called by vect_finish_replace_stmt and
1319 vect_finish_stmt_generation. Set the location of the new
1320 statement and create and return a stmt_vec_info for it. */
1322 static void
1325 {
1330 {
1333 /* While EH edges will generally prevent vectorization, stmt might
1334 e.g. be in a must-not-throw region. Ensure newly created stmts
1335 that could throw are part of the same region. */
1339 }
1340 else
1342 }
1344 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1345 which sets the same scalar result as STMT_INFO did. Create and return a
1346 stmt_vec_info for VEC_STMT. */
1348 void
1351 {
1359 }
1361 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1362 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1364 void
1368 {
1373 {
1377 {
1381 /* If we have an SSA vuse and insert a store, update virtual
1382 SSA form to avoid triggering the renamer. Do so only
1383 if we can easily see all uses - which is what almost always
1384 happens with the way vectorized stmts are inserted. */
1393 {
1397 }
1398 }
1399 }
1402 }
1404 /* We want to vectorize a call to combined function CFN with function
1405 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1406 as the types of all inputs. Check whether this is possible using
1407 an internal function, returning its code if so or IFN_LAST if not. */
1409 static internal_fn
1412 {
1413 internal_fn ifn;
1416 else
1419 {
1422 {
1427 /* The type size of both the vectype_in and vectype_out should be
1428 exactly the same when vectype_out isn't participating the optab.
1429 While there is no restriction for type size when vectype_out
1430 is part of the optab query. */
1435 OPTIMIZE_FOR_SPEED))
1437 }
1438 }
1440 }
1444 gimple_stmt_iterator *);
1446 /* Check whether a load or store statement in the loop described by
1447 LOOP_VINFO is possible in a loop using partial vectors. This is
1448 testing whether the vectorizer pass has the appropriate support,
1449 as well as whether the target does.
1451 VLS_TYPE says whether the statement is a load or store and VECTYPE
1452 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1453 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1454 says how the load or store is going to be implemented and GROUP_SIZE
1455 is the number of load or store statements in the containing group.
1456 If the access is a gather load or scatter store, GS_INFO describes
1457 its arguments. If the load or store is conditional, SCALAR_MASK is the
1458 condition under which it occurs.
1460 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1461 vectors is not supported, otherwise record the required rgroup control
1462 types. */
1464 static void
1466 slp_tree slp_node,
1467 vec_load_store_type vls_type,
1469 vect_memory_access_type
1470 memory_access_type,
1472 tree scalar_mask)
1473 {
1474 /* Invariant loads need no special support. */
1481 else
1489 {
1490 internal_fn ifn
1497 scalar_mask);
1498 else
1499 {
1502 "can't operate on partial vectors because"
1503 " the target doesn't have an appropriate"
1506 }
1508 }
1511 {
1512 internal_fn ifn = (is_load
1513 ? IFN_MASK_GATHER_LOAD
1515 internal_fn len_ifn = (is_load
1516 ? IFN_MASK_LEN_GATHER_LOAD
1528 scalar_mask);
1529 else
1530 {
1533 "can't operate on partial vectors because"
1534 " the target doesn't have an appropriate"
1537 }
1539 }
1543 {
1544 /* Element X of the data must come from iteration i * VF + X of the
1545 scalar loop. We need more work to support other mappings. */
1548 "can't operate on partial vectors because an"
1552 }
1555 {
1558 "can't operate on partial vectors when emulating"
1562 }
1564 /* We might load more scalars than we need for permuting SLP loads.
1565 We checked in get_group_load_store_type that the extra elements
1566 don't leak into a new vector. */
1568 {
1573 };
1577 machine_mode mask_mode;
1578 machine_mode vmode;
1581 {
1586 }
1589 {
1593 }
1596 {
1599 "can't operate on partial vectors because the"
1600 " target doesn't have the appropriate partial"
1603 }
1604 }
1606 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1607 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1608 that needs to be applied to all loads and stores in a vectorized loop.
1609 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1610 otherwise return VEC_MASK & LOOP_MASK.
1612 MASK_TYPE is the type of both masks. If new statements are needed,
1613 insert them before GSI. */
1615 static tree
1618 {
1634 }
1636 /* Determine whether we can use a gather load or scatter store to vectorize
1637 strided load or store STMT_INFO by truncating the current offset to a
1638 smaller width. We need to be able to construct an offset vector:
1640 { 0, X, X*2, X*3, ... }
1642 without loss of precision, where X is STMT_INFO's DR_STEP.
1644 Return true if this is possible, describing the gather load or scatter
1645 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1647 static bool
1651 {
1656 {
1657 /* ??? Perhaps we could use range information here? */
1662 }
1664 /* Get the number of bits in an element. */
1669 /* Set COUNT to the upper limit on the number of elements - 1.
1670 Start with the maximum vectorization factor. */
1673 /* Try lowering COUNT to the number of scalar latch iterations. */
1675 widest_int max_iters;
1680 /* Try scales of 1 and the element size. */
1684 {
1686 widest_int factor;
1690 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1697 /* Find the narrowest viable offset type. */
1702 /* See whether the target supports the operation with an offset
1703 no narrower than OFFSET_TYPE. */
1712 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1713 but we don't need to store that here. */
1721 }
1725 "truncating gather/scatter offset to %d bits"
1729 }
1731 /* Return true if we can use gather/scatter internal functions to
1732 vectorize STMT_INFO, which is a grouped or strided load or store.
1733 MASKED_P is true if load or store is conditional. When returning
1734 true, fill in GS_INFO with the information required to perform the
1735 operation. */
1737 static bool
1741 {
1756 "using gather/scatter for strided/grouped access,"
1760 }
1762 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1763 elements with a known constant step. Return -1 if that step
1764 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1766 static int
1768 {
1771 size_zero_node);
1772 }
1774 /* If the target supports a permute mask that reverses the elements in
1775 a vector of type VECTYPE, return that mask, otherwise return null. */
1777 static tree
1779 {
1782 /* The encoding has a single stepped pattern. */
1789 indices))
1792 }
1794 /* A subroutine of get_load_store_type, with a subset of the same
1795 arguments. Handle the case where STMT_INFO is a load or store that
1796 accesses consecutive elements with a negative step. Sets *POFFSET
1797 to the offset to be applied to the DR for the first access. */
1799 static vect_memory_access_type
1802 vec_load_store_type vls_type,
1804 {
1806 dr_alignment_support alignment_support_scheme;
1809 {
1814 }
1816 /* For backward running DRs the first access in vectype actually is
1817 N-1 elements before the address of the DR. */
1822 alignment_support_scheme
1826 {
1832 }
1835 {
1838 "negative step with invariant source;"
1841 }
1844 {
1850 }
1853 }
1855 /* STMT_INFO is either a masked or unconditional store. Return the value
1856 being stored. */
1858 tree
1860 {
1862 {
1865 }
1867 {
1872 }
1874 }
1876 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1878 This function returns a vector type which can be composed with NETLS pieces,
1879 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1880 same vector size as the return vector. It checks target whether supports
1881 pieces-size vector mode for construction firstly, if target fails to, check
1882 pieces-size scalar mode for construction further. It returns NULL_TREE if
1883 fails to find the available composition.
1885 For example, for (vtype=V16QI, nelts=4), we can probably get:
1886 - V16QI with PTYPE V4QI.
1887 - V4SI with PTYPE SI.
1888 - NULL_TREE. */
1890 static tree
1892 {
1900 /* When we are asked to compose the vector from its components let
1901 that happen directly. */
1903 {
1906 }
1911 {
1912 /* First check if vec_init optab supports construction from
1913 vector pieces directly. */
1916 machine_mode rmode;
1920 {
1923 }
1925 /* Otherwise check if exists an integer type of the same piece size and
1926 if vec_init optab supports construction from it directly. */
1931 {
1934 }
1935 }
1938 }
1940 /* A subroutine of get_load_store_type, with a subset of the same
1941 arguments. Handle the case where STMT_INFO is part of a grouped load
1942 or store.
1944 For stores, the statements in the group are all consecutive
1945 and there is no gap at the end. For loads, the statements in the
1946 group might not be consecutive; there can be gaps between statements
1947 as well as at the end. */
1949 static bool
1959 {
1962 stmt_vec_info first_stmt_info;
1966 {
1970 }
1971 else
1972 {
1976 }
1982 /* True if the vectorized statements would access beyond the last
1983 statement in the group. */
1986 /* True if we can cope with such overrun by peeling for gaps, so that
1987 there is at least one final scalar iteration after the vector loop. */
1990 && loop_vinfo
1993 /* There can only be a gap at the end of the group if the stride is
1994 known at compile time. */
1997 /* Stores can't yet have gaps. */
2001 {
2002 /* For SLP vectorization we directly vectorize a subchain
2003 without permutation. */
2005 first_dr_info
2008 {
2009 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2010 separated by the stride, until we have a complete vector.
2011 Fall back to scalar accesses if that isn't possible. */
2014 else
2016 }
2017 else
2018 {
2021 {
2023 "Grouped store with gaps requires"
2026 }
2027 /* An overrun is fine if the trailing elements are smaller
2028 than the alignment boundary B. Every vector access will
2029 be a multiple of B and so we are guaranteed to access a
2030 non-gap element in the same B-sized block. */
2033 vectype)
2037 /* If the gap splits the vector in half and the target
2038 can do half-vector operations avoid the epilogue peeling
2039 by simply loading half of the vector only. Usually
2040 the construction with an upper zero half will be elided. */
2041 dr_alignment_support alss;
2043 tree half_vtype;
2045 && !masked_p
2048 == dr_aligned
2057 {
2062 }
2065 {
2067 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2068 only correct for single element "interleaving" SLP. */
2071 else
2072 {
2073 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2074 separated by the stride, until we have a complete vector.
2075 Fall back to scalar accesses if that isn't possible. */
2078 else
2080 }
2081 }
2083 {
2086 /* Invariant accesses perform only component accesses, alignment
2087 is irrelevant for them. */
2089 }
2090 else
2093 /* When we have a contiguous access across loop iterations
2094 but the access in the loop doesn't cover the full vector
2095 we can end up with no gap recorded but still excess
2096 elements accessed, see PR103116. Make sure we peel for
2097 gaps if necessary and sufficient and give up if not.
2099 If there is a combination of the access not covering the full
2100 vector and a gap recorded then we may need to peel twice. */
2105 nunits))
2106 {
2111 /* Peeling for gaps assumes that a single scalar iteration
2112 is enough to make sure the last vector iteration doesn't
2113 access excess elements.
2114 ??? Enhancements include peeling multiple iterations
2115 or using masked loads with a static mask. */
2117 {
2120 "peeling for gaps insufficient for "
2123 }
2125 }
2126 }
2127 }
2128 else
2129 {
2130 /* We can always handle this case using elementwise accesses,
2131 but see if something more efficient is available. */
2134 /* If there is a gap at the end of the group then these optimizations
2135 would access excess elements in the last iteration. */
2137 /* An overrun is fine if the trailing elements are smaller than the
2138 alignment boundary B. Every vector access will be a multiple of B
2139 and so we are guaranteed to access a non-gap element in the
2140 same B-sized block. */
2142 && !masked_p
2150 {
2151 /* First cope with the degenerate case of a single-element
2152 vector. */
2154 ;
2156 else
2157 {
2158 /* Otherwise try using LOAD/STORE_LANES. */
2159 *lanes_ifn
2163 masked_p);
2165 {
2168 }
2170 /* If that fails, try using permuting loads. */
2173 single_element_p,
2174 group_size)
2176 {
2179 }
2180 }
2181 }
2183 /* As a last resort, trying using a gather load or scatter store.
2185 ??? Although the code can handle all group sizes correctly,
2186 it probably isn't a win to use separate strided accesses based
2187 on nearby locations. Or, even if it's a win over scalar code,
2188 it might not be a win over vectorizing at a lower VF, if that
2189 allows us to use contiguous accesses. */
2191 && single_element_p
2192 && loop_vinfo
2196 }
2200 {
2203 }
2204 else
2205 {
2207 *alignment_support_scheme
2210 }
2213 {
2214 /* STMT is the leader of the group. Check the operands of all the
2215 stmts of the group. */
2218 {
2222 {
2227 }
2229 }
2230 }
2233 {
2237 "Data access with gaps requires scalar "
2240 }
2243 }
2245 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2246 if there is a memory access type that the vectorized form can use,
2247 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2248 or scatters, fill in GS_INFO accordingly. In addition
2249 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2250 the target does not support the alignment scheme. *MISALIGNMENT
2251 is set according to the alignment of the access (including
2252 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2254 SLP says whether we're performing SLP rather than loop vectorization.
2255 MASKED_P is true if the statement is conditional on a vectorized mask.
2256 VECTYPE is the vector type that the vectorized statements will use.
2257 NCOPIES is the number of vector statements that will be needed. */
2259 static bool
2270 {
2276 {
2280 /* When using internal functions, we rely on pattern recognition
2281 to convert the type of the offset to the type that the target
2282 requires, with the result being a call to an internal function.
2283 If that failed for some reason (e.g. because another pattern
2284 took priority), just handle cases in which the offset already
2285 has the right type. */
2290 {
2296 }
2300 {
2306 }
2308 {
2314 {
2317 "unsupported vector types for emulated "
2320 }
2321 }
2322 /* Gather-scatter accesses perform only component accesses, alignment
2323 is irrelevant for them. */
2325 }
2327 {
2329 masked_p,
2331 alignment_support_scheme,
2334 }
2336 {
2342 else
2344 /* Alignment is irrelevant here. */
2346 }
2347 else
2348 {
2351 {
2354 /* Invariant accesses perform only component accesses, alignment
2355 is irrelevant for them. */
2357 }
2358 else
2359 {
2363 else
2367 *alignment_support_scheme
2371 }
2372 }
2377 {
2380 "Not using elementwise accesses due to variable "
2383 }
2386 {
2391 }
2393 /* FIXME: At the moment the cost model seems to underestimate the
2394 cost of using elementwise accesses. This check preserves the
2395 traditional behavior until that can be fixed. */
2404 {
2409 }
2411 }
2413 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2414 conditional operation STMT_INFO. When returning true, store the mask
2415 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2416 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2417 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2419 static bool
2424 {
2426 tree mask_vectype;
2427 slp_tree mask_node_1;
2430 {
2435 }
2438 {
2443 }
2445 /* If the caller is not prepared for adjusting an external/constant
2446 SLP mask vector type fail. */
2448 && !mask_node
2450 {
2455 }
2460 mask_node_1);
2463 {
2468 }
2472 {
2475 "vector mask type %T"
2480 }
2487 }
2489 /* Return true if stored value is suitable for vectorizing store
2490 statement STMT_INFO. When returning true, store the scalar stored
2491 in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2492 the type of the vectorized store value in
2493 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2495 static bool
2500 {
2503 {
2507 }
2509 op_no = vect_slp_child_index_for_operand
2513 tree rhs_vectype;
2516 {
2521 }
2523 /* In the case this is a store from a constant make sure
2524 native_encode_expr can handle it. */
2526 {
2531 }
2535 {
2540 }
2546 else
2549 }
2551 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2552 Note that we support masks with floating-point type, in which case the
2553 floats are interpreted as a bitmask. */
2555 static tree
2558 {
2563 {
2567 }
2569 {
2570 REAL_VALUE_TYPE r;
2578 }
2580 }
2582 /* Build an all-zero merge value of type VECTYPE while vectorizing
2583 STMT_INFO as a gather load. */
2585 static tree
2588 {
2589 tree merge;
2593 {
2594 REAL_VALUE_TYPE r;
2600 }
2601 else
2605 }
2607 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2608 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2609 the gather load operation. If the load is conditional, MASK is the
2610 vectorized condition, otherwise MASK is null. PTR is the base
2611 pointer and OFFSET is the vectorized offset. */
2618 {
2627 tree var;
2629 && (!mask
2635 {
2643 }
2648 {
2650 {
2655 else
2659 gassign *new_stmt
2664 {
2671 }
2674 }
2675 else
2676 {
2679 }
2680 }
2681 else
2682 {
2685 }
2692 {
2700 }
2703 }
2705 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2706 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2707 the scatter store operation. If the store is conditional, MASK is the
2708 unvectorized condition, otherwise MASK is null. */
2710 static void
2715 {
2721 poly_uint64 scatter_off_nunits
2724 /* FIXME: Keep the previous costing way in vect_model_store_cost by
2725 costing N scalar stores, but it should be tweaked to use target
2726 specific costs on related scatter store calls. */
2728 {
2742 "vect_model_store_cost: inside_cost = %d, "
2746 }
2752 {
2755 /* Currently gathers and scatters are only supported for
2756 fixed-length vectors. */
2765 }
2767 {
2770 /* Currently gathers and scatters are only supported for
2771 fixed-length vectors. */
2784 }
2785 else
2801 {
2802 gimple_seq seq;
2808 }
2812 {
2815 }
2823 {
2828 }
2841 {
2843 {
2847 else
2852 }
2854 {
2858 else
2863 }
2864 else
2865 {
2870 }
2873 {
2881 }
2884 {
2892 }
2895 {
2896 tree utype;
2899 {
2900 tree var
2902 gassign *new_stmt
2906 mask_op);
2909 }
2913 else
2917 gassign *new_stmt
2922 {
2928 }
2929 }
2931 gcall *new_stmt
2936 }
2938 }
2940 /* Prepare the base and offset in GS_INFO for vectorization.
2941 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2942 to the vectorized offset argument for the first copy of STMT_INFO.
2943 STMT_INFO is the statement described by GS_INFO and LOOP is the
2944 containing loop. */
2946 static void
2951 {
2955 {
2956 basic_block new_bb;
2960 }
2963 else
2964 {
2965 unsigned ncopies
2970 }
2971 }
2973 /* Prepare to implement a grouped or strided load or store using
2974 the gather load or scatter store operation described by GS_INFO.
2975 STMT_INFO is the load or store statement.
2977 Set *DATAREF_BUMP to the amount that should be added to the base
2978 address after each copy of the vectorized statement. Set *VEC_OFFSET
2979 to an invariant offset vector in which element I has the value
2980 I * DR_STEP / SCALE. */
2982 static void
2984 loop_vec_info loop_vinfo,
2989 {
2994 {
2995 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2996 ivtmp_8 = _31 * 16 (step in bytes);
2997 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2998 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2999 tree loop_len
3001 tree tmp
3004 loop_len);
3006 GSI_SAME_STMT);
3007 }
3008 else
3009 {
3010 tree bump
3015 }
3017 /* The offset given in GS_INFO can have pointer type, so use the element
3018 type of the vector instead. */
3021 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3026 /* Create {0, X, X*2, X*3, ...}. */
3030 }
3032 /* Prepare the pointer IVs which needs to be updated by a variable amount.
3033 Such variable amount is the outcome of .SELECT_VL. In this case, we can
3034 allow each iteration process the flexible number of elements as long as
3035 the number <= vf elments.
3037 Return data reference according to SELECT_VL.
3038 If new statements are needed, insert them before GSI. */
3040 static tree
3044 vect_memory_access_type memory_access_type)
3045 {
3049 /* gather/scatter never reach here. */
3052 /* When we support SELECT_VL pattern, we dynamic adjust
3053 the memory address by .SELECT_VL result.
3055 The result of .SELECT_VL is the number of elements to
3056 be processed of each iteration. So the memory address
3057 adjustment operation should be:
3059 addr = addr + .SELECT_VL (ARG..) * step;
3060 */
3061 tree loop_len
3064 /* Since the outcome of .SELECT_VL is element size, we should adjust
3065 it into bytesize so that it can be used in address pointer variable
3066 amount IVs adjustment. */
3073 }
3075 /* Return the amount that should be added to a vector pointer to move
3076 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3077 being vectorized and MEMORY_ACCESS_TYPE describes the type of
3078 vectorization. */
3080 static tree
3083 vect_memory_access_type memory_access_type,
3085 {
3093 memory_access_type);
3100 }
3102 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3104 static bool
3110 {
3120 /* Multiple types in SLP are handled by creating the appropriate number of
3121 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3122 case of SLP. */
3125 else
3139 /* The encoding uses one stepped pattern for each byte in the word. */
3151 {
3154 {
3159 }
3166 slp_node
3170 }
3174 /* Transform. */
3178 /* Arguments are ready. create the new vector stmt. */
3180 tree vop;
3182 {
3198 else
3200 }
3207 }
3209 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3210 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3211 in a single step. On success, store the binary pack code in
3212 *CONVERT_CODE. */
3214 static bool
3217 {
3222 code_helper code;
3232 }
3234 /* Function vectorizable_call.
3236 Check if STMT_INFO performs a function call that can be vectorized.
3237 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3238 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3239 Return true if STMT_INFO is vectorizable in this way. */
3241 static bool
3246 {
3248 tree vec_dest;
3249 tree scalar_dest;
3250 tree op;
3253 poly_uint64 nunits_in;
3254 poly_uint64 nunits_out;
3260 vect_unknown_def_type };
3268 tree lhs;
3277 /* Is STMT_INFO a vectorizable call? */
3285 /* Handled by vectorizable_load and vectorizable_store. */
3296 /* Process function arguments. */
3301 /* Bail out if the function has more than four arguments, we do not have
3302 interesting builtin functions to vectorize with more than two arguments
3303 except for fma. No arguments is also not good. */
3307 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3310 {
3313 }
3320 {
3322 {
3327 }
3331 {
3336 }
3338 /* We can only handle calls with arguments of the same type. */
3341 {
3346 }
3354 {
3359 }
3360 }
3361 /* If all arguments are external or constant defs, infer the vector type
3362 from the scalar type. */
3368 {
3374 }
3378 {
3383 }
3386 {
3391 }
3393 /* FORNOW */
3402 else
3405 /* We only handle functions that do not read or clobber memory. */
3407 {
3412 }
3414 /* For now, we only vectorize functions if a target specific builtin
3415 is available. TODO -- in some cases, it might be profitable to
3416 insert the calls for pieces of the vector, in order to be able
3417 to vectorize other operations in the loop. */
3422 /* First try using an internal function. */
3430 vectype_in);
3432 /* If that fails, try asking for a target-specific built-in function. */
3434 {
3441 }
3444 {
3446 && !slp_node
3447 && loop_vinfo
3452 {
3453 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3454 { 0, 1, 2, ... vf - 1 } vector. */
3456 }
3464 else
3465 {
3470 }
3471 }
3477 else
3480 /* Sanity check: make sure that at least one copy of the vectorized stmt
3481 needs to be generated. */
3491 {
3497 {
3502 }
3514 {
3518 OPTIMIZE_FOR_SPEED))
3521 OPTIMIZE_FOR_SPEED)))
3522 {
3525 "can't use a fully-masked loop because no"
3528 }
3529 else
3530 {
3531 unsigned int nvectors
3532 = (slp_node
3540 OPTIMIZE_FOR_SPEED))
3543 else
3545 scalar_mask);
3546 }
3547 }
3549 }
3551 /* Transform. */
3556 /* Handle def. */
3564 {
3566 {
3568 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3570 }
3573 }
3575 {
3578 }
3581 {
3586 {
3587 /* Build argument list for the vectorized call. */
3589 {
3595 /* Arguments are ready. Create the new vector stmt. */
3597 {
3600 {
3602 /* Always true for SLP. */
3607 }
3610 {
3613 }
3618 {
3619 /* We don't define any narrowing conditional functions
3620 at present. */
3623 gcall *call
3629 {
3632 }
3638 }
3639 else
3640 {
3642 {
3644 /* Always true for SLP. */
3646 tree len
3649 signed char biasval
3654 }
3656 {
3658 /* Always true for SLP. */
3666 }
3671 else
3678 }
3680 }
3682 }
3689 {
3692 {
3697 }
3699 }
3704 {
3707 signed char biasval
3712 }
3714 {
3720 }
3724 {
3726 tree new_var
3733 }
3735 {
3736 /* We don't define any narrowing conditional functions at
3737 present. */
3745 {
3748 }
3751 half_res);
3753 }
3754 else
3755 {
3759 else
3766 }
3771 }
3773 {
3776 }
3777 }
3779 {
3781 /* We don't define any narrowing conditional functions at present. */
3784 {
3785 /* Build argument list for the vectorized call. */
3788 else
3792 {
3798 /* Arguments are ready. Create the new vector stmt. */
3800 {
3804 {
3808 }
3812 else
3819 }
3821 }
3824 {
3827 {
3831 }
3837 }
3845 }
3851 {
3854 }
3855 }
3856 else
3857 /* No current target implements this case. */
3862 /* The call in STMT might prevent it from being removed in dce.
3863 We however cannot remove it here, due to the way the ssa name
3864 it defines is mapped to the new definition. So just replace
3865 rhs of the statement with something harmless. */
3873 gassign *new_stmt
3878 }
3881 struct simd_call_arg_info
3882 {
3883 tree vectype;
3884 tree op;
3885 HOST_WIDE_INT linear_step;
3889 };
3891 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3892 is linear within simd lane (but not within whole loop), note it in
3893 *ARGINFO. */
3895 static void
3898 {
3910 {
3911 tree t;
3915 {
3930 CASE_CONVERT:
3942 }
3948 {
3955 }
3956 }
3957 }
3959 /* Function vectorizable_simd_clone_call.
3961 Check if STMT_INFO performs a function call that can be vectorized
3962 by calling a simd clone of the function.
3963 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3964 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3965 Return true if STMT_INFO is vectorizable in this way. */
3967 static bool
3971 stmt_vector_for_cost *)
3972 {
3973 tree vec_dest;
3974 tree scalar_dest;
3977 tree vectype;
3978 poly_uint64 nunits;
3991 /* Is STMT a vectorizable call? */
3999 {
4005 }
4031 /* Process function arguments. */
4034 /* Bail out if the function has zero arguments. */
4045 {
4046 simd_call_arg_info thisarginfo;
4047 affine_iv iv;
4061 {
4066 }
4070 {
4071 /* With SLP we determine the vector type of constants/externals
4072 at analysis time, handling conflicts via
4073 vect_maybe_update_slp_op_vectype. At transform time
4074 we have a vector type recorded for SLP. */
4076 || !slp_node
4081 slp_node);
4082 }
4083 else
4086 /* For linear arguments, the analyze phase should have saved
4087 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
4090 {
4094 thisarginfo.simd_lane_linear
4096 /* If loop has been peeled for alignment, we need to adjust it. */
4100 {
4106 thisarginfo.op
4110 }
4111 }
4115 && loop_vinfo
4120 {
4123 }
4128 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4129 linear too. */
4132 && !vec_stmt
4135 && loop_vinfo
4140 }
4148 else
4151 {
4154 /* The number of arguments in the call and the number of parameters in
4155 the simdclone should match. However, when the simdclone is
4156 'inbranch', it could have one more paramater than nargs when using
4157 an inbranch simdclone to call a non-inbranch call, either in a
4158 non-masked loop using a all true constant mask, or inside a masked
4159 loop using it's mask. */
4162 simd_nargs--;
4177 {
4179 {
4210 /* FORNOW */
4214 /* While we can create a traditional data vector from
4215 an incoming integer mode mask we have no good way to
4216 force generate an integer mode mask from a traditional
4217 boolean vector input. */
4225 }
4229 {
4232 }
4236 }
4242 {
4244 SIMD_CLONE_ARG_TYPE_MASK);
4245 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4246 not in a branch, as we'd have to construct an all-true mask. */
4249 }
4251 {
4254 }
4255 }
4264 num_mask_args++;
4267 {
4271 {
4275 slp_node);
4280 }
4284 {
4289 }
4292 {
4295 {
4298 {
4299 /* FORNOW we only have partial support for vector-type masks
4300 that can't hold all of simdlen. */
4303 vect_location,
4304 "in-branch vector clones are not yet"
4307 }
4308 }
4310 {
4313 num_mask_args),
4315 {
4316 /* FORNOW we only have partial support for integer-type masks
4317 that represent the same number of lanes as the
4318 vectorized mask inputs. */
4321 vect_location,
4322 "in-branch vector clones are not yet "
4325 }
4326 }
4327 else
4328 {
4331 vect_location,
4332 "in-branch vector clones not supported"
4335 }
4336 }
4337 }
4343 else
4346 /* If the function isn't const, only allow it in simd loops where user
4347 has asserted that at least nunits consecutive iterations can be
4348 performed using SIMD instructions. */
4353 /* Sanity check: make sure that at least one copy of the vectorized stmt
4354 needs to be generated. */
4358 {
4362 {
4367 }
4368 /* When the original call is pure or const but the SIMD ABI dictates
4369 an aggregate return we will have to use a virtual definition and
4370 in a loop eventually even need to add a virtual PHI. That's
4371 not straight-forward so allow to fix this up via renaming. */
4376 /* ??? For SLP code-gen we end up inserting after the last
4377 vector argument def rather than at the original call position
4378 so automagic virtual operand updating doesn't work. */
4383 {
4385 {
4390 {
4400 }
4410 }
4411 }
4414 {
4418 "can't use a fully-masked loop because a"
4421 }
4425 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4426 dt, slp_node, cost_vec); */
4428 }
4430 /* Transform. */
4435 /* Handle def. */
4441 {
4445 {
4448 }
4449 }
4455 {
4458 }
4459 else
4462 {
4463 poly_uint64 callee_nelements;
4464 poly_uint64 caller_nelements;
4465 /* Build argument list for the vectorized call. */
4468 else
4472 {
4474 tree atype;
4477 {
4484 {
4486 {
4494 {
4501 }
4502 else
4503 {
4507 }
4509 vec_oprnd0
4513 gassign *new_stmt
4515 vec_oprnd0);
4519 }
4520 else
4521 {
4529 else
4532 {
4534 {
4538 op,
4542 }
4543 else
4549 vec_oprnd0);
4550 }
4553 atype))
4554 {
4556 vec_oprnd0);
4557 gassign *new_stmt
4559 vec_oprnd0);
4563 }
4564 else
4566 else
4567 {
4569 gassign *new_stmt
4571 vec_oprnd0);
4575 }
4576 }
4577 }
4581 {
4590 {
4592 {
4593 /* The mask type has fewer elements than simdlen. */
4595 /* FORNOW */
4597 }
4599 {
4600 /* The SIMD clone function has the same number of
4601 elements as the current function. */
4603 {
4607 op,
4610 }
4614 {
4615 vec_loop_masks *loop_masks
4617 tree loop_mask
4621 vec_oprnd0
4625 gsi);
4627 loop_mask });
4629 }
4630 vec_oprnd0
4634 gassign *new_stmt
4636 vec_oprnd0);
4640 }
4641 else
4642 {
4643 /* The mask type has more elements than simdlen. */
4645 /* FORNOW */
4647 }
4648 }
4649 }
4651 {
4653 /* Guess the number of lanes represented by atype. */
4654 poly_uint64 atype_subparts
4656 num_mask_args);
4659 {
4661 {
4665 op,
4668 }
4671 {
4672 /* The mask argument has fewer elements than the
4673 input vector. */
4674 /* FORNOW */
4676 }
4679 {
4680 /* The vector mask argument matches the input
4681 in the number of lanes, but not necessarily
4682 in the mode. */
4687 vec_oprnd0);
4688 gassign *new_stmt
4690 vec_oprnd0);
4694 {
4695 new_stmt
4697 NOP_EXPR,
4698 gimple_assign_lhs
4702 }
4704 }
4705 else
4706 {
4707 /* The mask argument has more elements than the
4708 input vector. */
4709 /* FORNOW */
4711 }
4712 }
4713 }
4714 else
4723 {
4724 gimple_seq stmts;
4729 {
4730 basic_block new_bb;
4734 }
4736 {
4739 }
4744 enum tree_code code
4749 poly_widest_int cst
4754 gassign *new_stmt
4759 UNKNOWN_LOCATION);
4762 }
4763 else
4764 {
4765 enum tree_code code
4770 poly_widest_int cst
4775 gassign *new_stmt
4780 }
4790 }
4791 }
4796 {
4799 tree mask;
4801 SIMD_CLONE_ARG_TYPE_MASK);
4807 {
4809 {
4813 }
4814 else
4819 {
4820 /* This means we are dealing with integer mask modes.
4821 First convert to an integer type with the same size as
4822 the current vector type. */
4823 unsigned HOST_WIDE_INT intermediate_size
4825 tree mid_int_type =
4828 new_stmt
4830 mask);
4832 /* Then zero-extend to the mask mode. */
4835 }
4837 {
4839 integer_one_node);
4841 integer_zero_node);
4845 }
4846 else
4854 }
4855 }
4859 {
4866 else
4869 }
4874 {
4876 {
4884 {
4885 tree t;
4887 {
4891 }
4892 else
4903 else
4905 }
4910 }
4912 {
4921 {
4926 {
4930 tem);
4935 }
4937 }
4938 else
4943 new_stmt
4952 else
4955 }
4957 {
4964 }
4966 {
4968 new_stmt
4971 }
4972 }
4978 else
4980 }
4983 {
4986 }
4989 /* Mark the clone as no longer being a candidate for GC. */
4992 /* The call in STMT might prevent it from being removed in dce.
4993 We however cannot remove it here, due to the way the ssa name
4994 it defines is mapped to the new definition. So just replace
4995 rhs of the statement with something harmless. */
5002 {
5006 }
5007 else
5013 }
5016 /* Function vect_gen_widened_results_half
5018 Create a vector stmt whose code, type, number of arguments, and result
5019 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5020 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5021 In the case that CODE is a CALL_EXPR, this means that a call to DECL
5022 needs to be created (DECL is a function-decl of a target-builtin).
5023 STMT_INFO is the original scalar stmt that we are vectorizing. */
5029 stmt_vec_info stmt_info)
5030 {
5032 tree new_temp;
5034 /* Generate half of the widened result: */
5043 }
5046 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5047 For multi-step conversions store the resulting vectors and call the function
5048 recursively. When NARROW_SRC_P is true, there's still a conversion after
5049 narrowing, don't store the vectors in the SLP_NODE or in vector info of
5050 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5052 static void
5055 stmt_vec_info stmt_info,
5060 {
5067 {
5068 /* Create demotion operation. */
5076 /* Store the resulting vector for next recursive call,
5077 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5079 else
5080 {
5081 /* This is the last step of the conversion sequence. Store the
5082 vectors in SLP_NODE or in vector info of the scalar statement
5083 (or in STMT_VINFO_RELATED_STMT chain). */
5086 else
5088 }
5089 }
5091 /* For multi-step demotion operations we first generate demotion operations
5092 from the source type to the intermediate types, and then combine the
5093 results (stored in VEC_OPRNDS) in demotion operation to the destination
5094 type. */
5096 {
5097 /* At each level of recursion we have half of the operands we had at the
5098 previous level. */
5104 narrow_src_p);
5105 }
5108 }
5111 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5112 and VEC_OPRNDS1, for a binary operation associated with scalar statement
5113 STMT_INFO. For multi-step conversions store the resulting vectors and
5114 call the function recursively. */
5116 static void
5122 code_helper ch1,
5124 {
5132 {
5135 else
5138 /* Generate the two halves of promotion operation. */
5141 stmt_info);
5144 stmt_info);
5146 {
5149 }
5150 else
5151 {
5154 }
5156 /* Store the results for the next step. */
5159 }
5163 }
5165 /* Create vectorized promotion stmts for widening stmts using only half the
5166 potential vector size for input. */
5167 static void
5173 code_helper code1,
5175 {
5185 {
5191 /* Widen the first vector input. */
5197 {
5198 /* Widen the second vector input. */
5202 /* Perform the operation. With both vector inputs widened. */
5204 }
5205 else
5206 {
5207 /* Perform the operation. With the single vector input widened. */
5209 }
5215 /* Store the results for the next step. */
5217 }
5221 }
5224 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5225 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5226 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5227 Return true if STMT_INFO is vectorizable in this way. */
5229 static bool
5234 {
5236 tree scalar_dest;
5242 tree new_temp;
5245 poly_uint64 nunits_in;
5246 poly_uint64 nunits_out;
5250 /* For conversions between floating point and integer, there're 2 NARROW
5251 cases. NARROW_SRC is for FLOAT_EXPR, means
5252 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5253 This is safe when the range of the source integer can fit into the lower
5254 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5255 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5256 For other conversions, when there's narrowing, NARROW_DST is used as
5257 default. */
5261 tree vop0;
5269 /* Is STMT a vectorizable conversion? */
5290 {
5293 }
5295 {
5298 }
5299 else
5312 /* Check types of lhs and rhs. */
5317 /* Check the operands of the operation. */
5321 {
5326 }
5341 {
5344 "type conversion to/from bit-precision unsupported."
5347 }
5350 {
5357 tree vectype1_in;
5360 {
5365 }
5366 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5367 OP1. */
5370 }
5372 /* If op0 is an external or constant def, infer the vector type
5373 from the scalar type. */
5379 {
5385 }
5389 {
5392 "can't convert between boolean and non "
5396 }
5403 else
5407 else
5408 {
5411 }
5413 /* Multiple types in SLP are handled by creating the appropriate number of
5414 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5415 case of SLP. */
5420 else
5423 /* Sanity check: make sure that at least one copy of the vectorized stmt
5424 needs to be generated. */
5430 opt_scalar_mode rhs_mode_iter;
5432 /* Supportable by target? */
5434 {
5443 {
5446 }
5448 /* For conversions between float and integer types try whether
5449 we can use intermediate signed integer types to support the
5450 conversion. */
5454 {
5458 scalar_mode intermediate_mode;
5460 {
5463 }
5464 else
5465 {
5470 }
5473 opt_scalar_mode mode_iter;
5475 {
5481 scalar_mode cvt_mode;
5486 cvt_type = build_nonstandard_integer_type
5489 /* Check if the intermediate type can hold OP0's range.
5490 When converting from float to integer this is not necessary
5491 because values that do not fit the (smaller) target type are
5492 unspecified anyway. */
5494 {
5505 }
5508 /* This should only happened for SLP as long as loop vectorizer
5509 only supports same-sized vector. */
5513 vectype_out,
5516 cvt_type,
5522 }
5525 {
5526 multi_step_cvt++;
5532 }
5533 }
5534 /* FALLTHRU */
5535 unsupported:
5543 {
5552 }
5557 {
5558 /* Binary widening operation can only be supported directly by the
5559 architecture. */
5562 }
5570 {
5575 cvt_type
5582 {
5589 }
5596 else
5600 cvt_type,
5604 {
5607 }
5608 }
5615 else
5616 {
5617 multi_step_cvt++;
5620 }
5634 {
5635 cvt_type
5643 else
5649 }
5650 /* If op0 can be represented with low precision integer,
5651 truncate it to cvt_type and the do FLOAT_EXPR. */
5653 {
5658 cvt_type
5676 {
5680 }
5681 }
5687 }
5690 {
5694 {
5699 }
5702 {
5707 }
5709 {
5711 /* The final packing step produces one vector result per copy. */
5712 unsigned int nvectors
5716 widen_arith);
5717 }
5718 else
5719 {
5721 /* The initial unpacking step produces two vector results
5722 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5723 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5724 unsigned int nvectors
5725 = (slp_node
5730 widen_arith);
5731 }
5734 }
5736 /* Transform. */
5742 {
5747 }
5749 /* In case of multi-step conversion, we first generate conversion operations
5750 to the intermediate types, and then from that types to the final one.
5751 We create vector destinations for the intermediate type (TYPES) received
5752 from supportable_*_operation, and store them in the correct order
5753 for future use in vect_create_vectorized_*_stmts (). */
5755 bool widen_or_narrow_float_p
5758 widen_or_narrow_float_p
5763 {
5766 {
5768 intermediate_type);
5770 }
5771 }
5775 widen_or_narrow_float_p
5780 {
5782 ;
5784 {
5788 }
5789 }
5792 {
5796 /* vec_dest is intermediate type operand when multi_step_cvt. */
5798 {
5801 }
5804 {
5805 /* Arguments are ready, create the new vector stmt. */
5808 {
5815 }
5823 else
5825 }
5829 /* In case the vectorization factor (VF) is bigger than the number
5830 of elements that we can fit in a vectype (nunits), we have to
5831 generate more than one vector stmt - i.e - we need to "unroll"
5832 the vector stmt by a factor VF/nunits. */
5838 {
5843 }
5844 /* Arguments are ready. Create the new vector stmts. */
5846 {
5850 {
5853 }
5857 op_type);
5858 else
5863 }
5866 {
5869 {
5873 }
5874 else
5879 else
5881 }
5886 /* In case the vectorization factor (VF) is bigger than the number
5887 of elements that we can fit in a vectype (nunits), we have to
5888 generate more than one vector stmt - i.e - we need to "unroll"
5889 the vector stmt by a factor VF/nunits. */
5892 /* Arguments are ready. Create the new vector stmts. */
5895 {
5900 }
5903 multi_step_cvt,
5907 /* After demoting op0 to cvt_type, convert it to dest. */
5909 {
5911 {
5912 /* Arguments are ready, create the new vector stmt. */
5914 gimple *new_stmt
5920 /* This is the last step of the conversion sequence. Store the
5921 vectors in SLP_NODE or in vector info of the scalar statement
5922 (or in STMT_VINFO_RELATED_STMT chain). */
5925 else
5927 }
5928 }
5930 }
5939 }
5941 /* Return true if we can assume from the scalar form of STMT_INFO that
5942 neither the scalar nor the vector forms will generate code. STMT_INFO
5943 is known not to involve a data reference. */
5945 bool
5947 {
5963 }
5965 /* Function vectorizable_assignment.
5967 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5968 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5969 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5970 Return true if STMT_INFO is vectorizable in this way. */
5972 static bool
5977 {
5978 tree vec_dest;
5979 tree scalar_dest;
5980 tree op;
5982 tree new_temp;
5988 tree vop;
5991 tree vectype_in;
6000 /* Is vectorizable assignment? */
6021 /* Multiple types in SLP are handled by creating the appropriate number of
6022 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6023 case of SLP. */
6026 else
6031 slp_tree slp_op;
6034 {
6039 }
6043 /* We can handle NOP_EXPR conversions that do not change the number
6044 of elements or the vector size. */
6047 && (!vectype_in
6054 {
6057 "can't convert between boolean and non "
6061 }
6063 /* We do not handle bit-precision changes. */
6070 /* But a conversion that does not change the bit-pattern is ok. */
6078 {
6081 "type conversion to/from bit-precision "
6084 }
6087 {
6090 {
6095 }
6100 cost_vec);
6102 }
6104 /* Transform. */
6108 /* Handle def. */
6111 /* Handle use. */
6114 /* Arguments are ready. create the new vector stmt. */
6116 {
6126 else
6128 }
6134 }
6137 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6138 either as shift by a scalar or by a vector. */
6140 bool
6142 {
6144 machine_mode vec_mode;
6145 optab optab;
6147 tree vectype;
6156 {
6162 }
6170 }
6173 /* Function vectorizable_shift.
6175 Check if STMT_INFO performs a shift operation that can be vectorized.
6176 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6177 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6178 Return true if STMT_INFO is vectorizable in this way. */
6180 static bool
6185 {
6186 tree vec_dest;
6187 tree scalar_dest;
6190 tree vectype;
6193 machine_mode vec_mode;
6194 tree new_temp;
6195 optab optab;
6197 machine_mode optab_op2_mode;
6200 poly_uint64 nunits_in;
6201 poly_uint64 nunits_out;
6202 tree vectype_out;
6203 tree op1_vectype;
6222 /* Is STMT a vectorizable binary/unary operation? */
6239 {
6244 }
6246 slp_tree slp_op0;
6249 {
6254 }
6255 /* If op0 is an external or constant def, infer the vector type
6256 from the scalar type. */
6262 {
6267 }
6274 stmt_vec_info op1_def_stmt_info;
6275 slp_tree slp_op1;
6278 {
6283 }
6285 /* Multiple types in SLP are handled by creating the appropriate number of
6286 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6287 case of SLP. */
6290 else
6295 /* Determine whether the shift amount is a vector, or scalar. If the
6296 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6306 {
6307 /* In SLP, need to check whether the shift count is the same,
6308 in loops if it is a constant or invariant, it is always
6309 a scalar shift. */
6311 {
6313 stmt_vec_info slpstmt_info;
6316 {
6320 }
6322 /* For internal SLP defs we have to make sure we see scalar stmts
6323 for all vector elements.
6324 ??? For different vectors we could resort to a different
6325 scalar shift operand but code-generation below simply always
6326 takes the first. */
6331 }
6333 /* If the shift amount is computed by a pattern stmt we cannot
6334 use the scalar amount directly thus give up and use a vector
6335 shift. */
6338 }
6339 else
6340 {
6345 }
6347 /* Vector shifted by vector. */
6350 {
6358 slp_op1);
6359 incompatible_op1_vectype_p
6365 && (!slp_node
6368 {
6371 "unusable type for last operand in"
6374 }
6375 }
6376 /* See if the machine has a vector shifted by scalar insn and if not
6377 then see if it has a vector shifted by vector insn. */
6378 else
6379 {
6383 {
6387 }
6388 else
6389 {
6394 {
6404 slp_op1);
6406 /* Unlike the other binary operators, shifts/rotates have
6407 the rhs being int, instead of the same type as the lhs,
6408 so make sure the scalar is the right type if we are
6409 dealing with vectors of long long/long/short/char. */
6410 incompatible_op1_vectype_p
6411 = (!op1_vectype
6416 {
6419 "unusable type for last operand in"
6422 }
6423 }
6424 }
6425 }
6427 /* Supportable by target? */
6429 {
6434 }
6438 {
6443 }
6444 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6449 {
6453 && (!incompatible_op1_vectype_p
6455 && !vect_maybe_update_slp_op_vectype
6458 {
6463 }
6464 /* Now adjust the constant shift amount in place. */
6466 && incompatible_op1_vectype_p
6468 {
6471 {
6477 }
6478 }
6484 }
6486 /* Transform. */
6493 {
6499 }
6501 /* Handle def. */
6505 {
6506 /* Vector shl and shr insn patterns can be defined with scalar
6507 operand 2 (shift operand). In this case, use constant or loop
6508 invariant op1 directly, without extending it to vector mode
6509 first. */
6512 {
6519 /* Store vec_oprnd1 for every vector stmt to be created.
6520 We check during the analysis that all the shift arguments
6521 are the same.
6522 TODO: Allow different constants for different vector
6523 stmts generated for an SLP instance. */
6527 }
6528 }
6530 {
6532 {
6533 /* If the argument was the same in all lanes create
6534 the correctly typed vector shift amount directly. */
6543 }
6545 /* The constant shift amount has been adjusted in place. */
6546 ;
6547 else
6549 }
6551 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6552 (a special case for certain kind of vector shifts); otherwise,
6553 operand 1 should be of a vector type (the usual case). */
6558 /* Arguments are ready. Create the new vector stmt. */
6560 {
6561 /* For internal defs where we need to use a scalar shift arg
6562 extract the first lane. */
6564 {
6567 gassign *new_stmt
6570 vop1,
6572 bitsize_zero_node));
6575 }
6576 else
6584 else
6586 }
6595 }
6597 /* Function vectorizable_operation.
6599 Check if STMT_INFO performs a binary, unary or ternary operation that can
6600 be vectorized.
6601 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6602 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6603 Return true if STMT_INFO is vectorizable in this way. */
6605 static bool
6610 {
6611 tree vec_dest;
6612 tree scalar_dest;
6614 tree vectype;
6617 machine_mode vec_mode;
6618 tree new_temp;
6620 optab optab;
6625 poly_uint64 nunits_in;
6626 poly_uint64 nunits_out;
6627 tree vectype_out;
6643 /* Is STMT a vectorizable binary/unary operation? */
6648 /* Loads and stores are handled in vectorizable_{load,store}. */
6654 /* Shifts are handled in vectorizable_shift. */
6661 /* Comparisons are handled in vectorizable_comparison. */
6665 /* Conditions are handled in vectorizable_condition. */
6669 /* For pointer addition and subtraction, we should use the normal
6670 plus and minus for the vector operation. */
6676 /* Support only unary or binary operations. */
6679 {
6683 op_type);
6685 }
6690 /* Most operations cannot handle bit-precision types without extra
6691 truncations. */
6695 /* Exception are bitwise binary operations. */
6699 {
6704 }
6706 slp_tree slp_op0;
6709 {
6714 }
6717 /* If op0 is an external or constant def, infer the vector type
6718 from the scalar type. */
6720 {
6721 /* For boolean type we cannot determine vectype by
6722 invariant value (don't know whether it is a vector
6723 of booleans or vector of integers). We use output
6724 vectype because operations on boolean don't change
6725 type. */
6727 {
6729 {
6734 }
6736 }
6737 else
6739 slp_node);
6740 }
6744 {
6751 }
6761 {
6764 {
6769 }
6775 }
6777 {
6780 {
6785 }
6791 }
6793 /* Multiple types in SLP are handled by creating the appropriate number of
6794 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6795 case of SLP. */
6797 {
6800 }
6801 else
6802 {
6805 }
6809 /* Reject attempts to combine mask types with nonmask types, e.g. if
6810 we have an AND between a (nonmask) boolean loaded from memory and
6811 a (mask) boolean result of a comparison.
6813 TODO: We could easily fix these cases up using pattern statements. */
6817 {
6822 }
6824 /* Supportable by target? */
6829 else
6830 {
6833 {
6838 }
6841 }
6845 {
6849 /* When vec_mode is not a vector mode and we verified ops we
6850 do not have to lower like AND are natively supported let
6851 those through even when the mode isn't word_mode. For
6852 ops we have to lower the lowering code assumes we are
6853 dealing with word_mode. */
6857 /* Check only during analysis. */
6859 {
6863 }
6868 }
6876 /* If operating on inactive elements could generate spurious traps,
6877 we need to restrict the operation to active lanes. Note that this
6878 specifically doesn't apply to unhoisted invariants, since they
6879 operate on the same value for every lane.
6881 Similarly, if this operation is part of a reduction, a fully-masked
6882 loop should only change the active lanes of the reduction chain,
6883 keeping the inactive lanes as-is. */
6888 {
6892 {
6895 OPTIMIZE_FOR_SPEED))
6900 OPTIMIZE_FOR_SPEED))
6903 else
6904 {
6907 "can't use a fully-masked loop because no"
6910 }
6911 }
6913 /* Put types on constant and invariant SLP children. */
6918 {
6923 }
6930 {
6931 /* The above vect_model_simple_cost call handles constants
6932 in the prologue and (mis-)costs one of the stmts as
6933 vector stmt. See below for the actual lowering that will
6934 be applied. */
6935 unsigned n
6938 {
6949 /* Bit operations do not have extra cost and are accounted
6950 as vector stmt by vect_model_simple_cost. */
6953 }
6955 {
6956 /* We also need to materialize two large constants. */
6961 }
6962 }
6964 }
6966 /* Transform. */
6975 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6976 vectors with unsigned elements, but the result is signed. So, we
6977 need to compute the MINUS_EXPR into vectype temporary and
6978 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6981 {
6984 }
6985 /* Handle def. */
6986 else
6989 /* In case the vectorization factor (VF) is bigger than the number
6990 of elements that we can fit in a vectype (nunits), we have to generate
6991 more than one vector stmt - i.e - we need to "unroll" the
6992 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6993 from one copy of the vector stmt to the next, in the field
6994 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6995 stages to find the correct vector defs to be used when vectorizing
6996 stmts that use the defs of the current stmt. The example below
6997 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6998 we need to create 4 vectorized stmts):
7000 before vectorization:
7001 RELATED_STMT VEC_STMT
7002 S1: x = memref - -
7003 S2: z = x + 1 - -
7005 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
7006 there):
7007 RELATED_STMT VEC_STMT
7008 VS1_0: vx0 = memref0 VS1_1 -
7009 VS1_1: vx1 = memref1 VS1_2 -
7010 VS1_2: vx2 = memref2 VS1_3 -
7011 VS1_3: vx3 = memref3 - -
7012 S1: x = load - VS1_0
7013 S2: z = x + 1 - -
7015 step2: vectorize stmt S2 (done here):
7016 To vectorize stmt S2 we first need to find the relevant vector
7017 def for the first operand 'x'. This is, as usual, obtained from
7018 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
7019 that defines 'x' (S1). This way we find the stmt VS1_0, and the
7020 relevant vector def 'vx0'. Having found 'vx0' we can generate
7021 the vector stmt VS2_0, and as usual, record it in the
7022 STMT_VINFO_VEC_STMT of stmt S2.
7023 When creating the second copy (VS2_1), we obtain the relevant vector
7024 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
7025 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
7026 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
7027 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
7028 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
7029 chain of stmts and pointers:
7030 RELATED_STMT VEC_STMT
7031 VS1_0: vx0 = memref0 VS1_1 -
7032 VS1_1: vx1 = memref1 VS1_2 -
7033 VS1_2: vx2 = memref2 VS1_3 -
7034 VS1_3: vx3 = memref3 - -
7035 S1: x = load - VS1_0
7036 VS2_0: vz0 = vx0 + v1 VS2_1 -
7037 VS2_1: vz1 = vx1 + v1 VS2_2 -
7038 VS2_2: vz2 = vx2 + v1 VS2_3 -
7039 VS2_3: vz3 = vx3 + v1 - -
7040 S2: z = x + 1 - VS2_0 */
7044 /* Arguments are ready. Create the new vector stmt. */
7046 {
7053 {
7054 /* Lower the operation. This follows vector lowering. */
7057 tree word_type
7061 tree high_bits
7070 {
7088 else
7093 {
7099 }
7108 }
7109 else
7110 {
7127 }
7129 signs);
7139 }
7141 {
7142 tree mask;
7146 else
7147 /* Dummy mask. */
7157 {
7158 /* Perform the operation on active elements only and take
7159 inactive elements from the reduction chain input. */
7162 }
7163 else
7164 {
7168 }
7170 {
7173 signed char biasval
7178 }
7179 gcall *call
7182 vops);
7188 }
7189 else
7190 {
7192 /* When combining two masks check if either of them is elsewhere
7193 combined with a loop mask, if that's the case we can mark that the
7194 new combined mask doesn't need to be combined with a loop mask. */
7198 {
7200 ncopies}))
7201 {
7207 }
7210 ncopies }))
7211 {
7217 }
7218 }
7227 /* Enter the combined value into the vector cond hash so we don't
7228 AND it with a loop mask again. */
7231 }
7234 {
7237 new_temp);
7242 }
7246 else
7248 }
7258 }
7260 /* A helper function to ensure data reference DR_INFO's base alignment. */
7262 static void
7264 {
7265 /* Alignment is only analyzed for the first element of a DR group,
7266 use that to look at base alignment we need to enforce. */
7273 {
7276 // We should only be able to increase the alignment of a base object if
7277 // we know what its new alignment should be at compile time.
7284 {
7287 }
7289 }
7290 }
7293 /* Function get_group_alias_ptr_type.
7295 Return the alias type for the group starting at FIRST_STMT_INFO. */
7297 static tree
7299 {
7305 {
7309 {
7314 }
7316 }
7318 }
7321 /* Function scan_operand_equal_p.
7323 Helper function for check_scan_store. Compare two references
7324 with .GOMP_SIMD_LANE bases. */
7326 static bool
7328 {
7333 {
7334 machine_mode mode;
7344 {
7350 {
7355 }
7356 }
7357 }
7364 {
7368 {
7371 {
7374 {
7380 {
7383 }
7384 }
7386 {
7389 }
7392 {
7396 }
7405 }
7409 }
7410 }
7412 }
7416 /* Normal permutation. */
7417 scan_store_kind_perm,
7419 /* Whole vector left shift permutation with zero init. */
7420 scan_store_kind_lshift_zero,
7422 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7423 scan_store_kind_lshift_cond
7424 };
7426 /* Function check_scan_store.
7428 Verify if we can perform the needed permutations or whole vector shifts.
7429 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7430 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7431 to do at each step. */
7433 static int
7436 {
7448 {
7454 {
7457 }
7458 else
7459 {
7464 }
7467 {
7472 {
7476 /* Whole vector shifts shift in zeros, so if init is all zero
7477 constant, there is no need to do anything further. */
7481 {
7486 }
7487 }
7489 }
7491 {
7496 }
7497 }
7500 }
7503 /* Function check_scan_store.
7505 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7507 static bool
7510 vect_memory_access_type memory_access_type)
7511 {
7514 tree ref_type;
7518 || mask
7530 {
7535 }
7537 /* We need to pattern match code built by OpenMP lowering and simplified
7538 by following optimizations into something we can handle.
7539 #pragma omp simd reduction(inscan,+:r)
7540 for (...)
7541 {
7542 r += something ();
7543 #pragma omp scan inclusive (r)
7544 use (r);
7545 }
7546 shall have body with:
7547 // Initialization for input phase, store the reduction initializer:
7548 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7549 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7550 D.2042[_21] = 0;
7551 // Actual input phase:
7552 ...
7553 r.0_5 = D.2042[_20];
7554 _6 = _4 + r.0_5;
7555 D.2042[_20] = _6;
7556 // Initialization for scan phase:
7557 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7558 _26 = D.2043[_25];
7559 _27 = D.2042[_25];
7560 _28 = _26 + _27;
7561 D.2043[_25] = _28;
7562 D.2042[_25] = _28;
7563 // Actual scan phase:
7564 ...
7565 r.1_8 = D.2042[_20];
7566 ...
7567 The "omp simd array" variable D.2042 holds the privatized copy used
7568 inside of the loop and D.2043 is another one that holds copies of
7569 the current original list item. The separate GOMP_SIMD_LANE ifn
7570 kinds are there in order to allow optimizing the initializer store
7571 and combiner sequence, e.g. if it is originally some C++ish user
7572 defined reduction, but allow the vectorizer to pattern recognize it
7573 and turn into the appropriate vectorized scan.
7575 For exclusive scan, this is slightly different:
7576 #pragma omp simd reduction(inscan,+:r)
7577 for (...)
7578 {
7579 use (r);
7580 #pragma omp scan exclusive (r)
7581 r += something ();
7582 }
7583 shall have body with:
7584 // Initialization for input phase, store the reduction initializer:
7585 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7586 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7587 D.2042[_21] = 0;
7588 // Actual input phase:
7589 ...
7590 r.0_5 = D.2042[_20];
7591 _6 = _4 + r.0_5;
7592 D.2042[_20] = _6;
7593 // Initialization for scan phase:
7594 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7595 _26 = D.2043[_25];
7596 D.2044[_25] = _26;
7597 _27 = D.2042[_25];
7598 _28 = _26 + _27;
7599 D.2043[_25] = _28;
7600 // Actual scan phase:
7601 ...
7602 r.1_8 = D.2044[_20];
7603 ... */
7606 {
7607 /* Match the D.2042[_21] = 0; store above. Just require that
7608 it is a constant or external definition store. */
7610 {
7611 fail_init:
7616 }
7626 /* These stores can be vectorized normally. */
7628 }
7631 {
7632 fail:
7637 }
7646 bool inscan_var_store
7650 {
7652 {
7653 use_operand_p use_p;
7654 imm_use_iterator iter;
7656 {
7663 || other_store_stmt
7667 }
7673 }
7674 }
7676 {
7677 use_operand_p use_p;
7678 imm_use_iterator iter;
7680 {
7687 }
7688 }
7689 else
7699 /* For pointer addition, we should use the normal plus for the vector
7700 operation. */
7702 {
7710 }
7738 {
7744 tree lrhs;
7747 else
7749 use_operand_p use_p;
7750 imm_use_iterator iter;
7752 {
7759 }
7760 }
7768 stmt_vec_info other_store_stmt_info
7781 {
7785 }
7796 {
7807 }
7826 {
7831 }
7839 /* The IL is as expected, now check if we can actually vectorize it.
7840 Inclusive scan:
7841 _26 = D.2043[_25];
7842 _27 = D.2042[_25];
7843 _28 = _26 + _27;
7844 D.2043[_25] = _28;
7845 D.2042[_25] = _28;
7846 should be vectorized as (where _40 is the vectorized rhs
7847 from the D.2042[_21] = 0; store):
7848 _30 = MEM <vector(8) int> [(int *)&D.2043];
7849 _31 = MEM <vector(8) int> [(int *)&D.2042];
7850 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7851 _33 = _31 + _32;
7852 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7853 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7854 _35 = _33 + _34;
7855 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7856 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7857 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7858 _37 = _35 + _36;
7859 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7860 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7861 _38 = _30 + _37;
7862 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7863 MEM <vector(8) int> [(int *)&D.2043] = _39;
7864 MEM <vector(8) int> [(int *)&D.2042] = _38;
7865 Exclusive scan:
7866 _26 = D.2043[_25];
7867 D.2044[_25] = _26;
7868 _27 = D.2042[_25];
7869 _28 = _26 + _27;
7870 D.2043[_25] = _28;
7871 should be vectorized as (where _40 is the vectorized rhs
7872 from the D.2042[_21] = 0; store):
7873 _30 = MEM <vector(8) int> [(int *)&D.2043];
7874 _31 = MEM <vector(8) int> [(int *)&D.2042];
7875 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7876 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7877 _34 = _32 + _33;
7878 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7879 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7880 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7881 _36 = _34 + _35;
7882 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7883 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7884 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7885 _38 = _36 + _37;
7886 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7887 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7888 _39 = _30 + _38;
7889 _50 = _31 + _39;
7890 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7891 MEM <vector(8) int> [(int *)&D.2044] = _39;
7892 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7903 }
7906 /* Function vectorizable_scan_store.
7908 Helper of vectorizable_score, arguments like on vectorizable_store.
7909 Handle only the transformation, checking is done in check_scan_store. */
7911 static bool
7915 {
7930 bool inscan_var_store
7934 {
7935 use_operand_p use_p;
7936 imm_use_iterator iter;
7938 {
7944 }
7945 }
7966 {
7970 }
7985 {
7992 else
7993 {
7998 }
8002 {
8009 }
8010 else
8012 }
8033 {
8044 {
8048 dataref_offset);
8054 }
8058 {
8061 (zero_vec
8071 {
8072 /* Whole vector shift shifted in zero bits, but if *init
8073 is not initializer_zerop, we need to replace those elements
8074 with elements from vec_oprnd1. */
8087 }
8089 /* For exclusive scan, perform the perms[i] permutation once
8090 more. */
8094 {
8098 }
8106 }
8114 /* For exclusive scan, new_temp computed above is the exclusive scan
8115 prefix sum. Turn it into inclusive prefix sum for the broadcast
8116 of the last element into orig. */
8118 {
8123 }
8132 {
8135 dataref_offset);
8140 }
8141 }
8145 {
8151 dataref_offset);
8156 }
8158 }
8161 /* Function vectorizable_store.
8163 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8164 that can be vectorized.
8165 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8166 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8167 Return true if STMT_INFO is vectorizable in this way. */
8169 static bool
8174 {
8175 tree data_ref;
8177 tree elem_type;
8180 machine_mode vec_mode;
8181 tree dummy;
8189 stmt_vec_info first_stmt_info;
8195 tree aggr_type;
8196 gather_scatter_info gs_info;
8197 poly_uint64 vf;
8198 vec_load_store_type vls_type;
8199 tree ref_type;
8208 /* Is vectorizable store? */
8212 {
8225 }
8226 else
8227 {
8238 mask_index = vect_slp_child_index_for_operand
8244 }
8246 /* Cannot have hybrid store SLP -- that would mean storing to the
8247 same location twice. */
8254 {
8257 }
8258 else
8261 /* Multiple types in SLP are handled by creating the appropriate number of
8262 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8263 case of SLP. */
8266 else
8271 /* FORNOW. This restriction should be relaxed. */
8273 {
8278 }
8280 tree op;
8281 slp_tree op_node;
8292 vect_memory_access_type memory_access_type;
8295 poly_int64 poffset;
8296 internal_fn lanes_ifn;
8304 {
8306 {
8311 }
8315 {
8320 }
8324 {
8329 }
8330 }
8331 else
8332 {
8333 /* FORNOW. In some cases can vectorize even if data-type not supported
8334 (e.g. - array initialization with 0). */
8337 }
8344 {
8348 }
8349 else
8350 {
8354 }
8357 {
8359 memory_access_type))
8361 }
8365 {
8373 mask);
8377 vectype))
8378 {
8383 }
8394 /* As function vect_transform_stmt shows, for interleaving stores
8395 the whole chain is vectorized when the last store in the chain
8396 is reached, the other stores in the group are skipped. So we
8397 want to only cost the last one here, but it's not trivial to
8398 get the last, as it's equivalent to use the first one for
8399 costing, use the first one instead. */
8401 && !slp
8404 }
8407 /* Transform. */
8412 {
8416 }
8418 {
8422 {
8433 "vect_model_store_cost: inside_cost = %d, "
8438 }
8440 }
8443 {
8444 /* FORNOW */
8448 {
8450 /* VEC_NUM is the number of vect stmts to be created for this
8451 group. */
8458 }
8459 else
8460 /* VEC_NUM is the number of vect stmts to be created for this
8461 group. */
8465 }
8466 else
8471 ncopies);
8473 /* Check if we need to update prologue cost for invariant,
8474 and update it accordingly if so. If it's not for
8475 interleaving store, we can just check vls_type; but if
8476 it's for interleaving store, need to check the def_type
8477 of the stored value since the current vls_type is just
8478 for first_stmt_info. */
8480 {
8485 {
8491 }
8496 };
8500 {
8502 gimple_stmt_iterator incr_gsi;
8505 tree offvar;
8506 tree ivstep;
8507 tree running_off;
8510 tree dr_offset;
8512 /* Checked by get_load_store_type. */
8519 stride_base
8520 = fold_build_pointer_plus
8527 /* For a store with loop-invariant (but other than power-of-2)
8528 stride (i.e. not a grouped access) like so:
8530 for (i = 0; i < n; i += stride)
8531 array[i] = ...;
8533 we generate a new induction variable and new stores from
8534 the components of the (vectorized) rhs:
8536 for (j = 0; ; j += VF*stride)
8537 vectemp = ...;
8538 tmp1 = vectemp[0];
8539 array[j] = tmp1;
8540 tmp2 = vectemp[1];
8541 array[j + stride] = tmp2;
8542 ...
8543 */
8550 {
8553 {
8559 /* First check if vec_extract optab doesn't support extraction
8560 of vector elts directly. */
8562 machine_mode vmode;
8569 {
8570 /* Try to avoid emitting an extract of vector elements
8571 by performing the extracts using an integer type of the
8572 same size, extracting from a vector of those and then
8573 re-interpreting it as the original vector type if
8574 supported. */
8575 unsigned lsize
8578 /* If we can't construct such a vector fall back to
8579 element extracts from the original vector type and
8580 element size stores. */
8588 {
8593 }
8594 /* Else fall back to vector extraction anyway.
8595 Fewer stores are more important than avoiding spilling
8596 of the vector we extract from. Compared to the
8597 construction case in vectorizable_load no store-forwarding
8598 issue exists here for reasonable archs. */
8599 }
8600 }
8603 {
8605 dr_alignment_support dr_align
8607 mis_align);
8610 {
8617 }
8618 }
8621 }
8624 {
8638 }
8643 /* For costing some adjacent vector stores, we'd like to cost with
8644 the total number of them once instead of cost each one by one. */
8647 {
8650 {
8652 {
8654 tree pos
8661 }
8662 }
8668 else
8671 unsigned HOST_WIDE_INT
8674 {
8676 {
8678 /* Pun the vector to extract from if necessary. */
8680 {
8682 tree cvt
8687 }
8688 }
8690 {
8692 {
8693 /* Only need vector extracting when there are more
8694 than one stores. */
8696 inside_cost
8699 /* Take a single lane vector type store as scalar
8700 store to avoid ICE like 110776. */
8703 n_adjacent_stores++;
8704 else
8705 inside_cost
8709 }
8713 /* Extract the i'th component. */
8721 GSI_SAME_STMT);
8729 /* And store it to *running_off. */
8736 {
8744 }
8747 {
8751 }
8752 }
8753 }
8758 }
8761 {
8768 "vect_model_store_cost: inside_cost = %d, "
8771 }
8774 }
8777 vec_loop_masks *loop_masks
8781 vec_loop_lens *loop_lens
8786 /* Shouldn't go with length-based approach if fully masked. */
8789 /* Targets with store-lane instructions must not require explicit
8790 realignment. vect_supportable_dr_alignment always returns either
8791 dr_aligned or dr_unaligned_supported for masked operations. */
8793 && !mask
8802 tree bump;
8805 {
8808 }
8810 {
8815 }
8816 else
8817 {
8820 else
8824 }
8829 /* In case the vectorization factor (VF) is bigger than the number
8830 of elements that we can fit in a vectype (nunits), we have to generate
8831 more than one vector stmt - i.e - we need to "unroll" the
8832 vector stmt by a factor VF/nunits. */
8834 /* In case of interleaving (non-unit grouped access):
8836 S1: &base + 2 = x2
8837 S2: &base = x0
8838 S3: &base + 1 = x1
8839 S4: &base + 3 = x3
8841 We create vectorized stores starting from base address (the access of the
8842 first stmt in the chain (S2 in the above example), when the last store stmt
8843 of the chain (S4) is reached:
8845 VS1: &base = vx2
8846 VS2: &base + vec_size*1 = vx0
8847 VS3: &base + vec_size*2 = vx1
8848 VS4: &base + vec_size*3 = vx3
8850 Then permutation statements are generated:
8852 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8853 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8854 ...
8856 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8857 (the order of the data-refs in the output of vect_permute_store_chain
8858 corresponds to the order of scalar stmts in the interleaving chain - see
8859 the documentation of vect_permute_store_chain()).
8861 In case of both multiple types and interleaving, above vector stores and
8862 permutation stmts are created for every copy. The result vector stmts are
8863 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8864 STMT_VINFO_RELATED_STMT for the next copies.
8865 */
8875 {
8878 /* For costing some adjacent vector stores, we'd like to cost with
8879 the total number of them once instead of cost each one by one. */
8882 {
8885 {
8886 /* For interleaved stores we collect vectorized defs for all
8887 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8888 as an input to vect_permute_store_chain(). */
8891 {
8892 /* Since gaps are not supported for interleaved stores,
8893 DR_GROUP_SIZE is the exact number of stmts in the
8894 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8898 else
8899 {
8905 }
8907 }
8910 {
8912 {
8915 mask_vectype);
8917 }
8919 /* We should have catched mismatched types earlier. */
8922 dataref_ptr
8926 }
8927 }
8929 {
8931 /* DR_CHAIN is then used as an input to
8932 vect_permute_store_chain(). */
8934 {
8937 }
8942 }
8945 {
8948 }
8950 /* Get an array into which we can store the individual vectors. */
8953 /* Invalidate the current contents of VEC_ARRAY. This should
8954 become an RTL clobber too, which prevents the vector registers
8955 from being upward-exposed. */
8958 /* Store the individual vectors into the array. */
8960 {
8963 i);
8964 }
8977 {
8981 else
8983 signed char biasval
8987 {
8990 }
8991 }
8995 {
8996 /* Emit:
8997 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8998 LEN, BIAS, VEC_ARRAY). */
9004 vec_array);
9005 }
9007 {
9008 /* Emit:
9009 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
9010 VEC_ARRAY). */
9016 }
9017 else
9018 {
9019 /* Emit:
9020 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
9024 }
9029 /* Record that VEC_ARRAY is now dead. */
9034 }
9037 {
9044 "vect_model_store_cost: inside_cost = %d, "
9047 }
9050 }
9053 {
9058 {
9061 {
9066 {
9067 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
9068 DR_CHAIN is of size 1. */
9076 {
9079 mask_vectype);
9081 }
9083 /* We should have catched mismatched types earlier. */
9090 else
9091 dataref_ptr
9095 bump);
9096 }
9097 }
9099 {
9108 }
9116 {
9123 }
9126 {
9128 {
9130 inside_cost
9134 }
9141 {
9145 else
9148 signed char biasval
9152 {
9155 }
9156 }
9165 call
9169 else
9176 }
9177 else
9178 {
9179 /* Emulated scatter. */
9182 {
9184 /* For emulated scatter N offset vector element extracts
9185 (we assume the scalar scaling and ptr + offset add is
9186 consumed by the load). */
9187 inside_cost
9190 /* N scalar stores plus extracting the elements. */
9191 inside_cost
9194 inside_cost
9198 }
9201 unsigned HOST_WIDE_INT const_offset_nunits
9207 unsigned HOST_WIDE_INT elt_size
9209 /* We support offset vectors with more elements
9210 than the data vector for now. */
9211 unsigned HOST_WIDE_INT factor
9220 {
9221 /* Compute the offsetted pointer. */
9224 tree idx
9229 tree ptr
9233 /* Extract the element to be stored. */
9234 tree elt
9240 tree ref
9244 }
9245 }
9249 }
9253 "vect_model_store_cost: inside_cost = %d, "
9258 }
9266 /* For costing some adjacent vector stores, we'd like to cost with
9267 the total number of them once instead of cost each one by one. */
9272 {
9275 {
9277 {
9278 /* Get vectorized arguments for SLP_NODE. */
9284 }
9285 else
9286 {
9287 /* For interleaved stores we collect vectorized defs for all the
9288 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9289 input to vect_permute_store_chain().
9291 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9292 is of size 1. */
9295 {
9296 /* Since gaps are not supported for interleaved stores,
9297 DR_GROUP_SIZE is the exact number of stmts in the chain.
9298 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9299 that there is no interleaving, DR_GROUP_SIZE is 1,
9300 and only one iteration of the loop will be executed. */
9304 else
9305 {
9311 }
9313 }
9315 {
9318 mask_vectype);
9320 }
9321 }
9323 /* We should have catched mismatched types earlier. */
9327 bool simd_lane_access_p
9330 && simd_lane_access_p
9331 && !loop_masks
9338 {
9341 }
9343 dataref_ptr
9348 }
9350 {
9352 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9353 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9354 of size 1. */
9356 {
9359 }
9364 else
9367 }
9371 {
9372 /* Permute. */
9375 {
9382 "vect_model_store_cost: "
9384 group_size);
9385 }
9386 else
9389 }
9393 {
9395 {
9399 /* For grouped stores vectorized defs are interleaved in
9400 vect_permute_store_chain(). */
9402 }
9405 {
9409 else
9410 {
9416 /* Generate the permute statement. */
9417 gimple *perm_stmt
9421 gsi);
9425 }
9426 }
9429 {
9430 n_adjacent_stores++;
9433 {
9437 }
9440 }
9456 /* Bump the vector pointer. */
9466 {
9469 }
9470 else
9475 misalign);
9478 /* Compute IFN when LOOP_LENS or final_mask valid. */
9483 {
9484 opt_machine_mode new_ovmode
9487 unsigned factor
9492 }
9494 {
9499 }
9502 {
9504 {
9505 /* Pass VF value to 'len' argument of
9506 MASK_LEN_STORE if LOOP_LENS is invalid. */
9508 }
9510 {
9511 /* Pass all ones value to 'mask' argument of
9512 MASK_LEN_STORE if final_mask is invalid. */
9515 }
9516 }
9518 {
9519 signed char biasval
9523 }
9525 /* Arguments are ready. Create the new vector stmt. */
9527 {
9530 /* Need conversion if it's wrapped with VnQI. */
9532 {
9533 tree new_vtype
9535 new_vmode);
9538 gassign *new_stmt
9542 }
9548 else
9555 }
9557 {
9559 gcall *call
9565 }
9566 else
9567 {
9568 data_ref
9570 dataref_offset ? dataref_offset
9573 ;
9574 else
9581 }
9589 }
9591 {
9595 }
9596 }
9599 {
9605 /* When vectorizing a store into the function result assign
9606 a penalty if the function returns in a multi-register location.
9607 In this case we assume we'll end up with having to spill the
9608 vector result and do piecewise loads as a conservative estimate. */
9614 {
9616 /* ??? Handle PARALLEL in some way. */
9618 {
9620 /* Assume that a single reg-reg move is possible and cheap,
9621 do not account for vector to gp register move cost. */
9623 {
9624 /* Spill. */
9625 prologue_cost
9628 /* Loads. */
9629 prologue_cost
9632 }
9633 }
9634 }
9637 "vect_model_store_cost: inside_cost = %d, "
9640 }
9643 }
9645 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9646 VECTOR_CST mask. No checks are made that the target platform supports the
9647 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9648 vect_gen_perm_mask_checked. */
9650 tree
9652 {
9653 tree mask_type;
9660 }
9662 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9663 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9665 tree
9667 {
9671 }
9673 /* Given a vector variable X and Y, that was generated for the scalar
9674 STMT_INFO, generate instructions to permute the vector elements of X and Y
9675 using permutation mask MASK_VEC, insert them at *GSI and return the
9676 permuted vector variable. */
9678 static tree
9682 {
9690 else
9694 /* Generate the permute statement. */
9699 }
9701 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9702 inserting them on the loops preheader edge. Returns true if we
9703 were successful in doing so (and thus STMT_INFO can be moved then),
9704 otherwise returns false. HOIST_P indicates if we want to hoist the
9705 definitions of all SSA uses, it would be false when we are costing. */
9707 static bool
9709 {
9710 ssa_op_iter i;
9711 tree op;
9715 {
9719 {
9720 /* Make sure we don't need to recurse. While we could do
9721 so in simple cases when there are more complex use webs
9722 we don't have an easy way to preserve stmt order to fulfil
9723 dependencies within them. */
9724 tree op2;
9725 ssa_op_iter i2;
9729 {
9734 }
9736 }
9737 }
9746 {
9750 {
9754 }
9755 }
9758 }
9760 /* vectorizable_load.
9762 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9763 that can be vectorized.
9764 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9765 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9766 Return true if STMT_INFO is vectorizable in this way. */
9768 static bool
9773 {
9774 tree scalar_dest;
9781 tree elem_type;
9782 /* Avoid false positive uninitialized warning, see PR110652. */
9784 machine_mode mode;
9785 tree dummy;
9792 poly_uint64 group_gap_adj;
9798 stmt_vec_info first_stmt_info;
9806 poly_uint64 vf;
9807 tree aggr_type;
9808 gather_scatter_info gs_info;
9809 tree ref_type;
9826 {
9841 }
9842 else
9843 {
9858 mask_index = vect_slp_child_index_for_operand
9864 }
9870 {
9874 }
9875 else
9878 /* Multiple types in SLP are handled by creating the appropriate number of
9879 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
9880 case of SLP. */
9883 else
9888 /* FORNOW. This restriction should be relaxed. */
9890 {
9895 }
9897 /* Invalidate assumptions made by dependence analysis when vectorization
9898 on the unrolled body effectively re-orders stmts. */
9903 {
9906 "cannot perform implicit CSE when unrolling "
9909 }
9914 /* FORNOW. In some cases can vectorize even if data-type not supported
9915 (e.g. - data copies). */
9917 {
9922 }
9924 /* Check if the load is a part of an interleaving chain. */
9926 {
9928 /* FORNOW */
9935 /* Refuse non-SLP vectorization of SLP-only groups. */
9937 {
9942 }
9944 /* Invalidate assumptions made by dependence analysis when vectorization
9945 on the unrolled body effectively re-orders stmts. */
9950 {
9953 "cannot perform implicit CSE when performing "
9956 }
9957 }
9958 else
9962 {
9966 {
9967 /* In BB vectorization we may not actually use a loaded vector
9968 accessing elements in excess of DR_GROUP_SIZE. */
9979 {
9982 "BB vectorization with gaps at the end of "
9985 }
9986 }
9992 {
9995 vect_location,
9998 }
9999 }
10001 vect_memory_access_type memory_access_type;
10004 poly_int64 poffset;
10005 internal_fn lanes_ifn;
10013 {
10015 {
10021 }
10024 {
10029 }
10033 {
10038 }
10039 }
10044 {
10046 && mask
10048 mask_vectype))
10049 {
10054 }
10064 mask);
10077 }
10087 /* Transform. */
10093 {
10095 /* If we have versioned for aliasing or the loop doesn't
10096 have any data dependencies that would preclude this,
10097 then we are sure this is a loop invariant load and
10098 thus we can insert it on the preheader edge.
10099 TODO: hoist_defs_of_uses should ideally be computed
10100 once at analysis time, remembered and used in the
10101 transform time. */
10103 && !nested_in_vect_loop
10106 {
10107 enum vect_cost_model_location cost_loc
10112 cost_loc);
10117 "vect_model_load_cost: inside_cost = %d, "
10121 }
10123 {
10133 tree vuse;
10136 else
10141 }
10142 /* These copies are all equivalent. */
10146 else
10147 {
10152 }
10157 else
10158 {
10162 }
10164 }
10168 {
10169 gimple_stmt_iterator incr_gsi;
10171 tree offvar;
10172 tree ivstep;
10173 tree running_off;
10176 /* Checked by get_load_store_type. */
10179 tree dr_offset;
10186 {
10189 }
10190 else
10191 {
10194 }
10197 {
10200 }
10201 else
10202 {
10204 cst_offset
10207 first_stmt_info));
10210 }
10213 {
10221 /* For a load with loop-invariant (but other than power-of-2)
10222 stride (i.e. not a grouped access) like so:
10224 for (i = 0; i < n; i += stride)
10225 ... = array[i];
10227 we generate a new induction variable and new accesses to
10228 form a new vector (or vectors, depending on ncopies):
10230 for (j = 0; ; j += VF*stride)
10231 tmp1 = array[j];
10232 tmp2 = array[j + stride];
10233 ...
10234 vectemp = {tmp1, tmp2, ...}
10235 */
10249 }
10259 {
10261 {
10262 /* First check if vec_init optab supports construction from vector
10263 elts directly. Otherwise avoid emitting a constructor of
10264 vector elements by performing the loads using an integer type
10265 of the same size, constructing a vector of those and then
10266 re-interpreting it as the original vector type. This avoids a
10267 huge runtime penalty due to the general inability to perform
10268 store forwarding from smaller stores to a larger load. */
10269 tree ptype;
10270 tree vtype
10275 {
10280 }
10281 }
10282 else
10283 {
10287 }
10289 }
10290 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10295 {
10296 /* For SLP permutation support we need to load the whole group,
10297 not only the number of vector stmts the permutation result
10298 fits in. */
10300 {
10301 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10302 variable VF. */
10306 }
10307 else
10309 }
10311 unsigned HOST_WIDE_INT
10314 /* For costing some adjacent vector loads, we'd like to cost with
10315 the total number of them once instead of cost each one by one. */
10318 {
10323 {
10325 {
10326 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10327 avoid ICE, see PR110776. */
10330 n_adjacent_loads++;
10331 else
10335 }
10349 {
10350 n_groups++;
10351 /* When doing SLP make sure to not load elements from
10352 the next vector iteration, those will not be accessed
10353 so just use the last element again. See PR107451. */
10355 {
10357 gimple *incr
10362 }
10364 }
10365 }
10368 {
10372 else
10373 {
10379 {
10380 new_stmt
10382 VIEW_CONVERT_EXPR,
10386 gsi);
10387 }
10388 }
10389 }
10392 {
10394 {
10397 else
10399 }
10400 else
10401 {
10405 }
10406 }
10407 }
10409 {
10412 {
10418 }
10419 else
10422 }
10425 {
10433 "vect_model_load_cost: inside_cost = %u, "
10435 inside_cost);
10436 }
10439 }
10447 {
10449 {
10452 }
10453 else
10454 {
10457 }
10458 /* For SLP vectorization we directly vectorize a subchain
10459 without permutation. */
10462 /* For BB vectorization always use the first stmt to base
10463 the data ref pointer on. */
10465 first_stmt_info_for_drptr
10468 /* Check if the chain of loads is already vectorized. */
10470 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10471 ??? But we can only do so if there is exactly one
10472 as we have no way to get at the rest. Leave the CSE
10473 opportunity alone.
10474 ??? With the group load eventually participating
10475 in multiple different permutations (having multiple
10476 slp nodes which refer to the same group) the CSE
10477 is even wrong code. See PR56270. */
10479 {
10482 }
10486 /* VEC_NUM is the number of vect stmts to be created for this group. */
10488 {
10490 /* If an SLP permutation is from N elements to N elements,
10491 and if one vector holds a whole number of N, we can load
10492 the inputs to the permutation in the same way as an
10493 unpermuted sequence. In other cases we need to load the
10494 whole group, not only the number of vector stmts the
10495 permutation result fits in. */
10500 {
10501 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10502 variable VF; see vect_transform_slp_perm_load. */
10507 }
10508 else
10509 {
10511 group_gap_adj
10513 }
10514 }
10515 else
10519 }
10520 else
10521 {
10529 }
10532 vec_loop_masks *loop_masks
10536 vec_loop_lens *loop_lens
10541 /* Shouldn't go with length-based approach if fully masked. */
10544 /* Targets with store-lane instructions must not require explicit
10545 realignment. vect_supportable_dr_alignment always returns either
10546 dr_aligned or dr_unaligned_supported for masked operations. */
10548 && !mask
10553 /* In case the vectorization factor (VF) is bigger than the number
10554 of elements that we can fit in a vectype (nunits), we have to generate
10555 more than one vector stmt - i.e - we need to "unroll" the
10556 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10557 from one copy of the vector stmt to the next, in the field
10558 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10559 stages to find the correct vector defs to be used when vectorizing
10560 stmts that use the defs of the current stmt. The example below
10561 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10562 need to create 4 vectorized stmts):
10564 before vectorization:
10565 RELATED_STMT VEC_STMT
10566 S1: x = memref - -
10567 S2: z = x + 1 - -
10569 step 1: vectorize stmt S1:
10570 We first create the vector stmt VS1_0, and, as usual, record a
10571 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10572 Next, we create the vector stmt VS1_1, and record a pointer to
10573 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10574 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10575 stmts and pointers:
10576 RELATED_STMT VEC_STMT
10577 VS1_0: vx0 = memref0 VS1_1 -
10578 VS1_1: vx1 = memref1 VS1_2 -
10579 VS1_2: vx2 = memref2 VS1_3 -
10580 VS1_3: vx3 = memref3 - -
10581 S1: x = load - VS1_0
10582 S2: z = x + 1 - -
10583 */
10585 /* In case of interleaving (non-unit grouped access):
10587 S1: x2 = &base + 2
10588 S2: x0 = &base
10589 S3: x1 = &base + 1
10590 S4: x3 = &base + 3
10592 Vectorized loads are created in the order of memory accesses
10593 starting from the access of the first stmt of the chain:
10595 VS1: vx0 = &base
10596 VS2: vx1 = &base + vec_size*1
10597 VS3: vx3 = &base + vec_size*2
10598 VS4: vx4 = &base + vec_size*3
10600 Then permutation statements are generated:
10602 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10603 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10604 ...
10606 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10607 (the order of the data-refs in the output of vect_permute_load_chain
10608 corresponds to the order of scalar stmts in the interleaving chain - see
10609 the documentation of vect_permute_load_chain()).
10610 The generation of permutation stmts and recording them in
10611 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10613 In case of both multiple types and interleaving, the vector loads and
10614 permutation stmts above are created for every copy. The result vector
10615 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10616 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10618 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10619 on a target that supports unaligned accesses (dr_unaligned_supported)
10620 we generate the following code:
10621 p = initial_addr;
10622 indx = 0;
10623 loop {
10624 p = p + indx * vectype_size;
10625 vec_dest = *(p);
10626 indx = indx + 1;
10627 }
10629 Otherwise, the data reference is potentially unaligned on a target that
10630 does not support unaligned accesses (dr_explicit_realign_optimized) -
10631 then generate the following code, in which the data in each iteration is
10632 obtained by two vector loads, one from the previous iteration, and one
10633 from the current iteration:
10634 p1 = initial_addr;
10635 msq_init = *(floor(p1))
10636 p2 = initial_addr + VS - 1;
10637 realignment_token = call target_builtin;
10638 indx = 0;
10639 loop {
10640 p2 = p2 + indx * vectype_size
10641 lsq = *(floor(p2))
10642 vec_dest = realign_load (msq, lsq, realignment_token)
10643 indx = indx + 1;
10644 msq = lsq;
10645 } */
10647 /* If the misalignment remains the same throughout the execution of the
10648 loop, we can create the init_addr and permutation mask at the loop
10649 preheader. Otherwise, it needs to be created inside the loop.
10650 This can only occur when vectorizing memory accesses in the inner-loop
10651 nested within an outer-loop that is being vectorized. */
10656 {
10659 }
10661 bool diff_first_stmt_info
10668 {
10669 /* If we have different first_stmt_info, we can't set up realignment
10670 here, since we can't guarantee first_stmt_info DR has been
10671 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10672 distance from first_stmt_info DR instead as below. */
10674 {
10681 {
10684 size_one_node);
10686 }
10687 }
10688 }
10689 else
10693 offset = (offset
10697 tree bump;
10700 {
10703 }
10705 {
10710 }
10711 else
10712 {
10715 else
10719 }
10724 {
10728 else
10731 }
10735 {
10741 /* For costing some adjacent vector loads, we'd like to cost with
10742 the total number of them once instead of cost each one by one. */
10745 {
10747 {
10748 /* An IFN_LOAD_LANES will load all its vector results,
10749 regardless of which ones we actually need. Account
10750 for the cost of unused results. */
10752 {
10755 do
10756 {
10759 }
10762 {
10765 "vect_model_load_cost: %d "
10767 gaps);
10769 alignment_support_scheme,
10773 }
10774 }
10775 n_adjacent_loads++;
10777 }
10779 /* 1. Create the vector or array pointer update chain. */
10781 dataref_ptr
10785 else
10786 {
10790 }
10807 {
10811 else
10813 signed char biasval
10817 {
10820 }
10821 }
10825 {
10826 /* Emit:
10827 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10828 VEC_MASK, LEN, BIAS). */
10834 }
10836 {
10837 /* Emit:
10838 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10839 VEC_MASK). */
10844 final_mask);
10845 }
10846 else
10847 {
10848 /* Emit:
10849 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10852 }
10858 /* Extract each vector into an SSA_NAME. */
10860 {
10864 }
10866 /* Record the mapping between SSA_NAMEs and statements. */
10869 /* Record that VEC_ARRAY is now dead. */
10875 }
10878 {
10886 "vect_model_load_cost: inside_cost = %u, "
10889 }
10892 }
10895 {
10902 {
10903 /* 1. Create the vector or array pointer update chain. */
10905 {
10910 else
10911 dataref_ptr
10915 }
10917 {
10922 }
10926 {
10931 {
10935 final_mask
10946 }
10948 /* 2. Create the vector-load in the loop. */
10951 {
10953 {
10955 inside_cost
10959 }
10966 {
10968 final_len
10972 else
10973 final_len
10976 signed char biasval
10980 {
10983 }
10984 }
10988 call
10997 else
11004 }
11006 {
11007 /* The builtin decls path for gather is legacy, x86 only. */
11010 {
11012 inside_cost
11016 }
11017 poly_uint64 offset_nunits
11020 {
11021 new_stmt = vect_build_one_gather_load_call
11024 final_mask);
11026 }
11028 {
11029 /* We have a offset vector with half the number of
11030 lanes but the builtins will produce full vectype
11031 data with just the lower lanes filled. */
11032 new_stmt = vect_build_one_gather_load_call
11035 final_mask);
11041 /* now put upper half of final_mask in final_mask low. */
11043 && !SCALAR_INT_MODE_P
11045 {
11052 tree perm_mask = vect_gen_perm_mask_checked
11055 VEC_PERM_EXPR,
11056 final_mask,
11057 final_mask,
11058 perm_mask);
11063 }
11065 {
11067 VEC_UNPACK_HI_EXPR,
11068 final_mask);
11069 final_mask = make_ssa_name
11074 }
11076 new_stmt = vect_build_one_gather_load_call
11078 dataref_ptr,
11080 final_mask);
11086 /* compose low + high. */
11093 tree perm_mask
11096 VEC_PERM_EXPR,
11099 }
11101 {
11102 /* We have a offset vector with double the number of
11103 lanes. Select the low/high part accordingly. */
11106 {
11113 tree perm_mask = vect_gen_perm_mask_checked
11116 VEC_PERM_EXPR,
11117 vec_offset,
11118 vec_offset,
11119 perm_mask);
11124 }
11125 new_stmt = vect_build_one_gather_load_call
11129 }
11130 else
11132 }
11133 else
11134 {
11135 /* Emulated gather-scatter. */
11139 {
11140 /* For emulated gathers N offset vector element
11141 offset add is consumed by the load). */
11145 /* N scalar loads plus gathering them into a
11146 vector. */
11147 inside_cost
11150 inside_cost
11154 }
11155 unsigned HOST_WIDE_INT const_offset_nunits
11161 /* We support offset vectors with more elements
11162 than the data vector for now. */
11163 unsigned HOST_WIDE_INT factor
11172 {
11175 tree idx
11180 scale);
11192 }
11197 }
11200 /* DATA_REF is null if we've already built the statement. */
11202 {
11205 }
11210 /* Store vector loads in the corresponding SLP_NODE. */
11213 }
11217 }
11224 "vect_model_load_cost: inside_cost = %u, "
11228 }
11232 /* For costing some adjacent vector loads, we'd like to cost with
11233 the total number of them once instead of cost each one by one. */
11236 {
11237 /* 1. Create the vector or array pointer update chain. */
11239 {
11240 bool simd_lane_access_p
11251 {
11254 }
11256 {
11257 dataref_ptr
11261 bump);
11262 /* Adjust the pointer by the difference to first_stmt. */
11263 data_reference_p ptrdr
11265 tree diff
11273 {
11277 alignment_support_scheme,
11280 }
11281 }
11282 else
11283 dataref_ptr
11285 at_loop,
11288 }
11290 {
11294 bump);
11295 else
11298 }
11305 {
11310 {
11324 }
11326 /* 2. Create the vector-load in the loop. */
11328 {
11331 {
11341 {
11342 align
11345 }
11346 else
11351 misalign);
11354 /* Compute IFN when LOOP_LENS or final_mask valid. */
11359 {
11360 opt_machine_mode new_ovmode
11363 unsigned factor
11368 }
11370 {
11375 }
11378 {
11380 {
11381 /* Pass VF value to 'len' argument of
11382 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11384 }
11386 {
11387 /* Pass all ones value to 'mask' argument of
11388 MASK_LEN_LOAD if final_mask is invalid. */
11391 }
11392 }
11394 {
11395 signed char biasval
11399 }
11402 {
11409 bias);
11410 else
11418 /* Need conversion if it's wrapped with VnQI. */
11420 {
11423 tree var
11427 gsi);
11431 }
11432 }
11434 {
11438 final_mask);
11442 }
11443 else
11444 {
11448 unsigned int vect_align
11450 unsigned int scalar_dr_size
11452 /* If there's no peeling for gaps but we have a gap
11453 with slp loads then load the lower half of the
11454 vector only. See get_group_load_store_type for
11455 when we apply this optimization. */
11457 && loop_vinfo
11462 {
11463 tree half_vtype;
11464 new_vtype
11469 }
11470 tree offset
11475 {
11476 unsigned HOST_WIDE_INT gap_offset
11480 }
11481 data_ref
11484 ;
11485 else
11490 {
11496 gsi);
11501 {
11505 }
11506 else
11507 {
11511 }
11516 else
11517 {
11524 vec_dest,
11526 }
11527 }
11528 }
11530 }
11532 {
11542 dr_explicit_realign,
11547 else
11549 // For explicit realign the target alignment should be
11550 // known at compile time.
11551 unsigned HOST_WIDE_INT align
11558 data_ref
11572 bump);
11578 else
11582 data_ref
11585 }
11587 {
11592 else
11594 // We should only be doing this if we know the target
11595 // alignment at compile time.
11596 unsigned HOST_WIDE_INT align
11606 }
11609 }
11611 /* One common place to cost the above vect load for different
11612 alignment support schemes. */
11614 {
11615 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11616 only need to take care of the first stmt, whose
11617 stmt_info is first_stmt_info, vec_num iterating on it
11618 will cover the cost for the remaining, it's consistent
11619 with transforming. For the prologue cost for realign,
11620 we only need to count it once for the whole group. */
11627 {
11628 /* Leave realign cases alone to keep them simple. */
11636 else
11637 n_adjacent_loads++;
11638 }
11639 }
11640 else
11641 {
11643 /* DATA_REF is null if we've already built the statement. */
11645 {
11648 }
11652 }
11654 /* 3. Handle explicit realignment if necessary/supported.
11655 Create in loop:
11656 vec_dest = realign_load (msq, lsq, realignment_token) */
11660 {
11672 {
11676 UNKNOWN_LOCATION);
11678 }
11679 }
11682 {
11686 else
11687 {
11692 }
11693 }
11695 /* Collect vector loads and later create their permutation in
11696 vect_transform_grouped_load (). */
11700 /* Store vector loads in the corresponding SLP_NODE. */
11704 /* With SLP permutation we load the gaps as well, without
11705 we need to skip the gaps after we manage to fully load
11706 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11710 && !slp_perm
11712 {
11713 poly_wide_int bump_val
11722 }
11723 }
11724 /* Bump the vector pointer to account for a gap or for excess
11725 elements loaded for a permuted SLP load. */
11729 {
11730 poly_wide_int bump_val
11737 }
11743 {
11745 /* For SLP we know we've seen all possible uses of dr_chain so
11746 direct vect_transform_slp_perm_load to DCE the unused parts.
11747 ??? This is a hack to prevent compile-time issues as seen
11748 in PR101120 and friends. */
11750 {
11755 }
11756 else
11757 {
11762 }
11763 }
11764 else
11765 {
11767 {
11769 /* We assume that the cost of a single load-lanes instruction
11770 is equivalent to the cost of DR_GROUP_SIZE separate loads.
11771 If a grouped access is instead being provided by a
11772 load-and-permute operation, include the cost of the
11773 permutes. */
11775 {
11776 /* Uses an even and odd extract operations or shuffle
11777 operations for each needed permute. */
11785 "vect_model_load_cost:"
11787 group_size);
11788 }
11790 {
11794 }
11795 }
11798 }
11800 }
11805 {
11816 "vect_model_load_cost: inside_cost = %u, "
11819 }
11822 }
11824 /* Function vect_is_simple_cond.
11826 Input:
11827 LOOP - the loop that is being vectorized.
11828 COND - Condition that is checked for simple use.
11830 Output:
11831 *COMP_VECTYPE - the vector type for the comparison.
11832 *DTS - The def types for the arguments of the comparison
11834 Returns whether a COND can be vectorized. Checks whether
11835 condition operands are supportable using vec_is_simple_use. */
11837 static bool
11841 {
11844 slp_tree slp_op;
11846 /* Mask case. */
11849 {
11852 || !*comp_vectype
11856 }
11865 {
11869 }
11873 else
11877 {
11881 }
11885 else
11894 /* Invariant comparison. */
11896 {
11900 else
11901 {
11902 /* If we can widen the comparison to match vectype do so. */
11904 && !slp_node
11907 scalar_type = build_nonstandard_integer_type
11910 slp_node);
11911 }
11912 }
11915 }
11917 /* vectorizable_condition.
11919 Check if STMT_INFO is conditional modify expression that can be vectorized.
11920 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
11921 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
11922 at GSI.
11924 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
11926 Return true if STMT_INFO is vectorizable in this way. */
11928 static bool
11933 {
11941 tree vec_compare;
11942 tree new_temp;
11957 tree vec_cmp_type;
11963 /* Is vectorizable conditional operation? */
11975 bool for_reduction
11978 {
11986 }
11987 else
11988 {
11991 }
11997 {
12000 }
12001 else
12002 {
12005 }
12041 {
12044 }
12046 /* For conditional reductions, the "then" value needs to be the candidate
12047 value calculated by this iteration while the "else" value needs to be
12048 the result carried over from previous iterations. If the COND_EXPR
12049 is the other way around, we need to swap it. */
12052 {
12055 else
12056 {
12061 else
12062 {
12064 /* Make sure we don't accidentally use the old condition. */
12066 }
12067 }
12069 }
12072 {
12073 /* Boolean values may have another representation in vectors
12074 and therefore we prefer bit operations over comparison for
12075 them (which also works for scalar masks). We store opcodes
12076 to use in bitop1 and bitop2. Statement is vectorized as
12077 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12078 depending on bitop1 and bitop2 arity. */
12080 {
12108 }
12110 }
12115 {
12120 }
12123 {
12125 {
12127 optab optab;
12134 {
12136 optab_default);
12139 }
12140 }
12144 /* Count one reduction-like operation per vector. */
12147 && (masked
12149 cond_code)
12151 ERROR_MARK))))
12155 && (!vect_maybe_update_slp_op_vectype
12158 && !vect_maybe_update_slp_op_vectype
12162 {
12167 }
12171 {
12173 {
12179 else
12183 }
12184 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12186 {
12189 "conditional reduction prevents the use"
12192 }
12193 }
12199 }
12201 /* Transform. */
12203 /* Handle def. */
12210 /* See whether another part of the vectorized code applies a loop
12211 mask to the condition, or to its inverse. */
12216 {
12219 }
12221 {
12224 else
12225 {
12229 else
12230 {
12235 {
12239 }
12240 else
12241 {
12242 /* Try the inverse of the current mask. We check if the
12243 inverse mask is live and if so we generate a negate of
12244 the current mask such that we still honor NaNs. */
12248 {
12253 }
12254 }
12255 }
12256 }
12257 }
12259 /* Handle cond expr. */
12264 reduction_type != EXTRACT_LAST_REDUCTION
12266 else
12271 reduction_type != EXTRACT_LAST_REDUCTION
12274 /* Arguments are ready. Create the new vector stmt. */
12276 {
12286 else
12287 {
12290 {
12295 }
12296 else
12297 {
12302 vec_cond_rhs);
12303 else
12304 new_stmt
12306 vec_cond_rhs);
12312 {
12313 /* Instead of doing ~x ? y : z do x ? z : y. */
12316 }
12317 else
12318 {
12321 new_stmt
12323 else
12324 new_stmt
12329 }
12330 }
12331 }
12333 /* If we decided to apply a loop mask to the result of the vector
12334 comparison, AND the comparison with the mask now. Later passes
12335 should then be able to reuse the AND results between mulitple
12336 vector statements.
12338 For example:
12339 for (int i = 0; i < 100; ++i)
12340 x[i] = y[i] ? z[i] : 10;
12342 results in following optimized GIMPLE:
12344 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12345 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12346 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12347 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12348 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12349 vect_iftmp.11_47, { 10, ... }>;
12351 instead of using a masked and unmasked forms of
12352 vec != { 0, ... } (masked in the MASK_LOAD,
12353 unmasked in the VEC_COND_EXPR). */
12355 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12356 in cases where that's necessary. */
12360 {
12362 {
12365 vec_compare);
12368 }
12371 {
12374 BIT_NOT_EXPR,
12375 vec_compare);
12378 }
12382 {
12384 {
12387 signed char biasval
12390 }
12391 else
12392 {
12395 }
12396 }
12398 {
12399 tree loop_mask
12403 gassign *g
12405 loop_mask);
12408 }
12409 }
12413 {
12417 new_stmt = gimple_build_call_internal
12420 else
12421 new_stmt = gimple_build_call_internal
12423 vec_then_clause);
12428 else
12429 {
12430 /* In this case we're moving the definition to later in the
12431 block. That doesn't matter because the only uses of the
12432 lhs are in phi statements. */
12436 }
12437 }
12438 else
12439 {
12444 }
12447 else
12449 }
12460 }
12462 /* Helper of vectorizable_comparison.
12464 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12465 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12466 comparison, put it in VEC_STMT, and insert it at GSI.
12468 Return true if STMT_INFO is vectorizable in this way. */
12470 static bool
12475 {
12479 tree new_temp;
12483 poly_uint64 nunits;
12490 tree mask_type;
12491 tree mask;
12504 else
12528 /* Invariant comparison. */
12530 {
12533 else
12535 slp_node);
12538 }
12542 /* Can't compare mask and non-mask types. */
12547 /* Boolean values may have another representation in vectors
12548 and therefore we prefer bit operations over comparison for
12549 them (which also works for scalar masks). We store opcodes
12550 to use in bitop1 and bitop2. Statement is vectorized as
12551 BITOP2 (rhs1 BITOP1 rhs2) or
12552 rhs1 BITOP2 (BITOP1 rhs2)
12553 depending on bitop1 and bitop2 arity. */
12556 {
12558 {
12561 }
12563 {
12566 }
12568 {
12572 }
12574 {
12578 }
12579 else
12580 {
12584 }
12585 }
12588 {
12590 {
12593 }
12594 else
12595 {
12597 optab optab;
12604 {
12608 }
12609 }
12611 /* Put types on constant and invariant SLP children. */
12615 {
12620 }
12626 }
12628 /* Transform. */
12630 /* Handle def. */
12640 /* Arguments are ready. Create the new vector stmt. */
12642 {
12648 {
12652 }
12653 else
12654 {
12657 else
12659 vec_rhs2);
12662 {
12666 else
12668 new_temp);
12670 }
12671 }
12674 else
12676 }
12685 }
12687 /* vectorizable_comparison.
12689 Check if STMT_INFO is comparison expression that can be vectorized.
12690 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12691 comparison, put it in VEC_STMT, and insert it at GSI.
12693 Return true if STMT_INFO is vectorizable in this way. */
12695 static bool
12700 {
12723 }
12725 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
12726 can handle all live statements in the node. Otherwise return true
12727 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
12728 VEC_STMT_P is as for vectorizable_live_operation. */
12730 static bool
12735 {
12737 {
12738 stmt_vec_info slp_stmt_info;
12741 {
12747 }
12748 }
12756 }
12758 /* Make sure the statement is vectorizable. */
12760 opt_result
12765 {
12769 gimple_seq pattern_def_seq;
12777 "not vectorized:"
12784 {
12785 gimple_stmt_iterator si;
12788 {
12789 stmt_vec_info pattern_def_stmt_info
12793 {
12794 /* Analyze def stmt of STMT if it's a pattern stmt. */
12800 opt_result res
12803 cost_vec);
12806 }
12807 }
12808 }
12810 /* Skip stmts that do not need to be vectorized. In loops this is expected
12811 to include:
12812 - the COND_EXPR which is the loop exit condition
12813 - any LABEL_EXPRs in the loop
12814 - computations that are used only for array indexing or loop control.
12815 In basic blocks we only analyze statements that are a part of some SLP
12816 instance, therefore, all the statements are relevant.
12818 Pattern statement needs to be analyzed instead of the original statement
12819 if the original statement is not relevant. Otherwise, we analyze both
12820 statements. In basic blocks we are called from some SLP instance
12821 traversal, don't analyze pattern stmts instead, the pattern stmts
12822 already will be part of SLP instance. */
12827 {
12829 && pattern_stmt_info
12832 {
12833 /* Analyze PATTERN_STMT instead of the original stmt. */
12839 }
12840 else
12841 {
12846 }
12847 }
12850 && pattern_stmt_info
12853 {
12854 /* Analyze PATTERN_STMT too. */
12860 opt_result res
12865 }
12868 {
12892 }
12899 {
12904 }
12907 {
12912 }
12918 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
12919 -mveclibabi= takes preference over library functions with
12920 the simd attribute. */
12923 cost_vec)
12940 cost_vec)
12945 else
12946 {
12952 cost_vec)
12958 cost_vec)
12966 cost_vec)
12968 }
12975 "not vectorized:"
12979 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
12980 need extra handling, except for vectorizable reductions. */
12988 "not vectorized:"
12993 }
12996 /* Function vect_transform_stmt.
12998 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13000 bool
13004 {
13016 {
13028 NULL);
13040 NULL);
13052 NULL);
13058 && !slp_node
13061 /* In case of interleaving, the whole chain is vectorized when the
13062 last store in the chain is reached. Store stmts before the last
13063 one are skipped, and there vec_stmt_info shouldn't be freed
13064 meanwhile. */
13065 ;
13066 else
13067 {
13072 }
13128 {
13133 }
13135 }
13141 {
13142 /* Handle stmts whose DEF is used outside the loop-nest that is
13143 being vectorized. */
13147 }
13153 }
13156 /* Remove a group of stores (for SLP or interleaving), free their
13157 stmt_vec_info. */
13159 void
13161 {
13165 {
13168 /* Free the attached stmt_vec_info and remove the stmt. */
13171 }
13172 }
13174 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13175 elements of type SCALAR_TYPE, or null if the target doesn't support
13176 such a type.
13178 If NUNITS is zero, return a vector type that contains elements of
13179 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13181 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13182 for this vectorization region and want to "autodetect" the best choice.
13183 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13184 and we want the new type to be interoperable with it. PREVAILING_MODE
13185 in this case can be a scalar integer mode or a vector mode; when it
13186 is a vector mode, the function acts like a tree-level version of
13187 related_vector_mode. */
13189 tree
13192 {
13194 scalar_mode inner_mode;
13195 machine_mode simd_mode;
13196 tree vectype;
13207 /* Interoperability between modes requires one to be a constant multiple
13208 of the other, so that the number of vectors required for each operation
13209 is a compile-time constant. */
13217 /* For vector types of elements whose mode precision doesn't
13218 match their types precision we use a element type of mode
13219 precision. The vectorization routines will have to make sure
13220 they support the proper result truncation/extension.
13221 We also make sure to build vector types with INTEGER_TYPE
13222 component type only. */
13229 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13230 When the component mode passes the above test simply use a type
13231 corresponding to that mode. The theory is that any use that
13232 would cause problems with this will disable vectorization anyway. */
13237 /* We can't build a vector type of elements with alignment bigger than
13238 their size. */
13243 /* If we felt back to using the mode fail if there was
13244 no scalar type for it. */
13248 /* If no prevailing mode was supplied, use the mode the target prefers.
13249 Otherwise lookup a vector mode based on the prevailing mode. */
13251 {
13255 {
13256 /* Traditional behavior is not to take the integer mode
13257 literally, but simply to use it as a way of determining
13258 the vector size. It is up to mode_for_vector to decide
13259 what the TYPE_MODE should be.
13261 Note that nunits == 1 is allowed in order to support single
13262 element vector types. */
13266 }
13267 }
13271 {
13272 /* Fall back to using mode_for_vector, mostly in the hope of being
13273 able to use an integer mode. */
13280 }
13284 /* In cases where the mode was chosen by mode_for_vector, check that
13285 the target actually supports the chosen mode, or that it at least
13286 allows the vector mode to be replaced by a like-sized integer. */
13291 /* Re-attach the address-space qualifier if we canonicalized the scalar
13292 type. */
13294 return build_qualified_type
13298 }
13300 /* Function get_vectype_for_scalar_type.
13302 Returns the vector type corresponding to SCALAR_TYPE as supported
13303 by the target. If GROUP_SIZE is nonzero and we're performing BB
13304 vectorization, make sure that the number of elements in the vector
13305 is no bigger than GROUP_SIZE. */
13307 tree
13310 {
13311 /* For BB vectorization, we should always have a group size once we've
13312 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13313 are tentative requests during things like early data reference
13314 analysis and pattern recognition. */
13317 else
13321 scalar_type);
13325 /* Register the natural choice of vector type, before the group size
13326 has been applied. */
13330 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13331 try again with an explicit number of elements. */
13333 && group_size
13335 {
13336 /* Start with the biggest number of units that fits within
13337 GROUP_SIZE and halve it until we find a valid vector type.
13338 Usually either the first attempt will succeed or all will
13339 fail (in the latter case because GROUP_SIZE is too small
13340 for the target), but it's possible that a target could have
13341 a hole between supported vector types.
13343 If GROUP_SIZE is not a power of 2, this has the effect of
13344 trying the largest power of 2 that fits within the group,
13345 even though the group is not a multiple of that vector size.
13346 The BB vectorizer will then try to carve up the group into
13347 smaller pieces. */
13349 do
13350 {
13354 }
13356 }
13359 }
13361 /* Return the vector type corresponding to SCALAR_TYPE as supported
13362 by the target. NODE, if nonnull, is the SLP tree node that will
13363 use the returned vector type. */
13365 tree
13367 {
13372 }
13374 /* Function get_mask_type_for_scalar_type.
13376 Returns the mask type corresponding to a result of comparison
13377 of vectors of specified SCALAR_TYPE as supported by target.
13378 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13379 make sure that the number of elements in the vector is no bigger
13380 than GROUP_SIZE. */
13382 tree
13385 {
13392 }
13394 /* Function get_mask_type_for_scalar_type.
13396 Returns the mask type corresponding to a result of comparison
13397 of vectors of specified SCALAR_TYPE as supported by target.
13398 NODE, if nonnull, is the SLP tree node that will use the returned
13399 vector type. */
13401 tree
13403 slp_tree node)
13404 {
13411 }
13413 /* Function get_same_sized_vectype
13415 Returns a vector type corresponding to SCALAR_TYPE of size
13416 VECTOR_TYPE if supported by the target. */
13418 tree
13420 {
13424 poly_uint64 nunits;
13431 }
13433 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13434 would not change the chosen vector modes. */
13436 bool
13438 {
13445 }
13447 /* Function vect_is_simple_use.
13449 Input:
13450 VINFO - the vect info of the loop or basic block that is being vectorized.
13451 OPERAND - operand in the loop or bb.
13452 Output:
13453 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13454 case OPERAND is an SSA_NAME that is defined in the vectorizable region
13455 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13456 the definition could be anywhere in the function
13457 DT - the type of definition
13459 Returns whether a stmt with OPERAND can be vectorized.
13460 For loops, supportable operands are constants, loop invariants, and operands
13461 that are defined by the current iteration of the loop. Unsupportable
13462 operands are those that are defined by a previous iteration of the loop (as
13463 is the case in reduction/induction computations).
13464 For basic blocks, supportable operands are constants and bb invariants.
13465 For now, operands defined outside the basic block are not supported. */
13467 bool
13470 {
13478 {
13484 else
13486 }
13496 else
13497 {
13502 else
13503 {
13509 }
13512 }
13515 {
13518 {
13549 }
13550 }
13553 {
13558 }
13561 }
13563 /* Function vect_is_simple_use.
13565 Same as vect_is_simple_use but also determines the vector operand
13566 type of OPERAND and stores it to *VECTYPE. If the definition of
13567 OPERAND is vect_uninitialized_def, vect_constant_def or
13568 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
13569 is responsible to compute the best suited vector type for the
13570 scalar operand. */
13572 bool
13576 {
13577 stmt_vec_info def_stmt_info;
13587 /* Now get a vector type if the def is internal, otherwise supply
13588 NULL_TREE and leave it up to the caller to figure out a proper
13589 type for the use stmt. */
13596 {
13602 }
13607 else
13611 }
13613 /* Function vect_is_simple_use.
13615 Same as vect_is_simple_use but determines the operand by operand
13616 position OPERAND from either STMT or SLP_NODE, filling in *OP
13617 and *SLP_DEF (when SLP_NODE is not NULL). */
13619 bool
13624 {
13626 {
13631 {
13634 }
13635 else
13636 {
13642 }
13643 }
13644 else
13645 {
13648 {
13651 {
13654 else
13656 }
13659 else
13661 }
13664 else
13667 }
13668 }
13670 /* If OP is not NULL and is external or constant update its vector
13671 type with VECTYPE. Returns true if successful or false if not,
13672 for example when conflicting vector types are present. */
13674 bool
13676 {
13681 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13682 should be handled by patters. Allow vect_constant_def for now. */
13688 }
13690 /* Function supportable_widening_operation
13692 Check whether an operation represented by the code CODE is a
13693 widening operation that is supported by the target platform in
13694 vector form (i.e., when operating on arguments of type VECTYPE_IN
13695 producing a result of type VECTYPE_OUT).
13697 Widening operations we currently support are NOP (CONVERT), FLOAT,
13698 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13699 are supported by the target platform either directly (via vector
13700 tree-codes), or via target builtins.
13702 Output:
13703 - CODE1 and CODE2 are codes of vector operations to be used when
13704 vectorizing the operation, if available.
13705 - MULTI_STEP_CVT determines the number of required intermediate steps in
13706 case of multi-step conversion (like char->short->int - in that case
13707 MULTI_STEP_CVT will be 1).
13708 - INTERM_TYPES contains the intermediate type required to perform the
13709 widening operation (short in the above example). */
13711 bool
13713 code_helper code,
13714 stmt_vec_info stmt_info,
13720 {
13723 machine_mode vec_mode;
13739 {
13741 /* Don't set c1 and c2 if code is not a tree_code. */
13745 /* The result of a vectorized widening operation usually requires
13746 two vectors (because the widened results do not fit into one vector).
13747 The generated vector results would normally be expected to be
13748 generated in the same order as in the original scalar computation,
13749 i.e. if 8 results are generated in each vector iteration, they are
13750 to be organized as follows:
13751 vect1: [res1,res2,res3,res4],
13752 vect2: [res5,res6,res7,res8].
13754 However, in the special case that the result of the widening
13755 operation is used in a reduction computation only, the order doesn't
13756 matter (because when vectorizing a reduction we change the order of
13757 the computation). Some targets can take advantage of this and
13758 generate more efficient code. For example, targets like Altivec,
13759 that support widen_mult using a sequence of {mult_even,mult_odd}
13760 generate the following vectors:
13761 vect1: [res1,res3,res5,res7],
13762 vect2: [res2,res4,res6,res8].
13764 When vectorizing outer-loops, we execute the inner-loop sequentially
13765 (each vectorized inner-loop iteration contributes to VF outer-loop
13766 iterations in parallel). We therefore don't allow to change the
13767 order of the computation in the inner-loop during outer-loop
13768 vectorization. */
13769 /* TODO: Another case in which order doesn't *really* matter is when we
13770 widen and then contract again, e.g. (short)((int)x * y >> 8).
13771 Normally, pack_trunc performs an even/odd permute, whereas the
13772 repack from an even/odd expansion would be an interleave, which
13773 would be significantly simpler for e.g. AVX2. */
13774 /* In any case, in order to avoid duplicating the code below, recurse
13775 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13776 are properly set up for the caller. If we fail, we'll continue with
13777 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13785 interm_types))
13786 {
13787 /* Elements in a vector with vect_used_by_reduction property cannot
13788 be reordered if the use chain with this property does not have the
13789 same operation. One such an example is s += a * b, where elements
13790 in a and b cannot be reordered. Here we check if the vector defined
13791 by STMT is only directly used in the reduction statement. */
13797 }
13813 /* Support the recursion induced just above. */
13823 CASE_CONVERT:
13840 }
13846 {
13847 /* The signedness is determined from output operand. */
13850 }
13856 {
13857 /* If the input and result modes are the same, a different optab
13858 is needed where we pass in the number of units in vectype. */
13861 }
13865 {
13866 /* If this is an internal fn then we must check whether the target
13867 supports either a low-high split or an even-odd split. */
13877 /* If we don't support low-high, then check for even-odd. */
13880 || !optab2
13882 {
13888 }
13889 }
13891 {
13893 {
13894 /* The signedness is determined from output operand. */
13897 }
13903 {
13904 /* If the input and result modes are the same, a different optab
13905 is needed where we pass in the number of units in vectype. */
13908 }
13909 else
13910 {
13913 }
13916 }
13928 {
13931 /* For scalar masks we may have different boolean
13932 vector types having the same QImode. Thus we
13933 add additional check for elements number. */
13937 }
13939 /* Check if it's a multi-step conversion that can be done using intermediate
13940 types. */
13948 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
13949 intermediate steps in promotion sequence. We try
13950 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
13951 not. */
13954 {
13957 intermediate_type
13960 {
13961 tree intermediate_element_type
13964 intermediate_type
13966 intermediate_mode);
13967 }
13968 else
13969 intermediate_type
13977 {
13978 /* If the input and result modes are the same, a different optab
13979 is needed where we pass in the number of units in vectype. */
13982 }
13983 else
13984 {
13987 }
14005 {
14011 }
14015 }
14019 }
14022 /* Function supportable_narrowing_operation
14024 Check whether an operation represented by the code CODE is a
14025 narrowing operation that is supported by the target platform in
14026 vector form (i.e., when operating on arguments of type VECTYPE_IN
14027 and producing a result of type VECTYPE_OUT).
14029 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14030 and FLOAT. This function checks if these operations are supported by
14031 the target platform directly via vector tree-codes.
14033 Output:
14034 - CODE1 is the code of a vector operation to be used when
14035 vectorizing the operation, if available.
14036 - MULTI_STEP_CVT determines the number of required intermediate steps in
14037 case of multi-step conversion (like int->short->char - in that case
14038 MULTI_STEP_CVT will be 1).
14039 - INTERM_TYPES contains the intermediate type required to perform the
14040 narrowing operation (short in the above example). */
14042 bool
14047 {
14048 machine_mode vec_mode;
14065 {
14066 CASE_CONVERT:
14074 else
14080 /* The signedness is determined from output operand. */
14091 }
14103 {
14106 /* For scalar masks we may have different boolean
14107 vector types having the same QImode. Thus we
14108 add additional check for elements number. */
14112 }
14117 /* Check if it's a multi-step conversion that can be done using intermediate
14118 types. */
14123 else
14126 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14127 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14128 costly than signed. */
14130 {
14133 intermediate_type
14135 interm_optab
14141 {
14145 }
14146 }
14148 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14149 intermediate steps in promotion sequence. We try
14150 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14153 {
14156 intermediate_type
14158 else
14159 intermediate_type
14167 else
14168 interm_optab
14170 optab_default);
14182 {
14188 }
14193 }
14197 }
14199 /* Generate and return a vector mask of MASK_TYPE such that
14200 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14201 Add the statements to SEQ. */
14203 tree
14206 {
14210 OPTIMIZE_FOR_SPEED));
14214 tree tmp;
14217 else
14222 }
14224 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14225 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14227 tree
14229 tree end_index)
14230 {
14233 }
14235 /* Try to compute the vector types required to vectorize STMT_INFO,
14236 returning true on success and false if vectorization isn't possible.
14237 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14238 take sure that the number of elements in the vectors is no bigger
14239 than GROUP_SIZE.
14241 On success:
14243 - Set *STMT_VECTYPE_OUT to:
14244 - NULL_TREE if the statement doesn't need to be vectorized;
14245 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14247 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14248 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14249 statement does not help to determine the overall number of units. */
14251 opt_result
14256 {
14259 /* For BB vectorization, we should always have a group size once we've
14260 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14261 are tentative requests during things like early data reference
14262 analysis and pattern recognition. */
14265 else
14272 /* MASK_STORE has no lhs, but is ok. */
14274 {
14276 {
14277 /* Ignore calls with no lhs. These must be calls to
14278 #pragma omp simd functions, and what vectorization factor
14279 it really needs can't be determined until
14280 vectorizable_simd_clone_call. */
14285 }
14289 }
14291 tree vectype;
14294 {
14299 }
14301 {
14310 }
14311 else
14312 {
14317 else
14321 {
14324 "get vectype for scalar type (group size %d):"
14326 else
14329 }
14333 "not vectorized:"
14335 scalar_type);
14339 }
14344 stmt);
14348 /* Don't try to compute scalar types if the stmt produces a boolean
14349 vector; use the existing vector type instead. */
14352 {
14353 /* The number of units is set according to the smallest scalar
14354 type (or the largest vector size, but we only support one
14355 vector size per vectorization). */
14359 {
14363 scalar_type);
14365 group_size);
14369 scalar_type);
14372 nunits_vectype);
14373 }
14374 }
14379 "Not vectorized: Incompatible number "
14384 {
14388 }
14392 }
14394 /* Generate and return statement sequence that sets vector length LEN that is:
14396 min_of_start_and_end = min (START_INDEX, END_INDEX);
14397 left_len = END_INDEX - min_of_start_and_end;
14398 rhs = min (left_len, LEN_LIMIT);
14399 LEN = rhs;
14401 Note: the cost of the code generated by this function is modeled
14402 by vect_estimate_min_profitable_iters, so changes here may need
14403 corresponding changes there. */
14405 gimple_seq
14407 {
14419 }