| blob | cabd4e3ae86473c4fdaf1315cdbf434914b68b7b |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
61 /* For lang_hooks.types.type_for_mode. */
64 /* Return the vectorized type for the given statement. */
66 tree
68 {
70 }
72 /* Return TRUE iff the given statement is in an inner loop relative to
73 the loop being vectorized. */
74 bool
76 {
88 }
90 /* Record the cost of a statement, either by directly informing the
91 target model or by saving it in a vector for later processing.
92 Return a preliminary estimate of the statement's cost. */
94 static unsigned
100 {
108 stmt_info_for_cost si
114 }
116 unsigned
121 {
124 }
126 unsigned
131 {
134 }
136 unsigned
140 {
145 }
147 /* Return a variable of type ELEM_TYPE[NELEMS]. */
149 static tree
151 {
154 }
156 /* ARRAY is an array of vectors created by create_vector_array.
157 Return an SSA_NAME for the vector in index N. The reference
158 is part of the vectorization of STMT_INFO and the vector is associated
159 with scalar destination SCALAR_DEST. */
161 static tree
165 {
182 }
184 /* ARRAY is an array of vectors created by create_vector_array.
185 Emit code to store SSA_NAME VECT in index N of the array.
186 The store is part of the vectorization of STMT_INFO. */
188 static void
192 {
193 tree array_ref;
202 }
204 /* PTR is a pointer to an array of type TYPE. Return a representation
205 of *PTR. The memory reference replaces those in FIRST_DR
206 (and its group). */
208 static tree
210 {
211 tree mem_ref;
214 /* Arrays have the same alignment as their type. */
217 }
219 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
220 Emit the clobber before *GSI. */
222 static void
225 {
229 }
231 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
233 /* Function vect_mark_relevant.
235 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
237 static void
240 {
249 /* If this stmt is an original stmt in a pattern, we might need to mark its
250 related pattern stmt instead of the original stmt. However, such stmts
251 may have their own uses that are not in any pattern, in such cases the
252 stmt itself should be marked. */
254 {
255 /* This is the last stmt in a sequence that was detected as a
256 pattern that can potentially be vectorized. Don't mark the stmt
257 as relevant/live because it's not going to be vectorized.
258 Instead mark the pattern-stmt that replaces it. */
262 "last stmt in pattern. don't mark"
272 {
275 "vec_stmt_relevant_p: forcing live pattern stmt "
278 }
284 }
292 {
297 }
300 }
303 /* Function is_simple_and_all_uses_invariant
305 Return true if STMT_INFO is simple and all uses of it are invariant. */
307 bool
309 loop_vec_info loop_vinfo)
310 {
311 tree op;
312 ssa_op_iter iter;
319 {
323 {
328 }
332 }
334 }
336 /* Function vect_stmt_relevant_p.
338 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
339 is "relevant for vectorization".
341 A stmt is considered "relevant for vectorization" if:
342 - it has uses outside the loop.
343 - it has vdefs (it alters memory).
344 - control stmts in the loop (except for the exit condition).
345 - it is an induction and we have multiple exits.
347 CHECKME: what other side effects would the vectorizer allow? */
349 static bool
352 {
354 ssa_op_iter op_iter;
355 imm_use_iterator imm_iter;
356 use_operand_p use_p;
357 def_operand_p def_p;
362 /* cond stmt other than loop exit cond. */
369 /* changing memory. */
373 {
378 }
380 /* uses outside the loop. */
382 {
384 {
387 {
395 /* We expect all such uses to be in the loop exit phis
396 (because of loop closed form) */
400 }
401 }
402 }
404 /* Check if it's an induction and multiple exits. In this case there will be
405 a usage later on after peeling which is needed for the alternate exit. */
408 {
411 "vec_stmt_relevant_p: induction forced for "
415 }
419 {
424 }
427 }
430 /* Function exist_non_indexing_operands_for_use_p
432 USE is one of the uses attached to STMT_INFO. Check if USE is
433 used in STMT_INFO for anything other than indexing an array. */
435 static bool
437 {
438 tree operand;
440 /* USE corresponds to some operand in STMT. If there is no data
441 reference in STMT, then any operand that corresponds to USE
442 is not indexing an array. */
446 /* STMT has a data_ref. FORNOW this means that its of one of
447 the following forms:
448 -1- ARRAY_REF = var
449 -2- var = ARRAY_REF
450 (This should have been verified in analyze_data_refs).
452 'var' in the second case corresponds to a def, not a use,
453 so USE cannot correspond to any operands that are not used
454 for array indexing.
456 Therefore, all we need to check is if STMT falls into the
457 first case, and whether var corresponds to USE. */
461 {
464 {
477 }
479 }
491 }
494 /*
495 Function process_use.
497 Inputs:
498 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
499 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
500 that defined USE. This is done by calling mark_relevant and passing it
501 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
502 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
503 be performed.
505 Outputs:
506 Generally, LIVE_P and RELEVANT are used to define the liveness and
507 relevance info of the DEF_STMT of this USE:
508 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
509 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
510 Exceptions:
511 - case 1: If USE is used only for address computations (e.g. array indexing),
512 which does not need to be directly vectorized, then the liveness/relevance
513 of the respective DEF_STMT is left unchanged.
514 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
515 we skip DEF_STMT cause it had already been processed.
516 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
517 "relevant" will be modified accordingly.
519 Return true if everything is as expected. Return false otherwise. */
521 static opt_result
525 {
526 stmt_vec_info dstmt_vinfo;
529 /* case 1: we are only interested in uses that need to be vectorized. Uses
530 that are used for address computation are not considered relevant. */
536 "not vectorized:"
545 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
546 We have to force the stmt live since the epilogue loop needs it to
547 continue computing the reduction. */
553 {
559 }
561 /* case 3a: outer-loop stmt defining an inner-loop stmt:
562 outer-loop-header-bb:
563 d = dstmt_vinfo
564 inner-loop:
565 stmt # use (d)
566 outer-loop-tail-bb:
567 ... */
569 {
575 {
596 }
597 }
599 /* case 3b: inner-loop stmt defining an outer-loop stmt:
600 outer-loop-header-bb:
601 ...
602 inner-loop:
603 d = dstmt_vinfo
604 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
605 stmt # use (d) */
607 {
613 {
631 }
632 }
633 /* We are also not interested in uses on loop PHI backedges that are
634 inductions. Otherwise we'll needlessly vectorize the IV increment
635 and cause hybrid SLP for SLP inductions. Unless the PHI is live
636 of course. */
643 {
648 }
653 }
656 /* Function vect_mark_stmts_to_be_vectorized.
658 Not all stmts in the loop need to be vectorized. For example:
660 for i...
661 for j...
662 1. T0 = i + j
663 2. T1 = a[T0]
665 3. j = j + 1
667 Stmt 1 and 3 do not need to be vectorized, because loop control and
668 addressing of vectorized data-refs are handled differently.
670 This pass detects such stmts. */
672 opt_result
674 {
678 gimple_stmt_iterator si;
680 basic_block bb;
688 /* 1. Init worklist. */
690 {
693 {
701 }
703 {
713 }
714 }
716 /* 2. Process_worklist */
718 {
719 use_operand_p use_p;
720 ssa_op_iter iter;
727 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
728 (DEF_STMT) as relevant/irrelevant according to the relevance property
729 of STMT. */
732 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
733 propagated as is to the DEF_STMTs of its USEs.
735 One exception is when STMT has been identified as defining a reduction
736 variable; in this case we set the relevance to vect_used_by_reduction.
737 This is because we distinguish between two kinds of relevant stmts -
738 those that are used by a reduction computation, and those that are
739 (also) used by a regular computation. This allows us later on to
740 identify stmts that are used solely by a reduction, and therefore the
741 order of the results that they produce does not have to be kept. */
744 {
773 }
776 {
777 /* Pattern statements are not inserted into the code, so
778 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
779 have to scan the RHS or function arguments instead. */
781 {
787 {
788 opt_result res
798 }
800 {
803 {
804 opt_result res
809 }
810 }
811 }
813 {
816 opt_result res
825 }
827 {
829 {
831 opt_result res
836 }
837 }
838 else
840 }
841 else
843 {
845 opt_result res
850 }
853 {
854 gather_scatter_info gs_info;
857 opt_result res
861 {
865 }
866 }
870 }
872 /* Function vect_model_simple_cost.
874 Models cost for simple operations, i.e. those that only emit ncopies of a
875 single op. Right now, this does not account for multiple insns that could
876 be generated for the single vector op. We will handle that shortly. */
878 static void
883 slp_tree node,
886 {
891 /* ??? Somehow we need to fix this at the callers. */
896 /* Cost the "broadcast" of a scalar operand in to a vector operand.
897 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
898 cost model. */
904 /* Pass the inside-of-loop statements to the target-specific cost model. */
910 "vect_model_simple_cost: inside_cost = %d, "
912 }
915 /* Model cost for type demotion and promotion operations. PWR is
916 normally zero for single-step promotions and demotions. It will be
917 one if two-step promotion/demotion is required, and so on. NCOPIES
918 is the number of vector results (and thus number of instructions)
919 for the narrowest end of the operation chain. Each additional
920 step doubles the number of instructions required. If WIDEN_ARITH
921 is true the stmt is doing widening arithmetic. */
923 static void
929 {
934 {
936 widen_arith
940 }
942 /* FORNOW: Assuming maximum 2 args per stmts. */
950 "vect_model_promotion_demotion_cost: inside_cost = %d, "
952 }
954 /* Returns true if the current function returns DECL. */
956 static bool
958 {
959 edge_iterator ei;
960 edge e;
962 {
968 /* We often end up with an aggregate copy to the result decl,
969 handle that case as well. First skip intermediate clobbers
970 though. */
972 do
973 {
975 }
981 }
983 }
985 /* Calculate cost of DR's memory access. */
986 void
988 dr_alignment_support alignment_support_scheme,
992 {
994 {
996 {
999 vect_body);
1005 }
1008 {
1009 /* Here, we assign an additional cost for the unaligned store. */
1015 "vect_model_store_cost: unaligned supported by "
1018 }
1021 {
1028 }
1032 }
1033 }
1035 /* Calculate cost of DR's memory access. */
1036 void
1038 dr_alignment_support alignment_support_scheme,
1045 {
1047 {
1049 {
1058 }
1060 {
1061 /* Here, we assign an additional cost for the unaligned load. */
1068 "vect_model_load_cost: unaligned supported by "
1072 }
1074 {
1080 /* FIXME: If the misalignment remains fixed across the iterations of
1081 the containing loop, the following cost should be added to the
1082 prologue costs. */
1092 }
1094 {
1097 "vect_model_load_cost: unaligned software "
1100 /* Unaligned software pipeline has a load of an address, an initial
1101 load, and possibly a mask operation to "prime" the loop. However,
1102 if this is an access in a group of loads, which provide grouped
1103 access, then the above cost should only be considered for one
1104 access in the group. Inside the loop, there is a load op
1105 and a realignment op. */
1108 {
1116 }
1125 "vect_model_load_cost: explicit realign optimized"
1129 }
1132 {
1139 }
1143 }
1144 }
1146 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1147 the loop preheader for the vectorized stmt STMT_VINFO. */
1149 static void
1152 {
1155 else
1161 }
1163 /* Function vect_init_vector.
1165 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1166 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1167 vector type a vector with all elements equal to VAL is created first.
1168 Place the initialization at GSI if it is not NULL. Otherwise, place the
1169 initialization at the loop preheader.
1170 Return the DEF of INIT_STMT.
1171 It will be used in the vectorization of STMT_INFO. */
1173 tree
1176 {
1178 tree new_temp;
1180 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1182 {
1185 {
1186 /* Scalar boolean value should be transformed into
1187 all zeros or all ones value before building a vector. */
1189 {
1195 else
1196 {
1202 }
1203 }
1204 else
1205 {
1210 else
1211 /* ??? Condition vectorization expects us to do
1212 promotion of invariant/external defs. */
1216 {
1220 }
1221 }
1222 }
1224 }
1230 }
1233 /* Function vect_get_vec_defs_for_operand.
1235 OP is an operand in STMT_VINFO. This function returns a vector of
1236 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1238 In the case that OP is an SSA_NAME which is defined in the loop, then
1239 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1241 In case OP is an invariant or constant, a new stmt that creates a vector def
1242 needs to be introduced. VECTYPE may be used to specify a required type for
1243 vector invariant. */
1245 void
1249 {
1259 stmt_vec_info def_stmt_info;
1268 {
1270 tree vector_type;
1277 else
1284 }
1285 else
1286 {
1292 }
1293 }
1296 /* Get vectorized definitions for OP0 and OP1. */
1298 void
1305 {
1307 {
1316 }
1317 else
1318 {
1331 }
1332 }
1334 void
1341 {
1347 }
1349 /* Helper function called by vect_finish_replace_stmt and
1350 vect_finish_stmt_generation. Set the location of the new
1351 statement and create and return a stmt_vec_info for it. */
1353 static void
1356 {
1361 {
1364 /* While EH edges will generally prevent vectorization, stmt might
1365 e.g. be in a must-not-throw region. Ensure newly created stmts
1366 that could throw are part of the same region. */
1370 }
1371 else
1373 }
1375 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1376 which sets the same scalar result as STMT_INFO did. Create and return a
1377 stmt_vec_info for VEC_STMT. */
1379 void
1382 {
1390 }
1392 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1393 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1395 void
1399 {
1404 {
1408 {
1412 /* If we have an SSA vuse and insert a store, update virtual
1413 SSA form to avoid triggering the renamer. Do so only
1414 if we can easily see all uses - which is what almost always
1415 happens with the way vectorized stmts are inserted. */
1424 {
1428 }
1429 }
1430 }
1433 }
1435 /* We want to vectorize a call to combined function CFN with function
1436 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1437 as the types of all inputs. Check whether this is possible using
1438 an internal function, returning its code if so or IFN_LAST if not. */
1440 static internal_fn
1443 {
1444 internal_fn ifn;
1447 else
1450 {
1453 {
1458 /* The type size of both the vectype_in and vectype_out should be
1459 exactly the same when vectype_out isn't participating the optab.
1460 While there is no restriction for type size when vectype_out
1461 is part of the optab query. */
1466 OPTIMIZE_FOR_SPEED))
1468 }
1469 }
1471 }
1475 gimple_stmt_iterator *);
1477 /* Check whether a load or store statement in the loop described by
1478 LOOP_VINFO is possible in a loop using partial vectors. This is
1479 testing whether the vectorizer pass has the appropriate support,
1480 as well as whether the target does.
1482 VLS_TYPE says whether the statement is a load or store and VECTYPE
1483 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1484 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1485 says how the load or store is going to be implemented and GROUP_SIZE
1486 is the number of load or store statements in the containing group.
1487 If the access is a gather load or scatter store, GS_INFO describes
1488 its arguments. If the load or store is conditional, SCALAR_MASK is the
1489 condition under which it occurs.
1491 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1492 vectors is not supported, otherwise record the required rgroup control
1493 types. */
1495 static void
1497 slp_tree slp_node,
1498 vec_load_store_type vls_type,
1500 vect_memory_access_type
1501 memory_access_type,
1503 tree scalar_mask)
1504 {
1505 /* Invariant loads need no special support. */
1512 else
1520 {
1521 internal_fn ifn
1528 scalar_mask);
1529 else
1530 {
1533 "can't operate on partial vectors because"
1534 " the target doesn't have an appropriate"
1537 }
1539 }
1542 {
1543 internal_fn ifn = (is_load
1544 ? IFN_MASK_GATHER_LOAD
1546 internal_fn len_ifn = (is_load
1547 ? IFN_MASK_LEN_GATHER_LOAD
1559 scalar_mask);
1560 else
1561 {
1564 "can't operate on partial vectors because"
1565 " the target doesn't have an appropriate"
1568 }
1570 }
1574 {
1575 /* Element X of the data must come from iteration i * VF + X of the
1576 scalar loop. We need more work to support other mappings. */
1579 "can't operate on partial vectors because an"
1583 }
1586 {
1589 "can't operate on partial vectors when emulating"
1593 }
1595 /* We might load more scalars than we need for permuting SLP loads.
1596 We checked in get_group_load_store_type that the extra elements
1597 don't leak into a new vector. */
1599 {
1604 };
1608 machine_mode mask_mode;
1609 machine_mode vmode;
1612 {
1617 }
1620 {
1624 }
1627 {
1630 "can't operate on partial vectors because the"
1631 " target doesn't have the appropriate partial"
1634 }
1635 }
1637 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1638 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1639 that needs to be applied to all loads and stores in a vectorized loop.
1640 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1641 otherwise return VEC_MASK & LOOP_MASK.
1643 MASK_TYPE is the type of both masks. If new statements are needed,
1644 insert them before GSI. */
1646 static tree
1649 {
1665 }
1667 /* Determine whether we can use a gather load or scatter store to vectorize
1668 strided load or store STMT_INFO by truncating the current offset to a
1669 smaller width. We need to be able to construct an offset vector:
1671 { 0, X, X*2, X*3, ... }
1673 without loss of precision, where X is STMT_INFO's DR_STEP.
1675 Return true if this is possible, describing the gather load or scatter
1676 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1678 static bool
1682 {
1687 {
1688 /* ??? Perhaps we could use range information here? */
1693 }
1695 /* Get the number of bits in an element. */
1700 /* Set COUNT to the upper limit on the number of elements - 1.
1701 Start with the maximum vectorization factor. */
1704 /* Try lowering COUNT to the number of scalar latch iterations. */
1706 widest_int max_iters;
1711 /* Try scales of 1 and the element size. */
1715 {
1717 widest_int factor;
1721 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1728 /* Find the narrowest viable offset type. */
1733 /* See whether the target supports the operation with an offset
1734 no narrower than OFFSET_TYPE. */
1743 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1744 but we don't need to store that here. */
1752 }
1756 "truncating gather/scatter offset to %d bits"
1760 }
1762 /* Return true if we can use gather/scatter internal functions to
1763 vectorize STMT_INFO, which is a grouped or strided load or store.
1764 MASKED_P is true if load or store is conditional. When returning
1765 true, fill in GS_INFO with the information required to perform the
1766 operation. */
1768 static bool
1772 {
1787 "using gather/scatter for strided/grouped access,"
1791 }
1793 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1794 elements with a known constant step. Return -1 if that step
1795 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1797 static int
1799 {
1802 size_zero_node);
1803 }
1805 /* If the target supports a permute mask that reverses the elements in
1806 a vector of type VECTYPE, return that mask, otherwise return null. */
1808 tree
1810 {
1813 /* The encoding has a single stepped pattern. */
1820 indices))
1823 }
1825 /* A subroutine of get_load_store_type, with a subset of the same
1826 arguments. Handle the case where STMT_INFO is a load or store that
1827 accesses consecutive elements with a negative step. Sets *POFFSET
1828 to the offset to be applied to the DR for the first access. */
1830 static vect_memory_access_type
1833 vec_load_store_type vls_type,
1835 {
1837 dr_alignment_support alignment_support_scheme;
1840 {
1845 }
1847 /* For backward running DRs the first access in vectype actually is
1848 N-1 elements before the address of the DR. */
1853 alignment_support_scheme
1857 {
1863 }
1866 {
1869 "negative step with invariant source;"
1872 }
1875 {
1881 }
1884 }
1886 /* STMT_INFO is either a masked or unconditional store. Return the value
1887 being stored. */
1889 tree
1891 {
1893 {
1896 }
1898 {
1903 }
1905 }
1907 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1909 This function returns a vector type which can be composed with NETLS pieces,
1910 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1911 same vector size as the return vector. It checks target whether supports
1912 pieces-size vector mode for construction firstly, if target fails to, check
1913 pieces-size scalar mode for construction further. It returns NULL_TREE if
1914 fails to find the available composition.
1916 For example, for (vtype=V16QI, nelts=4), we can probably get:
1917 - V16QI with PTYPE V4QI.
1918 - V4SI with PTYPE SI.
1919 - NULL_TREE. */
1921 static tree
1923 {
1931 /* When we are asked to compose the vector from its components let
1932 that happen directly. */
1934 {
1937 }
1942 {
1943 /* First check if vec_init optab supports construction from
1944 vector pieces directly. */
1947 machine_mode rmode;
1951 {
1954 }
1956 /* Otherwise check if exists an integer type of the same piece size and
1957 if vec_init optab supports construction from it directly. */
1962 {
1965 }
1966 }
1969 }
1971 /* A subroutine of get_load_store_type, with a subset of the same
1972 arguments. Handle the case where STMT_INFO is part of a grouped load
1973 or store.
1975 For stores, the statements in the group are all consecutive
1976 and there is no gap at the end. For loads, the statements in the
1977 group might not be consecutive; there can be gaps between statements
1978 as well as at the end. */
1980 static bool
1990 {
1993 stmt_vec_info first_stmt_info;
1997 {
2001 }
2002 else
2003 {
2007 }
2013 /* True if the vectorized statements would access beyond the last
2014 statement in the group. */
2017 /* True if we can cope with such overrun by peeling for gaps, so that
2018 there is at least one final scalar iteration after the vector loop. */
2021 && loop_vinfo
2024 /* There can only be a gap at the end of the group if the stride is
2025 known at compile time. */
2028 /* Stores can't yet have gaps. */
2032 {
2033 /* For SLP vectorization we directly vectorize a subchain
2034 without permutation. */
2036 first_dr_info
2039 {
2040 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2041 separated by the stride, until we have a complete vector.
2042 Fall back to scalar accesses if that isn't possible. */
2045 else
2047 }
2048 else
2049 {
2052 {
2054 "Grouped store with gaps requires"
2057 }
2058 /* An overrun is fine if the trailing elements are smaller
2059 than the alignment boundary B. Every vector access will
2060 be a multiple of B and so we are guaranteed to access a
2061 non-gap element in the same B-sized block. */
2064 vectype)
2068 /* If the gap splits the vector in half and the target
2069 can do half-vector operations avoid the epilogue peeling
2070 by simply loading half of the vector only. Usually
2071 the construction with an upper zero half will be elided. */
2072 dr_alignment_support alss;
2074 tree half_vtype;
2076 && !masked_p
2079 == dr_aligned
2088 {
2093 }
2096 {
2098 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2099 only correct for single element "interleaving" SLP. */
2102 else
2103 {
2104 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2105 separated by the stride, until we have a complete vector.
2106 Fall back to scalar accesses if that isn't possible. */
2109 else
2111 }
2112 }
2114 {
2117 /* Invariant accesses perform only component accesses, alignment
2118 is irrelevant for them. */
2120 }
2121 else
2124 /* When we have a contiguous access across loop iterations
2125 but the access in the loop doesn't cover the full vector
2126 we can end up with no gap recorded but still excess
2127 elements accessed, see PR103116. Make sure we peel for
2128 gaps if necessary and sufficient and give up if not.
2130 If there is a combination of the access not covering the full
2131 vector and a gap recorded then we may need to peel twice. */
2136 nunits))
2137 {
2142 /* Peeling for gaps assumes that a single scalar iteration
2143 is enough to make sure the last vector iteration doesn't
2144 access excess elements.
2145 ??? Enhancements include peeling multiple iterations
2146 or using masked loads with a static mask. */
2148 {
2151 "peeling for gaps insufficient for "
2154 }
2156 }
2157 }
2158 }
2159 else
2160 {
2161 /* We can always handle this case using elementwise accesses,
2162 but see if something more efficient is available. */
2165 /* If there is a gap at the end of the group then these optimizations
2166 would access excess elements in the last iteration. */
2168 /* An overrun is fine if the trailing elements are smaller than the
2169 alignment boundary B. Every vector access will be a multiple of B
2170 and so we are guaranteed to access a non-gap element in the
2171 same B-sized block. */
2173 && !masked_p
2181 {
2182 /* First cope with the degenerate case of a single-element
2183 vector. */
2185 ;
2187 else
2188 {
2189 /* Otherwise try using LOAD/STORE_LANES. */
2190 *lanes_ifn
2194 masked_p);
2196 {
2199 }
2201 /* If that fails, try using permuting loads. */
2204 single_element_p,
2205 group_size)
2207 {
2210 }
2211 }
2212 }
2214 /* As a last resort, trying using a gather load or scatter store.
2216 ??? Although the code can handle all group sizes correctly,
2217 it probably isn't a win to use separate strided accesses based
2218 on nearby locations. Or, even if it's a win over scalar code,
2219 it might not be a win over vectorizing at a lower VF, if that
2220 allows us to use contiguous accesses. */
2222 && single_element_p
2223 && loop_vinfo
2227 }
2231 {
2234 }
2235 else
2236 {
2238 *alignment_support_scheme
2241 }
2244 {
2245 /* STMT is the leader of the group. Check the operands of all the
2246 stmts of the group. */
2249 {
2253 {
2258 }
2260 }
2261 }
2264 {
2268 "Data access with gaps requires scalar "
2271 }
2274 }
2276 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2277 if there is a memory access type that the vectorized form can use,
2278 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2279 or scatters, fill in GS_INFO accordingly. In addition
2280 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2281 the target does not support the alignment scheme. *MISALIGNMENT
2282 is set according to the alignment of the access (including
2283 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2285 SLP says whether we're performing SLP rather than loop vectorization.
2286 MASKED_P is true if the statement is conditional on a vectorized mask.
2287 VECTYPE is the vector type that the vectorized statements will use.
2288 NCOPIES is the number of vector statements that will be needed. */
2290 static bool
2301 {
2307 {
2311 /* When using internal functions, we rely on pattern recognition
2312 to convert the type of the offset to the type that the target
2313 requires, with the result being a call to an internal function.
2314 If that failed for some reason (e.g. because another pattern
2315 took priority), just handle cases in which the offset already
2316 has the right type. */
2321 {
2327 }
2331 {
2337 }
2339 {
2345 {
2348 "unsupported vector types for emulated "
2351 }
2352 }
2353 /* Gather-scatter accesses perform only component accesses, alignment
2354 is irrelevant for them. */
2356 }
2358 {
2360 masked_p,
2362 alignment_support_scheme,
2365 }
2367 {
2373 else
2375 /* Alignment is irrelevant here. */
2377 }
2378 else
2379 {
2382 {
2385 /* Invariant accesses perform only component accesses, alignment
2386 is irrelevant for them. */
2388 }
2389 else
2390 {
2394 else
2398 *alignment_support_scheme
2402 }
2403 }
2408 {
2411 "Not using elementwise accesses due to variable "
2414 }
2417 {
2422 }
2424 /* FIXME: At the moment the cost model seems to underestimate the
2425 cost of using elementwise accesses. This check preserves the
2426 traditional behavior until that can be fixed. */
2435 {
2440 }
2442 }
2444 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2445 conditional operation STMT_INFO. When returning true, store the mask
2446 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2447 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2448 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2450 static bool
2455 {
2457 tree mask_vectype;
2458 slp_tree mask_node_1;
2461 {
2466 }
2469 {
2474 }
2476 /* If the caller is not prepared for adjusting an external/constant
2477 SLP mask vector type fail. */
2479 && !mask_node
2481 {
2486 }
2491 mask_node_1);
2494 {
2499 }
2503 {
2506 "vector mask type %T"
2511 }
2518 }
2520 /* Return true if stored value is suitable for vectorizing store
2521 statement STMT_INFO. When returning true, store the scalar stored
2522 in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2523 the type of the vectorized store value in
2524 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2526 static bool
2531 {
2534 {
2538 }
2540 op_no = vect_slp_child_index_for_operand
2544 tree rhs_vectype;
2547 {
2552 }
2554 /* In the case this is a store from a constant make sure
2555 native_encode_expr can handle it. */
2557 {
2562 }
2566 {
2571 }
2577 else
2580 }
2582 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2583 Note that we support masks with floating-point type, in which case the
2584 floats are interpreted as a bitmask. */
2586 static tree
2589 {
2594 {
2598 }
2600 {
2601 REAL_VALUE_TYPE r;
2609 }
2611 }
2613 /* Build an all-zero merge value of type VECTYPE while vectorizing
2614 STMT_INFO as a gather load. */
2616 static tree
2619 {
2620 tree merge;
2624 {
2625 REAL_VALUE_TYPE r;
2631 }
2632 else
2636 }
2638 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2639 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2640 the gather load operation. If the load is conditional, MASK is the
2641 vectorized condition, otherwise MASK is null. PTR is the base
2642 pointer and OFFSET is the vectorized offset. */
2649 {
2658 tree var;
2660 && (!mask
2666 {
2674 }
2679 {
2681 {
2686 else
2690 gassign *new_stmt
2695 {
2702 }
2705 }
2706 else
2707 {
2710 }
2711 }
2712 else
2713 {
2716 }
2723 {
2731 }
2734 }
2736 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2737 instructions before GSI. GS_INFO describes the scatter store operation.
2738 PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
2739 vectorized data to store.
2740 If the store is conditional, MASK is the vectorized condition, otherwise
2741 MASK is null. */
2748 {
2761 {
2764 tree utype;
2767 else
2771 gassign *new_stmt
2776 {
2782 }
2783 }
2784 else
2785 {
2788 }
2792 {
2800 }
2804 {
2812 }
2815 gcall *new_stmt
2818 }
2820 /* Prepare the base and offset in GS_INFO for vectorization.
2821 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2822 to the vectorized offset argument for the first copy of STMT_INFO.
2823 STMT_INFO is the statement described by GS_INFO and LOOP is the
2824 containing loop. */
2826 static void
2831 {
2835 {
2836 basic_block new_bb;
2840 }
2843 else
2844 {
2845 unsigned ncopies
2850 }
2851 }
2853 /* Prepare to implement a grouped or strided load or store using
2854 the gather load or scatter store operation described by GS_INFO.
2855 STMT_INFO is the load or store statement.
2857 Set *DATAREF_BUMP to the amount that should be added to the base
2858 address after each copy of the vectorized statement. Set *VEC_OFFSET
2859 to an invariant offset vector in which element I has the value
2860 I * DR_STEP / SCALE. */
2862 static void
2864 loop_vec_info loop_vinfo,
2869 {
2874 {
2875 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2876 ivtmp_8 = _31 * 16 (step in bytes);
2877 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2878 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2879 tree loop_len
2881 tree tmp
2884 loop_len);
2886 GSI_SAME_STMT);
2887 }
2888 else
2889 {
2890 tree bump
2895 }
2897 /* The offset given in GS_INFO can have pointer type, so use the element
2898 type of the vector instead. */
2901 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2906 /* Create {0, X, X*2, X*3, ...}. */
2910 }
2912 /* Prepare the pointer IVs which needs to be updated by a variable amount.
2913 Such variable amount is the outcome of .SELECT_VL. In this case, we can
2914 allow each iteration process the flexible number of elements as long as
2915 the number <= vf elments.
2917 Return data reference according to SELECT_VL.
2918 If new statements are needed, insert them before GSI. */
2920 static tree
2924 vect_memory_access_type memory_access_type)
2925 {
2929 /* gather/scatter never reach here. */
2932 /* When we support SELECT_VL pattern, we dynamic adjust
2933 the memory address by .SELECT_VL result.
2935 The result of .SELECT_VL is the number of elements to
2936 be processed of each iteration. So the memory address
2937 adjustment operation should be:
2939 addr = addr + .SELECT_VL (ARG..) * step;
2940 */
2941 tree loop_len
2944 /* Since the outcome of .SELECT_VL is element size, we should adjust
2945 it into bytesize so that it can be used in address pointer variable
2946 amount IVs adjustment. */
2953 }
2955 /* Return the amount that should be added to a vector pointer to move
2956 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
2957 being vectorized and MEMORY_ACCESS_TYPE describes the type of
2958 vectorization. */
2960 static tree
2963 vect_memory_access_type memory_access_type,
2965 {
2973 memory_access_type);
2980 }
2982 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
2984 static bool
2990 {
3000 /* Multiple types in SLP are handled by creating the appropriate number of
3001 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3002 case of SLP. */
3005 else
3011 {
3017 }
3028 /* The encoding uses one stepped pattern for each byte in the word. */
3040 {
3043 {
3048 }
3055 slp_node
3059 }
3063 /* Transform. */
3067 /* Arguments are ready. create the new vector stmt. */
3069 tree vop;
3071 {
3087 else
3089 }
3096 }
3098 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3099 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3100 in a single step. On success, store the binary pack code in
3101 *CONVERT_CODE. */
3103 static bool
3106 {
3111 code_helper code;
3121 }
3123 /* Function vectorizable_call.
3125 Check if STMT_INFO performs a function call that can be vectorized.
3126 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3127 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3128 Return true if STMT_INFO is vectorizable in this way. */
3130 static bool
3135 {
3137 tree vec_dest;
3138 tree scalar_dest;
3139 tree op;
3142 poly_uint64 nunits_in;
3143 poly_uint64 nunits_out;
3149 vect_unknown_def_type };
3157 tree lhs;
3167 /* Is STMT_INFO a vectorizable call? */
3175 /* Handled by vectorizable_load and vectorizable_store. */
3186 /* Process function arguments. */
3191 /* Bail out if the function has more than four arguments, we do not have
3192 interesting builtin functions to vectorize with more than two arguments
3193 except for fma. No arguments is also not good. */
3197 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3200 {
3203 }
3204 /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3205 argument just says whether it is well-defined at zero or not and what
3206 value should be returned for it. */
3208 {
3211 }
3218 {
3220 {
3225 }
3229 {
3234 }
3236 /* We can only handle calls with arguments of the same type. */
3239 {
3244 }
3252 {
3257 }
3258 }
3259 /* If all arguments are external or constant defs, infer the vector type
3260 from the scalar type. */
3266 {
3272 }
3276 {
3281 }
3284 {
3289 }
3291 /* FORNOW */
3300 else
3303 /* We only handle functions that do not read or clobber memory. */
3305 {
3310 }
3312 /* For now, we only vectorize functions if a target specific builtin
3313 is available. TODO -- in some cases, it might be profitable to
3314 insert the calls for pieces of the vector, in order to be able
3315 to vectorize other operations in the loop. */
3320 /* First try using an internal function. */
3328 vectype_in);
3330 /* If that fails, try asking for a target-specific built-in function. */
3332 {
3339 }
3342 {
3344 && !slp_node
3345 && loop_vinfo
3350 {
3351 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3352 { 0, 1, 2, ... vf - 1 } vector. */
3354 }
3362 else
3363 {
3368 }
3369 }
3375 else
3378 /* Sanity check: make sure that at least one copy of the vectorized stmt
3379 needs to be generated. */
3389 {
3395 {
3400 }
3412 {
3416 OPTIMIZE_FOR_SPEED))
3419 OPTIMIZE_FOR_SPEED)))
3420 {
3423 "can't use a fully-masked loop because no"
3426 }
3427 else
3428 {
3429 unsigned int nvectors
3430 = (slp_node
3438 OPTIMIZE_FOR_SPEED))
3441 else
3443 scalar_mask);
3444 }
3445 }
3447 }
3449 /* Transform. */
3454 /* Handle def. */
3462 {
3464 {
3466 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3468 }
3471 }
3473 {
3476 }
3481 {
3486 {
3487 /* Build argument list for the vectorized call. */
3489 {
3495 /* Arguments are ready. Create the new vector stmt. */
3497 {
3500 {
3502 /* Always true for SLP. */
3507 }
3510 {
3513 }
3521 {
3522 /* We don't define any narrowing conditional functions
3523 at present. */
3526 gcall *call
3532 {
3535 }
3541 }
3542 else
3543 {
3545 {
3547 /* Always true for SLP. */
3549 tree len
3552 signed char biasval
3557 }
3559 {
3561 /* Always true for SLP. */
3569 }
3574 else
3581 }
3583 }
3585 }
3592 {
3595 {
3600 }
3602 }
3609 {
3612 signed char biasval
3617 }
3619 {
3625 }
3629 {
3631 tree new_var
3638 }
3640 {
3641 /* We don't define any narrowing conditional functions at
3642 present. */
3650 {
3653 }
3656 half_res);
3658 }
3659 else
3660 {
3664 else
3671 }
3676 }
3678 {
3681 }
3682 }
3684 {
3686 /* We don't define any narrowing conditional functions at present. */
3689 {
3690 /* Build argument list for the vectorized call. */
3693 else
3697 {
3703 /* Arguments are ready. Create the new vector stmt. */
3705 {
3709 {
3713 }
3717 else
3724 }
3726 }
3729 {
3732 {
3736 }
3742 }
3750 }
3756 {
3759 }
3760 }
3761 else
3762 /* No current target implements this case. */
3767 /* The call in STMT might prevent it from being removed in dce.
3768 We however cannot remove it here, due to the way the ssa name
3769 it defines is mapped to the new definition. So just replace
3770 rhs of the statement with something harmless. */
3778 gassign *new_stmt
3783 }
3786 struct simd_call_arg_info
3787 {
3788 tree vectype;
3789 tree op;
3790 HOST_WIDE_INT linear_step;
3794 };
3796 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3797 is linear within simd lane (but not within whole loop), note it in
3798 *ARGINFO. */
3800 static void
3803 {
3815 {
3816 tree t;
3820 {
3835 CASE_CONVERT:
3847 }
3853 {
3860 }
3861 }
3862 }
3864 /* Function vectorizable_simd_clone_call.
3866 Check if STMT_INFO performs a function call that can be vectorized
3867 by calling a simd clone of the function.
3868 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3869 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3870 Return true if STMT_INFO is vectorizable in this way. */
3872 static bool
3876 stmt_vector_for_cost *)
3877 {
3878 tree vec_dest;
3879 tree scalar_dest;
3882 tree vectype;
3883 poly_uint64 nunits;
3896 /* Is STMT a vectorizable call? */
3904 {
3910 }
3936 /* Process function arguments. */
3939 /* Bail out if the function has zero arguments. */
3950 {
3951 simd_call_arg_info thisarginfo;
3952 affine_iv iv;
3966 {
3971 }
3975 {
3976 /* With SLP we determine the vector type of constants/externals
3977 at analysis time, handling conflicts via
3978 vect_maybe_update_slp_op_vectype. At transform time
3979 we have a vector type recorded for SLP. */
3981 || !slp_node
3986 slp_node);
3987 }
3988 else
3991 /* For linear arguments, the analyze phase should have saved
3992 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
3995 {
3999 thisarginfo.simd_lane_linear
4001 /* If loop has been peeled for alignment, we need to adjust it. */
4005 {
4011 thisarginfo.op
4015 }
4016 }
4020 && loop_vinfo
4025 {
4028 }
4033 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4034 linear too. */
4037 && !vec_stmt
4040 && loop_vinfo
4045 }
4053 else
4056 {
4059 /* The number of arguments in the call and the number of parameters in
4060 the simdclone should match. However, when the simdclone is
4061 'inbranch', it could have one more paramater than nargs when using
4062 an inbranch simdclone to call a non-inbranch call, either in a
4063 non-masked loop using a all true constant mask, or inside a masked
4064 loop using it's mask. */
4067 simd_nargs--;
4082 {
4084 {
4115 /* FORNOW */
4119 /* While we can create a traditional data vector from
4120 an incoming integer mode mask we have no good way to
4121 force generate an integer mode mask from a traditional
4122 boolean vector input. */
4130 }
4134 {
4137 }
4141 }
4147 {
4149 SIMD_CLONE_ARG_TYPE_MASK);
4150 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4151 not in a branch, as we'd have to construct an all-true mask. */
4154 }
4156 {
4159 }
4160 }
4169 num_mask_args++;
4172 {
4176 {
4180 slp_node);
4185 }
4189 {
4194 }
4197 {
4200 {
4203 {
4204 /* FORNOW we only have partial support for vector-type masks
4205 that can't hold all of simdlen. */
4208 vect_location,
4209 "in-branch vector clones are not yet"
4212 }
4213 }
4215 {
4218 num_mask_args),
4220 {
4221 /* FORNOW we only have partial support for integer-type masks
4222 that represent the same number of lanes as the
4223 vectorized mask inputs. */
4226 vect_location,
4227 "in-branch vector clones are not yet "
4230 }
4231 }
4232 else
4233 {
4236 vect_location,
4237 "in-branch vector clones not supported"
4240 }
4241 }
4242 }
4248 else
4251 /* If the function isn't const, only allow it in simd loops where user
4252 has asserted that at least nunits consecutive iterations can be
4253 performed using SIMD instructions. */
4258 /* Sanity check: make sure that at least one copy of the vectorized stmt
4259 needs to be generated. */
4263 {
4267 {
4272 }
4273 /* When the original call is pure or const but the SIMD ABI dictates
4274 an aggregate return we will have to use a virtual definition and
4275 in a loop eventually even need to add a virtual PHI. That's
4276 not straight-forward so allow to fix this up via renaming. */
4281 /* ??? For SLP code-gen we end up inserting after the last
4282 vector argument def rather than at the original call position
4283 so automagic virtual operand updating doesn't work. */
4288 {
4290 {
4295 {
4305 }
4315 }
4316 }
4319 {
4323 "can't use a fully-masked loop because a"
4326 }
4330 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4331 dt, slp_node, cost_vec); */
4333 }
4335 /* Transform. */
4340 /* Handle def. */
4346 {
4350 {
4353 }
4354 }
4360 {
4363 }
4364 else
4367 {
4368 poly_uint64 callee_nelements;
4369 poly_uint64 caller_nelements;
4370 /* Build argument list for the vectorized call. */
4373 else
4377 {
4379 tree atype;
4382 {
4389 {
4391 {
4399 {
4406 }
4407 else
4408 {
4412 }
4414 vec_oprnd0
4418 gassign *new_stmt
4420 vec_oprnd0);
4424 }
4425 else
4426 {
4434 else
4437 {
4439 {
4443 op,
4447 }
4448 else
4454 vec_oprnd0);
4455 }
4458 atype))
4459 {
4461 vec_oprnd0);
4462 gassign *new_stmt
4464 vec_oprnd0);
4468 }
4469 else
4471 else
4472 {
4474 gassign *new_stmt
4476 vec_oprnd0);
4480 }
4481 }
4482 }
4486 {
4495 {
4497 {
4498 /* The mask type has fewer elements than simdlen. */
4500 /* FORNOW */
4502 }
4504 {
4505 /* The SIMD clone function has the same number of
4506 elements as the current function. */
4508 {
4512 op,
4515 }
4519 {
4520 vec_loop_masks *loop_masks
4522 tree loop_mask
4526 vec_oprnd0
4530 gsi);
4532 loop_mask });
4534 }
4535 vec_oprnd0
4539 gassign *new_stmt
4541 vec_oprnd0);
4545 }
4546 else
4547 {
4548 /* The mask type has more elements than simdlen. */
4550 /* FORNOW */
4552 }
4553 }
4554 }
4556 {
4558 /* Guess the number of lanes represented by atype. */
4559 poly_uint64 atype_subparts
4561 num_mask_args);
4564 {
4566 {
4570 op,
4573 }
4576 {
4577 /* The mask argument has fewer elements than the
4578 input vector. */
4579 /* FORNOW */
4581 }
4584 {
4585 /* The vector mask argument matches the input
4586 in the number of lanes, but not necessarily
4587 in the mode. */
4592 vec_oprnd0);
4593 gassign *new_stmt
4595 vec_oprnd0);
4599 {
4600 new_stmt
4602 NOP_EXPR,
4603 gimple_assign_lhs
4607 }
4609 }
4610 else
4611 {
4612 /* The mask argument has more elements than the
4613 input vector. */
4614 /* FORNOW */
4616 }
4617 }
4618 }
4619 else
4628 {
4629 gimple_seq stmts;
4634 {
4635 basic_block new_bb;
4639 }
4641 {
4644 }
4649 enum tree_code code
4654 poly_widest_int cst
4659 gassign *new_stmt
4664 UNKNOWN_LOCATION);
4667 }
4668 else
4669 {
4670 enum tree_code code
4675 poly_widest_int cst
4680 gassign *new_stmt
4685 }
4695 }
4696 }
4701 {
4704 tree mask;
4706 SIMD_CLONE_ARG_TYPE_MASK);
4712 {
4714 {
4718 }
4719 else
4724 {
4725 /* This means we are dealing with integer mask modes.
4726 First convert to an integer type with the same size as
4727 the current vector type. */
4728 unsigned HOST_WIDE_INT intermediate_size
4730 tree mid_int_type =
4733 new_stmt
4735 mask);
4737 /* Then zero-extend to the mask mode. */
4740 }
4742 {
4744 integer_one_node);
4746 integer_zero_node);
4750 }
4751 else
4759 }
4760 }
4764 {
4771 else
4774 }
4779 {
4781 {
4789 {
4790 tree t;
4792 {
4796 }
4797 else
4808 else
4810 }
4815 }
4817 {
4826 {
4831 {
4835 tem);
4840 }
4842 }
4843 else
4848 new_stmt
4857 else
4860 }
4862 {
4869 }
4871 {
4873 new_stmt
4876 }
4877 }
4883 else
4885 }
4888 {
4891 }
4894 /* Mark the clone as no longer being a candidate for GC. */
4897 /* The call in STMT might prevent it from being removed in dce.
4898 We however cannot remove it here, due to the way the ssa name
4899 it defines is mapped to the new definition. So just replace
4900 rhs of the statement with something harmless. */
4907 {
4911 }
4912 else
4918 }
4921 /* Function vect_gen_widened_results_half
4923 Create a vector stmt whose code, type, number of arguments, and result
4924 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
4925 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
4926 In the case that CODE is a CALL_EXPR, this means that a call to DECL
4927 needs to be created (DECL is a function-decl of a target-builtin).
4928 STMT_INFO is the original scalar stmt that we are vectorizing. */
4934 stmt_vec_info stmt_info)
4935 {
4937 tree new_temp;
4939 /* Generate half of the widened result: */
4948 }
4951 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
4952 For multi-step conversions store the resulting vectors and call the function
4953 recursively. When NARROW_SRC_P is true, there's still a conversion after
4954 narrowing, don't store the vectors in the SLP_NODE or in vector info of
4955 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
4957 static void
4960 stmt_vec_info stmt_info,
4965 {
4972 {
4973 /* Create demotion operation. */
4981 /* Store the resulting vector for next recursive call,
4982 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
4984 else
4985 {
4986 /* This is the last step of the conversion sequence. Store the
4987 vectors in SLP_NODE or in vector info of the scalar statement
4988 (or in STMT_VINFO_RELATED_STMT chain). */
4991 else
4993 }
4994 }
4996 /* For multi-step demotion operations we first generate demotion operations
4997 from the source type to the intermediate types, and then combine the
4998 results (stored in VEC_OPRNDS) in demotion operation to the destination
4999 type. */
5001 {
5002 /* At each level of recursion we have half of the operands we had at the
5003 previous level. */
5009 narrow_src_p);
5010 }
5013 }
5016 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5017 and VEC_OPRNDS1, for a binary operation associated with scalar statement
5018 STMT_INFO. For multi-step conversions store the resulting vectors and
5019 call the function recursively. */
5021 static void
5027 code_helper ch1,
5029 {
5037 {
5040 else
5043 /* Generate the two halves of promotion operation. */
5046 stmt_info);
5049 stmt_info);
5051 {
5054 }
5055 else
5056 {
5059 }
5061 /* Store the results for the next step. */
5064 }
5068 }
5070 /* Create vectorized promotion stmts for widening stmts using only half the
5071 potential vector size for input. */
5072 static void
5078 code_helper code1,
5080 {
5090 {
5096 /* Widen the first vector input. */
5102 {
5103 /* Widen the second vector input. */
5107 /* Perform the operation. With both vector inputs widened. */
5109 }
5110 else
5111 {
5112 /* Perform the operation. With the single vector input widened. */
5114 }
5120 /* Store the results for the next step. */
5122 }
5126 }
5129 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5130 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5131 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5132 Return true if STMT_INFO is vectorizable in this way. */
5134 static bool
5139 {
5141 tree scalar_dest;
5147 tree new_temp;
5150 poly_uint64 nunits_in;
5151 poly_uint64 nunits_out;
5155 /* For conversions between floating point and integer, there're 2 NARROW
5156 cases. NARROW_SRC is for FLOAT_EXPR, means
5157 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5158 This is safe when the range of the source integer can fit into the lower
5159 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5160 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5161 For other conversions, when there's narrowing, NARROW_DST is used as
5162 default. */
5166 tree vop0;
5174 /* Is STMT a vectorizable conversion? */
5195 {
5198 }
5200 {
5203 }
5204 else
5217 /* Check types of lhs and rhs. */
5222 /* Check the operands of the operation. */
5226 {
5231 }
5246 {
5249 "type conversion to/from bit-precision unsupported."
5252 }
5255 {
5262 tree vectype1_in;
5265 {
5270 }
5271 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5272 OP1. */
5275 }
5277 /* If op0 is an external or constant def, infer the vector type
5278 from the scalar type. */
5284 {
5290 }
5294 {
5297 "can't convert between boolean and non "
5301 }
5308 else
5312 else
5313 {
5316 }
5318 /* Multiple types in SLP are handled by creating the appropriate number of
5319 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5320 case of SLP. */
5325 else
5328 /* Sanity check: make sure that at least one copy of the vectorized stmt
5329 needs to be generated. */
5335 opt_scalar_mode rhs_mode_iter;
5337 /* Supportable by target? */
5339 {
5348 {
5351 }
5353 /* For conversions between float and integer types try whether
5354 we can use intermediate signed integer types to support the
5355 conversion. */
5359 {
5363 scalar_mode intermediate_mode;
5365 {
5368 }
5369 else
5370 {
5375 }
5378 opt_scalar_mode mode_iter;
5380 {
5386 scalar_mode cvt_mode;
5391 cvt_type = build_nonstandard_integer_type
5394 /* Check if the intermediate type can hold OP0's range.
5395 When converting from float to integer this is not necessary
5396 because values that do not fit the (smaller) target type are
5397 unspecified anyway. */
5399 {
5410 }
5413 /* This should only happened for SLP as long as loop vectorizer
5414 only supports same-sized vector. */
5418 vectype_out,
5421 cvt_type,
5427 }
5430 {
5431 multi_step_cvt++;
5437 }
5438 }
5439 /* FALLTHRU */
5440 unsupported:
5448 {
5457 }
5462 {
5463 /* Binary widening operation can only be supported directly by the
5464 architecture. */
5467 }
5475 {
5480 cvt_type
5487 {
5494 }
5501 else
5505 cvt_type,
5509 {
5512 }
5513 }
5520 else
5521 {
5522 multi_step_cvt++;
5525 }
5539 {
5540 cvt_type
5548 else
5554 }
5555 /* If op0 can be represented with low precision integer,
5556 truncate it to cvt_type and the do FLOAT_EXPR. */
5558 {
5563 cvt_type
5581 {
5585 }
5586 }
5592 }
5595 {
5599 {
5604 }
5607 {
5612 }
5614 {
5616 /* The final packing step produces one vector result per copy. */
5617 unsigned int nvectors
5621 widen_arith);
5622 }
5623 else
5624 {
5626 /* The initial unpacking step produces two vector results
5627 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5628 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5629 unsigned int nvectors
5630 = (slp_node
5635 widen_arith);
5636 }
5639 }
5641 /* Transform. */
5647 {
5652 }
5654 /* In case of multi-step conversion, we first generate conversion operations
5655 to the intermediate types, and then from that types to the final one.
5656 We create vector destinations for the intermediate type (TYPES) received
5657 from supportable_*_operation, and store them in the correct order
5658 for future use in vect_create_vectorized_*_stmts (). */
5660 bool widen_or_narrow_float_p
5663 widen_or_narrow_float_p
5668 {
5671 {
5673 intermediate_type);
5675 }
5676 }
5680 widen_or_narrow_float_p
5685 {
5687 ;
5689 {
5693 }
5694 }
5697 {
5701 /* vec_dest is intermediate type operand when multi_step_cvt. */
5703 {
5706 }
5709 {
5710 /* Arguments are ready, create the new vector stmt. */
5713 {
5720 }
5728 else
5730 }
5734 /* In case the vectorization factor (VF) is bigger than the number
5735 of elements that we can fit in a vectype (nunits), we have to
5736 generate more than one vector stmt - i.e - we need to "unroll"
5737 the vector stmt by a factor VF/nunits. */
5743 {
5748 }
5749 /* Arguments are ready. Create the new vector stmts. */
5751 {
5755 {
5758 }
5762 op_type);
5763 else
5768 }
5771 {
5774 {
5778 }
5779 else
5784 else
5786 }
5791 /* In case the vectorization factor (VF) is bigger than the number
5792 of elements that we can fit in a vectype (nunits), we have to
5793 generate more than one vector stmt - i.e - we need to "unroll"
5794 the vector stmt by a factor VF/nunits. */
5797 /* Arguments are ready. Create the new vector stmts. */
5800 {
5805 }
5808 multi_step_cvt,
5812 /* After demoting op0 to cvt_type, convert it to dest. */
5814 {
5816 {
5817 /* Arguments are ready, create the new vector stmt. */
5819 gimple *new_stmt
5825 /* This is the last step of the conversion sequence. Store the
5826 vectors in SLP_NODE or in vector info of the scalar statement
5827 (or in STMT_VINFO_RELATED_STMT chain). */
5830 else
5832 }
5833 }
5835 }
5844 }
5846 /* Return true if we can assume from the scalar form of STMT_INFO that
5847 neither the scalar nor the vector forms will generate code. STMT_INFO
5848 is known not to involve a data reference. */
5850 bool
5852 {
5868 }
5870 /* Function vectorizable_assignment.
5872 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5873 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5874 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5875 Return true if STMT_INFO is vectorizable in this way. */
5877 static bool
5882 {
5883 tree vec_dest;
5884 tree scalar_dest;
5885 tree op;
5887 tree new_temp;
5893 tree vop;
5896 tree vectype_in;
5905 /* Is vectorizable assignment? */
5926 /* Multiple types in SLP are handled by creating the appropriate number of
5927 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5928 case of SLP. */
5931 else
5936 slp_tree slp_op;
5939 {
5944 }
5948 /* We can handle NOP_EXPR conversions that do not change the number
5949 of elements or the vector size. */
5952 && (!vectype_in
5959 {
5962 "can't convert between boolean and non "
5966 }
5968 /* We do not handle bit-precision changes. */
5975 /* But a conversion that does not change the bit-pattern is ok. */
5983 {
5986 "type conversion to/from bit-precision "
5989 }
5992 {
5995 {
6000 }
6005 cost_vec);
6007 }
6009 /* Transform. */
6013 /* Handle def. */
6016 /* Handle use. */
6019 /* Arguments are ready. create the new vector stmt. */
6021 {
6031 else
6033 }
6039 }
6042 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6043 either as shift by a scalar or by a vector. */
6045 bool
6047 {
6049 machine_mode vec_mode;
6050 optab optab;
6052 tree vectype;
6061 {
6067 }
6075 }
6078 /* Function vectorizable_shift.
6080 Check if STMT_INFO performs a shift operation that can be vectorized.
6081 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6082 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6083 Return true if STMT_INFO is vectorizable in this way. */
6085 static bool
6090 {
6091 tree vec_dest;
6092 tree scalar_dest;
6095 tree vectype;
6098 machine_mode vec_mode;
6099 tree new_temp;
6100 optab optab;
6102 machine_mode optab_op2_mode;
6105 poly_uint64 nunits_in;
6106 poly_uint64 nunits_out;
6107 tree vectype_out;
6108 tree op1_vectype;
6127 /* Is STMT a vectorizable binary/unary operation? */
6144 {
6149 }
6151 slp_tree slp_op0;
6154 {
6159 }
6160 /* If op0 is an external or constant def, infer the vector type
6161 from the scalar type. */
6167 {
6172 }
6179 stmt_vec_info op1_def_stmt_info;
6180 slp_tree slp_op1;
6183 {
6188 }
6190 /* Multiple types in SLP are handled by creating the appropriate number of
6191 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6192 case of SLP. */
6195 else
6200 /* Determine whether the shift amount is a vector, or scalar. If the
6201 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6211 {
6212 /* In SLP, need to check whether the shift count is the same,
6213 in loops if it is a constant or invariant, it is always
6214 a scalar shift. */
6216 {
6218 stmt_vec_info slpstmt_info;
6221 {
6225 }
6227 /* For internal SLP defs we have to make sure we see scalar stmts
6228 for all vector elements.
6229 ??? For different vectors we could resort to a different
6230 scalar shift operand but code-generation below simply always
6231 takes the first. */
6236 }
6238 /* If the shift amount is computed by a pattern stmt we cannot
6239 use the scalar amount directly thus give up and use a vector
6240 shift. */
6243 }
6244 else
6245 {
6250 }
6252 /* Vector shifted by vector. */
6255 {
6263 slp_op1);
6264 incompatible_op1_vectype_p
6270 && (!slp_node
6273 {
6276 "unusable type for last operand in"
6279 }
6280 }
6281 /* See if the machine has a vector shifted by scalar insn and if not
6282 then see if it has a vector shifted by vector insn. */
6283 else
6284 {
6288 {
6292 }
6293 else
6294 {
6299 {
6309 slp_op1);
6311 /* Unlike the other binary operators, shifts/rotates have
6312 the rhs being int, instead of the same type as the lhs,
6313 so make sure the scalar is the right type if we are
6314 dealing with vectors of long long/long/short/char. */
6315 incompatible_op1_vectype_p
6316 = (!op1_vectype
6321 {
6324 "unusable type for last operand in"
6327 }
6328 }
6329 }
6330 }
6332 /* Supportable by target? */
6334 {
6339 }
6343 {
6348 }
6349 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6354 {
6358 && (!incompatible_op1_vectype_p
6360 && !vect_maybe_update_slp_op_vectype
6363 {
6368 }
6369 /* Now adjust the constant shift amount in place. */
6371 && incompatible_op1_vectype_p
6373 {
6376 {
6382 }
6383 }
6389 }
6391 /* Transform. */
6398 {
6404 }
6406 /* Handle def. */
6410 {
6411 /* Vector shl and shr insn patterns can be defined with scalar
6412 operand 2 (shift operand). In this case, use constant or loop
6413 invariant op1 directly, without extending it to vector mode
6414 first. */
6417 {
6424 /* Store vec_oprnd1 for every vector stmt to be created.
6425 We check during the analysis that all the shift arguments
6426 are the same.
6427 TODO: Allow different constants for different vector
6428 stmts generated for an SLP instance. */
6432 }
6433 }
6435 {
6437 {
6438 /* If the argument was the same in all lanes create
6439 the correctly typed vector shift amount directly. */
6448 }
6450 /* The constant shift amount has been adjusted in place. */
6451 ;
6452 else
6454 }
6456 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6457 (a special case for certain kind of vector shifts); otherwise,
6458 operand 1 should be of a vector type (the usual case). */
6463 /* Arguments are ready. Create the new vector stmt. */
6465 {
6466 /* For internal defs where we need to use a scalar shift arg
6467 extract the first lane. */
6469 {
6472 gassign *new_stmt
6475 vop1,
6477 bitsize_zero_node));
6480 }
6481 else
6489 else
6491 }
6500 }
6502 /* Function vectorizable_operation.
6504 Check if STMT_INFO performs a binary, unary or ternary operation that can
6505 be vectorized.
6506 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6507 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6508 Return true if STMT_INFO is vectorizable in this way. */
6510 static bool
6515 {
6516 tree vec_dest;
6517 tree scalar_dest;
6519 tree vectype;
6522 machine_mode vec_mode;
6523 tree new_temp;
6525 optab optab;
6530 poly_uint64 nunits_in;
6531 poly_uint64 nunits_out;
6532 tree vectype_out;
6548 /* Is STMT a vectorizable binary/unary operation? */
6553 /* Loads and stores are handled in vectorizable_{load,store}. */
6559 /* Shifts are handled in vectorizable_shift. */
6566 /* Comparisons are handled in vectorizable_comparison. */
6570 /* Conditions are handled in vectorizable_condition. */
6574 /* For pointer addition and subtraction, we should use the normal
6575 plus and minus for the vector operation. */
6581 /* Support only unary or binary operations. */
6584 {
6588 op_type);
6590 }
6595 /* Most operations cannot handle bit-precision types without extra
6596 truncations. */
6600 /* Exception are bitwise binary operations. */
6604 {
6609 }
6611 slp_tree slp_op0;
6614 {
6619 }
6622 /* If op0 is an external or constant def, infer the vector type
6623 from the scalar type. */
6625 {
6626 /* For boolean type we cannot determine vectype by
6627 invariant value (don't know whether it is a vector
6628 of booleans or vector of integers). We use output
6629 vectype because operations on boolean don't change
6630 type. */
6632 {
6634 {
6639 }
6641 }
6642 else
6644 slp_node);
6645 }
6649 {
6656 }
6666 {
6669 {
6674 }
6680 }
6682 {
6685 {
6690 }
6696 }
6698 /* Multiple types in SLP are handled by creating the appropriate number of
6699 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6700 case of SLP. */
6702 {
6705 }
6706 else
6707 {
6710 }
6714 /* Reject attempts to combine mask types with nonmask types, e.g. if
6715 we have an AND between a (nonmask) boolean loaded from memory and
6716 a (mask) boolean result of a comparison.
6718 TODO: We could easily fix these cases up using pattern statements. */
6722 {
6727 }
6729 /* Supportable by target? */
6734 else
6735 {
6738 {
6743 }
6746 }
6750 {
6754 /* When vec_mode is not a vector mode and we verified ops we
6755 do not have to lower like AND are natively supported let
6756 those through even when the mode isn't word_mode. For
6757 ops we have to lower the lowering code assumes we are
6758 dealing with word_mode. */
6762 /* Check only during analysis. */
6764 {
6768 }
6773 }
6781 /* If operating on inactive elements could generate spurious traps,
6782 we need to restrict the operation to active lanes. Note that this
6783 specifically doesn't apply to unhoisted invariants, since they
6784 operate on the same value for every lane.
6786 Similarly, if this operation is part of a reduction, a fully-masked
6787 loop should only change the active lanes of the reduction chain,
6788 keeping the inactive lanes as-is. */
6793 {
6797 {
6800 OPTIMIZE_FOR_SPEED))
6805 OPTIMIZE_FOR_SPEED))
6808 else
6809 {
6812 "can't use a fully-masked loop because no"
6815 }
6816 }
6818 /* Put types on constant and invariant SLP children. */
6823 {
6828 }
6835 {
6836 /* The above vect_model_simple_cost call handles constants
6837 in the prologue and (mis-)costs one of the stmts as
6838 vector stmt. See below for the actual lowering that will
6839 be applied. */
6840 unsigned n
6843 {
6854 /* Bit operations do not have extra cost and are accounted
6855 as vector stmt by vect_model_simple_cost. */
6858 }
6860 {
6861 /* We also need to materialize two large constants. */
6866 }
6867 }
6869 }
6871 /* Transform. */
6880 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6881 vectors with unsigned elements, but the result is signed. So, we
6882 need to compute the MINUS_EXPR into vectype temporary and
6883 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6886 {
6889 }
6890 /* Handle def. */
6891 else
6894 /* In case the vectorization factor (VF) is bigger than the number
6895 of elements that we can fit in a vectype (nunits), we have to generate
6896 more than one vector stmt - i.e - we need to "unroll" the
6897 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6898 from one copy of the vector stmt to the next, in the field
6899 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6900 stages to find the correct vector defs to be used when vectorizing
6901 stmts that use the defs of the current stmt. The example below
6902 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6903 we need to create 4 vectorized stmts):
6905 before vectorization:
6906 RELATED_STMT VEC_STMT
6907 S1: x = memref - -
6908 S2: z = x + 1 - -
6910 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6911 there):
6912 RELATED_STMT VEC_STMT
6913 VS1_0: vx0 = memref0 VS1_1 -
6914 VS1_1: vx1 = memref1 VS1_2 -
6915 VS1_2: vx2 = memref2 VS1_3 -
6916 VS1_3: vx3 = memref3 - -
6917 S1: x = load - VS1_0
6918 S2: z = x + 1 - -
6920 step2: vectorize stmt S2 (done here):
6921 To vectorize stmt S2 we first need to find the relevant vector
6922 def for the first operand 'x'. This is, as usual, obtained from
6923 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
6924 that defines 'x' (S1). This way we find the stmt VS1_0, and the
6925 relevant vector def 'vx0'. Having found 'vx0' we can generate
6926 the vector stmt VS2_0, and as usual, record it in the
6927 STMT_VINFO_VEC_STMT of stmt S2.
6928 When creating the second copy (VS2_1), we obtain the relevant vector
6929 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
6930 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
6931 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
6932 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
6933 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
6934 chain of stmts and pointers:
6935 RELATED_STMT VEC_STMT
6936 VS1_0: vx0 = memref0 VS1_1 -
6937 VS1_1: vx1 = memref1 VS1_2 -
6938 VS1_2: vx2 = memref2 VS1_3 -
6939 VS1_3: vx3 = memref3 - -
6940 S1: x = load - VS1_0
6941 VS2_0: vz0 = vx0 + v1 VS2_1 -
6942 VS2_1: vz1 = vx1 + v1 VS2_2 -
6943 VS2_2: vz2 = vx2 + v1 VS2_3 -
6944 VS2_3: vz3 = vx3 + v1 - -
6945 S2: z = x + 1 - VS2_0 */
6949 /* Arguments are ready. Create the new vector stmt. */
6951 {
6958 {
6959 /* Lower the operation. This follows vector lowering. */
6962 tree word_type
6966 tree high_bits
6975 {
6993 else
6998 {
7004 }
7013 }
7014 else
7015 {
7032 }
7034 signs);
7044 }
7046 {
7047 tree mask;
7051 else
7052 /* Dummy mask. */
7062 {
7063 /* Perform the operation on active elements only and take
7064 inactive elements from the reduction chain input. */
7067 }
7068 else
7069 {
7073 }
7075 {
7078 signed char biasval
7083 }
7084 gcall *call
7087 vops);
7093 }
7094 else
7095 {
7097 /* When combining two masks check if either of them is elsewhere
7098 combined with a loop mask, if that's the case we can mark that the
7099 new combined mask doesn't need to be combined with a loop mask. */
7103 {
7105 ncopies}))
7106 {
7112 }
7115 ncopies }))
7116 {
7122 }
7123 }
7132 /* Enter the combined value into the vector cond hash so we don't
7133 AND it with a loop mask again. */
7136 }
7139 {
7142 new_temp);
7147 }
7151 else
7153 }
7163 }
7165 /* A helper function to ensure data reference DR_INFO's base alignment. */
7167 static void
7169 {
7170 /* Alignment is only analyzed for the first element of a DR group,
7171 use that to look at base alignment we need to enforce. */
7178 {
7181 // We should only be able to increase the alignment of a base object if
7182 // we know what its new alignment should be at compile time.
7189 {
7192 }
7194 }
7195 }
7198 /* Function get_group_alias_ptr_type.
7200 Return the alias type for the group starting at FIRST_STMT_INFO. */
7202 static tree
7204 {
7210 {
7214 {
7219 }
7221 }
7223 }
7226 /* Function scan_operand_equal_p.
7228 Helper function for check_scan_store. Compare two references
7229 with .GOMP_SIMD_LANE bases. */
7231 static bool
7233 {
7238 {
7239 machine_mode mode;
7249 {
7255 {
7260 }
7261 }
7262 }
7269 {
7273 {
7276 {
7279 {
7285 {
7288 }
7289 }
7291 {
7294 }
7297 {
7301 }
7310 }
7314 }
7315 }
7317 }
7321 /* Normal permutation. */
7322 scan_store_kind_perm,
7324 /* Whole vector left shift permutation with zero init. */
7325 scan_store_kind_lshift_zero,
7327 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7328 scan_store_kind_lshift_cond
7329 };
7331 /* Function check_scan_store.
7333 Verify if we can perform the needed permutations or whole vector shifts.
7334 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7335 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7336 to do at each step. */
7338 static int
7341 {
7353 {
7359 {
7362 }
7363 else
7364 {
7369 }
7372 {
7377 {
7381 /* Whole vector shifts shift in zeros, so if init is all zero
7382 constant, there is no need to do anything further. */
7386 {
7391 }
7392 }
7394 }
7396 {
7401 }
7402 }
7405 }
7408 /* Function check_scan_store.
7410 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7412 static bool
7415 vect_memory_access_type memory_access_type)
7416 {
7419 tree ref_type;
7423 || mask
7435 {
7440 }
7442 /* We need to pattern match code built by OpenMP lowering and simplified
7443 by following optimizations into something we can handle.
7444 #pragma omp simd reduction(inscan,+:r)
7445 for (...)
7446 {
7447 r += something ();
7448 #pragma omp scan inclusive (r)
7449 use (r);
7450 }
7451 shall have body with:
7452 // Initialization for input phase, store the reduction initializer:
7453 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7454 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7455 D.2042[_21] = 0;
7456 // Actual input phase:
7457 ...
7458 r.0_5 = D.2042[_20];
7459 _6 = _4 + r.0_5;
7460 D.2042[_20] = _6;
7461 // Initialization for scan phase:
7462 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7463 _26 = D.2043[_25];
7464 _27 = D.2042[_25];
7465 _28 = _26 + _27;
7466 D.2043[_25] = _28;
7467 D.2042[_25] = _28;
7468 // Actual scan phase:
7469 ...
7470 r.1_8 = D.2042[_20];
7471 ...
7472 The "omp simd array" variable D.2042 holds the privatized copy used
7473 inside of the loop and D.2043 is another one that holds copies of
7474 the current original list item. The separate GOMP_SIMD_LANE ifn
7475 kinds are there in order to allow optimizing the initializer store
7476 and combiner sequence, e.g. if it is originally some C++ish user
7477 defined reduction, but allow the vectorizer to pattern recognize it
7478 and turn into the appropriate vectorized scan.
7480 For exclusive scan, this is slightly different:
7481 #pragma omp simd reduction(inscan,+:r)
7482 for (...)
7483 {
7484 use (r);
7485 #pragma omp scan exclusive (r)
7486 r += something ();
7487 }
7488 shall have body with:
7489 // Initialization for input phase, store the reduction initializer:
7490 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7491 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7492 D.2042[_21] = 0;
7493 // Actual input phase:
7494 ...
7495 r.0_5 = D.2042[_20];
7496 _6 = _4 + r.0_5;
7497 D.2042[_20] = _6;
7498 // Initialization for scan phase:
7499 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7500 _26 = D.2043[_25];
7501 D.2044[_25] = _26;
7502 _27 = D.2042[_25];
7503 _28 = _26 + _27;
7504 D.2043[_25] = _28;
7505 // Actual scan phase:
7506 ...
7507 r.1_8 = D.2044[_20];
7508 ... */
7511 {
7512 /* Match the D.2042[_21] = 0; store above. Just require that
7513 it is a constant or external definition store. */
7515 {
7516 fail_init:
7521 }
7531 /* These stores can be vectorized normally. */
7533 }
7536 {
7537 fail:
7542 }
7551 bool inscan_var_store
7555 {
7557 {
7558 use_operand_p use_p;
7559 imm_use_iterator iter;
7561 {
7568 || other_store_stmt
7572 }
7578 }
7579 }
7581 {
7582 use_operand_p use_p;
7583 imm_use_iterator iter;
7585 {
7592 }
7593 }
7594 else
7604 /* For pointer addition, we should use the normal plus for the vector
7605 operation. */
7607 {
7615 }
7643 {
7649 tree lrhs;
7652 else
7654 use_operand_p use_p;
7655 imm_use_iterator iter;
7657 {
7664 }
7665 }
7673 stmt_vec_info other_store_stmt_info
7686 {
7690 }
7701 {
7712 }
7731 {
7736 }
7744 /* The IL is as expected, now check if we can actually vectorize it.
7745 Inclusive scan:
7746 _26 = D.2043[_25];
7747 _27 = D.2042[_25];
7748 _28 = _26 + _27;
7749 D.2043[_25] = _28;
7750 D.2042[_25] = _28;
7751 should be vectorized as (where _40 is the vectorized rhs
7752 from the D.2042[_21] = 0; store):
7753 _30 = MEM <vector(8) int> [(int *)&D.2043];
7754 _31 = MEM <vector(8) int> [(int *)&D.2042];
7755 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7756 _33 = _31 + _32;
7757 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7758 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7759 _35 = _33 + _34;
7760 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7761 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7762 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7763 _37 = _35 + _36;
7764 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7765 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7766 _38 = _30 + _37;
7767 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7768 MEM <vector(8) int> [(int *)&D.2043] = _39;
7769 MEM <vector(8) int> [(int *)&D.2042] = _38;
7770 Exclusive scan:
7771 _26 = D.2043[_25];
7772 D.2044[_25] = _26;
7773 _27 = D.2042[_25];
7774 _28 = _26 + _27;
7775 D.2043[_25] = _28;
7776 should be vectorized as (where _40 is the vectorized rhs
7777 from the D.2042[_21] = 0; store):
7778 _30 = MEM <vector(8) int> [(int *)&D.2043];
7779 _31 = MEM <vector(8) int> [(int *)&D.2042];
7780 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7781 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7782 _34 = _32 + _33;
7783 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7784 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7785 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7786 _36 = _34 + _35;
7787 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7788 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7789 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7790 _38 = _36 + _37;
7791 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7792 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7793 _39 = _30 + _38;
7794 _50 = _31 + _39;
7795 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7796 MEM <vector(8) int> [(int *)&D.2044] = _39;
7797 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7808 }
7811 /* Function vectorizable_scan_store.
7813 Helper of vectorizable_score, arguments like on vectorizable_store.
7814 Handle only the transformation, checking is done in check_scan_store. */
7816 static bool
7820 {
7835 bool inscan_var_store
7839 {
7840 use_operand_p use_p;
7841 imm_use_iterator iter;
7843 {
7849 }
7850 }
7871 {
7875 }
7890 {
7897 else
7898 {
7903 }
7907 {
7914 }
7915 else
7917 }
7938 {
7949 {
7953 dataref_offset);
7959 }
7963 {
7966 (zero_vec
7976 {
7977 /* Whole vector shift shifted in zero bits, but if *init
7978 is not initializer_zerop, we need to replace those elements
7979 with elements from vec_oprnd1. */
7992 }
7994 /* For exclusive scan, perform the perms[i] permutation once
7995 more. */
7999 {
8003 }
8011 }
8019 /* For exclusive scan, new_temp computed above is the exclusive scan
8020 prefix sum. Turn it into inclusive prefix sum for the broadcast
8021 of the last element into orig. */
8023 {
8028 }
8037 {
8040 dataref_offset);
8045 }
8046 }
8050 {
8056 dataref_offset);
8061 }
8063 }
8066 /* Function vectorizable_store.
8068 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8069 that can be vectorized.
8070 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8071 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8072 Return true if STMT_INFO is vectorizable in this way. */
8074 static bool
8079 {
8080 tree data_ref;
8082 tree elem_type;
8085 machine_mode vec_mode;
8086 tree dummy;
8094 stmt_vec_info first_stmt_info;
8100 tree aggr_type;
8101 gather_scatter_info gs_info;
8102 poly_uint64 vf;
8103 vec_load_store_type vls_type;
8104 tree ref_type;
8113 /* Is vectorizable store? */
8118 {
8131 }
8132 else
8133 {
8144 mask_index = vect_slp_child_index_for_operand
8151 }
8153 /* Cannot have hybrid store SLP -- that would mean storing to the
8154 same location twice. */
8161 {
8164 }
8165 else
8168 /* Multiple types in SLP are handled by creating the appropriate number of
8169 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8170 case of SLP. */
8173 else
8178 /* FORNOW. This restriction should be relaxed. */
8180 {
8185 }
8187 tree op;
8188 slp_tree op_node;
8199 vect_memory_access_type memory_access_type;
8202 poly_int64 poffset;
8203 internal_fn lanes_ifn;
8211 {
8213 {
8218 }
8222 {
8227 }
8231 {
8236 }
8237 }
8238 else
8239 {
8240 /* FORNOW. In some cases can vectorize even if data-type not supported
8241 (e.g. - array initialization with 0). */
8244 }
8251 {
8255 }
8256 else
8257 {
8261 }
8264 {
8266 memory_access_type))
8268 }
8272 {
8280 mask);
8284 || (mask
8286 mask_vectype))))
8287 {
8292 }
8303 /* As function vect_transform_stmt shows, for interleaving stores
8304 the whole chain is vectorized when the last store in the chain
8305 is reached, the other stores in the group are skipped. So we
8306 want to only cost the last one here, but it's not trivial to
8307 get the last, as it's equivalent to use the first one for
8308 costing, use the first one instead. */
8310 && !slp
8313 }
8316 /* Transform. */
8321 {
8325 {
8336 "vect_model_store_cost: inside_cost = %d, "
8341 }
8343 }
8346 {
8347 /* FORNOW */
8351 {
8353 /* VEC_NUM is the number of vect stmts to be created for this
8354 group. */
8361 }
8362 else
8363 /* VEC_NUM is the number of vect stmts to be created for this
8364 group. */
8368 }
8369 else
8374 ncopies);
8376 /* Check if we need to update prologue cost for invariant,
8377 and update it accordingly if so. If it's not for
8378 interleaving store, we can just check vls_type; but if
8379 it's for interleaving store, need to check the def_type
8380 of the stored value since the current vls_type is just
8381 for first_stmt_info. */
8383 {
8388 {
8394 }
8399 };
8403 {
8405 gimple_stmt_iterator incr_gsi;
8408 tree offvar;
8409 tree ivstep;
8410 tree running_off;
8413 tree dr_offset;
8415 /* Checked by get_load_store_type. */
8422 stride_base
8423 = fold_build_pointer_plus
8430 /* For a store with loop-invariant (but other than power-of-2)
8431 stride (i.e. not a grouped access) like so:
8433 for (i = 0; i < n; i += stride)
8434 array[i] = ...;
8436 we generate a new induction variable and new stores from
8437 the components of the (vectorized) rhs:
8439 for (j = 0; ; j += VF*stride)
8440 vectemp = ...;
8441 tmp1 = vectemp[0];
8442 array[j] = tmp1;
8443 tmp2 = vectemp[1];
8444 array[j + stride] = tmp2;
8445 ...
8446 */
8453 {
8456 {
8462 /* First check if vec_extract optab doesn't support extraction
8463 of vector elts directly. */
8465 machine_mode vmode;
8472 {
8473 /* Try to avoid emitting an extract of vector elements
8474 by performing the extracts using an integer type of the
8475 same size, extracting from a vector of those and then
8476 re-interpreting it as the original vector type if
8477 supported. */
8478 unsigned lsize
8481 /* If we can't construct such a vector fall back to
8482 element extracts from the original vector type and
8483 element size stores. */
8491 {
8496 }
8497 /* Else fall back to vector extraction anyway.
8498 Fewer stores are more important than avoiding spilling
8499 of the vector we extract from. Compared to the
8500 construction case in vectorizable_load no store-forwarding
8501 issue exists here for reasonable archs. */
8502 }
8503 }
8506 {
8508 dr_alignment_support dr_align
8510 mis_align);
8513 {
8520 }
8521 }
8524 }
8527 {
8541 }
8546 /* For costing some adjacent vector stores, we'd like to cost with
8547 the total number of them once instead of cost each one by one. */
8550 {
8553 {
8555 {
8557 tree pos
8564 }
8565 }
8571 else
8574 unsigned HOST_WIDE_INT
8577 {
8579 {
8581 /* Pun the vector to extract from if necessary. */
8583 {
8585 tree cvt
8590 }
8591 }
8593 {
8595 {
8596 /* Only need vector extracting when there are more
8597 than one stores. */
8599 inside_cost
8602 /* Take a single lane vector type store as scalar
8603 store to avoid ICE like 110776. */
8606 n_adjacent_stores++;
8607 else
8608 inside_cost
8612 }
8616 /* Extract the i'th component. */
8624 GSI_SAME_STMT);
8632 /* And store it to *running_off. */
8639 {
8647 }
8650 {
8654 }
8655 }
8656 }
8661 }
8664 {
8671 "vect_model_store_cost: inside_cost = %d, "
8674 }
8677 }
8680 vec_loop_masks *loop_masks
8684 vec_loop_lens *loop_lens
8689 /* Shouldn't go with length-based approach if fully masked. */
8692 /* Targets with store-lane instructions must not require explicit
8693 realignment. vect_supportable_dr_alignment always returns either
8694 dr_aligned or dr_unaligned_supported for masked operations. */
8696 && !mask
8705 tree bump;
8708 {
8711 }
8713 {
8718 }
8719 else
8720 {
8723 else
8727 }
8732 /* In case the vectorization factor (VF) is bigger than the number
8733 of elements that we can fit in a vectype (nunits), we have to generate
8734 more than one vector stmt - i.e - we need to "unroll" the
8735 vector stmt by a factor VF/nunits. */
8737 /* In case of interleaving (non-unit grouped access):
8739 S1: &base + 2 = x2
8740 S2: &base = x0
8741 S3: &base + 1 = x1
8742 S4: &base + 3 = x3
8744 We create vectorized stores starting from base address (the access of the
8745 first stmt in the chain (S2 in the above example), when the last store stmt
8746 of the chain (S4) is reached:
8748 VS1: &base = vx2
8749 VS2: &base + vec_size*1 = vx0
8750 VS3: &base + vec_size*2 = vx1
8751 VS4: &base + vec_size*3 = vx3
8753 Then permutation statements are generated:
8755 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8756 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8757 ...
8759 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8760 (the order of the data-refs in the output of vect_permute_store_chain
8761 corresponds to the order of scalar stmts in the interleaving chain - see
8762 the documentation of vect_permute_store_chain()).
8764 In case of both multiple types and interleaving, above vector stores and
8765 permutation stmts are created for every copy. The result vector stmts are
8766 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8767 STMT_VINFO_RELATED_STMT for the next copies.
8768 */
8778 {
8781 /* For costing some adjacent vector stores, we'd like to cost with
8782 the total number of them once instead of cost each one by one. */
8785 {
8788 {
8789 /* For interleaved stores we collect vectorized defs for all
8790 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8791 as an input to vect_permute_store_chain(). */
8794 {
8795 /* Since gaps are not supported for interleaved stores,
8796 DR_GROUP_SIZE is the exact number of stmts in the
8797 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8801 else
8802 {
8808 }
8810 }
8813 {
8815 {
8818 mask_vectype);
8820 }
8822 /* We should have catched mismatched types earlier. */
8825 dataref_ptr
8829 }
8830 }
8832 {
8834 /* DR_CHAIN is then used as an input to
8835 vect_permute_store_chain(). */
8837 {
8840 }
8845 }
8848 {
8851 }
8853 /* Get an array into which we can store the individual vectors. */
8856 /* Invalidate the current contents of VEC_ARRAY. This should
8857 become an RTL clobber too, which prevents the vector registers
8858 from being upward-exposed. */
8861 /* Store the individual vectors into the array. */
8863 {
8866 i);
8867 }
8880 {
8884 else
8886 signed char biasval
8890 {
8893 }
8894 }
8898 {
8899 /* Emit:
8900 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8901 LEN, BIAS, VEC_ARRAY). */
8907 vec_array);
8908 }
8910 {
8911 /* Emit:
8912 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8913 VEC_ARRAY). */
8919 }
8920 else
8921 {
8922 /* Emit:
8923 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8927 }
8932 /* Record that VEC_ARRAY is now dead. */
8937 }
8940 {
8947 "vect_model_store_cost: inside_cost = %d, "
8950 }
8953 }
8956 {
8961 {
8964 {
8969 {
8970 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8971 DR_CHAIN is of size 1. */
8975 else
8979 {
8982 else
8984 ncopies,
8986 mask_vectype);
8987 }
8993 else
8994 dataref_ptr
8998 bump);
8999 }
9000 }
9002 {
9007 }
9011 {
9013 {
9017 /* We should have catched mismatched types earlier. */
9020 }
9026 {
9034 }
9037 {
9039 {
9041 inside_cost
9045 }
9052 {
9057 else
9059 signed char biasval
9063 {
9066 }
9067 }
9071 call = gimple_build_call_internal
9076 call = gimple_build_call_internal
9079 else
9086 }
9088 {
9089 /* The builtin decls path for scatter is legacy, x86 only. */
9091 && (!final_mask
9092 || SCALAR_INT_MODE_P
9095 {
9097 inside_cost
9101 }
9102 poly_uint64 offset_nunits
9105 {
9106 new_stmt = vect_build_one_scatter_store_call
9112 }
9114 {
9115 /* We have a offset vector with half the number of
9116 lanes but the builtins will store full vectype
9117 data from the lower lanes. */
9118 new_stmt = vect_build_one_scatter_store_call
9120 dataref_ptr,
9131 tree perm_mask
9135 perm_mask);
9141 {
9143 VEC_UNPACK_HI_EXPR,
9144 final_mask);
9145 final_mask = make_ssa_name
9150 }
9151 new_stmt = vect_build_one_scatter_store_call
9153 dataref_ptr,
9158 }
9160 {
9161 /* We have a offset vector with double the number of
9162 lanes. Select the low/high part accordingly. */
9165 {
9172 tree perm_mask = vect_gen_perm_mask_checked
9175 VEC_PERM_EXPR,
9176 vec_offset,
9177 vec_offset,
9178 perm_mask);
9183 }
9184 new_stmt = vect_build_one_scatter_store_call
9190 }
9191 else
9193 }
9194 else
9195 {
9196 /* Emulated scatter. */
9199 {
9201 /* For emulated scatter N offset vector element extracts
9202 (we assume the scalar scaling and ptr + offset add is
9203 consumed by the load). */
9204 inside_cost
9207 /* N scalar stores plus extracting the elements. */
9208 inside_cost
9211 inside_cost
9215 }
9218 unsigned HOST_WIDE_INT const_offset_nunits
9224 unsigned HOST_WIDE_INT elt_size
9226 /* We support offset vectors with more elements
9227 than the data vector for now. */
9228 unsigned HOST_WIDE_INT factor
9231 unsigned elt_offset
9238 {
9239 /* Compute the offsetted pointer. */
9242 tree idx
9248 tree ptr
9253 /* Extract the element to be stored. */
9254 tree elt
9261 tree ref
9267 }
9270 }
9271 }
9274 }
9281 "vect_model_store_cost: inside_cost = %d, "
9286 }
9294 /* For costing some adjacent vector stores, we'd like to cost with
9295 the total number of them once instead of cost each one by one. */
9300 {
9303 {
9305 {
9306 /* Get vectorized arguments for SLP_NODE. */
9312 }
9313 else
9314 {
9315 /* For interleaved stores we collect vectorized defs for all the
9316 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9317 input to vect_permute_store_chain().
9319 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9320 is of size 1. */
9323 {
9324 /* Since gaps are not supported for interleaved stores,
9325 DR_GROUP_SIZE is the exact number of stmts in the chain.
9326 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9327 that there is no interleaving, DR_GROUP_SIZE is 1,
9328 and only one iteration of the loop will be executed. */
9332 else
9333 {
9339 }
9341 }
9343 {
9346 mask_vectype);
9348 }
9349 }
9351 /* We should have catched mismatched types earlier. */
9355 bool simd_lane_access_p
9358 && simd_lane_access_p
9359 && !loop_masks
9366 {
9369 }
9371 dataref_ptr
9376 }
9378 {
9380 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9381 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9382 of size 1. */
9384 {
9387 }
9392 else
9395 }
9399 {
9400 /* Permute. */
9403 {
9410 "vect_model_store_cost: "
9412 group_size);
9413 }
9414 else
9417 }
9421 {
9423 {
9427 /* For grouped stores vectorized defs are interleaved in
9428 vect_permute_store_chain(). */
9430 }
9433 {
9437 else
9438 {
9444 /* Generate the permute statement. */
9445 gimple *perm_stmt
9449 gsi);
9453 }
9454 }
9457 {
9458 n_adjacent_stores++;
9461 {
9465 }
9468 }
9484 /* Bump the vector pointer. */
9494 {
9497 }
9498 else
9503 misalign);
9506 /* Compute IFN when LOOP_LENS or final_mask valid. */
9511 {
9512 opt_machine_mode new_ovmode
9515 unsigned factor
9520 }
9522 {
9527 }
9530 {
9532 {
9533 /* Pass VF value to 'len' argument of
9534 MASK_LEN_STORE if LOOP_LENS is invalid. */
9536 }
9538 {
9539 /* Pass all ones value to 'mask' argument of
9540 MASK_LEN_STORE if final_mask is invalid. */
9543 }
9544 }
9546 {
9547 signed char biasval
9551 }
9553 /* Arguments are ready. Create the new vector stmt. */
9555 {
9558 /* Need conversion if it's wrapped with VnQI. */
9560 {
9561 tree new_vtype
9563 new_vmode);
9566 gassign *new_stmt
9570 }
9576 else
9583 }
9585 {
9587 gcall *call
9593 }
9594 else
9595 {
9596 data_ref
9598 dataref_offset ? dataref_offset
9601 ;
9602 else
9609 }
9617 }
9619 {
9623 }
9624 }
9627 {
9633 /* When vectorizing a store into the function result assign
9634 a penalty if the function returns in a multi-register location.
9635 In this case we assume we'll end up with having to spill the
9636 vector result and do piecewise loads as a conservative estimate. */
9642 {
9644 /* ??? Handle PARALLEL in some way. */
9646 {
9648 /* Assume that a single reg-reg move is possible and cheap,
9649 do not account for vector to gp register move cost. */
9651 {
9652 /* Spill. */
9653 prologue_cost
9656 /* Loads. */
9657 prologue_cost
9660 }
9661 }
9662 }
9665 "vect_model_store_cost: inside_cost = %d, "
9668 }
9671 }
9673 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9674 VECTOR_CST mask. No checks are made that the target platform supports the
9675 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9676 vect_gen_perm_mask_checked. */
9678 tree
9680 {
9681 tree mask_type;
9688 }
9690 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9691 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9693 tree
9695 {
9699 }
9701 /* Given a vector variable X and Y, that was generated for the scalar
9702 STMT_INFO, generate instructions to permute the vector elements of X and Y
9703 using permutation mask MASK_VEC, insert them at *GSI and return the
9704 permuted vector variable. */
9706 static tree
9710 {
9718 else
9722 /* Generate the permute statement. */
9727 }
9729 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9730 inserting them on the loops preheader edge. Returns true if we
9731 were successful in doing so (and thus STMT_INFO can be moved then),
9732 otherwise returns false. HOIST_P indicates if we want to hoist the
9733 definitions of all SSA uses, it would be false when we are costing. */
9735 static bool
9737 {
9738 ssa_op_iter i;
9739 tree op;
9743 {
9747 {
9748 /* Make sure we don't need to recurse. While we could do
9749 so in simple cases when there are more complex use webs
9750 we don't have an easy way to preserve stmt order to fulfil
9751 dependencies within them. */
9752 tree op2;
9753 ssa_op_iter i2;
9757 {
9762 }
9764 }
9765 }
9774 {
9778 {
9782 }
9783 }
9786 }
9788 /* vectorizable_load.
9790 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9791 that can be vectorized.
9792 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9793 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9794 Return true if STMT_INFO is vectorizable in this way. */
9796 static bool
9801 {
9802 tree scalar_dest;
9809 tree elem_type;
9810 /* Avoid false positive uninitialized warning, see PR110652. */
9812 machine_mode mode;
9813 tree dummy;
9820 poly_uint64 group_gap_adj;
9826 stmt_vec_info first_stmt_info;
9834 poly_uint64 vf;
9835 tree aggr_type;
9836 gather_scatter_info gs_info;
9837 tree ref_type;
9854 {
9869 }
9870 else
9871 {
9886 mask_index = vect_slp_child_index_for_operand
9892 }
9898 {
9902 }
9903 else
9906 /* Multiple types in SLP are handled by creating the appropriate number of
9907 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
9908 case of SLP. */
9911 else
9916 /* FORNOW. This restriction should be relaxed. */
9918 {
9923 }
9925 /* Invalidate assumptions made by dependence analysis when vectorization
9926 on the unrolled body effectively re-orders stmts. */
9931 {
9934 "cannot perform implicit CSE when unrolling "
9937 }
9942 /* FORNOW. In some cases can vectorize even if data-type not supported
9943 (e.g. - data copies). */
9945 {
9950 }
9952 /* Check if the load is a part of an interleaving chain. */
9954 {
9956 /* FORNOW */
9963 /* Refuse non-SLP vectorization of SLP-only groups. */
9965 {
9970 }
9972 /* Invalidate assumptions made by dependence analysis when vectorization
9973 on the unrolled body effectively re-orders stmts. */
9978 {
9981 "cannot perform implicit CSE when performing "
9984 }
9985 }
9986 else
9990 {
9994 {
9995 /* In BB vectorization we may not actually use a loaded vector
9996 accessing elements in excess of DR_GROUP_SIZE. */
10007 {
10010 "BB vectorization with gaps at the end of "
10013 }
10014 }
10020 {
10023 vect_location,
10026 }
10027 }
10029 vect_memory_access_type memory_access_type;
10032 poly_int64 poffset;
10033 internal_fn lanes_ifn;
10041 {
10043 {
10049 }
10052 {
10057 }
10061 {
10066 }
10067 }
10072 {
10074 && mask
10076 mask_vectype))
10077 {
10082 }
10092 mask);
10105 }
10115 /* Transform. */
10121 {
10123 /* If we have versioned for aliasing or the loop doesn't
10124 have any data dependencies that would preclude this,
10125 then we are sure this is a loop invariant load and
10126 thus we can insert it on the preheader edge.
10127 TODO: hoist_defs_of_uses should ideally be computed
10128 once at analysis time, remembered and used in the
10129 transform time. */
10131 && !nested_in_vect_loop
10134 {
10135 enum vect_cost_model_location cost_loc
10140 cost_loc);
10145 "vect_model_load_cost: inside_cost = %d, "
10149 }
10151 {
10161 tree vuse;
10164 else
10169 }
10170 /* These copies are all equivalent. */
10174 else
10175 {
10180 }
10185 else
10186 {
10190 }
10192 }
10196 {
10197 gimple_stmt_iterator incr_gsi;
10199 tree offvar;
10200 tree ivstep;
10201 tree running_off;
10204 /* Checked by get_load_store_type. */
10207 tree dr_offset;
10214 {
10217 }
10218 else
10219 {
10222 }
10225 {
10228 }
10229 else
10230 {
10232 cst_offset
10235 first_stmt_info));
10238 }
10241 {
10249 /* For a load with loop-invariant (but other than power-of-2)
10250 stride (i.e. not a grouped access) like so:
10252 for (i = 0; i < n; i += stride)
10253 ... = array[i];
10255 we generate a new induction variable and new accesses to
10256 form a new vector (or vectors, depending on ncopies):
10258 for (j = 0; ; j += VF*stride)
10259 tmp1 = array[j];
10260 tmp2 = array[j + stride];
10261 ...
10262 vectemp = {tmp1, tmp2, ...}
10263 */
10277 }
10287 {
10289 {
10290 /* First check if vec_init optab supports construction from vector
10291 elts directly. Otherwise avoid emitting a constructor of
10292 vector elements by performing the loads using an integer type
10293 of the same size, constructing a vector of those and then
10294 re-interpreting it as the original vector type. This avoids a
10295 huge runtime penalty due to the general inability to perform
10296 store forwarding from smaller stores to a larger load. */
10297 tree ptype;
10298 tree vtype
10303 {
10308 }
10309 }
10310 else
10311 {
10315 }
10317 }
10318 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10323 {
10324 /* For SLP permutation support we need to load the whole group,
10325 not only the number of vector stmts the permutation result
10326 fits in. */
10328 {
10329 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10330 variable VF. */
10334 }
10335 else
10337 }
10339 unsigned HOST_WIDE_INT
10342 /* For costing some adjacent vector loads, we'd like to cost with
10343 the total number of them once instead of cost each one by one. */
10346 {
10351 {
10353 {
10354 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10355 avoid ICE, see PR110776. */
10358 n_adjacent_loads++;
10359 else
10363 }
10377 {
10378 n_groups++;
10379 /* When doing SLP make sure to not load elements from
10380 the next vector iteration, those will not be accessed
10381 so just use the last element again. See PR107451. */
10383 {
10385 gimple *incr
10390 }
10392 }
10393 }
10396 {
10400 else
10401 {
10407 {
10408 new_stmt
10410 VIEW_CONVERT_EXPR,
10414 gsi);
10415 }
10416 }
10417 }
10420 {
10422 {
10425 else
10427 }
10428 else
10429 {
10433 }
10434 }
10435 }
10437 {
10440 {
10446 }
10447 else
10450 }
10453 {
10461 "vect_model_load_cost: inside_cost = %u, "
10463 inside_cost);
10464 }
10467 }
10475 {
10477 {
10480 }
10481 else
10482 {
10485 }
10486 /* For SLP vectorization we directly vectorize a subchain
10487 without permutation. */
10490 /* For BB vectorization always use the first stmt to base
10491 the data ref pointer on. */
10493 first_stmt_info_for_drptr
10496 /* Check if the chain of loads is already vectorized. */
10498 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10499 ??? But we can only do so if there is exactly one
10500 as we have no way to get at the rest. Leave the CSE
10501 opportunity alone.
10502 ??? With the group load eventually participating
10503 in multiple different permutations (having multiple
10504 slp nodes which refer to the same group) the CSE
10505 is even wrong code. See PR56270. */
10507 {
10510 }
10514 /* VEC_NUM is the number of vect stmts to be created for this group. */
10516 {
10518 /* If an SLP permutation is from N elements to N elements,
10519 and if one vector holds a whole number of N, we can load
10520 the inputs to the permutation in the same way as an
10521 unpermuted sequence. In other cases we need to load the
10522 whole group, not only the number of vector stmts the
10523 permutation result fits in. */
10528 {
10529 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10530 variable VF; see vect_transform_slp_perm_load. */
10535 }
10536 else
10537 {
10539 group_gap_adj
10541 }
10542 }
10543 else
10547 }
10548 else
10549 {
10557 }
10560 vec_loop_masks *loop_masks
10564 vec_loop_lens *loop_lens
10569 /* Shouldn't go with length-based approach if fully masked. */
10572 /* Targets with store-lane instructions must not require explicit
10573 realignment. vect_supportable_dr_alignment always returns either
10574 dr_aligned or dr_unaligned_supported for masked operations. */
10576 && !mask
10581 /* In case the vectorization factor (VF) is bigger than the number
10582 of elements that we can fit in a vectype (nunits), we have to generate
10583 more than one vector stmt - i.e - we need to "unroll" the
10584 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10585 from one copy of the vector stmt to the next, in the field
10586 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10587 stages to find the correct vector defs to be used when vectorizing
10588 stmts that use the defs of the current stmt. The example below
10589 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10590 need to create 4 vectorized stmts):
10592 before vectorization:
10593 RELATED_STMT VEC_STMT
10594 S1: x = memref - -
10595 S2: z = x + 1 - -
10597 step 1: vectorize stmt S1:
10598 We first create the vector stmt VS1_0, and, as usual, record a
10599 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10600 Next, we create the vector stmt VS1_1, and record a pointer to
10601 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10602 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10603 stmts and pointers:
10604 RELATED_STMT VEC_STMT
10605 VS1_0: vx0 = memref0 VS1_1 -
10606 VS1_1: vx1 = memref1 VS1_2 -
10607 VS1_2: vx2 = memref2 VS1_3 -
10608 VS1_3: vx3 = memref3 - -
10609 S1: x = load - VS1_0
10610 S2: z = x + 1 - -
10611 */
10613 /* In case of interleaving (non-unit grouped access):
10615 S1: x2 = &base + 2
10616 S2: x0 = &base
10617 S3: x1 = &base + 1
10618 S4: x3 = &base + 3
10620 Vectorized loads are created in the order of memory accesses
10621 starting from the access of the first stmt of the chain:
10623 VS1: vx0 = &base
10624 VS2: vx1 = &base + vec_size*1
10625 VS3: vx3 = &base + vec_size*2
10626 VS4: vx4 = &base + vec_size*3
10628 Then permutation statements are generated:
10630 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10631 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10632 ...
10634 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10635 (the order of the data-refs in the output of vect_permute_load_chain
10636 corresponds to the order of scalar stmts in the interleaving chain - see
10637 the documentation of vect_permute_load_chain()).
10638 The generation of permutation stmts and recording them in
10639 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10641 In case of both multiple types and interleaving, the vector loads and
10642 permutation stmts above are created for every copy. The result vector
10643 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10644 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10646 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10647 on a target that supports unaligned accesses (dr_unaligned_supported)
10648 we generate the following code:
10649 p = initial_addr;
10650 indx = 0;
10651 loop {
10652 p = p + indx * vectype_size;
10653 vec_dest = *(p);
10654 indx = indx + 1;
10655 }
10657 Otherwise, the data reference is potentially unaligned on a target that
10658 does not support unaligned accesses (dr_explicit_realign_optimized) -
10659 then generate the following code, in which the data in each iteration is
10660 obtained by two vector loads, one from the previous iteration, and one
10661 from the current iteration:
10662 p1 = initial_addr;
10663 msq_init = *(floor(p1))
10664 p2 = initial_addr + VS - 1;
10665 realignment_token = call target_builtin;
10666 indx = 0;
10667 loop {
10668 p2 = p2 + indx * vectype_size
10669 lsq = *(floor(p2))
10670 vec_dest = realign_load (msq, lsq, realignment_token)
10671 indx = indx + 1;
10672 msq = lsq;
10673 } */
10675 /* If the misalignment remains the same throughout the execution of the
10676 loop, we can create the init_addr and permutation mask at the loop
10677 preheader. Otherwise, it needs to be created inside the loop.
10678 This can only occur when vectorizing memory accesses in the inner-loop
10679 nested within an outer-loop that is being vectorized. */
10684 {
10687 }
10689 bool diff_first_stmt_info
10696 {
10697 /* If we have different first_stmt_info, we can't set up realignment
10698 here, since we can't guarantee first_stmt_info DR has been
10699 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10700 distance from first_stmt_info DR instead as below. */
10702 {
10709 {
10712 size_one_node);
10714 }
10715 }
10716 }
10717 else
10721 offset = (offset
10725 tree bump;
10728 {
10731 }
10733 {
10738 }
10739 else
10740 {
10743 else
10747 }
10752 {
10756 else
10759 }
10763 {
10769 /* For costing some adjacent vector loads, we'd like to cost with
10770 the total number of them once instead of cost each one by one. */
10773 {
10775 {
10776 /* An IFN_LOAD_LANES will load all its vector results,
10777 regardless of which ones we actually need. Account
10778 for the cost of unused results. */
10780 {
10783 do
10784 {
10787 }
10790 {
10793 "vect_model_load_cost: %d "
10795 gaps);
10797 alignment_support_scheme,
10801 }
10802 }
10803 n_adjacent_loads++;
10805 }
10807 /* 1. Create the vector or array pointer update chain. */
10809 dataref_ptr
10813 else
10814 {
10818 }
10835 {
10839 else
10841 signed char biasval
10845 {
10848 }
10849 }
10853 {
10854 /* Emit:
10855 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10856 VEC_MASK, LEN, BIAS). */
10862 }
10864 {
10865 /* Emit:
10866 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10867 VEC_MASK). */
10872 final_mask);
10873 }
10874 else
10875 {
10876 /* Emit:
10877 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10880 }
10886 /* Extract each vector into an SSA_NAME. */
10888 {
10892 }
10894 /* Record the mapping between SSA_NAMEs and statements. */
10897 /* Record that VEC_ARRAY is now dead. */
10903 }
10906 {
10914 "vect_model_load_cost: inside_cost = %u, "
10917 }
10920 }
10923 {
10930 {
10931 /* 1. Create the vector or array pointer update chain. */
10933 {
10938 else
10939 dataref_ptr
10943 }
10945 {
10950 }
10954 {
10959 {
10963 final_mask
10974 }
10976 /* 2. Create the vector-load in the loop. */
10979 {
10981 {
10983 inside_cost
10987 }
10994 {
10996 final_len
11000 else
11001 final_len
11004 signed char biasval
11008 {
11011 }
11012 }
11016 call
11025 else
11032 }
11034 {
11035 /* The builtin decls path for gather is legacy, x86 only. */
11038 {
11040 inside_cost
11044 }
11045 poly_uint64 offset_nunits
11048 {
11049 new_stmt = vect_build_one_gather_load_call
11052 final_mask);
11054 }
11056 {
11057 /* We have a offset vector with half the number of
11058 lanes but the builtins will produce full vectype
11059 data with just the lower lanes filled. */
11060 new_stmt = vect_build_one_gather_load_call
11063 final_mask);
11069 /* now put upper half of final_mask in final_mask low. */
11071 && !SCALAR_INT_MODE_P
11073 {
11080 tree perm_mask = vect_gen_perm_mask_checked
11083 VEC_PERM_EXPR,
11084 final_mask,
11085 final_mask,
11086 perm_mask);
11091 }
11093 {
11095 VEC_UNPACK_HI_EXPR,
11096 final_mask);
11097 final_mask = make_ssa_name
11102 }
11104 new_stmt = vect_build_one_gather_load_call
11106 dataref_ptr,
11108 final_mask);
11114 /* compose low + high. */
11121 tree perm_mask
11124 VEC_PERM_EXPR,
11127 }
11129 {
11130 /* We have a offset vector with double the number of
11131 lanes. Select the low/high part accordingly. */
11134 {
11141 tree perm_mask = vect_gen_perm_mask_checked
11144 VEC_PERM_EXPR,
11145 vec_offset,
11146 vec_offset,
11147 perm_mask);
11152 }
11153 new_stmt = vect_build_one_gather_load_call
11157 }
11158 else
11160 }
11161 else
11162 {
11163 /* Emulated gather-scatter. */
11167 {
11168 /* For emulated gathers N offset vector element
11169 offset add is consumed by the load). */
11173 /* N scalar loads plus gathering them into a
11174 vector. */
11175 inside_cost
11178 inside_cost
11182 }
11183 unsigned HOST_WIDE_INT const_offset_nunits
11189 /* We support offset vectors with more elements
11190 than the data vector for now. */
11191 unsigned HOST_WIDE_INT factor
11194 unsigned elt_offset
11201 {
11204 tree idx
11209 scale);
11221 }
11226 }
11229 /* DATA_REF is null if we've already built the statement. */
11231 {
11234 }
11239 /* Store vector loads in the corresponding SLP_NODE. */
11242 }
11246 }
11253 "vect_model_load_cost: inside_cost = %u, "
11257 }
11261 /* For costing some adjacent vector loads, we'd like to cost with
11262 the total number of them once instead of cost each one by one. */
11265 {
11266 /* 1. Create the vector or array pointer update chain. */
11268 {
11269 bool simd_lane_access_p
11280 {
11283 }
11285 {
11286 dataref_ptr
11290 bump);
11291 /* Adjust the pointer by the difference to first_stmt. */
11292 data_reference_p ptrdr
11294 tree diff
11302 {
11306 alignment_support_scheme,
11309 }
11310 }
11311 else
11312 dataref_ptr
11314 at_loop,
11317 }
11319 {
11323 bump);
11324 else
11327 }
11334 {
11339 {
11353 }
11355 /* 2. Create the vector-load in the loop. */
11357 {
11360 {
11370 {
11371 align
11374 }
11375 else
11380 misalign);
11383 /* Compute IFN when LOOP_LENS or final_mask valid. */
11388 {
11389 opt_machine_mode new_ovmode
11392 unsigned factor
11397 }
11399 {
11404 }
11407 {
11409 {
11410 /* Pass VF value to 'len' argument of
11411 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11413 }
11415 {
11416 /* Pass all ones value to 'mask' argument of
11417 MASK_LEN_LOAD if final_mask is invalid. */
11420 }
11421 }
11423 {
11424 signed char biasval
11428 }
11431 {
11438 bias);
11439 else
11447 /* Need conversion if it's wrapped with VnQI. */
11449 {
11452 tree var
11456 gsi);
11460 }
11461 }
11463 {
11467 final_mask);
11471 }
11472 else
11473 {
11477 unsigned int vect_align
11479 unsigned int scalar_dr_size
11481 /* If there's no peeling for gaps but we have a gap
11482 with slp loads then load the lower half of the
11483 vector only. See get_group_load_store_type for
11484 when we apply this optimization. */
11486 && loop_vinfo
11491 {
11492 tree half_vtype;
11493 new_vtype
11498 }
11499 /* Try to use a single smaller load when we are about
11500 to load excess elements compared to the unrolled
11501 scalar loop.
11502 ??? This should cover the above case as well. */
11505 {
11508 /* DR will be unused. */
11511 tree_to_poly_uint64
11513 /* Aligned access to excess elements is OK if
11514 at least one element is accessed in the
11515 scalar loop. */
11516 ;
11517 else
11518 {
11519 auto remain
11522 /* remain should now be > 0 and < nunits. */
11525 {
11526 tree ptype;
11527 new_vtype
11529 num,
11533 }
11534 /* Else use multiple loads or a masked load? */
11535 }
11536 }
11537 tree offset
11541 ;
11544 {
11545 poly_uint64 gap_offset
11550 }
11552 {
11553 data_ref
11556 ;
11557 else
11561 }
11565 {
11571 gsi);
11574 /* We've computed 'num' above to statically two
11575 or via constant_multiple_p. */
11576 unsigned num
11579 tree_to_poly_uint64
11584 {
11589 }
11590 else
11591 {
11596 }
11601 else
11602 {
11609 vec_dest,
11611 }
11612 }
11613 }
11615 }
11617 {
11627 dr_explicit_realign,
11632 else
11634 // For explicit realign the target alignment should be
11635 // known at compile time.
11636 unsigned HOST_WIDE_INT align
11643 data_ref
11657 bump);
11663 else
11667 data_ref
11670 }
11672 {
11677 else
11679 // We should only be doing this if we know the target
11680 // alignment at compile time.
11681 unsigned HOST_WIDE_INT align
11691 }
11694 }
11696 /* One common place to cost the above vect load for different
11697 alignment support schemes. */
11699 {
11700 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11701 only need to take care of the first stmt, whose
11702 stmt_info is first_stmt_info, vec_num iterating on it
11703 will cover the cost for the remaining, it's consistent
11704 with transforming. For the prologue cost for realign,
11705 we only need to count it once for the whole group. */
11712 {
11713 /* Leave realign cases alone to keep them simple. */
11721 else
11722 n_adjacent_loads++;
11723 }
11724 }
11725 else
11726 {
11728 /* DATA_REF is null if we've already built the statement. */
11730 {
11733 }
11737 }
11739 /* 3. Handle explicit realignment if necessary/supported.
11740 Create in loop:
11741 vec_dest = realign_load (msq, lsq, realignment_token) */
11745 {
11757 {
11761 UNKNOWN_LOCATION);
11763 }
11764 }
11767 {
11771 else
11772 {
11777 }
11778 }
11780 /* Collect vector loads and later create their permutation in
11781 vect_transform_grouped_load (). */
11785 /* Store vector loads in the corresponding SLP_NODE. */
11789 /* With SLP permutation we load the gaps as well, without
11790 we need to skip the gaps after we manage to fully load
11791 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11795 && !slp_perm
11797 {
11798 poly_wide_int bump_val
11807 }
11808 }
11809 /* Bump the vector pointer to account for a gap or for excess
11810 elements loaded for a permuted SLP load. */
11814 {
11815 poly_wide_int bump_val
11822 }
11828 {
11830 /* For SLP we know we've seen all possible uses of dr_chain so
11831 direct vect_transform_slp_perm_load to DCE the unused parts.
11832 ??? This is a hack to prevent compile-time issues as seen
11833 in PR101120 and friends. */
11835 {
11840 }
11841 else
11842 {
11847 }
11848 }
11849 else
11850 {
11852 {
11854 /* We assume that the cost of a single load-lanes instruction
11855 is equivalent to the cost of DR_GROUP_SIZE separate loads.
11856 If a grouped access is instead being provided by a
11857 load-and-permute operation, include the cost of the
11858 permutes. */
11860 {
11861 /* Uses an even and odd extract operations or shuffle
11862 operations for each needed permute. */
11870 "vect_model_load_cost:"
11872 group_size);
11873 }
11875 {
11879 }
11880 }
11883 }
11885 }
11890 {
11901 "vect_model_load_cost: inside_cost = %u, "
11904 }
11907 }
11909 /* Function vect_is_simple_cond.
11911 Input:
11912 LOOP - the loop that is being vectorized.
11913 COND - Condition that is checked for simple use.
11915 Output:
11916 *COMP_VECTYPE - the vector type for the comparison.
11917 *DTS - The def types for the arguments of the comparison
11919 Returns whether a COND can be vectorized. Checks whether
11920 condition operands are supportable using vec_is_simple_use. */
11922 static bool
11926 {
11929 slp_tree slp_op;
11931 /* Mask case. */
11934 {
11937 || !*comp_vectype
11941 }
11950 {
11954 }
11958 else
11962 {
11966 }
11970 else
11979 /* Invariant comparison. */
11981 {
11985 else
11986 {
11987 /* If we can widen the comparison to match vectype do so. */
11989 && !slp_node
11992 scalar_type = build_nonstandard_integer_type
11995 slp_node);
11996 }
11997 }
12000 }
12002 /* vectorizable_condition.
12004 Check if STMT_INFO is conditional modify expression that can be vectorized.
12005 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12006 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
12007 at GSI.
12009 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12011 Return true if STMT_INFO is vectorizable in this way. */
12013 static bool
12018 {
12026 tree vec_compare;
12027 tree new_temp;
12042 tree vec_cmp_type;
12048 /* Is vectorizable conditional operation? */
12060 bool for_reduction
12063 {
12071 }
12072 else
12073 {
12076 }
12082 {
12085 }
12086 else
12087 {
12090 }
12126 {
12129 }
12131 /* For conditional reductions, the "then" value needs to be the candidate
12132 value calculated by this iteration while the "else" value needs to be
12133 the result carried over from previous iterations. If the COND_EXPR
12134 is the other way around, we need to swap it. */
12137 {
12140 else
12141 {
12146 else
12147 {
12149 /* Make sure we don't accidentally use the old condition. */
12151 }
12152 }
12154 }
12157 {
12158 /* Boolean values may have another representation in vectors
12159 and therefore we prefer bit operations over comparison for
12160 them (which also works for scalar masks). We store opcodes
12161 to use in bitop1 and bitop2. Statement is vectorized as
12162 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12163 depending on bitop1 and bitop2 arity. */
12165 {
12193 }
12195 }
12200 {
12205 }
12208 {
12210 {
12212 optab optab;
12219 {
12221 optab_default);
12224 }
12225 }
12229 /* Count one reduction-like operation per vector. */
12232 && (masked
12234 cond_code)
12236 ERROR_MARK))))
12240 && (!vect_maybe_update_slp_op_vectype
12243 && !vect_maybe_update_slp_op_vectype
12247 {
12252 }
12256 {
12258 {
12264 else
12268 }
12269 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12271 {
12274 "conditional reduction prevents the use"
12277 }
12278 }
12284 }
12286 /* Transform. */
12288 /* Handle def. */
12295 /* See whether another part of the vectorized code applies a loop
12296 mask to the condition, or to its inverse. */
12301 {
12304 }
12306 {
12309 else
12310 {
12314 else
12315 {
12320 {
12324 }
12325 else
12326 {
12327 /* Try the inverse of the current mask. We check if the
12328 inverse mask is live and if so we generate a negate of
12329 the current mask such that we still honor NaNs. */
12333 {
12338 }
12339 }
12340 }
12341 }
12342 }
12344 /* Handle cond expr. */
12349 reduction_type != EXTRACT_LAST_REDUCTION
12351 else
12356 reduction_type != EXTRACT_LAST_REDUCTION
12359 /* Arguments are ready. Create the new vector stmt. */
12361 {
12371 else
12372 {
12375 {
12380 }
12381 else
12382 {
12387 vec_cond_rhs);
12388 else
12389 new_stmt
12391 vec_cond_rhs);
12397 {
12398 /* Instead of doing ~x ? y : z do x ? z : y. */
12401 }
12402 else
12403 {
12406 new_stmt
12408 else
12409 new_stmt
12414 }
12415 }
12416 }
12418 /* If we decided to apply a loop mask to the result of the vector
12419 comparison, AND the comparison with the mask now. Later passes
12420 should then be able to reuse the AND results between mulitple
12421 vector statements.
12423 For example:
12424 for (int i = 0; i < 100; ++i)
12425 x[i] = y[i] ? z[i] : 10;
12427 results in following optimized GIMPLE:
12429 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12430 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12431 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12432 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12433 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12434 vect_iftmp.11_47, { 10, ... }>;
12436 instead of using a masked and unmasked forms of
12437 vec != { 0, ... } (masked in the MASK_LOAD,
12438 unmasked in the VEC_COND_EXPR). */
12440 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12441 in cases where that's necessary. */
12445 {
12447 {
12450 vec_compare);
12453 }
12456 {
12459 BIT_NOT_EXPR,
12460 vec_compare);
12463 }
12467 {
12469 {
12472 signed char biasval
12475 }
12476 else
12477 {
12480 }
12481 }
12483 {
12484 tree loop_mask
12488 gassign *g
12490 loop_mask);
12493 }
12494 }
12498 {
12502 new_stmt = gimple_build_call_internal
12505 else
12506 new_stmt = gimple_build_call_internal
12508 vec_then_clause);
12513 else
12514 {
12515 /* In this case we're moving the definition to later in the
12516 block. That doesn't matter because the only uses of the
12517 lhs are in phi statements. */
12521 }
12522 }
12523 else
12524 {
12529 }
12532 else
12534 }
12545 }
12547 /* Helper of vectorizable_comparison.
12549 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12550 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12551 comparison, put it in VEC_STMT, and insert it at GSI.
12553 Return true if STMT_INFO is vectorizable in this way. */
12555 static bool
12560 {
12564 tree new_temp;
12568 poly_uint64 nunits;
12575 tree mask_type;
12589 else
12613 /* Invariant comparison. */
12615 {
12618 else
12620 slp_node);
12623 }
12627 /* Can't compare mask and non-mask types. */
12632 /* Boolean values may have another representation in vectors
12633 and therefore we prefer bit operations over comparison for
12634 them (which also works for scalar masks). We store opcodes
12635 to use in bitop1 and bitop2. Statement is vectorized as
12636 BITOP2 (rhs1 BITOP1 rhs2) or
12637 rhs1 BITOP2 (BITOP1 rhs2)
12638 depending on bitop1 and bitop2 arity. */
12641 {
12643 {
12646 }
12648 {
12651 }
12653 {
12657 }
12659 {
12663 }
12664 else
12665 {
12669 }
12670 }
12673 {
12675 {
12678 }
12679 else
12680 {
12682 optab optab;
12689 {
12693 }
12694 }
12696 /* Put types on constant and invariant SLP children. */
12700 {
12705 }
12711 }
12713 /* Transform. */
12715 /* Handle def. */
12726 /* Arguments are ready. Create the new vector stmt. */
12728 {
12734 else
12737 {
12741 }
12742 else
12743 {
12746 else
12748 vec_rhs2);
12751 {
12755 else
12757 new_temp);
12759 }
12760 }
12763 else
12765 }
12774 }
12776 /* vectorizable_comparison.
12778 Check if STMT_INFO is comparison expression that can be vectorized.
12779 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12780 comparison, put it in VEC_STMT, and insert it at GSI.
12782 Return true if STMT_INFO is vectorizable in this way. */
12784 static bool
12789 {
12812 }
12814 /* Check to see if the current early break given in STMT_INFO is valid for
12815 vectorization. */
12817 static bool
12821 {
12838 slp_tree slp_op0;
12839 tree op0;
12843 {
12848 }
12858 else
12864 /* Now build the new conditional. Pattern gimple_conds get dropped during
12865 codegen so we must replace the original insn. */
12868 /* When vectorizing we assume that if the branch edge is taken that we're
12869 exiting the loop. This is not however always the case as the compiler will
12870 rewrite conditions to always be a comparison against 0. To do this it
12871 sometimes flips the edges. This is fine for scalar, but for vector we
12872 then have to flip the test, as we're still assuming that if you take the
12873 branch edge that we found the exit condition. i.e. we need to know whether
12874 we are generating a `forall` or an `exist` condition. */
12885 {
12890 }
12892 /* Analyze only. */
12894 {
12896 {
12899 "can't vectorize early exit because the "
12900 "target doesn't support flag setting vector "
12903 }
12907 {
12910 "can't vectorize early exit because the "
12911 "target does not support boolean vector %s "
12914 vectype);
12916 }
12923 {
12925 OPTIMIZE_FOR_SPEED))
12927 else
12929 }
12933 }
12935 /* Tranform. */
12955 else
12956 {
12961 }
12963 /* Determine if we need to reduce the final value. */
12965 {
12966 /* We build the reductions in a way to maintain as much parallelism as
12967 possible. */
12970 /* Mask the statements as we queue them up. Normally we loop over
12971 vec_num, but since we inspect the exact results of vectorization
12972 we don't need to and instead can just use the stmts themselves. */
12975 {
12976 tree stmt_mask
12978 i);
12979 stmt_mask
12983 }
12984 else
12988 {
12996 }
12997 }
12998 else
12999 {
13002 {
13003 tree mask
13007 }
13008 }
13017 else
13024 }
13026 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13027 can handle all live statements in the node. Otherwise return true
13028 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13029 VEC_STMT_P is as for vectorizable_live_operation. */
13031 static bool
13036 {
13039 {
13040 stmt_vec_info slp_stmt_info;
13043 {
13045 || (loop_vinfo
13053 }
13054 }
13064 }
13066 /* Make sure the statement is vectorizable. */
13068 opt_result
13073 {
13077 gimple_seq pattern_def_seq;
13085 "not vectorized:"
13092 {
13093 gimple_stmt_iterator si;
13096 {
13097 stmt_vec_info pattern_def_stmt_info
13101 {
13102 /* Analyze def stmt of STMT if it's a pattern stmt. */
13108 opt_result res
13111 cost_vec);
13114 }
13115 }
13116 }
13118 /* Skip stmts that do not need to be vectorized. In loops this is expected
13119 to include:
13120 - the COND_EXPR which is the loop exit condition
13121 - any LABEL_EXPRs in the loop
13122 - computations that are used only for array indexing or loop control.
13123 In basic blocks we only analyze statements that are a part of some SLP
13124 instance, therefore, all the statements are relevant.
13126 Pattern statement needs to be analyzed instead of the original statement
13127 if the original statement is not relevant. Otherwise, we analyze both
13128 statements. In basic blocks we are called from some SLP instance
13129 traversal, don't analyze pattern stmts instead, the pattern stmts
13130 already will be part of SLP instance. */
13135 {
13137 && pattern_stmt_info
13140 {
13141 /* Analyze PATTERN_STMT instead of the original stmt. */
13147 }
13148 else
13149 {
13154 }
13155 }
13158 && pattern_stmt_info
13161 {
13162 /* Analyze PATTERN_STMT too. */
13168 opt_result res
13173 }
13176 {
13201 }
13208 {
13214 }
13217 {
13222 }
13228 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13229 -mveclibabi= takes preference over library functions with
13230 the simd attribute. */
13233 cost_vec)
13250 cost_vec)
13256 cost_vec));
13257 else
13258 {
13264 cost_vec)
13270 cost_vec)
13278 cost_vec)
13281 cost_vec));
13283 }
13290 "not vectorized:"
13294 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13295 need extra handling, except for vectorizable reductions. */
13303 "not vectorized:"
13308 }
13311 /* Function vect_transform_stmt.
13313 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13315 bool
13319 {
13331 {
13343 NULL);
13355 NULL);
13367 NULL);
13373 && !slp_node
13376 /* In case of interleaving, the whole chain is vectorized when the
13377 last store in the chain is reached. Store stmts before the last
13378 one are skipped, and there vec_stmt_info shouldn't be freed
13379 meanwhile. */
13380 ;
13381 else
13382 {
13387 }
13449 {
13454 }
13456 }
13462 {
13463 /* Handle stmts whose DEF is used outside the loop-nest that is
13464 being vectorized. */
13468 }
13474 }
13477 /* Remove a group of stores (for SLP or interleaving), free their
13478 stmt_vec_info. */
13480 void
13482 {
13486 {
13489 /* Free the attached stmt_vec_info and remove the stmt. */
13492 }
13493 }
13495 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13496 elements of type SCALAR_TYPE, or null if the target doesn't support
13497 such a type.
13499 If NUNITS is zero, return a vector type that contains elements of
13500 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13502 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13503 for this vectorization region and want to "autodetect" the best choice.
13504 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13505 and we want the new type to be interoperable with it. PREVAILING_MODE
13506 in this case can be a scalar integer mode or a vector mode; when it
13507 is a vector mode, the function acts like a tree-level version of
13508 related_vector_mode. */
13510 tree
13513 {
13515 scalar_mode inner_mode;
13516 machine_mode simd_mode;
13517 tree vectype;
13528 /* Interoperability between modes requires one to be a constant multiple
13529 of the other, so that the number of vectors required for each operation
13530 is a compile-time constant. */
13538 /* For vector types of elements whose mode precision doesn't
13539 match their types precision we use a element type of mode
13540 precision. The vectorization routines will have to make sure
13541 they support the proper result truncation/extension.
13542 We also make sure to build vector types with INTEGER_TYPE
13543 component type only. */
13550 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13551 When the component mode passes the above test simply use a type
13552 corresponding to that mode. The theory is that any use that
13553 would cause problems with this will disable vectorization anyway. */
13558 /* We can't build a vector type of elements with alignment bigger than
13559 their size. */
13564 /* If we felt back to using the mode fail if there was
13565 no scalar type for it. */
13569 /* If no prevailing mode was supplied, use the mode the target prefers.
13570 Otherwise lookup a vector mode based on the prevailing mode. */
13572 {
13576 {
13577 /* Traditional behavior is not to take the integer mode
13578 literally, but simply to use it as a way of determining
13579 the vector size. It is up to mode_for_vector to decide
13580 what the TYPE_MODE should be.
13582 Note that nunits == 1 is allowed in order to support single
13583 element vector types. */
13587 }
13588 }
13592 {
13593 /* Fall back to using mode_for_vector, mostly in the hope of being
13594 able to use an integer mode. */
13601 }
13605 /* In cases where the mode was chosen by mode_for_vector, check that
13606 the target actually supports the chosen mode, or that it at least
13607 allows the vector mode to be replaced by a like-sized integer. */
13612 /* Re-attach the address-space qualifier if we canonicalized the scalar
13613 type. */
13615 return build_qualified_type
13619 }
13621 /* Function get_vectype_for_scalar_type.
13623 Returns the vector type corresponding to SCALAR_TYPE as supported
13624 by the target. If GROUP_SIZE is nonzero and we're performing BB
13625 vectorization, make sure that the number of elements in the vector
13626 is no bigger than GROUP_SIZE. */
13628 tree
13631 {
13632 /* For BB vectorization, we should always have a group size once we've
13633 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13634 are tentative requests during things like early data reference
13635 analysis and pattern recognition. */
13638 else
13642 scalar_type);
13646 /* Register the natural choice of vector type, before the group size
13647 has been applied. */
13651 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13652 try again with an explicit number of elements. */
13654 && group_size
13656 {
13657 /* Start with the biggest number of units that fits within
13658 GROUP_SIZE and halve it until we find a valid vector type.
13659 Usually either the first attempt will succeed or all will
13660 fail (in the latter case because GROUP_SIZE is too small
13661 for the target), but it's possible that a target could have
13662 a hole between supported vector types.
13664 If GROUP_SIZE is not a power of 2, this has the effect of
13665 trying the largest power of 2 that fits within the group,
13666 even though the group is not a multiple of that vector size.
13667 The BB vectorizer will then try to carve up the group into
13668 smaller pieces. */
13670 do
13671 {
13675 }
13677 }
13680 }
13682 /* Return the vector type corresponding to SCALAR_TYPE as supported
13683 by the target. NODE, if nonnull, is the SLP tree node that will
13684 use the returned vector type. */
13686 tree
13688 {
13693 }
13695 /* Function get_mask_type_for_scalar_type.
13697 Returns the mask type corresponding to a result of comparison
13698 of vectors of specified SCALAR_TYPE as supported by target.
13699 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13700 make sure that the number of elements in the vector is no bigger
13701 than GROUP_SIZE. */
13703 tree
13706 {
13713 }
13715 /* Function get_mask_type_for_scalar_type.
13717 Returns the mask type corresponding to a result of comparison
13718 of vectors of specified SCALAR_TYPE as supported by target.
13719 NODE, if nonnull, is the SLP tree node that will use the returned
13720 vector type. */
13722 tree
13724 slp_tree node)
13725 {
13732 }
13734 /* Function get_same_sized_vectype
13736 Returns a vector type corresponding to SCALAR_TYPE of size
13737 VECTOR_TYPE if supported by the target. */
13739 tree
13741 {
13745 poly_uint64 nunits;
13752 }
13754 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13755 would not change the chosen vector modes. */
13757 bool
13759 {
13766 }
13768 /* Function vect_is_simple_use.
13770 Input:
13771 VINFO - the vect info of the loop or basic block that is being vectorized.
13772 OPERAND - operand in the loop or bb.
13773 Output:
13774 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13775 case OPERAND is an SSA_NAME that is defined in the vectorizable region
13776 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13777 the definition could be anywhere in the function
13778 DT - the type of definition
13780 Returns whether a stmt with OPERAND can be vectorized.
13781 For loops, supportable operands are constants, loop invariants, and operands
13782 that are defined by the current iteration of the loop. Unsupportable
13783 operands are those that are defined by a previous iteration of the loop (as
13784 is the case in reduction/induction computations).
13785 For basic blocks, supportable operands are constants and bb invariants.
13786 For now, operands defined outside the basic block are not supported. */
13788 bool
13791 {
13799 {
13805 else
13807 }
13817 else
13818 {
13823 else
13824 {
13830 }
13833 }
13836 {
13839 {
13873 }
13874 }
13877 {
13882 }
13885 }
13887 /* Function vect_is_simple_use.
13889 Same as vect_is_simple_use but also determines the vector operand
13890 type of OPERAND and stores it to *VECTYPE. If the definition of
13891 OPERAND is vect_uninitialized_def, vect_constant_def or
13892 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
13893 is responsible to compute the best suited vector type for the
13894 scalar operand. */
13896 bool
13900 {
13901 stmt_vec_info def_stmt_info;
13911 /* Now get a vector type if the def is internal, otherwise supply
13912 NULL_TREE and leave it up to the caller to figure out a proper
13913 type for the use stmt. */
13920 {
13926 }
13931 else
13935 }
13937 /* Function vect_is_simple_use.
13939 Same as vect_is_simple_use but determines the operand by operand
13940 position OPERAND from either STMT or SLP_NODE, filling in *OP
13941 and *SLP_DEF (when SLP_NODE is not NULL). */
13943 bool
13948 {
13950 {
13955 {
13958 }
13959 else
13960 {
13966 }
13967 }
13968 else
13969 {
13972 {
13975 {
13978 else
13980 }
13983 else
13985 }
13990 else
13993 }
13994 }
13996 /* If OP is not NULL and is external or constant update its vector
13997 type with VECTYPE. Returns true if successful or false if not,
13998 for example when conflicting vector types are present. */
14000 bool
14002 {
14007 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
14008 should be handled by patters. Allow vect_constant_def for now. */
14014 }
14016 /* Function supportable_widening_operation
14018 Check whether an operation represented by the code CODE is a
14019 widening operation that is supported by the target platform in
14020 vector form (i.e., when operating on arguments of type VECTYPE_IN
14021 producing a result of type VECTYPE_OUT).
14023 Widening operations we currently support are NOP (CONVERT), FLOAT,
14024 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
14025 are supported by the target platform either directly (via vector
14026 tree-codes), or via target builtins.
14028 Output:
14029 - CODE1 and CODE2 are codes of vector operations to be used when
14030 vectorizing the operation, if available.
14031 - MULTI_STEP_CVT determines the number of required intermediate steps in
14032 case of multi-step conversion (like char->short->int - in that case
14033 MULTI_STEP_CVT will be 1).
14034 - INTERM_TYPES contains the intermediate type required to perform the
14035 widening operation (short in the above example). */
14037 bool
14039 code_helper code,
14040 stmt_vec_info stmt_info,
14046 {
14049 machine_mode vec_mode;
14065 {
14067 /* Don't set c1 and c2 if code is not a tree_code. */
14071 /* The result of a vectorized widening operation usually requires
14072 two vectors (because the widened results do not fit into one vector).
14073 The generated vector results would normally be expected to be
14074 generated in the same order as in the original scalar computation,
14075 i.e. if 8 results are generated in each vector iteration, they are
14076 to be organized as follows:
14077 vect1: [res1,res2,res3,res4],
14078 vect2: [res5,res6,res7,res8].
14080 However, in the special case that the result of the widening
14081 operation is used in a reduction computation only, the order doesn't
14082 matter (because when vectorizing a reduction we change the order of
14083 the computation). Some targets can take advantage of this and
14084 generate more efficient code. For example, targets like Altivec,
14085 that support widen_mult using a sequence of {mult_even,mult_odd}
14086 generate the following vectors:
14087 vect1: [res1,res3,res5,res7],
14088 vect2: [res2,res4,res6,res8].
14090 When vectorizing outer-loops, we execute the inner-loop sequentially
14091 (each vectorized inner-loop iteration contributes to VF outer-loop
14092 iterations in parallel). We therefore don't allow to change the
14093 order of the computation in the inner-loop during outer-loop
14094 vectorization. */
14095 /* TODO: Another case in which order doesn't *really* matter is when we
14096 widen and then contract again, e.g. (short)((int)x * y >> 8).
14097 Normally, pack_trunc performs an even/odd permute, whereas the
14098 repack from an even/odd expansion would be an interleave, which
14099 would be significantly simpler for e.g. AVX2. */
14100 /* In any case, in order to avoid duplicating the code below, recurse
14101 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
14102 are properly set up for the caller. If we fail, we'll continue with
14103 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
14111 interm_types))
14112 {
14113 /* Elements in a vector with vect_used_by_reduction property cannot
14114 be reordered if the use chain with this property does not have the
14115 same operation. One such an example is s += a * b, where elements
14116 in a and b cannot be reordered. Here we check if the vector defined
14117 by STMT is only directly used in the reduction statement. */
14123 }
14139 /* Support the recursion induced just above. */
14149 CASE_CONVERT:
14166 }
14172 {
14173 /* The signedness is determined from output operand. */
14176 }
14182 {
14183 /* If the input and result modes are the same, a different optab
14184 is needed where we pass in the number of units in vectype. */
14187 }
14191 {
14192 /* If this is an internal fn then we must check whether the target
14193 supports either a low-high split or an even-odd split. */
14203 /* If we don't support low-high, then check for even-odd. */
14206 || !optab2
14208 {
14214 }
14215 }
14217 {
14219 {
14220 /* The signedness is determined from output operand. */
14223 }
14229 {
14230 /* If the input and result modes are the same, a different optab
14231 is needed where we pass in the number of units in vectype. */
14234 }
14235 else
14236 {
14239 }
14242 }
14254 {
14257 /* For scalar masks we may have different boolean
14258 vector types having the same QImode. Thus we
14259 add additional check for elements number. */
14263 }
14265 /* Check if it's a multi-step conversion that can be done using intermediate
14266 types. */
14274 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14275 intermediate steps in promotion sequence. We try
14276 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14277 not. */
14280 {
14283 intermediate_type
14286 {
14287 tree intermediate_element_type
14290 intermediate_type
14292 intermediate_mode);
14293 }
14294 else
14295 intermediate_type
14303 {
14304 /* If the input and result modes are the same, a different optab
14305 is needed where we pass in the number of units in vectype. */
14308 }
14309 else
14310 {
14313 }
14331 {
14337 }
14341 }
14345 }
14348 /* Function supportable_narrowing_operation
14350 Check whether an operation represented by the code CODE is a
14351 narrowing operation that is supported by the target platform in
14352 vector form (i.e., when operating on arguments of type VECTYPE_IN
14353 and producing a result of type VECTYPE_OUT).
14355 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14356 and FLOAT. This function checks if these operations are supported by
14357 the target platform directly via vector tree-codes.
14359 Output:
14360 - CODE1 is the code of a vector operation to be used when
14361 vectorizing the operation, if available.
14362 - MULTI_STEP_CVT determines the number of required intermediate steps in
14363 case of multi-step conversion (like int->short->char - in that case
14364 MULTI_STEP_CVT will be 1).
14365 - INTERM_TYPES contains the intermediate type required to perform the
14366 narrowing operation (short in the above example). */
14368 bool
14373 {
14374 machine_mode vec_mode;
14391 {
14392 CASE_CONVERT:
14400 else
14406 /* The signedness is determined from output operand. */
14417 }
14429 {
14432 /* For scalar masks we may have different boolean
14433 vector types having the same QImode. Thus we
14434 add additional check for elements number. */
14438 }
14443 /* Check if it's a multi-step conversion that can be done using intermediate
14444 types. */
14449 else
14452 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14453 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14454 costly than signed. */
14456 {
14459 intermediate_type
14461 interm_optab
14467 {
14471 }
14472 }
14474 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14475 intermediate steps in promotion sequence. We try
14476 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14479 {
14482 intermediate_type
14484 else
14485 intermediate_type
14493 else
14494 interm_optab
14496 optab_default);
14508 {
14514 }
14519 }
14523 }
14525 /* Generate and return a vector mask of MASK_TYPE such that
14526 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14527 Add the statements to SEQ. */
14529 tree
14532 {
14536 OPTIMIZE_FOR_SPEED));
14540 tree tmp;
14543 else
14548 }
14550 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14551 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14553 tree
14555 tree end_index)
14556 {
14559 }
14561 /* Try to compute the vector types required to vectorize STMT_INFO,
14562 returning true on success and false if vectorization isn't possible.
14563 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14564 take sure that the number of elements in the vectors is no bigger
14565 than GROUP_SIZE.
14567 On success:
14569 - Set *STMT_VECTYPE_OUT to:
14570 - NULL_TREE if the statement doesn't need to be vectorized;
14571 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14573 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14574 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14575 statement does not help to determine the overall number of units. */
14577 opt_result
14582 {
14585 /* For BB vectorization, we should always have a group size once we've
14586 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14587 are tentative requests during things like early data reference
14588 analysis and pattern recognition. */
14591 else
14598 /* Allow vector conditionals through here. */
14600 /* MASK_STORE has no lhs, but is ok. */
14602 {
14604 {
14605 /* Ignore calls with no lhs. These must be calls to
14606 #pragma omp simd functions, and what vectorization factor
14607 it really needs can't be determined until
14608 vectorizable_simd_clone_call. */
14613 }
14617 }
14619 tree vectype;
14622 {
14627 }
14629 {
14638 }
14639 else
14640 {
14641 /* If we got here with a gcond it means that the target had no available vector
14642 mode for the scalar type. We can't vectorize so abort. */
14645 "not vectorized:"
14647 scalar_type);
14653 else
14657 {
14660 "get vectype for scalar type (group size %d):"
14662 else
14665 }
14669 "not vectorized:"
14671 scalar_type);
14675 }
14680 stmt);
14684 /* Don't try to compute scalar types if the stmt produces a boolean
14685 vector; use the existing vector type instead. */
14688 {
14689 /* The number of units is set according to the smallest scalar
14690 type (or the largest vector size, but we only support one
14691 vector size per vectorization). */
14695 {
14699 scalar_type);
14701 group_size);
14705 scalar_type);
14708 nunits_vectype);
14709 }
14710 }
14715 "Not vectorized: Incompatible number "
14720 {
14724 }
14728 }
14730 /* Generate and return statement sequence that sets vector length LEN that is:
14732 min_of_start_and_end = min (START_INDEX, END_INDEX);
14733 left_len = END_INDEX - min_of_start_and_end;
14734 rhs = min (left_len, LEN_LIMIT);
14735 LEN = rhs;
14737 Note: the cost of the code generated by this function is modeled
14738 by vect_estimate_min_profitable_iters, so changes here may need
14739 corresponding changes there. */
14741 gimple_seq
14743 {
14755 }