| blob | 7e571968a59d7f0cc9833ca930f5f2975c8dcd21 |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
61 /* For lang_hooks.types.type_for_mode. */
64 /* Return the vectorized type for the given statement. */
66 tree
68 {
70 }
72 /* Return TRUE iff the given statement is in an inner loop relative to
73 the loop being vectorized. */
74 bool
76 {
88 }
90 /* Record the cost of a statement, either by directly informing the
91 target model or by saving it in a vector for later processing.
92 Return a preliminary estimate of the statement's cost. */
94 static unsigned
100 {
108 stmt_info_for_cost si
114 }
116 unsigned
121 {
124 }
126 unsigned
131 {
134 }
136 unsigned
140 {
145 }
147 /* Return a variable of type ELEM_TYPE[NELEMS]. */
149 static tree
151 {
154 }
156 /* ARRAY is an array of vectors created by create_vector_array.
157 Return an SSA_NAME for the vector in index N. The reference
158 is part of the vectorization of STMT_INFO and the vector is associated
159 with scalar destination SCALAR_DEST. */
161 static tree
165 {
182 }
184 /* ARRAY is an array of vectors created by create_vector_array.
185 Emit code to store SSA_NAME VECT in index N of the array.
186 The store is part of the vectorization of STMT_INFO. */
188 static void
192 {
193 tree array_ref;
202 }
204 /* PTR is a pointer to an array of type TYPE. Return a representation
205 of *PTR. The memory reference replaces those in FIRST_DR
206 (and its group). */
208 static tree
210 {
211 tree mem_ref;
214 /* Arrays have the same alignment as their type. */
217 }
219 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
220 Emit the clobber before *GSI. */
222 static void
225 {
229 }
231 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
233 /* Function vect_mark_relevant.
235 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
237 static void
240 {
249 /* If this stmt is an original stmt in a pattern, we might need to mark its
250 related pattern stmt instead of the original stmt. However, such stmts
251 may have their own uses that are not in any pattern, in such cases the
252 stmt itself should be marked. */
254 {
255 /* This is the last stmt in a sequence that was detected as a
256 pattern that can potentially be vectorized. Don't mark the stmt
257 as relevant/live because it's not going to be vectorized.
258 Instead mark the pattern-stmt that replaces it. */
262 "last stmt in pattern. don't mark"
272 {
275 "vec_stmt_relevant_p: forcing live pattern stmt "
278 }
284 }
292 {
297 }
300 }
303 /* Function is_simple_and_all_uses_invariant
305 Return true if STMT_INFO is simple and all uses of it are invariant. */
307 bool
309 loop_vec_info loop_vinfo)
310 {
311 tree op;
312 ssa_op_iter iter;
319 {
323 {
328 }
332 }
334 }
336 /* Function vect_stmt_relevant_p.
338 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
339 is "relevant for vectorization".
341 A stmt is considered "relevant for vectorization" if:
342 - it has uses outside the loop.
343 - it has vdefs (it alters memory).
344 - control stmts in the loop (except for the exit condition).
345 - it is an induction and we have multiple exits.
347 CHECKME: what other side effects would the vectorizer allow? */
349 static bool
352 {
354 ssa_op_iter op_iter;
355 imm_use_iterator imm_iter;
356 use_operand_p use_p;
357 def_operand_p def_p;
362 /* cond stmt other than loop exit cond. */
369 /* changing memory. */
373 {
378 }
380 /* uses outside the loop. */
382 {
384 {
387 {
395 /* We expect all such uses to be in the loop exit phis
396 (because of loop closed form) */
400 }
401 }
402 }
404 /* Check if it's an induction and multiple exits. In this case there will be
405 a usage later on after peeling which is needed for the alternate exit. */
408 {
411 "vec_stmt_relevant_p: induction forced for "
415 }
419 {
424 }
427 }
430 /* Function exist_non_indexing_operands_for_use_p
432 USE is one of the uses attached to STMT_INFO. Check if USE is
433 used in STMT_INFO for anything other than indexing an array. */
435 static bool
437 {
438 tree operand;
440 /* USE corresponds to some operand in STMT. If there is no data
441 reference in STMT, then any operand that corresponds to USE
442 is not indexing an array. */
446 /* STMT has a data_ref. FORNOW this means that its of one of
447 the following forms:
448 -1- ARRAY_REF = var
449 -2- var = ARRAY_REF
450 (This should have been verified in analyze_data_refs).
452 'var' in the second case corresponds to a def, not a use,
453 so USE cannot correspond to any operands that are not used
454 for array indexing.
456 Therefore, all we need to check is if STMT falls into the
457 first case, and whether var corresponds to USE. */
461 {
464 {
477 }
479 }
491 }
494 /*
495 Function process_use.
497 Inputs:
498 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
499 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
500 that defined USE. This is done by calling mark_relevant and passing it
501 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
502 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
503 be performed.
505 Outputs:
506 Generally, LIVE_P and RELEVANT are used to define the liveness and
507 relevance info of the DEF_STMT of this USE:
508 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
509 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
510 Exceptions:
511 - case 1: If USE is used only for address computations (e.g. array indexing),
512 which does not need to be directly vectorized, then the liveness/relevance
513 of the respective DEF_STMT is left unchanged.
514 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
515 we skip DEF_STMT cause it had already been processed.
516 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
517 "relevant" will be modified accordingly.
519 Return true if everything is as expected. Return false otherwise. */
521 static opt_result
525 {
526 stmt_vec_info dstmt_vinfo;
529 /* case 1: we are only interested in uses that need to be vectorized. Uses
530 that are used for address computation are not considered relevant. */
536 "not vectorized:"
545 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
546 We have to force the stmt live since the epilogue loop needs it to
547 continue computing the reduction. */
553 {
559 }
561 /* case 3a: outer-loop stmt defining an inner-loop stmt:
562 outer-loop-header-bb:
563 d = dstmt_vinfo
564 inner-loop:
565 stmt # use (d)
566 outer-loop-tail-bb:
567 ... */
569 {
575 {
596 }
597 }
599 /* case 3b: inner-loop stmt defining an outer-loop stmt:
600 outer-loop-header-bb:
601 ...
602 inner-loop:
603 d = dstmt_vinfo
604 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
605 stmt # use (d) */
607 {
613 {
631 }
632 }
633 /* We are also not interested in uses on loop PHI backedges that are
634 inductions. Otherwise we'll needlessly vectorize the IV increment
635 and cause hybrid SLP for SLP inductions. Unless the PHI is live
636 of course. */
643 {
648 }
653 }
656 /* Function vect_mark_stmts_to_be_vectorized.
658 Not all stmts in the loop need to be vectorized. For example:
660 for i...
661 for j...
662 1. T0 = i + j
663 2. T1 = a[T0]
665 3. j = j + 1
667 Stmt 1 and 3 do not need to be vectorized, because loop control and
668 addressing of vectorized data-refs are handled differently.
670 This pass detects such stmts. */
672 opt_result
674 {
678 gimple_stmt_iterator si;
680 basic_block bb;
688 /* 1. Init worklist. */
690 {
693 {
701 }
703 {
713 }
714 }
716 /* 2. Process_worklist */
718 {
719 use_operand_p use_p;
720 ssa_op_iter iter;
727 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
728 (DEF_STMT) as relevant/irrelevant according to the relevance property
729 of STMT. */
732 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
733 propagated as is to the DEF_STMTs of its USEs.
735 One exception is when STMT has been identified as defining a reduction
736 variable; in this case we set the relevance to vect_used_by_reduction.
737 This is because we distinguish between two kinds of relevant stmts -
738 those that are used by a reduction computation, and those that are
739 (also) used by a regular computation. This allows us later on to
740 identify stmts that are used solely by a reduction, and therefore the
741 order of the results that they produce does not have to be kept. */
744 {
773 }
776 {
777 /* Pattern statements are not inserted into the code, so
778 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
779 have to scan the RHS or function arguments instead. */
781 {
787 {
788 opt_result res
798 }
800 {
803 {
804 opt_result res
809 }
810 }
811 }
813 {
816 opt_result res
825 }
827 {
829 {
831 opt_result res
836 }
837 }
838 else
840 }
841 else
843 {
845 opt_result res
850 }
853 {
854 gather_scatter_info gs_info;
857 opt_result res
861 {
865 }
866 }
870 }
872 /* Function vect_model_simple_cost.
874 Models cost for simple operations, i.e. those that only emit ncopies of a
875 single op. Right now, this does not account for multiple insns that could
876 be generated for the single vector op. We will handle that shortly. */
878 static void
883 slp_tree node,
886 {
891 /* ??? Somehow we need to fix this at the callers. */
896 /* Cost the "broadcast" of a scalar operand in to a vector operand.
897 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
898 cost model. */
904 /* Pass the inside-of-loop statements to the target-specific cost model. */
910 "vect_model_simple_cost: inside_cost = %d, "
912 }
915 /* Model cost for type demotion and promotion operations. PWR is
916 normally zero for single-step promotions and demotions. It will be
917 one if two-step promotion/demotion is required, and so on. NCOPIES
918 is the number of vector results (and thus number of instructions)
919 for the narrowest end of the operation chain. Each additional
920 step doubles the number of instructions required. If WIDEN_ARITH
921 is true the stmt is doing widening arithmetic. */
923 static void
929 {
934 {
936 widen_arith
940 }
942 /* FORNOW: Assuming maximum 2 args per stmts. */
950 "vect_model_promotion_demotion_cost: inside_cost = %d, "
952 }
954 /* Returns true if the current function returns DECL. */
956 static bool
958 {
959 edge_iterator ei;
960 edge e;
962 {
968 /* We often end up with an aggregate copy to the result decl,
969 handle that case as well. First skip intermediate clobbers
970 though. */
972 do
973 {
975 }
981 }
983 }
985 /* Calculate cost of DR's memory access. */
986 void
988 dr_alignment_support alignment_support_scheme,
992 {
994 {
996 {
999 vect_body);
1005 }
1008 {
1009 /* Here, we assign an additional cost for the unaligned store. */
1015 "vect_model_store_cost: unaligned supported by "
1018 }
1021 {
1028 }
1032 }
1033 }
1035 /* Calculate cost of DR's memory access. */
1036 void
1038 dr_alignment_support alignment_support_scheme,
1045 {
1047 {
1049 {
1058 }
1060 {
1061 /* Here, we assign an additional cost for the unaligned load. */
1068 "vect_model_load_cost: unaligned supported by "
1072 }
1074 {
1080 /* FIXME: If the misalignment remains fixed across the iterations of
1081 the containing loop, the following cost should be added to the
1082 prologue costs. */
1092 }
1094 {
1097 "vect_model_load_cost: unaligned software "
1100 /* Unaligned software pipeline has a load of an address, an initial
1101 load, and possibly a mask operation to "prime" the loop. However,
1102 if this is an access in a group of loads, which provide grouped
1103 access, then the above cost should only be considered for one
1104 access in the group. Inside the loop, there is a load op
1105 and a realignment op. */
1108 {
1116 }
1125 "vect_model_load_cost: explicit realign optimized"
1129 }
1132 {
1139 }
1143 }
1144 }
1146 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1147 the loop preheader for the vectorized stmt STMT_VINFO. */
1149 static void
1152 {
1155 else
1161 }
1163 /* Function vect_init_vector.
1165 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1166 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1167 vector type a vector with all elements equal to VAL is created first.
1168 Place the initialization at GSI if it is not NULL. Otherwise, place the
1169 initialization at the loop preheader.
1170 Return the DEF of INIT_STMT.
1171 It will be used in the vectorization of STMT_INFO. */
1173 tree
1176 {
1178 tree new_temp;
1180 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1182 {
1185 {
1186 /* Scalar boolean value should be transformed into
1187 all zeros or all ones value before building a vector. */
1189 {
1195 else
1196 {
1202 }
1203 }
1204 else
1205 {
1210 else
1211 /* ??? Condition vectorization expects us to do
1212 promotion of invariant/external defs. */
1216 {
1220 }
1221 }
1222 }
1224 }
1230 }
1233 /* Function vect_get_vec_defs_for_operand.
1235 OP is an operand in STMT_VINFO. This function returns a vector of
1236 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1238 In the case that OP is an SSA_NAME which is defined in the loop, then
1239 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1241 In case OP is an invariant or constant, a new stmt that creates a vector def
1242 needs to be introduced. VECTYPE may be used to specify a required type for
1243 vector invariant. */
1245 void
1249 {
1259 stmt_vec_info def_stmt_info;
1268 {
1270 tree vector_type;
1277 else
1284 }
1285 else
1286 {
1292 }
1293 }
1296 /* Get vectorized definitions for OP0 and OP1. */
1298 void
1305 {
1307 {
1316 }
1317 else
1318 {
1331 }
1332 }
1334 void
1341 {
1347 }
1349 /* Helper function called by vect_finish_replace_stmt and
1350 vect_finish_stmt_generation. Set the location of the new
1351 statement and create and return a stmt_vec_info for it. */
1353 static void
1356 {
1361 {
1364 /* While EH edges will generally prevent vectorization, stmt might
1365 e.g. be in a must-not-throw region. Ensure newly created stmts
1366 that could throw are part of the same region. */
1370 }
1371 else
1373 }
1375 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1376 which sets the same scalar result as STMT_INFO did. Create and return a
1377 stmt_vec_info for VEC_STMT. */
1379 void
1382 {
1390 }
1392 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1393 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1395 void
1399 {
1404 {
1408 {
1412 /* If we have an SSA vuse and insert a store, update virtual
1413 SSA form to avoid triggering the renamer. Do so only
1414 if we can easily see all uses - which is what almost always
1415 happens with the way vectorized stmts are inserted. */
1424 {
1428 }
1429 }
1430 }
1433 }
1435 /* We want to vectorize a call to combined function CFN with function
1436 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1437 as the types of all inputs. Check whether this is possible using
1438 an internal function, returning its code if so or IFN_LAST if not. */
1440 static internal_fn
1443 {
1444 internal_fn ifn;
1447 else
1450 {
1453 {
1458 /* The type size of both the vectype_in and vectype_out should be
1459 exactly the same when vectype_out isn't participating the optab.
1460 While there is no restriction for type size when vectype_out
1461 is part of the optab query. */
1466 OPTIMIZE_FOR_SPEED))
1468 }
1469 }
1471 }
1475 gimple_stmt_iterator *);
1477 /* Check whether a load or store statement in the loop described by
1478 LOOP_VINFO is possible in a loop using partial vectors. This is
1479 testing whether the vectorizer pass has the appropriate support,
1480 as well as whether the target does.
1482 VLS_TYPE says whether the statement is a load or store and VECTYPE
1483 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1484 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1485 says how the load or store is going to be implemented and GROUP_SIZE
1486 is the number of load or store statements in the containing group.
1487 If the access is a gather load or scatter store, GS_INFO describes
1488 its arguments. If the load or store is conditional, SCALAR_MASK is the
1489 condition under which it occurs.
1491 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1492 vectors is not supported, otherwise record the required rgroup control
1493 types. */
1495 static void
1497 slp_tree slp_node,
1498 vec_load_store_type vls_type,
1500 vect_memory_access_type
1501 memory_access_type,
1503 tree scalar_mask)
1504 {
1505 /* Invariant loads need no special support. */
1512 else
1520 {
1521 internal_fn ifn
1528 scalar_mask);
1529 else
1530 {
1533 "can't operate on partial vectors because"
1534 " the target doesn't have an appropriate"
1537 }
1539 }
1542 {
1543 internal_fn ifn = (is_load
1544 ? IFN_MASK_GATHER_LOAD
1546 internal_fn len_ifn = (is_load
1547 ? IFN_MASK_LEN_GATHER_LOAD
1559 scalar_mask);
1560 else
1561 {
1564 "can't operate on partial vectors because"
1565 " the target doesn't have an appropriate"
1568 }
1570 }
1574 {
1575 /* Element X of the data must come from iteration i * VF + X of the
1576 scalar loop. We need more work to support other mappings. */
1579 "can't operate on partial vectors because an"
1583 }
1586 {
1589 "can't operate on partial vectors when emulating"
1593 }
1595 /* We might load more scalars than we need for permuting SLP loads.
1596 We checked in get_group_load_store_type that the extra elements
1597 don't leak into a new vector. */
1599 {
1604 };
1608 machine_mode mask_mode;
1609 machine_mode vmode;
1612 {
1617 }
1620 {
1624 }
1627 {
1630 "can't operate on partial vectors because the"
1631 " target doesn't have the appropriate partial"
1634 }
1635 }
1637 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1638 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1639 that needs to be applied to all loads and stores in a vectorized loop.
1640 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1641 otherwise return VEC_MASK & LOOP_MASK.
1643 MASK_TYPE is the type of both masks. If new statements are needed,
1644 insert them before GSI. */
1646 static tree
1649 {
1665 }
1667 /* Determine whether we can use a gather load or scatter store to vectorize
1668 strided load or store STMT_INFO by truncating the current offset to a
1669 smaller width. We need to be able to construct an offset vector:
1671 { 0, X, X*2, X*3, ... }
1673 without loss of precision, where X is STMT_INFO's DR_STEP.
1675 Return true if this is possible, describing the gather load or scatter
1676 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1678 static bool
1682 {
1687 {
1688 /* ??? Perhaps we could use range information here? */
1693 }
1695 /* Get the number of bits in an element. */
1700 /* Set COUNT to the upper limit on the number of elements - 1.
1701 Start with the maximum vectorization factor. */
1704 /* Try lowering COUNT to the number of scalar latch iterations. */
1706 widest_int max_iters;
1711 /* Try scales of 1 and the element size. */
1715 {
1717 widest_int factor;
1721 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1728 /* Find the narrowest viable offset type. */
1733 /* See whether the target supports the operation with an offset
1734 no narrower than OFFSET_TYPE. */
1743 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1744 but we don't need to store that here. */
1752 }
1756 "truncating gather/scatter offset to %d bits"
1760 }
1762 /* Return true if we can use gather/scatter internal functions to
1763 vectorize STMT_INFO, which is a grouped or strided load or store.
1764 MASKED_P is true if load or store is conditional. When returning
1765 true, fill in GS_INFO with the information required to perform the
1766 operation. */
1768 static bool
1772 {
1787 "using gather/scatter for strided/grouped access,"
1791 }
1793 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1794 elements with a known constant step. Return -1 if that step
1795 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1797 static int
1799 {
1802 size_zero_node);
1803 }
1805 /* If the target supports a permute mask that reverses the elements in
1806 a vector of type VECTYPE, return that mask, otherwise return null. */
1808 tree
1810 {
1813 /* The encoding has a single stepped pattern. */
1820 indices))
1823 }
1825 /* A subroutine of get_load_store_type, with a subset of the same
1826 arguments. Handle the case where STMT_INFO is a load or store that
1827 accesses consecutive elements with a negative step. Sets *POFFSET
1828 to the offset to be applied to the DR for the first access. */
1830 static vect_memory_access_type
1833 vec_load_store_type vls_type,
1835 {
1837 dr_alignment_support alignment_support_scheme;
1840 {
1845 }
1847 /* For backward running DRs the first access in vectype actually is
1848 N-1 elements before the address of the DR. */
1853 alignment_support_scheme
1857 {
1863 }
1866 {
1869 "negative step with invariant source;"
1872 }
1875 {
1881 }
1884 }
1886 /* STMT_INFO is either a masked or unconditional store. Return the value
1887 being stored. */
1889 tree
1891 {
1893 {
1896 }
1898 {
1903 }
1905 }
1907 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1909 This function returns a vector type which can be composed with NETLS pieces,
1910 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1911 same vector size as the return vector. It checks target whether supports
1912 pieces-size vector mode for construction firstly, if target fails to, check
1913 pieces-size scalar mode for construction further. It returns NULL_TREE if
1914 fails to find the available composition.
1916 For example, for (vtype=V16QI, nelts=4), we can probably get:
1917 - V16QI with PTYPE V4QI.
1918 - V4SI with PTYPE SI.
1919 - NULL_TREE. */
1921 static tree
1923 {
1931 /* When we are asked to compose the vector from its components let
1932 that happen directly. */
1934 {
1937 }
1942 {
1943 /* First check if vec_init optab supports construction from
1944 vector pieces directly. */
1947 machine_mode rmode;
1951 {
1954 }
1956 /* Otherwise check if exists an integer type of the same piece size and
1957 if vec_init optab supports construction from it directly. */
1962 {
1965 }
1966 }
1969 }
1971 /* A subroutine of get_load_store_type, with a subset of the same
1972 arguments. Handle the case where STMT_INFO is part of a grouped load
1973 or store.
1975 For stores, the statements in the group are all consecutive
1976 and there is no gap at the end. For loads, the statements in the
1977 group might not be consecutive; there can be gaps between statements
1978 as well as at the end. */
1980 static bool
1990 {
1993 stmt_vec_info first_stmt_info;
1997 {
2001 }
2002 else
2003 {
2007 }
2013 /* True if the vectorized statements would access beyond the last
2014 statement in the group. */
2017 /* True if we can cope with such overrun by peeling for gaps, so that
2018 there is at least one final scalar iteration after the vector loop. */
2021 && loop_vinfo
2024 /* There can only be a gap at the end of the group if the stride is
2025 known at compile time. */
2028 /* Stores can't yet have gaps. */
2032 {
2033 /* For SLP vectorization we directly vectorize a subchain
2034 without permutation. */
2036 first_dr_info
2039 {
2040 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2041 separated by the stride, until we have a complete vector.
2042 Fall back to scalar accesses if that isn't possible. */
2045 else
2047 }
2048 else
2049 {
2052 {
2054 "Grouped store with gaps requires"
2057 }
2058 /* An overrun is fine if the trailing elements are smaller
2059 than the alignment boundary B. Every vector access will
2060 be a multiple of B and so we are guaranteed to access a
2061 non-gap element in the same B-sized block. */
2064 vectype)
2068 /* If the gap splits the vector in half and the target
2069 can do half-vector operations avoid the epilogue peeling
2070 by simply loading half of the vector only. Usually
2071 the construction with an upper zero half will be elided. */
2072 dr_alignment_support alss;
2074 tree half_vtype;
2076 && !masked_p
2079 == dr_aligned
2088 {
2093 }
2096 {
2098 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2099 only correct for single element "interleaving" SLP. */
2102 else
2103 {
2104 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2105 separated by the stride, until we have a complete vector.
2106 Fall back to scalar accesses if that isn't possible. */
2109 else
2111 }
2112 }
2114 {
2117 /* Invariant accesses perform only component accesses, alignment
2118 is irrelevant for them. */
2120 }
2121 else
2124 /* When we have a contiguous access across loop iterations
2125 but the access in the loop doesn't cover the full vector
2126 we can end up with no gap recorded but still excess
2127 elements accessed, see PR103116. Make sure we peel for
2128 gaps if necessary and sufficient and give up if not.
2130 If there is a combination of the access not covering the full
2131 vector and a gap recorded then we may need to peel twice. */
2136 nunits))
2137 {
2142 /* Peeling for gaps assumes that a single scalar iteration
2143 is enough to make sure the last vector iteration doesn't
2144 access excess elements.
2145 ??? Enhancements include peeling multiple iterations
2146 or using masked loads with a static mask. */
2148 {
2151 "peeling for gaps insufficient for "
2154 }
2156 }
2157 }
2158 }
2159 else
2160 {
2161 /* We can always handle this case using elementwise accesses,
2162 but see if something more efficient is available. */
2165 /* If there is a gap at the end of the group then these optimizations
2166 would access excess elements in the last iteration. */
2168 /* An overrun is fine if the trailing elements are smaller than the
2169 alignment boundary B. Every vector access will be a multiple of B
2170 and so we are guaranteed to access a non-gap element in the
2171 same B-sized block. */
2173 && !masked_p
2181 {
2182 /* First cope with the degenerate case of a single-element
2183 vector. */
2185 ;
2187 else
2188 {
2189 /* Otherwise try using LOAD/STORE_LANES. */
2190 *lanes_ifn
2194 masked_p);
2196 {
2199 }
2201 /* If that fails, try using permuting loads. */
2204 single_element_p,
2205 group_size)
2207 {
2210 }
2211 }
2212 }
2214 /* As a last resort, trying using a gather load or scatter store.
2216 ??? Although the code can handle all group sizes correctly,
2217 it probably isn't a win to use separate strided accesses based
2218 on nearby locations. Or, even if it's a win over scalar code,
2219 it might not be a win over vectorizing at a lower VF, if that
2220 allows us to use contiguous accesses. */
2222 && single_element_p
2223 && loop_vinfo
2227 }
2231 {
2234 }
2235 else
2236 {
2238 *alignment_support_scheme
2241 }
2244 {
2245 /* STMT is the leader of the group. Check the operands of all the
2246 stmts of the group. */
2249 {
2253 {
2258 }
2260 }
2261 }
2264 {
2268 "Data access with gaps requires scalar "
2271 }
2274 }
2276 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2277 if there is a memory access type that the vectorized form can use,
2278 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2279 or scatters, fill in GS_INFO accordingly. In addition
2280 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2281 the target does not support the alignment scheme. *MISALIGNMENT
2282 is set according to the alignment of the access (including
2283 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2285 SLP says whether we're performing SLP rather than loop vectorization.
2286 MASKED_P is true if the statement is conditional on a vectorized mask.
2287 VECTYPE is the vector type that the vectorized statements will use.
2288 NCOPIES is the number of vector statements that will be needed. */
2290 static bool
2301 {
2307 {
2311 /* When using internal functions, we rely on pattern recognition
2312 to convert the type of the offset to the type that the target
2313 requires, with the result being a call to an internal function.
2314 If that failed for some reason (e.g. because another pattern
2315 took priority), just handle cases in which the offset already
2316 has the right type. */
2321 {
2327 }
2331 {
2337 }
2339 {
2345 {
2348 "unsupported vector types for emulated "
2351 }
2352 }
2353 /* Gather-scatter accesses perform only component accesses, alignment
2354 is irrelevant for them. */
2356 }
2358 {
2360 masked_p,
2362 alignment_support_scheme,
2365 }
2367 {
2373 else
2375 /* Alignment is irrelevant here. */
2377 }
2378 else
2379 {
2382 {
2385 /* Invariant accesses perform only component accesses, alignment
2386 is irrelevant for them. */
2388 }
2389 else
2390 {
2394 else
2398 *alignment_support_scheme
2402 }
2403 }
2408 {
2411 "Not using elementwise accesses due to variable "
2414 }
2417 {
2422 }
2424 /* FIXME: At the moment the cost model seems to underestimate the
2425 cost of using elementwise accesses. This check preserves the
2426 traditional behavior until that can be fixed. */
2435 {
2440 }
2442 }
2444 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2445 conditional operation STMT_INFO. When returning true, store the mask
2446 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2447 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2448 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2450 static bool
2455 {
2457 tree mask_vectype;
2458 slp_tree mask_node_1;
2461 {
2466 }
2469 {
2474 }
2476 /* If the caller is not prepared for adjusting an external/constant
2477 SLP mask vector type fail. */
2479 && !mask_node
2481 {
2486 }
2491 mask_node_1);
2494 {
2499 }
2503 {
2506 "vector mask type %T"
2511 }
2518 }
2520 /* Return true if stored value is suitable for vectorizing store
2521 statement STMT_INFO. When returning true, store the scalar stored
2522 in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2523 the type of the vectorized store value in
2524 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2526 static bool
2531 {
2534 {
2538 }
2540 op_no = vect_slp_child_index_for_operand
2544 tree rhs_vectype;
2547 {
2552 }
2554 /* In the case this is a store from a constant make sure
2555 native_encode_expr can handle it. */
2557 {
2562 }
2566 {
2571 }
2577 else
2580 }
2582 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2583 Note that we support masks with floating-point type, in which case the
2584 floats are interpreted as a bitmask. */
2586 static tree
2589 {
2594 {
2598 }
2600 {
2601 REAL_VALUE_TYPE r;
2609 }
2611 }
2613 /* Build an all-zero merge value of type VECTYPE while vectorizing
2614 STMT_INFO as a gather load. */
2616 static tree
2619 {
2620 tree merge;
2624 {
2625 REAL_VALUE_TYPE r;
2631 }
2632 else
2636 }
2638 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2639 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2640 the gather load operation. If the load is conditional, MASK is the
2641 vectorized condition, otherwise MASK is null. PTR is the base
2642 pointer and OFFSET is the vectorized offset. */
2649 {
2658 tree var;
2660 && (!mask
2666 {
2674 }
2679 {
2681 {
2686 else
2690 gassign *new_stmt
2695 {
2702 }
2705 }
2706 else
2707 {
2710 }
2711 }
2712 else
2713 {
2716 }
2723 {
2731 }
2734 }
2736 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2737 instructions before GSI. GS_INFO describes the scatter store operation.
2738 PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
2739 vectorized data to store.
2740 If the store is conditional, MASK is the vectorized condition, otherwise
2741 MASK is null. */
2748 {
2761 {
2764 tree utype;
2767 else
2771 gassign *new_stmt
2776 {
2782 }
2783 }
2784 else
2785 {
2788 }
2792 {
2800 }
2804 {
2812 }
2815 gcall *new_stmt
2818 }
2820 /* Prepare the base and offset in GS_INFO for vectorization.
2821 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2822 to the vectorized offset argument for the first copy of STMT_INFO.
2823 STMT_INFO is the statement described by GS_INFO and LOOP is the
2824 containing loop. */
2826 static void
2831 {
2835 {
2836 basic_block new_bb;
2840 }
2843 else
2844 {
2845 unsigned ncopies
2850 }
2851 }
2853 /* Prepare to implement a grouped or strided load or store using
2854 the gather load or scatter store operation described by GS_INFO.
2855 STMT_INFO is the load or store statement.
2857 Set *DATAREF_BUMP to the amount that should be added to the base
2858 address after each copy of the vectorized statement. Set *VEC_OFFSET
2859 to an invariant offset vector in which element I has the value
2860 I * DR_STEP / SCALE. */
2862 static void
2864 loop_vec_info loop_vinfo,
2869 {
2874 {
2875 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2876 ivtmp_8 = _31 * 16 (step in bytes);
2877 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2878 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2879 tree loop_len
2881 tree tmp
2884 loop_len);
2886 GSI_SAME_STMT);
2887 }
2888 else
2889 {
2890 tree bump
2895 }
2897 /* The offset given in GS_INFO can have pointer type, so use the element
2898 type of the vector instead. */
2901 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2906 /* Create {0, X, X*2, X*3, ...}. */
2910 }
2912 /* Prepare the pointer IVs which needs to be updated by a variable amount.
2913 Such variable amount is the outcome of .SELECT_VL. In this case, we can
2914 allow each iteration process the flexible number of elements as long as
2915 the number <= vf elments.
2917 Return data reference according to SELECT_VL.
2918 If new statements are needed, insert them before GSI. */
2920 static tree
2924 vect_memory_access_type memory_access_type)
2925 {
2929 /* gather/scatter never reach here. */
2932 /* When we support SELECT_VL pattern, we dynamic adjust
2933 the memory address by .SELECT_VL result.
2935 The result of .SELECT_VL is the number of elements to
2936 be processed of each iteration. So the memory address
2937 adjustment operation should be:
2939 addr = addr + .SELECT_VL (ARG..) * step;
2940 */
2941 tree loop_len
2944 /* Since the outcome of .SELECT_VL is element size, we should adjust
2945 it into bytesize so that it can be used in address pointer variable
2946 amount IVs adjustment. */
2953 }
2955 /* Return the amount that should be added to a vector pointer to move
2956 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
2957 being vectorized and MEMORY_ACCESS_TYPE describes the type of
2958 vectorization. */
2960 static tree
2963 vect_memory_access_type memory_access_type,
2965 {
2973 memory_access_type);
2980 }
2982 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
2984 static bool
2990 {
3000 /* Multiple types in SLP are handled by creating the appropriate number of
3001 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3002 case of SLP. */
3005 else
3011 {
3017 }
3028 /* The encoding uses one stepped pattern for each byte in the word. */
3040 {
3043 {
3048 }
3055 slp_node
3059 }
3063 /* Transform. */
3067 /* Arguments are ready. create the new vector stmt. */
3069 tree vop;
3071 {
3087 else
3089 }
3096 }
3098 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3099 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3100 in a single step. On success, store the binary pack code in
3101 *CONVERT_CODE. */
3103 static bool
3106 {
3111 code_helper code;
3121 }
3123 /* Function vectorizable_call.
3125 Check if STMT_INFO performs a function call that can be vectorized.
3126 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3127 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3128 Return true if STMT_INFO is vectorizable in this way. */
3130 static bool
3135 {
3137 tree vec_dest;
3138 tree scalar_dest;
3139 tree op;
3142 poly_uint64 nunits_in;
3143 poly_uint64 nunits_out;
3149 vect_unknown_def_type };
3157 tree lhs;
3167 /* Is STMT_INFO a vectorizable call? */
3175 /* Handled by vectorizable_load and vectorizable_store. */
3186 /* Process function arguments. */
3191 /* Bail out if the function has more than four arguments, we do not have
3192 interesting builtin functions to vectorize with more than two arguments
3193 except for fma. No arguments is also not good. */
3197 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3200 {
3203 }
3204 /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3205 argument just says whether it is well-defined at zero or not and what
3206 value should be returned for it. */
3208 {
3211 }
3218 {
3220 {
3225 }
3229 {
3234 }
3236 /* We can only handle calls with arguments of the same type. */
3239 {
3244 }
3252 {
3257 }
3258 }
3259 /* If all arguments are external or constant defs, infer the vector type
3260 from the scalar type. */
3266 {
3272 }
3276 {
3281 }
3284 {
3289 }
3291 /* FORNOW */
3300 else
3303 /* We only handle functions that do not read or clobber memory. */
3305 {
3310 }
3312 /* For now, we only vectorize functions if a target specific builtin
3313 is available. TODO -- in some cases, it might be profitable to
3314 insert the calls for pieces of the vector, in order to be able
3315 to vectorize other operations in the loop. */
3320 /* First try using an internal function. */
3328 vectype_in);
3330 /* If that fails, try asking for a target-specific built-in function. */
3332 {
3339 }
3342 {
3344 && !slp_node
3345 && loop_vinfo
3350 {
3351 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3352 { 0, 1, 2, ... vf - 1 } vector. */
3354 }
3362 else
3363 {
3368 }
3369 }
3375 else
3378 /* Sanity check: make sure that at least one copy of the vectorized stmt
3379 needs to be generated. */
3389 {
3395 {
3400 }
3412 {
3416 OPTIMIZE_FOR_SPEED))
3419 OPTIMIZE_FOR_SPEED)))
3420 {
3423 "can't use a fully-masked loop because no"
3426 }
3427 else
3428 {
3429 unsigned int nvectors
3430 = (slp_node
3438 OPTIMIZE_FOR_SPEED))
3441 else
3443 scalar_mask);
3444 }
3445 }
3447 }
3449 /* Transform. */
3454 /* Handle def. */
3462 {
3464 {
3466 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3468 }
3471 }
3473 {
3476 }
3481 {
3486 {
3487 /* Build argument list for the vectorized call. */
3489 {
3495 /* Arguments are ready. Create the new vector stmt. */
3497 {
3500 {
3502 /* Always true for SLP. */
3507 }
3510 {
3513 }
3521 {
3522 /* We don't define any narrowing conditional functions
3523 at present. */
3526 gcall *call
3532 {
3535 }
3541 }
3542 else
3543 {
3545 {
3547 /* Always true for SLP. */
3549 tree len
3552 signed char biasval
3557 }
3559 {
3561 /* Always true for SLP. */
3569 }
3574 else
3581 }
3583 }
3585 }
3592 {
3595 {
3600 }
3602 }
3609 {
3612 signed char biasval
3617 }
3619 {
3625 }
3629 {
3631 tree new_var
3638 }
3640 {
3641 /* We don't define any narrowing conditional functions at
3642 present. */
3650 {
3653 }
3656 half_res);
3658 }
3659 else
3660 {
3664 else
3671 }
3676 }
3678 {
3681 }
3682 }
3684 {
3686 /* We don't define any narrowing conditional functions at present. */
3689 {
3690 /* Build argument list for the vectorized call. */
3693 else
3697 {
3703 /* Arguments are ready. Create the new vector stmt. */
3705 {
3709 {
3713 }
3717 else
3724 }
3726 }
3729 {
3732 {
3736 }
3742 }
3750 }
3756 {
3759 }
3760 }
3761 else
3762 /* No current target implements this case. */
3767 /* The call in STMT might prevent it from being removed in dce.
3768 We however cannot remove it here, due to the way the ssa name
3769 it defines is mapped to the new definition. So just replace
3770 rhs of the statement with something harmless. */
3778 gassign *new_stmt
3783 }
3786 struct simd_call_arg_info
3787 {
3788 tree vectype;
3789 tree op;
3790 HOST_WIDE_INT linear_step;
3794 };
3796 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3797 is linear within simd lane (but not within whole loop), note it in
3798 *ARGINFO. */
3800 static void
3803 {
3815 {
3816 tree t;
3820 {
3835 CASE_CONVERT:
3847 }
3853 {
3860 }
3861 }
3862 }
3864 /* Function vectorizable_simd_clone_call.
3866 Check if STMT_INFO performs a function call that can be vectorized
3867 by calling a simd clone of the function.
3868 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3869 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3870 Return true if STMT_INFO is vectorizable in this way. */
3872 static bool
3876 stmt_vector_for_cost *)
3877 {
3878 tree vec_dest;
3879 tree scalar_dest;
3882 tree vectype;
3883 poly_uint64 nunits;
3896 /* Is STMT a vectorizable call? */
3904 {
3910 }
3936 /* Process function arguments. */
3939 /* Bail out if the function has zero arguments. */
3950 {
3951 simd_call_arg_info thisarginfo;
3952 affine_iv iv;
3966 {
3971 }
3975 {
3976 /* With SLP we determine the vector type of constants/externals
3977 at analysis time, handling conflicts via
3978 vect_maybe_update_slp_op_vectype. At transform time
3979 we have a vector type recorded for SLP. */
3981 || !slp_node
3986 slp_node);
3987 }
3988 else
3991 /* For linear arguments, the analyze phase should have saved
3992 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
3995 {
3999 thisarginfo.simd_lane_linear
4001 /* If loop has been peeled for alignment, we need to adjust it. */
4005 {
4011 thisarginfo.op
4015 }
4016 }
4020 && loop_vinfo
4025 {
4028 }
4033 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4034 linear too. */
4037 && !vec_stmt
4040 && loop_vinfo
4045 }
4053 else
4056 {
4059 /* The number of arguments in the call and the number of parameters in
4060 the simdclone should match. However, when the simdclone is
4061 'inbranch', it could have one more paramater than nargs when using
4062 an inbranch simdclone to call a non-inbranch call, either in a
4063 non-masked loop using a all true constant mask, or inside a masked
4064 loop using it's mask. */
4067 simd_nargs--;
4082 {
4084 {
4115 /* FORNOW */
4119 /* While we can create a traditional data vector from
4120 an incoming integer mode mask we have no good way to
4121 force generate an integer mode mask from a traditional
4122 boolean vector input. */
4130 }
4134 {
4137 }
4141 }
4147 {
4149 SIMD_CLONE_ARG_TYPE_MASK);
4150 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4151 not in a branch, as we'd have to construct an all-true mask. */
4154 }
4156 {
4159 }
4160 }
4169 num_mask_args++;
4172 {
4176 {
4180 slp_node);
4185 }
4189 {
4194 }
4197 {
4200 {
4203 {
4204 /* FORNOW we only have partial support for vector-type masks
4205 that can't hold all of simdlen. */
4208 vect_location,
4209 "in-branch vector clones are not yet"
4212 }
4215 {
4218 vect_location,
4219 "cannot compute mask argument for"
4222 }
4223 }
4225 {
4228 num_mask_args),
4230 {
4231 /* FORNOW we only have partial support for integer-type masks
4232 that represent the same number of lanes as the
4233 vectorized mask inputs. */
4236 vect_location,
4237 "in-branch vector clones are not yet "
4240 }
4241 }
4242 else
4243 {
4246 vect_location,
4247 "in-branch vector clones not supported"
4250 }
4251 }
4252 }
4258 else
4261 /* If the function isn't const, only allow it in simd loops where user
4262 has asserted that at least nunits consecutive iterations can be
4263 performed using SIMD instructions. */
4268 /* Sanity check: make sure that at least one copy of the vectorized stmt
4269 needs to be generated. */
4273 {
4277 {
4282 }
4283 /* When the original call is pure or const but the SIMD ABI dictates
4284 an aggregate return we will have to use a virtual definition and
4285 in a loop eventually even need to add a virtual PHI. That's
4286 not straight-forward so allow to fix this up via renaming. */
4291 /* ??? For SLP code-gen we end up inserting after the last
4292 vector argument def rather than at the original call position
4293 so automagic virtual operand updating doesn't work. */
4298 {
4300 {
4305 {
4315 }
4325 }
4326 }
4329 {
4333 "can't use a fully-masked loop because a"
4336 }
4340 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4341 dt, slp_node, cost_vec); */
4343 }
4345 /* Transform. */
4350 /* Handle def. */
4356 {
4360 {
4363 }
4364 }
4370 {
4373 }
4374 else
4377 {
4378 poly_uint64 callee_nelements;
4379 poly_uint64 caller_nelements;
4380 /* Build argument list for the vectorized call. */
4383 else
4387 {
4389 tree atype;
4392 {
4399 {
4401 {
4409 {
4416 }
4417 else
4418 {
4422 }
4424 vec_oprnd0
4428 gassign *new_stmt
4430 vec_oprnd0);
4434 }
4435 else
4436 {
4444 else
4447 {
4449 {
4453 op,
4457 }
4458 else
4464 vec_oprnd0);
4465 }
4468 atype))
4469 {
4471 vec_oprnd0);
4472 gassign *new_stmt
4474 vec_oprnd0);
4478 }
4479 else
4481 else
4482 {
4484 gassign *new_stmt
4486 vec_oprnd0);
4490 }
4491 }
4492 }
4496 {
4505 {
4507 {
4508 /* The mask type has fewer elements than simdlen. */
4510 /* FORNOW */
4512 }
4514 {
4515 /* The SIMD clone function has the same number of
4516 elements as the current function. */
4518 {
4522 op,
4525 }
4529 {
4530 vec_loop_masks *loop_masks
4532 tree loop_mask
4536 vec_oprnd0
4540 gsi);
4542 loop_mask });
4544 }
4545 vec_oprnd0
4549 gassign *new_stmt
4551 vec_oprnd0);
4555 }
4556 else
4557 {
4558 /* The mask type has more elements than simdlen. */
4560 /* FORNOW */
4562 }
4563 }
4564 }
4566 {
4568 /* Guess the number of lanes represented by atype. */
4569 poly_uint64 atype_subparts
4571 num_mask_args);
4574 {
4576 {
4580 op,
4583 }
4586 {
4587 /* The mask argument has fewer elements than the
4588 input vector. */
4589 /* FORNOW */
4591 }
4594 {
4595 /* The vector mask argument matches the input
4596 in the number of lanes, but not necessarily
4597 in the mode. */
4602 vec_oprnd0);
4603 gassign *new_stmt
4605 vec_oprnd0);
4609 {
4610 new_stmt
4612 NOP_EXPR,
4613 gimple_assign_lhs
4617 }
4619 }
4620 else
4621 {
4622 /* The mask argument has more elements than the
4623 input vector. */
4624 /* FORNOW */
4626 }
4627 }
4628 }
4629 else
4638 {
4639 gimple_seq stmts;
4644 {
4645 basic_block new_bb;
4649 }
4651 {
4654 }
4659 enum tree_code code
4664 poly_widest_int cst
4669 gassign *new_stmt
4674 UNKNOWN_LOCATION);
4677 }
4678 else
4679 {
4680 enum tree_code code
4685 poly_widest_int cst
4690 gassign *new_stmt
4695 }
4705 }
4706 }
4711 {
4714 tree mask;
4716 SIMD_CLONE_ARG_TYPE_MASK);
4722 {
4724 {
4728 }
4729 else
4734 {
4735 /* This means we are dealing with integer mask modes.
4736 First convert to an integer type with the same size as
4737 the current vector type. */
4738 unsigned HOST_WIDE_INT intermediate_size
4740 tree mid_int_type =
4743 new_stmt
4745 mask);
4747 /* Then zero-extend to the mask mode. */
4750 }
4752 {
4754 integer_one_node);
4756 integer_zero_node);
4760 }
4761 else
4769 }
4770 }
4774 {
4781 else
4784 }
4789 {
4791 {
4799 {
4800 tree t;
4802 {
4806 }
4807 else
4818 else
4820 }
4825 }
4827 {
4836 {
4841 {
4845 tem);
4850 }
4852 }
4853 else
4858 new_stmt
4867 else
4870 }
4872 {
4879 }
4881 {
4883 new_stmt
4886 }
4887 }
4893 else
4895 }
4898 {
4901 }
4904 /* Mark the clone as no longer being a candidate for GC. */
4907 /* The call in STMT might prevent it from being removed in dce.
4908 We however cannot remove it here, due to the way the ssa name
4909 it defines is mapped to the new definition. So just replace
4910 rhs of the statement with something harmless. */
4917 {
4921 }
4922 else
4928 }
4931 /* Function vect_gen_widened_results_half
4933 Create a vector stmt whose code, type, number of arguments, and result
4934 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
4935 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
4936 In the case that CODE is a CALL_EXPR, this means that a call to DECL
4937 needs to be created (DECL is a function-decl of a target-builtin).
4938 STMT_INFO is the original scalar stmt that we are vectorizing. */
4944 stmt_vec_info stmt_info)
4945 {
4947 tree new_temp;
4949 /* Generate half of the widened result: */
4958 }
4961 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
4962 For multi-step conversions store the resulting vectors and call the function
4963 recursively. When NARROW_SRC_P is true, there's still a conversion after
4964 narrowing, don't store the vectors in the SLP_NODE or in vector info of
4965 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
4967 static void
4970 stmt_vec_info stmt_info,
4975 {
4982 {
4983 /* Create demotion operation. */
4991 /* Store the resulting vector for next recursive call,
4992 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
4994 else
4995 {
4996 /* This is the last step of the conversion sequence. Store the
4997 vectors in SLP_NODE or in vector info of the scalar statement
4998 (or in STMT_VINFO_RELATED_STMT chain). */
5001 else
5003 }
5004 }
5006 /* For multi-step demotion operations we first generate demotion operations
5007 from the source type to the intermediate types, and then combine the
5008 results (stored in VEC_OPRNDS) in demotion operation to the destination
5009 type. */
5011 {
5012 /* At each level of recursion we have half of the operands we had at the
5013 previous level. */
5019 narrow_src_p);
5020 }
5023 }
5026 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5027 and VEC_OPRNDS1, for a binary operation associated with scalar statement
5028 STMT_INFO. For multi-step conversions store the resulting vectors and
5029 call the function recursively. */
5031 static void
5037 code_helper ch1,
5039 {
5047 {
5050 else
5053 /* Generate the two halves of promotion operation. */
5056 stmt_info);
5059 stmt_info);
5061 {
5064 }
5065 else
5066 {
5069 }
5071 /* Store the results for the next step. */
5074 }
5078 }
5080 /* Create vectorized promotion stmts for widening stmts using only half the
5081 potential vector size for input. */
5082 static void
5088 code_helper code1,
5090 {
5100 {
5106 /* Widen the first vector input. */
5112 {
5113 /* Widen the second vector input. */
5117 /* Perform the operation. With both vector inputs widened. */
5119 }
5120 else
5121 {
5122 /* Perform the operation. With the single vector input widened. */
5124 }
5130 /* Store the results for the next step. */
5132 }
5136 }
5139 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5140 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5141 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5142 Return true if STMT_INFO is vectorizable in this way. */
5144 static bool
5149 {
5151 tree scalar_dest;
5157 tree new_temp;
5160 poly_uint64 nunits_in;
5161 poly_uint64 nunits_out;
5165 /* For conversions between floating point and integer, there're 2 NARROW
5166 cases. NARROW_SRC is for FLOAT_EXPR, means
5167 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5168 This is safe when the range of the source integer can fit into the lower
5169 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5170 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5171 For other conversions, when there's narrowing, NARROW_DST is used as
5172 default. */
5176 tree vop0;
5184 /* Is STMT a vectorizable conversion? */
5205 {
5208 }
5210 {
5213 }
5214 else
5227 /* Check types of lhs and rhs. */
5232 /* Check the operands of the operation. */
5236 {
5241 }
5256 {
5259 "type conversion to/from bit-precision unsupported."
5262 }
5265 {
5272 tree vectype1_in;
5275 {
5280 }
5281 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5282 OP1. */
5285 }
5287 /* If op0 is an external or constant def, infer the vector type
5288 from the scalar type. */
5294 {
5300 }
5304 {
5307 "can't convert between boolean and non "
5311 }
5318 else
5322 else
5323 {
5326 }
5328 /* Multiple types in SLP are handled by creating the appropriate number of
5329 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5330 case of SLP. */
5335 else
5338 /* Sanity check: make sure that at least one copy of the vectorized stmt
5339 needs to be generated. */
5345 opt_scalar_mode rhs_mode_iter;
5347 /* Supportable by target? */
5349 {
5358 {
5361 }
5363 /* For conversions between float and integer types try whether
5364 we can use intermediate signed integer types to support the
5365 conversion. */
5369 {
5373 scalar_mode intermediate_mode;
5375 {
5378 }
5379 else
5380 {
5385 }
5388 opt_scalar_mode mode_iter;
5390 {
5396 scalar_mode cvt_mode;
5401 cvt_type = build_nonstandard_integer_type
5404 /* Check if the intermediate type can hold OP0's range.
5405 When converting from float to integer this is not necessary
5406 because values that do not fit the (smaller) target type are
5407 unspecified anyway. */
5409 {
5420 }
5423 /* This should only happened for SLP as long as loop vectorizer
5424 only supports same-sized vector. */
5428 vectype_out,
5431 cvt_type,
5437 }
5440 {
5441 multi_step_cvt++;
5447 }
5448 }
5449 /* FALLTHRU */
5450 unsupported:
5458 {
5467 }
5472 {
5473 /* Binary widening operation can only be supported directly by the
5474 architecture. */
5477 }
5485 {
5490 cvt_type
5497 {
5504 }
5511 else
5515 cvt_type,
5519 {
5522 }
5523 }
5530 else
5531 {
5532 multi_step_cvt++;
5535 }
5549 {
5550 cvt_type
5558 else
5564 }
5565 /* If op0 can be represented with low precision integer,
5566 truncate it to cvt_type and the do FLOAT_EXPR. */
5568 {
5573 cvt_type
5591 {
5595 }
5596 }
5602 }
5605 {
5609 {
5614 }
5617 {
5622 }
5624 {
5626 /* The final packing step produces one vector result per copy. */
5627 unsigned int nvectors
5631 widen_arith);
5632 }
5633 else
5634 {
5636 /* The initial unpacking step produces two vector results
5637 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5638 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5639 unsigned int nvectors
5640 = (slp_node
5645 widen_arith);
5646 }
5649 }
5651 /* Transform. */
5657 {
5662 }
5664 /* In case of multi-step conversion, we first generate conversion operations
5665 to the intermediate types, and then from that types to the final one.
5666 We create vector destinations for the intermediate type (TYPES) received
5667 from supportable_*_operation, and store them in the correct order
5668 for future use in vect_create_vectorized_*_stmts (). */
5670 bool widen_or_narrow_float_p
5673 widen_or_narrow_float_p
5678 {
5681 {
5683 intermediate_type);
5685 }
5686 }
5690 widen_or_narrow_float_p
5695 {
5697 ;
5699 {
5703 }
5704 }
5707 {
5711 /* vec_dest is intermediate type operand when multi_step_cvt. */
5713 {
5716 }
5719 {
5720 /* Arguments are ready, create the new vector stmt. */
5723 {
5730 }
5738 else
5740 }
5744 /* In case the vectorization factor (VF) is bigger than the number
5745 of elements that we can fit in a vectype (nunits), we have to
5746 generate more than one vector stmt - i.e - we need to "unroll"
5747 the vector stmt by a factor VF/nunits. */
5753 {
5758 }
5759 /* Arguments are ready. Create the new vector stmts. */
5761 {
5765 {
5768 }
5772 op_type);
5773 else
5778 }
5781 {
5784 {
5788 }
5789 else
5794 else
5796 }
5801 /* In case the vectorization factor (VF) is bigger than the number
5802 of elements that we can fit in a vectype (nunits), we have to
5803 generate more than one vector stmt - i.e - we need to "unroll"
5804 the vector stmt by a factor VF/nunits. */
5807 /* Arguments are ready. Create the new vector stmts. */
5810 {
5815 }
5818 multi_step_cvt,
5822 /* After demoting op0 to cvt_type, convert it to dest. */
5824 {
5826 {
5827 /* Arguments are ready, create the new vector stmt. */
5829 gimple *new_stmt
5835 /* This is the last step of the conversion sequence. Store the
5836 vectors in SLP_NODE or in vector info of the scalar statement
5837 (or in STMT_VINFO_RELATED_STMT chain). */
5840 else
5842 }
5843 }
5845 }
5854 }
5856 /* Return true if we can assume from the scalar form of STMT_INFO that
5857 neither the scalar nor the vector forms will generate code. STMT_INFO
5858 is known not to involve a data reference. */
5860 bool
5862 {
5878 }
5880 /* Function vectorizable_assignment.
5882 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5883 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5884 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5885 Return true if STMT_INFO is vectorizable in this way. */
5887 static bool
5892 {
5893 tree vec_dest;
5894 tree scalar_dest;
5895 tree op;
5897 tree new_temp;
5903 tree vop;
5906 tree vectype_in;
5915 /* Is vectorizable assignment? */
5936 /* Multiple types in SLP are handled by creating the appropriate number of
5937 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5938 case of SLP. */
5941 else
5946 slp_tree slp_op;
5949 {
5954 }
5958 /* We can handle VIEW_CONVERT conversions that do not change the number
5959 of elements or the vector size or other conversions when the component
5960 mode keeps the same. */
5972 {
5975 "can't convert between boolean and non "
5979 }
5981 /* We do not handle bit-precision changes. */
5988 /* But a conversion that does not change the bit-pattern is ok. */
5996 {
5999 "type conversion to/from bit-precision "
6002 }
6005 {
6008 {
6013 }
6018 cost_vec);
6020 }
6022 /* Transform. */
6026 /* Handle def. */
6029 /* Handle use. */
6032 /* Arguments are ready. create the new vector stmt. */
6034 {
6044 else
6046 }
6052 }
6055 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6056 either as shift by a scalar or by a vector. */
6058 bool
6060 {
6062 machine_mode vec_mode;
6063 optab optab;
6065 tree vectype;
6074 {
6080 }
6088 }
6091 /* Function vectorizable_shift.
6093 Check if STMT_INFO performs a shift operation that can be vectorized.
6094 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6095 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6096 Return true if STMT_INFO is vectorizable in this way. */
6098 static bool
6103 {
6104 tree vec_dest;
6105 tree scalar_dest;
6108 tree vectype;
6111 machine_mode vec_mode;
6112 tree new_temp;
6113 optab optab;
6115 machine_mode optab_op2_mode;
6118 poly_uint64 nunits_in;
6119 poly_uint64 nunits_out;
6120 tree vectype_out;
6121 tree op1_vectype;
6140 /* Is STMT a vectorizable binary/unary operation? */
6157 {
6162 }
6164 slp_tree slp_op0;
6167 {
6172 }
6173 /* If op0 is an external or constant def, infer the vector type
6174 from the scalar type. */
6180 {
6185 }
6192 stmt_vec_info op1_def_stmt_info;
6193 slp_tree slp_op1;
6196 {
6201 }
6203 /* Multiple types in SLP are handled by creating the appropriate number of
6204 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6205 case of SLP. */
6208 else
6213 /* Determine whether the shift amount is a vector, or scalar. If the
6214 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6224 {
6225 /* In SLP, need to check whether the shift count is the same,
6226 in loops if it is a constant or invariant, it is always
6227 a scalar shift. */
6229 {
6231 stmt_vec_info slpstmt_info;
6234 {
6238 }
6240 /* For internal SLP defs we have to make sure we see scalar stmts
6241 for all vector elements.
6242 ??? For different vectors we could resort to a different
6243 scalar shift operand but code-generation below simply always
6244 takes the first. */
6249 }
6251 /* If the shift amount is computed by a pattern stmt we cannot
6252 use the scalar amount directly thus give up and use a vector
6253 shift. */
6256 }
6257 else
6258 {
6263 }
6265 /* Vector shifted by vector. */
6268 {
6276 slp_op1);
6277 incompatible_op1_vectype_p
6283 && (!slp_node
6286 {
6289 "unusable type for last operand in"
6292 }
6293 }
6294 /* See if the machine has a vector shifted by scalar insn and if not
6295 then see if it has a vector shifted by vector insn. */
6296 else
6297 {
6301 {
6305 }
6306 else
6307 {
6312 {
6322 slp_op1);
6324 /* Unlike the other binary operators, shifts/rotates have
6325 the rhs being int, instead of the same type as the lhs,
6326 so make sure the scalar is the right type if we are
6327 dealing with vectors of long long/long/short/char. */
6328 incompatible_op1_vectype_p
6329 = (!op1_vectype
6334 {
6337 "unusable type for last operand in"
6340 }
6341 }
6342 }
6343 }
6345 /* Supportable by target? */
6347 {
6352 }
6356 {
6361 }
6362 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6367 {
6371 && (!incompatible_op1_vectype_p
6373 && !vect_maybe_update_slp_op_vectype
6376 {
6381 }
6382 /* Now adjust the constant shift amount in place. */
6384 && incompatible_op1_vectype_p
6386 {
6389 {
6395 }
6396 }
6402 }
6404 /* Transform. */
6411 {
6417 }
6419 /* Handle def. */
6423 {
6424 /* Vector shl and shr insn patterns can be defined with scalar
6425 operand 2 (shift operand). In this case, use constant or loop
6426 invariant op1 directly, without extending it to vector mode
6427 first. */
6430 {
6437 /* Store vec_oprnd1 for every vector stmt to be created.
6438 We check during the analysis that all the shift arguments
6439 are the same.
6440 TODO: Allow different constants for different vector
6441 stmts generated for an SLP instance. */
6445 }
6446 }
6448 {
6450 {
6451 /* If the argument was the same in all lanes create
6452 the correctly typed vector shift amount directly. */
6461 }
6463 /* The constant shift amount has been adjusted in place. */
6464 ;
6465 else
6467 }
6469 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6470 (a special case for certain kind of vector shifts); otherwise,
6471 operand 1 should be of a vector type (the usual case). */
6476 /* Arguments are ready. Create the new vector stmt. */
6478 {
6479 /* For internal defs where we need to use a scalar shift arg
6480 extract the first lane. */
6482 {
6485 gassign *new_stmt
6488 vop1,
6490 bitsize_zero_node));
6493 }
6494 else
6502 else
6504 }
6513 }
6515 /* Function vectorizable_operation.
6517 Check if STMT_INFO performs a binary, unary or ternary operation that can
6518 be vectorized.
6519 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6520 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6521 Return true if STMT_INFO is vectorizable in this way. */
6523 static bool
6528 {
6529 tree vec_dest;
6530 tree scalar_dest;
6532 tree vectype;
6535 machine_mode vec_mode;
6536 tree new_temp;
6538 optab optab;
6543 poly_uint64 nunits_in;
6544 poly_uint64 nunits_out;
6545 tree vectype_out;
6561 /* Is STMT a vectorizable binary/unary operation? */
6566 /* Loads and stores are handled in vectorizable_{load,store}. */
6572 /* Shifts are handled in vectorizable_shift. */
6579 /* Comparisons are handled in vectorizable_comparison. */
6583 /* Conditions are handled in vectorizable_condition. */
6587 /* For pointer addition and subtraction, we should use the normal
6588 plus and minus for the vector operation. */
6594 /* Support only unary or binary operations. */
6597 {
6601 op_type);
6603 }
6608 /* Most operations cannot handle bit-precision types without extra
6609 truncations. */
6613 /* Exception are bitwise binary operations. */
6617 {
6622 }
6624 slp_tree slp_op0;
6627 {
6632 }
6635 /* If op0 is an external or constant def, infer the vector type
6636 from the scalar type. */
6638 {
6639 /* For boolean type we cannot determine vectype by
6640 invariant value (don't know whether it is a vector
6641 of booleans or vector of integers). We use output
6642 vectype because operations on boolean don't change
6643 type. */
6645 {
6647 {
6652 }
6654 }
6655 else
6657 slp_node);
6658 }
6662 {
6669 }
6680 {
6683 {
6688 }
6696 }
6698 {
6701 {
6706 }
6714 }
6716 /* Multiple types in SLP are handled by creating the appropriate number of
6717 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6718 case of SLP. */
6720 {
6723 }
6724 else
6725 {
6728 }
6732 /* Reject attempts to combine mask types with nonmask types, e.g. if
6733 we have an AND between a (nonmask) boolean loaded from memory and
6734 a (mask) boolean result of a comparison.
6736 TODO: We could easily fix these cases up using pattern statements. */
6740 {
6745 }
6747 /* Supportable by target? */
6752 else
6753 {
6756 {
6761 }
6764 }
6768 {
6772 /* When vec_mode is not a vector mode and we verified ops we
6773 do not have to lower like AND are natively supported let
6774 those through even when the mode isn't word_mode. For
6775 ops we have to lower the lowering code assumes we are
6776 dealing with word_mode. */
6781 /* Check only during analysis. */
6783 {
6787 }
6792 }
6800 /* If operating on inactive elements could generate spurious traps,
6801 we need to restrict the operation to active lanes. Note that this
6802 specifically doesn't apply to unhoisted invariants, since they
6803 operate on the same value for every lane.
6805 Similarly, if this operation is part of a reduction, a fully-masked
6806 loop should only change the active lanes of the reduction chain,
6807 keeping the inactive lanes as-is. */
6812 {
6816 {
6819 OPTIMIZE_FOR_SPEED))
6824 OPTIMIZE_FOR_SPEED))
6827 else
6828 {
6831 "can't use a fully-masked loop because no"
6834 }
6835 }
6837 /* Put types on constant and invariant SLP children. */
6842 {
6847 }
6854 {
6855 /* The above vect_model_simple_cost call handles constants
6856 in the prologue and (mis-)costs one of the stmts as
6857 vector stmt. See below for the actual lowering that will
6858 be applied. */
6859 unsigned n
6862 {
6873 /* Bit operations do not have extra cost and are accounted
6874 as vector stmt by vect_model_simple_cost. */
6877 }
6879 {
6880 /* We also need to materialize two large constants. */
6885 }
6886 }
6888 }
6890 /* Transform. */
6899 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6900 vectors with unsigned elements, but the result is signed. So, we
6901 need to compute the MINUS_EXPR into vectype temporary and
6902 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6905 {
6908 }
6909 /* Handle def. */
6910 else
6913 /* In case the vectorization factor (VF) is bigger than the number
6914 of elements that we can fit in a vectype (nunits), we have to generate
6915 more than one vector stmt - i.e - we need to "unroll" the
6916 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6917 from one copy of the vector stmt to the next, in the field
6918 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6919 stages to find the correct vector defs to be used when vectorizing
6920 stmts that use the defs of the current stmt. The example below
6921 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6922 we need to create 4 vectorized stmts):
6924 before vectorization:
6925 RELATED_STMT VEC_STMT
6926 S1: x = memref - -
6927 S2: z = x + 1 - -
6929 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6930 there):
6931 RELATED_STMT VEC_STMT
6932 VS1_0: vx0 = memref0 VS1_1 -
6933 VS1_1: vx1 = memref1 VS1_2 -
6934 VS1_2: vx2 = memref2 VS1_3 -
6935 VS1_3: vx3 = memref3 - -
6936 S1: x = load - VS1_0
6937 S2: z = x + 1 - -
6939 step2: vectorize stmt S2 (done here):
6940 To vectorize stmt S2 we first need to find the relevant vector
6941 def for the first operand 'x'. This is, as usual, obtained from
6942 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
6943 that defines 'x' (S1). This way we find the stmt VS1_0, and the
6944 relevant vector def 'vx0'. Having found 'vx0' we can generate
6945 the vector stmt VS2_0, and as usual, record it in the
6946 STMT_VINFO_VEC_STMT of stmt S2.
6947 When creating the second copy (VS2_1), we obtain the relevant vector
6948 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
6949 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
6950 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
6951 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
6952 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
6953 chain of stmts and pointers:
6954 RELATED_STMT VEC_STMT
6955 VS1_0: vx0 = memref0 VS1_1 -
6956 VS1_1: vx1 = memref1 VS1_2 -
6957 VS1_2: vx2 = memref2 VS1_3 -
6958 VS1_3: vx3 = memref3 - -
6959 S1: x = load - VS1_0
6960 VS2_0: vz0 = vx0 + v1 VS2_1 -
6961 VS2_1: vz1 = vx1 + v1 VS2_2 -
6962 VS2_2: vz2 = vx2 + v1 VS2_3 -
6963 VS2_3: vz3 = vx3 + v1 - -
6964 S2: z = x + 1 - VS2_0 */
6968 /* Arguments are ready. Create the new vector stmt. */
6970 {
6977 {
6978 /* Lower the operation. This follows vector lowering. */
6981 tree word_type
6985 tree high_bits
6994 {
7012 else
7017 {
7023 }
7032 }
7033 else
7034 {
7051 }
7053 signs);
7063 }
7065 {
7066 tree mask;
7070 else
7071 /* Dummy mask. */
7081 {
7082 /* Perform the operation on active elements only and take
7083 inactive elements from the reduction chain input. */
7086 }
7087 else
7088 {
7092 }
7094 {
7097 signed char biasval
7102 }
7103 gcall *call
7106 vops);
7112 }
7113 else
7114 {
7116 /* When combining two masks check if either of them is elsewhere
7117 combined with a loop mask, if that's the case we can mark that the
7118 new combined mask doesn't need to be combined with a loop mask. */
7122 {
7124 ncopies}))
7125 {
7131 }
7134 ncopies }))
7135 {
7141 }
7142 }
7151 /* Enter the combined value into the vector cond hash so we don't
7152 AND it with a loop mask again. */
7155 }
7158 {
7161 new_temp);
7166 }
7170 else
7172 }
7182 }
7184 /* A helper function to ensure data reference DR_INFO's base alignment. */
7186 static void
7188 {
7189 /* Alignment is only analyzed for the first element of a DR group,
7190 use that to look at base alignment we need to enforce. */
7197 {
7200 // We should only be able to increase the alignment of a base object if
7201 // we know what its new alignment should be at compile time.
7208 {
7211 }
7213 }
7214 }
7217 /* Function get_group_alias_ptr_type.
7219 Return the alias type for the group starting at FIRST_STMT_INFO. */
7221 static tree
7223 {
7229 {
7233 {
7238 }
7240 }
7242 }
7245 /* Function scan_operand_equal_p.
7247 Helper function for check_scan_store. Compare two references
7248 with .GOMP_SIMD_LANE bases. */
7250 static bool
7252 {
7257 {
7258 machine_mode mode;
7268 {
7274 {
7279 }
7280 }
7281 }
7288 {
7292 {
7295 {
7298 {
7304 {
7307 }
7308 }
7310 {
7313 }
7316 {
7320 }
7329 }
7333 }
7334 }
7336 }
7340 /* Normal permutation. */
7341 scan_store_kind_perm,
7343 /* Whole vector left shift permutation with zero init. */
7344 scan_store_kind_lshift_zero,
7346 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7347 scan_store_kind_lshift_cond
7348 };
7350 /* Function check_scan_store.
7352 Verify if we can perform the needed permutations or whole vector shifts.
7353 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7354 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7355 to do at each step. */
7357 static int
7360 {
7372 {
7378 {
7381 }
7382 else
7383 {
7388 }
7391 {
7396 {
7400 /* Whole vector shifts shift in zeros, so if init is all zero
7401 constant, there is no need to do anything further. */
7405 {
7410 }
7411 }
7413 }
7415 {
7420 }
7421 }
7424 }
7427 /* Function check_scan_store.
7429 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7431 static bool
7434 vect_memory_access_type memory_access_type)
7435 {
7438 tree ref_type;
7442 || mask
7454 {
7459 }
7461 /* We need to pattern match code built by OpenMP lowering and simplified
7462 by following optimizations into something we can handle.
7463 #pragma omp simd reduction(inscan,+:r)
7464 for (...)
7465 {
7466 r += something ();
7467 #pragma omp scan inclusive (r)
7468 use (r);
7469 }
7470 shall have body with:
7471 // Initialization for input phase, store the reduction initializer:
7472 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7473 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7474 D.2042[_21] = 0;
7475 // Actual input phase:
7476 ...
7477 r.0_5 = D.2042[_20];
7478 _6 = _4 + r.0_5;
7479 D.2042[_20] = _6;
7480 // Initialization for scan phase:
7481 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7482 _26 = D.2043[_25];
7483 _27 = D.2042[_25];
7484 _28 = _26 + _27;
7485 D.2043[_25] = _28;
7486 D.2042[_25] = _28;
7487 // Actual scan phase:
7488 ...
7489 r.1_8 = D.2042[_20];
7490 ...
7491 The "omp simd array" variable D.2042 holds the privatized copy used
7492 inside of the loop and D.2043 is another one that holds copies of
7493 the current original list item. The separate GOMP_SIMD_LANE ifn
7494 kinds are there in order to allow optimizing the initializer store
7495 and combiner sequence, e.g. if it is originally some C++ish user
7496 defined reduction, but allow the vectorizer to pattern recognize it
7497 and turn into the appropriate vectorized scan.
7499 For exclusive scan, this is slightly different:
7500 #pragma omp simd reduction(inscan,+:r)
7501 for (...)
7502 {
7503 use (r);
7504 #pragma omp scan exclusive (r)
7505 r += something ();
7506 }
7507 shall have body with:
7508 // Initialization for input phase, store the reduction initializer:
7509 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7510 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7511 D.2042[_21] = 0;
7512 // Actual input phase:
7513 ...
7514 r.0_5 = D.2042[_20];
7515 _6 = _4 + r.0_5;
7516 D.2042[_20] = _6;
7517 // Initialization for scan phase:
7518 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7519 _26 = D.2043[_25];
7520 D.2044[_25] = _26;
7521 _27 = D.2042[_25];
7522 _28 = _26 + _27;
7523 D.2043[_25] = _28;
7524 // Actual scan phase:
7525 ...
7526 r.1_8 = D.2044[_20];
7527 ... */
7530 {
7531 /* Match the D.2042[_21] = 0; store above. Just require that
7532 it is a constant or external definition store. */
7534 {
7535 fail_init:
7540 }
7550 /* These stores can be vectorized normally. */
7552 }
7555 {
7556 fail:
7561 }
7570 bool inscan_var_store
7574 {
7576 {
7577 use_operand_p use_p;
7578 imm_use_iterator iter;
7580 {
7587 || other_store_stmt
7591 }
7597 }
7598 }
7600 {
7601 use_operand_p use_p;
7602 imm_use_iterator iter;
7604 {
7611 }
7612 }
7613 else
7623 /* For pointer addition, we should use the normal plus for the vector
7624 operation. */
7626 {
7634 }
7662 {
7668 tree lrhs;
7671 else
7673 use_operand_p use_p;
7674 imm_use_iterator iter;
7676 {
7683 }
7684 }
7692 stmt_vec_info other_store_stmt_info
7705 {
7709 }
7720 {
7731 }
7750 {
7755 }
7763 /* The IL is as expected, now check if we can actually vectorize it.
7764 Inclusive scan:
7765 _26 = D.2043[_25];
7766 _27 = D.2042[_25];
7767 _28 = _26 + _27;
7768 D.2043[_25] = _28;
7769 D.2042[_25] = _28;
7770 should be vectorized as (where _40 is the vectorized rhs
7771 from the D.2042[_21] = 0; store):
7772 _30 = MEM <vector(8) int> [(int *)&D.2043];
7773 _31 = MEM <vector(8) int> [(int *)&D.2042];
7774 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7775 _33 = _31 + _32;
7776 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7777 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7778 _35 = _33 + _34;
7779 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7780 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7781 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7782 _37 = _35 + _36;
7783 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7784 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7785 _38 = _30 + _37;
7786 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7787 MEM <vector(8) int> [(int *)&D.2043] = _39;
7788 MEM <vector(8) int> [(int *)&D.2042] = _38;
7789 Exclusive scan:
7790 _26 = D.2043[_25];
7791 D.2044[_25] = _26;
7792 _27 = D.2042[_25];
7793 _28 = _26 + _27;
7794 D.2043[_25] = _28;
7795 should be vectorized as (where _40 is the vectorized rhs
7796 from the D.2042[_21] = 0; store):
7797 _30 = MEM <vector(8) int> [(int *)&D.2043];
7798 _31 = MEM <vector(8) int> [(int *)&D.2042];
7799 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7800 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7801 _34 = _32 + _33;
7802 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7803 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7804 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7805 _36 = _34 + _35;
7806 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7807 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7808 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7809 _38 = _36 + _37;
7810 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7811 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7812 _39 = _30 + _38;
7813 _50 = _31 + _39;
7814 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7815 MEM <vector(8) int> [(int *)&D.2044] = _39;
7816 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7827 }
7830 /* Function vectorizable_scan_store.
7832 Helper of vectorizable_score, arguments like on vectorizable_store.
7833 Handle only the transformation, checking is done in check_scan_store. */
7835 static bool
7839 {
7854 bool inscan_var_store
7858 {
7859 use_operand_p use_p;
7860 imm_use_iterator iter;
7862 {
7868 }
7869 }
7890 {
7894 }
7909 {
7916 else
7917 {
7922 }
7926 {
7933 }
7934 else
7936 }
7957 {
7968 {
7972 dataref_offset);
7978 }
7982 {
7985 (zero_vec
7995 {
7996 /* Whole vector shift shifted in zero bits, but if *init
7997 is not initializer_zerop, we need to replace those elements
7998 with elements from vec_oprnd1. */
8011 }
8013 /* For exclusive scan, perform the perms[i] permutation once
8014 more. */
8018 {
8022 }
8030 }
8038 /* For exclusive scan, new_temp computed above is the exclusive scan
8039 prefix sum. Turn it into inclusive prefix sum for the broadcast
8040 of the last element into orig. */
8042 {
8047 }
8056 {
8059 dataref_offset);
8064 }
8065 }
8069 {
8075 dataref_offset);
8080 }
8082 }
8085 /* Function vectorizable_store.
8087 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8088 that can be vectorized.
8089 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8090 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8091 Return true if STMT_INFO is vectorizable in this way. */
8093 static bool
8098 {
8099 tree data_ref;
8101 tree elem_type;
8104 machine_mode vec_mode;
8105 tree dummy;
8113 stmt_vec_info first_stmt_info;
8119 tree aggr_type;
8120 gather_scatter_info gs_info;
8121 poly_uint64 vf;
8122 vec_load_store_type vls_type;
8123 tree ref_type;
8132 /* Is vectorizable store? */
8137 {
8150 }
8151 else
8152 {
8163 mask_index = vect_slp_child_index_for_operand
8170 }
8172 /* Cannot have hybrid store SLP -- that would mean storing to the
8173 same location twice. */
8180 {
8183 }
8184 else
8187 /* Multiple types in SLP are handled by creating the appropriate number of
8188 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8189 case of SLP. */
8192 else
8197 /* FORNOW. This restriction should be relaxed. */
8199 {
8204 }
8206 tree op;
8207 slp_tree op_node;
8218 vect_memory_access_type memory_access_type;
8221 poly_int64 poffset;
8222 internal_fn lanes_ifn;
8230 {
8232 {
8237 }
8241 {
8246 }
8250 {
8255 }
8256 }
8257 else
8258 {
8259 /* FORNOW. In some cases can vectorize even if data-type not supported
8260 (e.g. - array initialization with 0). */
8263 }
8270 {
8274 }
8275 else
8276 {
8280 }
8283 {
8285 memory_access_type))
8287 }
8291 {
8299 mask);
8303 || (mask
8305 mask_vectype))))
8306 {
8311 }
8322 /* As function vect_transform_stmt shows, for interleaving stores
8323 the whole chain is vectorized when the last store in the chain
8324 is reached, the other stores in the group are skipped. So we
8325 want to only cost the last one here, but it's not trivial to
8326 get the last, as it's equivalent to use the first one for
8327 costing, use the first one instead. */
8329 && !slp
8332 }
8335 /* Transform. */
8340 {
8344 {
8355 "vect_model_store_cost: inside_cost = %d, "
8360 }
8362 }
8365 {
8366 /* FORNOW */
8370 {
8372 /* VEC_NUM is the number of vect stmts to be created for this
8373 group. */
8380 }
8381 else
8382 /* VEC_NUM is the number of vect stmts to be created for this
8383 group. */
8387 }
8388 else
8393 ncopies);
8395 /* Check if we need to update prologue cost for invariant,
8396 and update it accordingly if so. If it's not for
8397 interleaving store, we can just check vls_type; but if
8398 it's for interleaving store, need to check the def_type
8399 of the stored value since the current vls_type is just
8400 for first_stmt_info. */
8402 {
8407 {
8413 }
8418 };
8422 {
8424 gimple_stmt_iterator incr_gsi;
8427 tree offvar;
8428 tree ivstep;
8429 tree running_off;
8432 tree dr_offset;
8434 /* Checked by get_load_store_type. */
8441 stride_base
8442 = fold_build_pointer_plus
8449 /* For a store with loop-invariant (but other than power-of-2)
8450 stride (i.e. not a grouped access) like so:
8452 for (i = 0; i < n; i += stride)
8453 array[i] = ...;
8455 we generate a new induction variable and new stores from
8456 the components of the (vectorized) rhs:
8458 for (j = 0; ; j += VF*stride)
8459 vectemp = ...;
8460 tmp1 = vectemp[0];
8461 array[j] = tmp1;
8462 tmp2 = vectemp[1];
8463 array[j + stride] = tmp2;
8464 ...
8465 */
8472 {
8475 {
8481 /* First check if vec_extract optab doesn't support extraction
8482 of vector elts directly. */
8484 machine_mode vmode;
8491 {
8492 /* Try to avoid emitting an extract of vector elements
8493 by performing the extracts using an integer type of the
8494 same size, extracting from a vector of those and then
8495 re-interpreting it as the original vector type if
8496 supported. */
8497 unsigned lsize
8500 /* If we can't construct such a vector fall back to
8501 element extracts from the original vector type and
8502 element size stores. */
8510 {
8515 }
8516 /* Else fall back to vector extraction anyway.
8517 Fewer stores are more important than avoiding spilling
8518 of the vector we extract from. Compared to the
8519 construction case in vectorizable_load no store-forwarding
8520 issue exists here for reasonable archs. */
8521 }
8522 }
8525 {
8527 dr_alignment_support dr_align
8529 mis_align);
8532 {
8539 }
8540 }
8543 }
8546 {
8560 }
8565 /* For costing some adjacent vector stores, we'd like to cost with
8566 the total number of them once instead of cost each one by one. */
8569 {
8572 {
8574 {
8576 tree pos
8583 }
8584 }
8590 else
8593 unsigned HOST_WIDE_INT
8596 {
8598 {
8600 /* Pun the vector to extract from if necessary. */
8602 {
8604 tree cvt
8609 }
8610 }
8612 {
8614 {
8615 /* Only need vector extracting when there are more
8616 than one stores. */
8618 inside_cost
8621 /* Take a single lane vector type store as scalar
8622 store to avoid ICE like 110776. */
8625 n_adjacent_stores++;
8626 else
8627 inside_cost
8631 }
8635 /* Extract the i'th component. */
8643 GSI_SAME_STMT);
8651 /* And store it to *running_off. */
8658 {
8666 }
8669 {
8673 }
8674 }
8675 }
8680 }
8683 {
8690 "vect_model_store_cost: inside_cost = %d, "
8693 }
8696 }
8699 vec_loop_masks *loop_masks
8703 vec_loop_lens *loop_lens
8708 /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8709 are some difference here. We cannot enable both the lens and masks
8710 during transform but it is allowed during analysis.
8711 Shouldn't go with length-based approach if fully masked. */
8713 /* The cost_vec is NULL during transfrom. */
8716 /* Targets with store-lane instructions must not require explicit
8717 realignment. vect_supportable_dr_alignment always returns either
8718 dr_aligned or dr_unaligned_supported for masked operations. */
8720 && !mask
8729 tree bump;
8732 {
8735 }
8737 {
8742 }
8743 else
8744 {
8747 else
8751 }
8756 /* In case the vectorization factor (VF) is bigger than the number
8757 of elements that we can fit in a vectype (nunits), we have to generate
8758 more than one vector stmt - i.e - we need to "unroll" the
8759 vector stmt by a factor VF/nunits. */
8761 /* In case of interleaving (non-unit grouped access):
8763 S1: &base + 2 = x2
8764 S2: &base = x0
8765 S3: &base + 1 = x1
8766 S4: &base + 3 = x3
8768 We create vectorized stores starting from base address (the access of the
8769 first stmt in the chain (S2 in the above example), when the last store stmt
8770 of the chain (S4) is reached:
8772 VS1: &base = vx2
8773 VS2: &base + vec_size*1 = vx0
8774 VS3: &base + vec_size*2 = vx1
8775 VS4: &base + vec_size*3 = vx3
8777 Then permutation statements are generated:
8779 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8780 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8781 ...
8783 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8784 (the order of the data-refs in the output of vect_permute_store_chain
8785 corresponds to the order of scalar stmts in the interleaving chain - see
8786 the documentation of vect_permute_store_chain()).
8788 In case of both multiple types and interleaving, above vector stores and
8789 permutation stmts are created for every copy. The result vector stmts are
8790 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8791 STMT_VINFO_RELATED_STMT for the next copies.
8792 */
8802 {
8805 /* For costing some adjacent vector stores, we'd like to cost with
8806 the total number of them once instead of cost each one by one. */
8809 {
8812 {
8813 /* For interleaved stores we collect vectorized defs for all
8814 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8815 as an input to vect_permute_store_chain(). */
8818 {
8819 /* Since gaps are not supported for interleaved stores,
8820 DR_GROUP_SIZE is the exact number of stmts in the
8821 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8825 else
8826 {
8832 }
8834 }
8837 {
8839 {
8842 mask_vectype);
8844 }
8846 /* We should have catched mismatched types earlier. */
8849 dataref_ptr
8853 }
8854 }
8856 {
8858 /* DR_CHAIN is then used as an input to
8859 vect_permute_store_chain(). */
8861 {
8864 }
8869 }
8872 {
8875 }
8877 /* Get an array into which we can store the individual vectors. */
8880 /* Invalidate the current contents of VEC_ARRAY. This should
8881 become an RTL clobber too, which prevents the vector registers
8882 from being upward-exposed. */
8885 /* Store the individual vectors into the array. */
8887 {
8890 i);
8891 }
8904 {
8908 else
8910 signed char biasval
8914 {
8917 }
8918 }
8922 {
8923 /* Emit:
8924 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8925 LEN, BIAS, VEC_ARRAY). */
8931 vec_array);
8932 }
8934 {
8935 /* Emit:
8936 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8937 VEC_ARRAY). */
8943 }
8944 else
8945 {
8946 /* Emit:
8947 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8951 }
8956 /* Record that VEC_ARRAY is now dead. */
8961 }
8964 {
8971 "vect_model_store_cost: inside_cost = %d, "
8974 }
8977 }
8980 {
8985 {
8988 {
8993 {
8994 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8995 DR_CHAIN is of size 1. */
8999 else
9003 {
9006 else
9008 ncopies,
9010 mask_vectype);
9011 }
9017 else
9018 dataref_ptr
9022 bump);
9023 }
9024 }
9026 {
9031 }
9035 {
9037 {
9041 /* We should have catched mismatched types earlier. */
9044 }
9050 {
9058 }
9061 {
9063 {
9065 inside_cost
9069 }
9076 {
9081 else
9083 signed char biasval
9087 {
9090 }
9091 }
9095 call = gimple_build_call_internal
9100 call = gimple_build_call_internal
9103 else
9110 }
9112 {
9113 /* The builtin decls path for scatter is legacy, x86 only. */
9115 && (!final_mask
9116 || SCALAR_INT_MODE_P
9119 {
9121 inside_cost
9125 }
9126 poly_uint64 offset_nunits
9129 {
9130 new_stmt = vect_build_one_scatter_store_call
9136 }
9138 {
9139 /* We have a offset vector with half the number of
9140 lanes but the builtins will store full vectype
9141 data from the lower lanes. */
9142 new_stmt = vect_build_one_scatter_store_call
9144 dataref_ptr,
9155 tree perm_mask
9159 perm_mask);
9165 {
9167 VEC_UNPACK_HI_EXPR,
9168 final_mask);
9169 final_mask = make_ssa_name
9174 }
9175 new_stmt = vect_build_one_scatter_store_call
9177 dataref_ptr,
9182 }
9184 {
9185 /* We have a offset vector with double the number of
9186 lanes. Select the low/high part accordingly. */
9189 {
9196 tree perm_mask = vect_gen_perm_mask_checked
9199 VEC_PERM_EXPR,
9200 vec_offset,
9201 vec_offset,
9202 perm_mask);
9207 }
9208 new_stmt = vect_build_one_scatter_store_call
9214 }
9215 else
9217 }
9218 else
9219 {
9220 /* Emulated scatter. */
9223 {
9225 /* For emulated scatter N offset vector element extracts
9226 (we assume the scalar scaling and ptr + offset add is
9227 consumed by the load). */
9228 inside_cost
9231 /* N scalar stores plus extracting the elements. */
9232 inside_cost
9235 inside_cost
9239 }
9242 unsigned HOST_WIDE_INT const_offset_nunits
9248 unsigned HOST_WIDE_INT elt_size
9250 /* We support offset vectors with more elements
9251 than the data vector for now. */
9252 unsigned HOST_WIDE_INT factor
9255 unsigned elt_offset
9262 {
9263 /* Compute the offsetted pointer. */
9266 tree idx
9272 tree ptr
9277 /* Extract the element to be stored. */
9278 tree elt
9285 tree ref
9291 }
9294 }
9295 }
9298 }
9305 "vect_model_store_cost: inside_cost = %d, "
9310 }
9318 /* For costing some adjacent vector stores, we'd like to cost with
9319 the total number of them once instead of cost each one by one. */
9324 {
9327 {
9329 {
9330 /* Get vectorized arguments for SLP_NODE. */
9336 }
9337 else
9338 {
9339 /* For interleaved stores we collect vectorized defs for all the
9340 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9341 input to vect_permute_store_chain().
9343 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9344 is of size 1. */
9347 {
9348 /* Since gaps are not supported for interleaved stores,
9349 DR_GROUP_SIZE is the exact number of stmts in the chain.
9350 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9351 that there is no interleaving, DR_GROUP_SIZE is 1,
9352 and only one iteration of the loop will be executed. */
9356 else
9357 {
9363 }
9365 }
9367 {
9370 mask_vectype);
9372 }
9373 }
9375 /* We should have catched mismatched types earlier. */
9379 bool simd_lane_access_p
9382 && simd_lane_access_p
9383 && !loop_masks
9390 {
9393 }
9395 dataref_ptr
9400 }
9402 {
9404 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9405 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9406 of size 1. */
9408 {
9411 }
9416 else
9419 }
9423 {
9424 /* Permute. */
9427 {
9434 "vect_model_store_cost: "
9436 group_size);
9437 }
9438 else
9441 }
9445 {
9447 {
9451 /* For grouped stores vectorized defs are interleaved in
9452 vect_permute_store_chain(). */
9454 }
9457 {
9461 else
9462 {
9468 /* Generate the permute statement. */
9469 gimple *perm_stmt
9473 gsi);
9477 }
9478 }
9481 {
9482 n_adjacent_stores++;
9485 {
9489 }
9492 }
9508 /* Bump the vector pointer. */
9518 {
9521 }
9522 else
9527 misalign);
9530 /* Compute IFN when LOOP_LENS or final_mask valid. */
9535 {
9536 opt_machine_mode new_ovmode
9539 unsigned factor
9544 }
9546 {
9551 }
9554 {
9556 {
9557 /* Pass VF value to 'len' argument of
9558 MASK_LEN_STORE if LOOP_LENS is invalid. */
9560 }
9562 {
9563 /* Pass all ones value to 'mask' argument of
9564 MASK_LEN_STORE if final_mask is invalid. */
9567 }
9568 }
9570 {
9571 signed char biasval
9575 }
9577 /* Arguments are ready. Create the new vector stmt. */
9579 {
9582 /* Need conversion if it's wrapped with VnQI. */
9584 {
9585 tree new_vtype
9587 new_vmode);
9590 gassign *new_stmt
9594 }
9600 else
9607 }
9609 {
9611 gcall *call
9617 }
9618 else
9619 {
9620 data_ref
9622 dataref_offset ? dataref_offset
9625 ;
9626 else
9633 }
9641 }
9643 {
9647 }
9648 }
9651 {
9657 /* When vectorizing a store into the function result assign
9658 a penalty if the function returns in a multi-register location.
9659 In this case we assume we'll end up with having to spill the
9660 vector result and do piecewise loads as a conservative estimate. */
9666 {
9668 /* ??? Handle PARALLEL in some way. */
9670 {
9672 /* Assume that a single reg-reg move is possible and cheap,
9673 do not account for vector to gp register move cost. */
9675 {
9676 /* Spill. */
9677 prologue_cost
9680 /* Loads. */
9681 prologue_cost
9684 }
9685 }
9686 }
9689 "vect_model_store_cost: inside_cost = %d, "
9692 }
9695 }
9697 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9698 VECTOR_CST mask. No checks are made that the target platform supports the
9699 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9700 vect_gen_perm_mask_checked. */
9702 tree
9704 {
9705 tree mask_type;
9712 }
9714 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9715 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9717 tree
9719 {
9723 }
9725 /* Given a vector variable X and Y, that was generated for the scalar
9726 STMT_INFO, generate instructions to permute the vector elements of X and Y
9727 using permutation mask MASK_VEC, insert them at *GSI and return the
9728 permuted vector variable. */
9730 static tree
9734 {
9742 else
9746 /* Generate the permute statement. */
9751 }
9753 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9754 inserting them on the loops preheader edge. Returns true if we
9755 were successful in doing so (and thus STMT_INFO can be moved then),
9756 otherwise returns false. HOIST_P indicates if we want to hoist the
9757 definitions of all SSA uses, it would be false when we are costing. */
9759 static bool
9761 {
9762 ssa_op_iter i;
9763 tree op;
9767 {
9771 {
9772 /* Make sure we don't need to recurse. While we could do
9773 so in simple cases when there are more complex use webs
9774 we don't have an easy way to preserve stmt order to fulfil
9775 dependencies within them. */
9776 tree op2;
9777 ssa_op_iter i2;
9781 {
9786 }
9788 }
9789 }
9798 {
9802 {
9806 }
9807 }
9810 }
9812 /* vectorizable_load.
9814 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9815 that can be vectorized.
9816 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9817 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9818 Return true if STMT_INFO is vectorizable in this way. */
9820 static bool
9825 {
9826 tree scalar_dest;
9833 tree elem_type;
9834 /* Avoid false positive uninitialized warning, see PR110652. */
9836 machine_mode mode;
9837 tree dummy;
9844 poly_uint64 group_gap_adj;
9850 stmt_vec_info first_stmt_info;
9858 poly_uint64 vf;
9859 tree aggr_type;
9860 gather_scatter_info gs_info;
9861 tree ref_type;
9878 {
9893 }
9894 else
9895 {
9910 mask_index = vect_slp_child_index_for_operand
9916 }
9922 {
9926 }
9927 else
9930 /* Multiple types in SLP are handled by creating the appropriate number of
9931 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
9932 case of SLP. */
9935 else
9940 /* FORNOW. This restriction should be relaxed. */
9942 {
9947 }
9949 /* Invalidate assumptions made by dependence analysis when vectorization
9950 on the unrolled body effectively re-orders stmts. */
9955 {
9958 "cannot perform implicit CSE when unrolling "
9961 }
9966 /* FORNOW. In some cases can vectorize even if data-type not supported
9967 (e.g. - data copies). */
9969 {
9974 }
9976 /* Check if the load is a part of an interleaving chain. */
9978 {
9980 /* FORNOW */
9987 /* Refuse non-SLP vectorization of SLP-only groups. */
9989 {
9994 }
9996 /* Invalidate assumptions made by dependence analysis when vectorization
9997 on the unrolled body effectively re-orders stmts. */
10002 {
10005 "cannot perform implicit CSE when performing "
10008 }
10009 }
10010 else
10014 {
10018 {
10019 /* In BB vectorization we may not actually use a loaded vector
10020 accessing elements in excess of DR_GROUP_SIZE. */
10031 {
10034 "BB vectorization with gaps at the end of "
10037 }
10038 }
10044 {
10047 vect_location,
10050 }
10051 }
10053 vect_memory_access_type memory_access_type;
10056 poly_int64 poffset;
10057 internal_fn lanes_ifn;
10065 {
10067 {
10073 }
10076 {
10081 }
10085 {
10090 }
10093 {
10098 }
10099 }
10104 {
10106 && mask
10108 mask_vectype))
10109 {
10114 }
10124 mask);
10137 }
10147 /* Transform. */
10153 {
10155 /* If we have versioned for aliasing or the loop doesn't
10156 have any data dependencies that would preclude this,
10157 then we are sure this is a loop invariant load and
10158 thus we can insert it on the preheader edge.
10159 TODO: hoist_defs_of_uses should ideally be computed
10160 once at analysis time, remembered and used in the
10161 transform time. */
10163 && !nested_in_vect_loop
10166 {
10167 enum vect_cost_model_location cost_loc
10172 cost_loc);
10177 "vect_model_load_cost: inside_cost = %d, "
10181 }
10183 {
10193 tree vuse;
10196 else
10201 }
10202 /* These copies are all equivalent. */
10206 else
10207 {
10212 }
10217 else
10218 {
10222 }
10224 }
10228 {
10229 gimple_stmt_iterator incr_gsi;
10231 tree offvar;
10232 tree ivstep;
10233 tree running_off;
10236 /* Checked by get_load_store_type. */
10239 tree dr_offset;
10246 {
10249 }
10250 else
10251 {
10254 }
10257 {
10260 }
10261 else
10262 {
10264 cst_offset
10267 first_stmt_info));
10270 }
10273 {
10281 /* For a load with loop-invariant (but other than power-of-2)
10282 stride (i.e. not a grouped access) like so:
10284 for (i = 0; i < n; i += stride)
10285 ... = array[i];
10287 we generate a new induction variable and new accesses to
10288 form a new vector (or vectors, depending on ncopies):
10290 for (j = 0; ; j += VF*stride)
10291 tmp1 = array[j];
10292 tmp2 = array[j + stride];
10293 ...
10294 vectemp = {tmp1, tmp2, ...}
10295 */
10309 }
10319 {
10321 {
10322 /* First check if vec_init optab supports construction from vector
10323 elts directly. Otherwise avoid emitting a constructor of
10324 vector elements by performing the loads using an integer type
10325 of the same size, constructing a vector of those and then
10326 re-interpreting it as the original vector type. This avoids a
10327 huge runtime penalty due to the general inability to perform
10328 store forwarding from smaller stores to a larger load. */
10329 tree ptype;
10330 tree vtype
10335 {
10340 }
10341 }
10342 else
10343 {
10347 }
10349 }
10350 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10355 {
10356 /* For SLP permutation support we need to load the whole group,
10357 not only the number of vector stmts the permutation result
10358 fits in. */
10360 {
10361 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10362 variable VF. */
10366 }
10367 else
10369 }
10371 unsigned HOST_WIDE_INT
10374 /* For costing some adjacent vector loads, we'd like to cost with
10375 the total number of them once instead of cost each one by one. */
10378 {
10383 {
10385 {
10386 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10387 avoid ICE, see PR110776. */
10390 n_adjacent_loads++;
10391 else
10395 }
10409 {
10410 n_groups++;
10411 /* When doing SLP make sure to not load elements from
10412 the next vector iteration, those will not be accessed
10413 so just use the last element again. See PR107451. */
10415 {
10417 gimple *incr
10422 }
10424 }
10425 }
10428 {
10432 else
10433 {
10439 {
10440 new_stmt
10442 VIEW_CONVERT_EXPR,
10446 gsi);
10447 }
10448 }
10449 }
10452 {
10454 {
10457 else
10459 }
10460 else
10461 {
10465 }
10466 }
10467 }
10469 {
10472 {
10478 }
10479 else
10482 }
10485 {
10493 "vect_model_load_cost: inside_cost = %u, "
10495 inside_cost);
10496 }
10499 }
10507 {
10509 {
10512 }
10513 else
10514 {
10517 }
10518 /* For SLP vectorization we directly vectorize a subchain
10519 without permutation. */
10522 /* For BB vectorization always use the first stmt to base
10523 the data ref pointer on. */
10525 first_stmt_info_for_drptr
10528 /* Check if the chain of loads is already vectorized. */
10530 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10531 ??? But we can only do so if there is exactly one
10532 as we have no way to get at the rest. Leave the CSE
10533 opportunity alone.
10534 ??? With the group load eventually participating
10535 in multiple different permutations (having multiple
10536 slp nodes which refer to the same group) the CSE
10537 is even wrong code. See PR56270. */
10539 {
10542 }
10546 /* VEC_NUM is the number of vect stmts to be created for this group. */
10548 {
10550 /* If an SLP permutation is from N elements to N elements,
10551 and if one vector holds a whole number of N, we can load
10552 the inputs to the permutation in the same way as an
10553 unpermuted sequence. In other cases we need to load the
10554 whole group, not only the number of vector stmts the
10555 permutation result fits in. */
10560 {
10561 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10562 variable VF; see vect_transform_slp_perm_load. */
10567 }
10568 else
10569 {
10571 group_gap_adj
10573 }
10574 }
10575 else
10579 }
10580 else
10581 {
10589 }
10592 vec_loop_masks *loop_masks
10596 vec_loop_lens *loop_lens
10601 /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10602 are some difference here. We cannot enable both the lens and masks
10603 during transform but it is allowed during analysis.
10604 Shouldn't go with length-based approach if fully masked. */
10606 /* The cost_vec is NULL during transfrom. */
10609 /* Targets with store-lane instructions must not require explicit
10610 realignment. vect_supportable_dr_alignment always returns either
10611 dr_aligned or dr_unaligned_supported for masked operations. */
10613 && !mask
10618 /* In case the vectorization factor (VF) is bigger than the number
10619 of elements that we can fit in a vectype (nunits), we have to generate
10620 more than one vector stmt - i.e - we need to "unroll" the
10621 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10622 from one copy of the vector stmt to the next, in the field
10623 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10624 stages to find the correct vector defs to be used when vectorizing
10625 stmts that use the defs of the current stmt. The example below
10626 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10627 need to create 4 vectorized stmts):
10629 before vectorization:
10630 RELATED_STMT VEC_STMT
10631 S1: x = memref - -
10632 S2: z = x + 1 - -
10634 step 1: vectorize stmt S1:
10635 We first create the vector stmt VS1_0, and, as usual, record a
10636 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10637 Next, we create the vector stmt VS1_1, and record a pointer to
10638 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10639 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10640 stmts and pointers:
10641 RELATED_STMT VEC_STMT
10642 VS1_0: vx0 = memref0 VS1_1 -
10643 VS1_1: vx1 = memref1 VS1_2 -
10644 VS1_2: vx2 = memref2 VS1_3 -
10645 VS1_3: vx3 = memref3 - -
10646 S1: x = load - VS1_0
10647 S2: z = x + 1 - -
10648 */
10650 /* In case of interleaving (non-unit grouped access):
10652 S1: x2 = &base + 2
10653 S2: x0 = &base
10654 S3: x1 = &base + 1
10655 S4: x3 = &base + 3
10657 Vectorized loads are created in the order of memory accesses
10658 starting from the access of the first stmt of the chain:
10660 VS1: vx0 = &base
10661 VS2: vx1 = &base + vec_size*1
10662 VS3: vx3 = &base + vec_size*2
10663 VS4: vx4 = &base + vec_size*3
10665 Then permutation statements are generated:
10667 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10668 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10669 ...
10671 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10672 (the order of the data-refs in the output of vect_permute_load_chain
10673 corresponds to the order of scalar stmts in the interleaving chain - see
10674 the documentation of vect_permute_load_chain()).
10675 The generation of permutation stmts and recording them in
10676 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10678 In case of both multiple types and interleaving, the vector loads and
10679 permutation stmts above are created for every copy. The result vector
10680 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10681 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10683 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10684 on a target that supports unaligned accesses (dr_unaligned_supported)
10685 we generate the following code:
10686 p = initial_addr;
10687 indx = 0;
10688 loop {
10689 p = p + indx * vectype_size;
10690 vec_dest = *(p);
10691 indx = indx + 1;
10692 }
10694 Otherwise, the data reference is potentially unaligned on a target that
10695 does not support unaligned accesses (dr_explicit_realign_optimized) -
10696 then generate the following code, in which the data in each iteration is
10697 obtained by two vector loads, one from the previous iteration, and one
10698 from the current iteration:
10699 p1 = initial_addr;
10700 msq_init = *(floor(p1))
10701 p2 = initial_addr + VS - 1;
10702 realignment_token = call target_builtin;
10703 indx = 0;
10704 loop {
10705 p2 = p2 + indx * vectype_size
10706 lsq = *(floor(p2))
10707 vec_dest = realign_load (msq, lsq, realignment_token)
10708 indx = indx + 1;
10709 msq = lsq;
10710 } */
10712 /* If the misalignment remains the same throughout the execution of the
10713 loop, we can create the init_addr and permutation mask at the loop
10714 preheader. Otherwise, it needs to be created inside the loop.
10715 This can only occur when vectorizing memory accesses in the inner-loop
10716 nested within an outer-loop that is being vectorized. */
10721 {
10724 }
10726 bool diff_first_stmt_info
10733 {
10734 /* If we have different first_stmt_info, we can't set up realignment
10735 here, since we can't guarantee first_stmt_info DR has been
10736 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10737 distance from first_stmt_info DR instead as below. */
10739 {
10746 {
10749 size_one_node);
10751 }
10752 }
10753 }
10754 else
10758 offset = (offset
10762 tree bump;
10765 {
10768 }
10770 {
10775 }
10776 else
10777 {
10780 else
10784 }
10789 {
10793 else
10796 }
10800 {
10806 /* For costing some adjacent vector loads, we'd like to cost with
10807 the total number of them once instead of cost each one by one. */
10810 {
10812 {
10813 /* An IFN_LOAD_LANES will load all its vector results,
10814 regardless of which ones we actually need. Account
10815 for the cost of unused results. */
10817 {
10820 do
10821 {
10824 }
10827 {
10830 "vect_model_load_cost: %d "
10832 gaps);
10834 alignment_support_scheme,
10838 }
10839 }
10840 n_adjacent_loads++;
10842 }
10844 /* 1. Create the vector or array pointer update chain. */
10846 dataref_ptr
10850 else
10851 {
10855 }
10872 {
10876 else
10878 signed char biasval
10882 {
10885 }
10886 }
10890 {
10891 /* Emit:
10892 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10893 VEC_MASK, LEN, BIAS). */
10899 }
10901 {
10902 /* Emit:
10903 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10904 VEC_MASK). */
10909 final_mask);
10910 }
10911 else
10912 {
10913 /* Emit:
10914 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10917 }
10923 /* Extract each vector into an SSA_NAME. */
10925 {
10929 }
10931 /* Record the mapping between SSA_NAMEs and statements. */
10934 /* Record that VEC_ARRAY is now dead. */
10940 }
10943 {
10951 "vect_model_load_cost: inside_cost = %u, "
10954 }
10957 }
10960 {
10967 {
10968 /* 1. Create the vector or array pointer update chain. */
10970 {
10975 else
10976 dataref_ptr
10980 }
10982 {
10987 }
10991 {
10996 {
11000 final_mask
11011 }
11013 /* 2. Create the vector-load in the loop. */
11016 {
11018 {
11020 inside_cost
11024 }
11031 {
11033 final_len
11037 else
11038 final_len
11041 signed char biasval
11045 {
11048 }
11049 }
11053 call
11062 else
11069 }
11071 {
11072 /* The builtin decls path for gather is legacy, x86 only. */
11075 {
11077 inside_cost
11081 }
11082 poly_uint64 offset_nunits
11085 {
11086 new_stmt = vect_build_one_gather_load_call
11089 final_mask);
11091 }
11093 {
11094 /* We have a offset vector with half the number of
11095 lanes but the builtins will produce full vectype
11096 data with just the lower lanes filled. */
11097 new_stmt = vect_build_one_gather_load_call
11100 final_mask);
11106 /* now put upper half of final_mask in final_mask low. */
11108 && !SCALAR_INT_MODE_P
11110 {
11117 tree perm_mask = vect_gen_perm_mask_checked
11120 VEC_PERM_EXPR,
11121 final_mask,
11122 final_mask,
11123 perm_mask);
11128 }
11130 {
11132 VEC_UNPACK_HI_EXPR,
11133 final_mask);
11134 final_mask = make_ssa_name
11139 }
11141 new_stmt = vect_build_one_gather_load_call
11143 dataref_ptr,
11145 final_mask);
11151 /* compose low + high. */
11158 tree perm_mask
11161 VEC_PERM_EXPR,
11164 }
11166 {
11167 /* We have a offset vector with double the number of
11168 lanes. Select the low/high part accordingly. */
11171 {
11178 tree perm_mask = vect_gen_perm_mask_checked
11181 VEC_PERM_EXPR,
11182 vec_offset,
11183 vec_offset,
11184 perm_mask);
11189 }
11190 new_stmt = vect_build_one_gather_load_call
11194 }
11195 else
11197 }
11198 else
11199 {
11200 /* Emulated gather-scatter. */
11204 {
11205 /* For emulated gathers N offset vector element
11206 offset add is consumed by the load). */
11210 /* N scalar loads plus gathering them into a
11211 vector. */
11212 inside_cost
11215 inside_cost
11219 }
11220 unsigned HOST_WIDE_INT const_offset_nunits
11226 /* We support offset vectors with more elements
11227 than the data vector for now. */
11228 unsigned HOST_WIDE_INT factor
11231 unsigned elt_offset
11238 {
11241 tree idx
11246 scale);
11258 }
11263 }
11266 /* DATA_REF is null if we've already built the statement. */
11268 {
11271 }
11276 /* Store vector loads in the corresponding SLP_NODE. */
11279 }
11283 }
11290 "vect_model_load_cost: inside_cost = %u, "
11294 }
11298 /* For costing some adjacent vector loads, we'd like to cost with
11299 the total number of them once instead of cost each one by one. */
11302 {
11303 /* 1. Create the vector or array pointer update chain. */
11305 {
11306 bool simd_lane_access_p
11317 {
11320 }
11322 {
11323 dataref_ptr
11327 bump);
11328 /* Adjust the pointer by the difference to first_stmt. */
11329 data_reference_p ptrdr
11331 tree diff
11339 {
11343 alignment_support_scheme,
11346 }
11347 }
11348 else
11349 dataref_ptr
11351 at_loop,
11354 }
11356 {
11360 bump);
11361 else
11364 }
11371 {
11376 {
11390 }
11392 /* 2. Create the vector-load in the loop. */
11394 {
11397 {
11407 {
11408 align
11411 }
11412 else
11417 misalign);
11420 /* Compute IFN when LOOP_LENS or final_mask valid. */
11425 {
11426 opt_machine_mode new_ovmode
11429 unsigned factor
11434 }
11436 {
11441 }
11444 {
11446 {
11447 /* Pass VF value to 'len' argument of
11448 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11450 }
11452 {
11453 /* Pass all ones value to 'mask' argument of
11454 MASK_LEN_LOAD if final_mask is invalid. */
11457 }
11458 }
11460 {
11461 signed char biasval
11465 }
11468 {
11475 bias);
11476 else
11484 /* Need conversion if it's wrapped with VnQI. */
11486 {
11489 tree var
11493 gsi);
11497 }
11498 }
11500 {
11504 final_mask);
11508 }
11509 else
11510 {
11514 unsigned int vect_align
11516 unsigned int scalar_dr_size
11518 /* If there's no peeling for gaps but we have a gap
11519 with slp loads then load the lower half of the
11520 vector only. See get_group_load_store_type for
11521 when we apply this optimization. */
11523 && loop_vinfo
11528 {
11529 tree half_vtype;
11530 new_vtype
11535 }
11536 /* Try to use a single smaller load when we are about
11537 to load excess elements compared to the unrolled
11538 scalar loop.
11539 ??? This should cover the above case as well. */
11542 {
11545 /* DR will be unused. */
11548 tree_to_poly_uint64
11550 /* Aligned access to excess elements is OK if
11551 at least one element is accessed in the
11552 scalar loop. */
11553 ;
11554 else
11555 {
11556 auto remain
11559 /* remain should now be > 0 and < nunits. */
11562 {
11563 tree ptype;
11564 new_vtype
11566 num,
11570 }
11571 /* Else use multiple loads or a masked load? */
11572 }
11573 }
11574 tree offset
11578 ;
11581 {
11582 poly_uint64 gap_offset
11587 }
11589 {
11590 data_ref
11593 ;
11594 else
11598 }
11602 {
11608 gsi);
11611 /* We've computed 'num' above to statically two
11612 or via constant_multiple_p. */
11613 unsigned num
11616 tree_to_poly_uint64
11621 {
11626 }
11627 else
11628 {
11633 }
11638 else
11639 {
11646 vec_dest,
11648 }
11649 }
11650 }
11652 }
11654 {
11664 dr_explicit_realign,
11669 else
11671 // For explicit realign the target alignment should be
11672 // known at compile time.
11673 unsigned HOST_WIDE_INT align
11680 data_ref
11694 bump);
11700 else
11704 data_ref
11707 }
11709 {
11714 else
11716 // We should only be doing this if we know the target
11717 // alignment at compile time.
11718 unsigned HOST_WIDE_INT align
11728 }
11731 }
11733 /* One common place to cost the above vect load for different
11734 alignment support schemes. */
11736 {
11737 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11738 only need to take care of the first stmt, whose
11739 stmt_info is first_stmt_info, vec_num iterating on it
11740 will cover the cost for the remaining, it's consistent
11741 with transforming. For the prologue cost for realign,
11742 we only need to count it once for the whole group. */
11749 {
11750 /* Leave realign cases alone to keep them simple. */
11758 else
11759 n_adjacent_loads++;
11760 }
11761 }
11762 else
11763 {
11765 /* DATA_REF is null if we've already built the statement. */
11767 {
11770 }
11774 }
11776 /* 3. Handle explicit realignment if necessary/supported.
11777 Create in loop:
11778 vec_dest = realign_load (msq, lsq, realignment_token) */
11782 {
11794 {
11798 UNKNOWN_LOCATION);
11800 }
11801 }
11804 {
11808 else
11809 {
11814 }
11815 }
11817 /* Collect vector loads and later create their permutation in
11818 vect_transform_grouped_load (). */
11822 /* Store vector loads in the corresponding SLP_NODE. */
11826 /* With SLP permutation we load the gaps as well, without
11827 we need to skip the gaps after we manage to fully load
11828 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11832 && !slp_perm
11834 {
11835 poly_wide_int bump_val
11844 }
11845 }
11846 /* Bump the vector pointer to account for a gap or for excess
11847 elements loaded for a permuted SLP load. */
11851 {
11852 poly_wide_int bump_val
11859 }
11865 {
11867 /* For SLP we know we've seen all possible uses of dr_chain so
11868 direct vect_transform_slp_perm_load to DCE the unused parts.
11869 ??? This is a hack to prevent compile-time issues as seen
11870 in PR101120 and friends. */
11872 {
11877 }
11878 else
11879 {
11884 }
11885 }
11886 else
11887 {
11889 {
11891 /* We assume that the cost of a single load-lanes instruction
11892 is equivalent to the cost of DR_GROUP_SIZE separate loads.
11893 If a grouped access is instead being provided by a
11894 load-and-permute operation, include the cost of the
11895 permutes. */
11897 {
11898 /* Uses an even and odd extract operations or shuffle
11899 operations for each needed permute. */
11907 "vect_model_load_cost:"
11909 group_size);
11910 }
11912 {
11916 }
11917 }
11920 }
11922 }
11927 {
11938 "vect_model_load_cost: inside_cost = %u, "
11941 }
11944 }
11946 /* Function vect_is_simple_cond.
11948 Input:
11949 LOOP - the loop that is being vectorized.
11950 COND - Condition that is checked for simple use.
11952 Output:
11953 *COMP_VECTYPE - the vector type for the comparison.
11954 *DTS - The def types for the arguments of the comparison
11956 Returns whether a COND can be vectorized. Checks whether
11957 condition operands are supportable using vec_is_simple_use. */
11959 static bool
11963 {
11966 slp_tree slp_op;
11968 /* Mask case. */
11971 {
11974 || !*comp_vectype
11978 }
11987 {
11991 }
11995 else
11999 {
12003 }
12007 else
12016 /* Invariant comparison. */
12018 {
12022 else
12023 {
12024 /* If we can widen the comparison to match vectype do so. */
12026 && !slp_node
12029 scalar_type = build_nonstandard_integer_type
12032 slp_node);
12033 }
12034 }
12037 }
12039 /* vectorizable_condition.
12041 Check if STMT_INFO is conditional modify expression that can be vectorized.
12042 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12043 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
12044 at GSI.
12046 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12048 Return true if STMT_INFO is vectorizable in this way. */
12050 static bool
12055 {
12063 tree vec_compare;
12064 tree new_temp;
12079 tree vec_cmp_type;
12085 /* Is vectorizable conditional operation? */
12097 bool for_reduction
12100 {
12108 }
12109 else
12110 {
12113 }
12119 {
12122 }
12123 else
12124 {
12127 }
12163 {
12166 }
12168 /* For conditional reductions, the "then" value needs to be the candidate
12169 value calculated by this iteration while the "else" value needs to be
12170 the result carried over from previous iterations. If the COND_EXPR
12171 is the other way around, we need to swap it. */
12174 {
12177 else
12178 {
12183 else
12184 {
12186 /* Make sure we don't accidentally use the old condition. */
12188 }
12189 }
12191 }
12194 {
12195 /* Boolean values may have another representation in vectors
12196 and therefore we prefer bit operations over comparison for
12197 them (which also works for scalar masks). We store opcodes
12198 to use in bitop1 and bitop2. Statement is vectorized as
12199 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12200 depending on bitop1 and bitop2 arity. */
12202 {
12230 }
12232 }
12237 {
12242 }
12245 {
12247 {
12249 optab optab;
12256 {
12258 optab_default);
12261 }
12262 }
12266 /* Count one reduction-like operation per vector. */
12269 && (masked
12271 cond_code)
12273 ERROR_MARK))))
12277 && (!vect_maybe_update_slp_op_vectype
12280 && !vect_maybe_update_slp_op_vectype
12284 {
12289 }
12293 {
12295 {
12301 else
12305 }
12306 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12308 {
12311 "conditional reduction prevents the use"
12314 }
12315 }
12321 }
12323 /* Transform. */
12325 /* Handle def. */
12332 /* See whether another part of the vectorized code applies a loop
12333 mask to the condition, or to its inverse. */
12338 {
12341 }
12343 {
12346 else
12347 {
12351 else
12352 {
12357 {
12361 }
12362 else
12363 {
12364 /* Try the inverse of the current mask. We check if the
12365 inverse mask is live and if so we generate a negate of
12366 the current mask such that we still honor NaNs. */
12370 {
12375 }
12376 }
12377 }
12378 }
12379 }
12381 /* Handle cond expr. */
12386 reduction_type != EXTRACT_LAST_REDUCTION
12388 else
12393 reduction_type != EXTRACT_LAST_REDUCTION
12396 /* Arguments are ready. Create the new vector stmt. */
12398 {
12408 else
12409 {
12412 {
12417 }
12418 else
12419 {
12424 vec_cond_rhs);
12425 else
12426 new_stmt
12428 vec_cond_rhs);
12434 {
12435 /* Instead of doing ~x ? y : z do x ? z : y. */
12438 }
12439 else
12440 {
12443 new_stmt
12445 else
12446 new_stmt
12451 }
12452 }
12453 }
12455 /* If we decided to apply a loop mask to the result of the vector
12456 comparison, AND the comparison with the mask now. Later passes
12457 should then be able to reuse the AND results between mulitple
12458 vector statements.
12460 For example:
12461 for (int i = 0; i < 100; ++i)
12462 x[i] = y[i] ? z[i] : 10;
12464 results in following optimized GIMPLE:
12466 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12467 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12468 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12469 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12470 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12471 vect_iftmp.11_47, { 10, ... }>;
12473 instead of using a masked and unmasked forms of
12474 vec != { 0, ... } (masked in the MASK_LOAD,
12475 unmasked in the VEC_COND_EXPR). */
12477 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12478 in cases where that's necessary. */
12482 {
12484 {
12487 vec_compare);
12490 }
12493 {
12496 BIT_NOT_EXPR,
12497 vec_compare);
12500 }
12504 {
12506 {
12509 signed char biasval
12512 }
12513 else
12514 {
12517 }
12518 }
12520 {
12521 tree loop_mask
12525 gassign *g
12527 loop_mask);
12530 }
12531 }
12535 {
12539 new_stmt = gimple_build_call_internal
12542 else
12543 new_stmt = gimple_build_call_internal
12545 vec_then_clause);
12550 else
12551 {
12552 /* In this case we're moving the definition to later in the
12553 block. That doesn't matter because the only uses of the
12554 lhs are in phi statements. */
12558 }
12559 }
12560 else
12561 {
12566 }
12569 else
12571 }
12582 }
12584 /* Helper of vectorizable_comparison.
12586 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12587 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12588 comparison, put it in VEC_STMT, and insert it at GSI.
12590 Return true if STMT_INFO is vectorizable in this way. */
12592 static bool
12597 {
12601 tree new_temp;
12605 poly_uint64 nunits;
12612 tree mask_type;
12626 else
12650 /* Invariant comparison. */
12652 {
12655 else
12657 slp_node);
12660 }
12664 /* Can't compare mask and non-mask types. */
12669 /* Boolean values may have another representation in vectors
12670 and therefore we prefer bit operations over comparison for
12671 them (which also works for scalar masks). We store opcodes
12672 to use in bitop1 and bitop2. Statement is vectorized as
12673 BITOP2 (rhs1 BITOP1 rhs2) or
12674 rhs1 BITOP2 (BITOP1 rhs2)
12675 depending on bitop1 and bitop2 arity. */
12678 {
12680 {
12683 }
12685 {
12688 }
12690 {
12694 }
12696 {
12700 }
12701 else
12702 {
12706 }
12707 }
12710 {
12712 {
12715 }
12716 else
12717 {
12719 optab optab;
12726 {
12730 }
12731 }
12733 /* Put types on constant and invariant SLP children. */
12737 {
12742 }
12748 }
12750 /* Transform. */
12752 /* Handle def. */
12763 /* Arguments are ready. Create the new vector stmt. */
12765 {
12771 else
12774 {
12778 }
12779 else
12780 {
12783 else
12785 vec_rhs2);
12788 {
12792 else
12794 new_temp);
12796 }
12797 }
12800 else
12802 }
12811 }
12813 /* vectorizable_comparison.
12815 Check if STMT_INFO is comparison expression that can be vectorized.
12816 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12817 comparison, put it in VEC_STMT, and insert it at GSI.
12819 Return true if STMT_INFO is vectorizable in this way. */
12821 static bool
12826 {
12849 }
12851 /* Check to see if the current early break given in STMT_INFO is valid for
12852 vectorization. */
12854 static bool
12858 {
12875 slp_tree slp_op0;
12876 tree op0;
12880 {
12885 }
12895 else
12901 /* Now build the new conditional. Pattern gimple_conds get dropped during
12902 codegen so we must replace the original insn. */
12905 /* When vectorizing we assume that if the branch edge is taken that we're
12906 exiting the loop. This is not however always the case as the compiler will
12907 rewrite conditions to always be a comparison against 0. To do this it
12908 sometimes flips the edges. This is fine for scalar, but for vector we
12909 then have to flip the test, as we're still assuming that if you take the
12910 branch edge that we found the exit condition. i.e. we need to know whether
12911 we are generating a `forall` or an `exist` condition. */
12922 {
12927 }
12929 /* Analyze only. */
12931 {
12933 {
12936 "can't vectorize early exit because the "
12937 "target doesn't support flag setting vector "
12940 }
12944 {
12947 "can't vectorize early exit because the "
12948 "target does not support boolean vector %s "
12951 vectype);
12953 }
12960 {
12962 OPTIMIZE_FOR_SPEED))
12964 else
12966 }
12970 }
12972 /* Tranform. */
12992 else
12993 {
12998 }
13000 /* Determine if we need to reduce the final value. */
13002 {
13003 /* We build the reductions in a way to maintain as much parallelism as
13004 possible. */
13007 /* Mask the statements as we queue them up. Normally we loop over
13008 vec_num, but since we inspect the exact results of vectorization
13009 we don't need to and instead can just use the stmts themselves. */
13012 {
13013 tree stmt_mask
13015 i);
13016 stmt_mask
13020 }
13021 else
13025 {
13033 }
13034 }
13035 else
13036 {
13039 {
13040 tree mask
13044 }
13045 }
13054 else
13061 }
13063 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13064 can handle all live statements in the node. Otherwise return true
13065 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13066 VEC_STMT_P is as for vectorizable_live_operation. */
13068 static bool
13073 {
13076 {
13077 stmt_vec_info slp_stmt_info;
13080 {
13082 || (loop_vinfo
13090 }
13091 }
13101 }
13103 /* Make sure the statement is vectorizable. */
13105 opt_result
13110 {
13114 gimple_seq pattern_def_seq;
13122 "not vectorized:"
13129 {
13130 gimple_stmt_iterator si;
13133 {
13134 stmt_vec_info pattern_def_stmt_info
13138 {
13139 /* Analyze def stmt of STMT if it's a pattern stmt. */
13145 opt_result res
13148 cost_vec);
13151 }
13152 }
13153 }
13155 /* Skip stmts that do not need to be vectorized. In loops this is expected
13156 to include:
13157 - the COND_EXPR which is the loop exit condition
13158 - any LABEL_EXPRs in the loop
13159 - computations that are used only for array indexing or loop control.
13160 In basic blocks we only analyze statements that are a part of some SLP
13161 instance, therefore, all the statements are relevant.
13163 Pattern statement needs to be analyzed instead of the original statement
13164 if the original statement is not relevant. Otherwise, we analyze both
13165 statements. In basic blocks we are called from some SLP instance
13166 traversal, don't analyze pattern stmts instead, the pattern stmts
13167 already will be part of SLP instance. */
13172 {
13174 && pattern_stmt_info
13177 {
13178 /* Analyze PATTERN_STMT instead of the original stmt. */
13184 }
13185 else
13186 {
13191 }
13192 }
13195 && pattern_stmt_info
13198 {
13199 /* Analyze PATTERN_STMT too. */
13205 opt_result res
13210 }
13213 {
13238 }
13245 {
13251 }
13254 {
13259 }
13265 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13266 -mveclibabi= takes preference over library functions with
13267 the simd attribute. */
13270 cost_vec)
13287 cost_vec)
13293 cost_vec));
13294 else
13295 {
13301 cost_vec)
13307 cost_vec)
13315 cost_vec)
13318 cost_vec));
13320 }
13327 "not vectorized:"
13331 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13332 need extra handling, except for vectorizable reductions. */
13340 "not vectorized:"
13345 }
13348 /* Function vect_transform_stmt.
13350 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13352 bool
13356 {
13368 {
13380 NULL);
13392 NULL);
13404 NULL);
13410 && !slp_node
13413 /* In case of interleaving, the whole chain is vectorized when the
13414 last store in the chain is reached. Store stmts before the last
13415 one are skipped, and there vec_stmt_info shouldn't be freed
13416 meanwhile. */
13417 ;
13418 else
13419 {
13424 }
13486 {
13491 }
13493 }
13499 {
13500 /* Handle stmts whose DEF is used outside the loop-nest that is
13501 being vectorized. */
13505 }
13511 }
13514 /* Remove a group of stores (for SLP or interleaving), free their
13515 stmt_vec_info. */
13517 void
13519 {
13523 {
13526 /* Free the attached stmt_vec_info and remove the stmt. */
13529 }
13530 }
13532 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13533 elements of type SCALAR_TYPE, or null if the target doesn't support
13534 such a type.
13536 If NUNITS is zero, return a vector type that contains elements of
13537 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13539 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13540 for this vectorization region and want to "autodetect" the best choice.
13541 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13542 and we want the new type to be interoperable with it. PREVAILING_MODE
13543 in this case can be a scalar integer mode or a vector mode; when it
13544 is a vector mode, the function acts like a tree-level version of
13545 related_vector_mode. */
13547 tree
13550 {
13552 scalar_mode inner_mode;
13553 machine_mode simd_mode;
13554 tree vectype;
13565 /* Interoperability between modes requires one to be a constant multiple
13566 of the other, so that the number of vectors required for each operation
13567 is a compile-time constant. */
13575 /* For vector types of elements whose mode precision doesn't
13576 match their types precision we use a element type of mode
13577 precision. The vectorization routines will have to make sure
13578 they support the proper result truncation/extension.
13579 We also make sure to build vector types with INTEGER_TYPE
13580 component type only. */
13587 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13588 When the component mode passes the above test simply use a type
13589 corresponding to that mode. The theory is that any use that
13590 would cause problems with this will disable vectorization anyway. */
13595 /* We can't build a vector type of elements with alignment bigger than
13596 their size. */
13601 /* If we felt back to using the mode fail if there was
13602 no scalar type for it. */
13606 /* If no prevailing mode was supplied, use the mode the target prefers.
13607 Otherwise lookup a vector mode based on the prevailing mode. */
13609 {
13613 {
13614 /* Traditional behavior is not to take the integer mode
13615 literally, but simply to use it as a way of determining
13616 the vector size. It is up to mode_for_vector to decide
13617 what the TYPE_MODE should be.
13619 Note that nunits == 1 is allowed in order to support single
13620 element vector types. */
13624 }
13625 }
13629 {
13630 /* Fall back to using mode_for_vector, mostly in the hope of being
13631 able to use an integer mode. */
13638 }
13642 /* In cases where the mode was chosen by mode_for_vector, check that
13643 the target actually supports the chosen mode, or that it at least
13644 allows the vector mode to be replaced by a like-sized integer. */
13649 /* Re-attach the address-space qualifier if we canonicalized the scalar
13650 type. */
13652 return build_qualified_type
13656 }
13658 /* Function get_vectype_for_scalar_type.
13660 Returns the vector type corresponding to SCALAR_TYPE as supported
13661 by the target. If GROUP_SIZE is nonzero and we're performing BB
13662 vectorization, make sure that the number of elements in the vector
13663 is no bigger than GROUP_SIZE. */
13665 tree
13668 {
13669 /* For BB vectorization, we should always have a group size once we've
13670 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13671 are tentative requests during things like early data reference
13672 analysis and pattern recognition. */
13675 else
13679 scalar_type);
13683 /* Register the natural choice of vector type, before the group size
13684 has been applied. */
13688 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13689 try again with an explicit number of elements. */
13691 && group_size
13693 {
13694 /* Start with the biggest number of units that fits within
13695 GROUP_SIZE and halve it until we find a valid vector type.
13696 Usually either the first attempt will succeed or all will
13697 fail (in the latter case because GROUP_SIZE is too small
13698 for the target), but it's possible that a target could have
13699 a hole between supported vector types.
13701 If GROUP_SIZE is not a power of 2, this has the effect of
13702 trying the largest power of 2 that fits within the group,
13703 even though the group is not a multiple of that vector size.
13704 The BB vectorizer will then try to carve up the group into
13705 smaller pieces. */
13707 do
13708 {
13712 }
13714 }
13717 }
13719 /* Return the vector type corresponding to SCALAR_TYPE as supported
13720 by the target. NODE, if nonnull, is the SLP tree node that will
13721 use the returned vector type. */
13723 tree
13725 {
13730 }
13732 /* Function get_mask_type_for_scalar_type.
13734 Returns the mask type corresponding to a result of comparison
13735 of vectors of specified SCALAR_TYPE as supported by target.
13736 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13737 make sure that the number of elements in the vector is no bigger
13738 than GROUP_SIZE. */
13740 tree
13743 {
13750 }
13752 /* Function get_mask_type_for_scalar_type.
13754 Returns the mask type corresponding to a result of comparison
13755 of vectors of specified SCALAR_TYPE as supported by target.
13756 NODE, if nonnull, is the SLP tree node that will use the returned
13757 vector type. */
13759 tree
13761 slp_tree node)
13762 {
13769 }
13771 /* Function get_same_sized_vectype
13773 Returns a vector type corresponding to SCALAR_TYPE of size
13774 VECTOR_TYPE if supported by the target. */
13776 tree
13778 {
13782 poly_uint64 nunits;
13789 }
13791 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13792 would not change the chosen vector modes. */
13794 bool
13796 {
13803 }
13805 /* Function vect_is_simple_use.
13807 Input:
13808 VINFO - the vect info of the loop or basic block that is being vectorized.
13809 OPERAND - operand in the loop or bb.
13810 Output:
13811 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13812 case OPERAND is an SSA_NAME that is defined in the vectorizable region
13813 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13814 the definition could be anywhere in the function
13815 DT - the type of definition
13817 Returns whether a stmt with OPERAND can be vectorized.
13818 For loops, supportable operands are constants, loop invariants, and operands
13819 that are defined by the current iteration of the loop. Unsupportable
13820 operands are those that are defined by a previous iteration of the loop (as
13821 is the case in reduction/induction computations).
13822 For basic blocks, supportable operands are constants and bb invariants.
13823 For now, operands defined outside the basic block are not supported. */
13825 bool
13828 {
13836 {
13842 else
13844 }
13854 else
13855 {
13860 else
13861 {
13867 }
13870 }
13873 {
13876 {
13910 }
13911 }
13914 {
13919 }
13922 }
13924 /* Function vect_is_simple_use.
13926 Same as vect_is_simple_use but also determines the vector operand
13927 type of OPERAND and stores it to *VECTYPE. If the definition of
13928 OPERAND is vect_uninitialized_def, vect_constant_def or
13929 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
13930 is responsible to compute the best suited vector type for the
13931 scalar operand. */
13933 bool
13937 {
13938 stmt_vec_info def_stmt_info;
13948 /* Now get a vector type if the def is internal, otherwise supply
13949 NULL_TREE and leave it up to the caller to figure out a proper
13950 type for the use stmt. */
13957 {
13963 }
13968 else
13972 }
13974 /* Function vect_is_simple_use.
13976 Same as vect_is_simple_use but determines the operand by operand
13977 position OPERAND from either STMT or SLP_NODE, filling in *OP
13978 and *SLP_DEF (when SLP_NODE is not NULL). */
13980 bool
13985 {
13987 {
13992 {
13995 }
13996 else
13997 {
14003 }
14004 }
14005 else
14006 {
14009 {
14012 {
14015 else
14017 }
14020 else
14022 }
14027 else
14030 }
14031 }
14033 /* If OP is not NULL and is external or constant update its vector
14034 type with VECTYPE. Returns true if successful or false if not,
14035 for example when conflicting vector types are present. */
14037 bool
14039 {
14044 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
14045 should be handled by patters. Allow vect_constant_def for now. */
14051 }
14053 /* Function supportable_widening_operation
14055 Check whether an operation represented by the code CODE is a
14056 widening operation that is supported by the target platform in
14057 vector form (i.e., when operating on arguments of type VECTYPE_IN
14058 producing a result of type VECTYPE_OUT).
14060 Widening operations we currently support are NOP (CONVERT), FLOAT,
14061 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
14062 are supported by the target platform either directly (via vector
14063 tree-codes), or via target builtins.
14065 Output:
14066 - CODE1 and CODE2 are codes of vector operations to be used when
14067 vectorizing the operation, if available.
14068 - MULTI_STEP_CVT determines the number of required intermediate steps in
14069 case of multi-step conversion (like char->short->int - in that case
14070 MULTI_STEP_CVT will be 1).
14071 - INTERM_TYPES contains the intermediate type required to perform the
14072 widening operation (short in the above example). */
14074 bool
14076 code_helper code,
14077 stmt_vec_info stmt_info,
14083 {
14086 machine_mode vec_mode;
14102 {
14104 /* Don't set c1 and c2 if code is not a tree_code. */
14108 /* The result of a vectorized widening operation usually requires
14109 two vectors (because the widened results do not fit into one vector).
14110 The generated vector results would normally be expected to be
14111 generated in the same order as in the original scalar computation,
14112 i.e. if 8 results are generated in each vector iteration, they are
14113 to be organized as follows:
14114 vect1: [res1,res2,res3,res4],
14115 vect2: [res5,res6,res7,res8].
14117 However, in the special case that the result of the widening
14118 operation is used in a reduction computation only, the order doesn't
14119 matter (because when vectorizing a reduction we change the order of
14120 the computation). Some targets can take advantage of this and
14121 generate more efficient code. For example, targets like Altivec,
14122 that support widen_mult using a sequence of {mult_even,mult_odd}
14123 generate the following vectors:
14124 vect1: [res1,res3,res5,res7],
14125 vect2: [res2,res4,res6,res8].
14127 When vectorizing outer-loops, we execute the inner-loop sequentially
14128 (each vectorized inner-loop iteration contributes to VF outer-loop
14129 iterations in parallel). We therefore don't allow to change the
14130 order of the computation in the inner-loop during outer-loop
14131 vectorization. */
14132 /* TODO: Another case in which order doesn't *really* matter is when we
14133 widen and then contract again, e.g. (short)((int)x * y >> 8).
14134 Normally, pack_trunc performs an even/odd permute, whereas the
14135 repack from an even/odd expansion would be an interleave, which
14136 would be significantly simpler for e.g. AVX2. */
14137 /* In any case, in order to avoid duplicating the code below, recurse
14138 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
14139 are properly set up for the caller. If we fail, we'll continue with
14140 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
14148 interm_types))
14149 {
14150 /* Elements in a vector with vect_used_by_reduction property cannot
14151 be reordered if the use chain with this property does not have the
14152 same operation. One such an example is s += a * b, where elements
14153 in a and b cannot be reordered. Here we check if the vector defined
14154 by STMT is only directly used in the reduction statement. */
14160 }
14176 /* Support the recursion induced just above. */
14186 CASE_CONVERT:
14203 }
14209 {
14210 /* The signedness is determined from output operand. */
14213 }
14219 {
14220 /* If the input and result modes are the same, a different optab
14221 is needed where we pass in the number of units in vectype. */
14224 }
14228 {
14229 /* If this is an internal fn then we must check whether the target
14230 supports either a low-high split or an even-odd split. */
14240 /* If we don't support low-high, then check for even-odd. */
14243 || !optab2
14245 {
14251 }
14252 }
14254 {
14256 {
14257 /* The signedness is determined from output operand. */
14260 }
14266 {
14267 /* If the input and result modes are the same, a different optab
14268 is needed where we pass in the number of units in vectype. */
14271 }
14272 else
14273 {
14276 }
14279 }
14291 {
14294 /* For scalar masks we may have different boolean
14295 vector types having the same QImode. Thus we
14296 add additional check for elements number. */
14300 }
14302 /* Check if it's a multi-step conversion that can be done using intermediate
14303 types. */
14311 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14312 intermediate steps in promotion sequence. We try
14313 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14314 not. */
14317 {
14320 intermediate_type
14323 {
14324 tree intermediate_element_type
14327 intermediate_type
14329 intermediate_mode);
14330 }
14331 else
14332 intermediate_type
14340 {
14341 /* If the input and result modes are the same, a different optab
14342 is needed where we pass in the number of units in vectype. */
14345 }
14346 else
14347 {
14350 }
14368 {
14374 }
14378 }
14382 }
14385 /* Function supportable_narrowing_operation
14387 Check whether an operation represented by the code CODE is a
14388 narrowing operation that is supported by the target platform in
14389 vector form (i.e., when operating on arguments of type VECTYPE_IN
14390 and producing a result of type VECTYPE_OUT).
14392 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14393 and FLOAT. This function checks if these operations are supported by
14394 the target platform directly via vector tree-codes.
14396 Output:
14397 - CODE1 is the code of a vector operation to be used when
14398 vectorizing the operation, if available.
14399 - MULTI_STEP_CVT determines the number of required intermediate steps in
14400 case of multi-step conversion (like int->short->char - in that case
14401 MULTI_STEP_CVT will be 1).
14402 - INTERM_TYPES contains the intermediate type required to perform the
14403 narrowing operation (short in the above example). */
14405 bool
14410 {
14411 machine_mode vec_mode;
14428 {
14429 CASE_CONVERT:
14437 else
14443 /* The signedness is determined from output operand. */
14454 }
14466 {
14469 /* For scalar masks we may have different boolean
14470 vector types having the same QImode. Thus we
14471 add additional check for elements number. */
14475 }
14480 /* Check if it's a multi-step conversion that can be done using intermediate
14481 types. */
14486 else
14489 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14490 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14491 costly than signed. */
14493 {
14496 intermediate_type
14498 interm_optab
14504 {
14508 }
14509 }
14511 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14512 intermediate steps in promotion sequence. We try
14513 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14516 {
14519 intermediate_type
14521 else
14522 intermediate_type
14530 else
14531 interm_optab
14533 optab_default);
14545 {
14551 }
14556 }
14560 }
14562 /* Generate and return a vector mask of MASK_TYPE such that
14563 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14564 Add the statements to SEQ. */
14566 tree
14569 {
14573 OPTIMIZE_FOR_SPEED));
14577 tree tmp;
14580 else
14585 }
14587 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14588 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14590 tree
14592 tree end_index)
14593 {
14596 }
14598 /* Try to compute the vector types required to vectorize STMT_INFO,
14599 returning true on success and false if vectorization isn't possible.
14600 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14601 take sure that the number of elements in the vectors is no bigger
14602 than GROUP_SIZE.
14604 On success:
14606 - Set *STMT_VECTYPE_OUT to:
14607 - NULL_TREE if the statement doesn't need to be vectorized;
14608 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14610 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14611 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14612 statement does not help to determine the overall number of units. */
14614 opt_result
14619 {
14622 /* For BB vectorization, we should always have a group size once we've
14623 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14624 are tentative requests during things like early data reference
14625 analysis and pattern recognition. */
14628 else
14635 /* Allow vector conditionals through here. */
14637 /* MASK_STORE has no lhs, but is ok. */
14639 {
14641 {
14642 /* Ignore calls with no lhs. These must be calls to
14643 #pragma omp simd functions, and what vectorization factor
14644 it really needs can't be determined until
14645 vectorizable_simd_clone_call. */
14650 }
14654 }
14656 tree vectype;
14659 {
14664 }
14666 {
14675 }
14676 else
14677 {
14678 /* If we got here with a gcond it means that the target had no available vector
14679 mode for the scalar type. We can't vectorize so abort. */
14682 "not vectorized:"
14684 scalar_type);
14690 else
14694 {
14697 "get vectype for scalar type (group size %d):"
14699 else
14702 }
14706 "not vectorized:"
14708 scalar_type);
14712 }
14717 stmt);
14721 /* Don't try to compute scalar types if the stmt produces a boolean
14722 vector; use the existing vector type instead. */
14725 {
14726 /* The number of units is set according to the smallest scalar
14727 type (or the largest vector size, but we only support one
14728 vector size per vectorization). */
14732 {
14736 scalar_type);
14738 group_size);
14742 scalar_type);
14745 nunits_vectype);
14746 }
14747 }
14752 "Not vectorized: Incompatible number "
14757 {
14761 }
14765 }
14767 /* Generate and return statement sequence that sets vector length LEN that is:
14769 min_of_start_and_end = min (START_INDEX, END_INDEX);
14770 left_len = END_INDEX - min_of_start_and_end;
14771 rhs = min (left_len, LEN_LIMIT);
14772 LEN = rhs;
14774 Note: the cost of the code generated by this function is modeled
14775 by vect_estimate_min_profitable_iters, so changes here may need
14776 corresponding changes there. */
14778 gimple_seq
14780 {
14792 }