| blob | 337b24c51f953667ad4146002052ce40f34b7529 |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2023 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
61 /* For lang_hooks.types.type_for_mode. */
64 /* Return the vectorized type for the given statement. */
66 tree
68 {
70 }
72 /* Return TRUE iff the given statement is in an inner loop relative to
73 the loop being vectorized. */
74 bool
76 {
88 }
90 /* Record the cost of a statement, either by directly informing the
91 target model or by saving it in a vector for later processing.
92 Return a preliminary estimate of the statement's cost. */
94 static unsigned
100 {
108 stmt_info_for_cost si
114 }
116 unsigned
121 {
124 }
126 unsigned
131 {
134 }
136 unsigned
140 {
145 }
147 /* Return a variable of type ELEM_TYPE[NELEMS]. */
149 static tree
151 {
154 }
156 /* ARRAY is an array of vectors created by create_vector_array.
157 Return an SSA_NAME for the vector in index N. The reference
158 is part of the vectorization of STMT_INFO and the vector is associated
159 with scalar destination SCALAR_DEST. */
161 static tree
165 {
182 }
184 /* ARRAY is an array of vectors created by create_vector_array.
185 Emit code to store SSA_NAME VECT in index N of the array.
186 The store is part of the vectorization of STMT_INFO. */
188 static void
192 {
193 tree array_ref;
202 }
204 /* PTR is a pointer to an array of type TYPE. Return a representation
205 of *PTR. The memory reference replaces those in FIRST_DR
206 (and its group). */
208 static tree
210 {
211 tree mem_ref;
214 /* Arrays have the same alignment as their type. */
217 }
219 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
220 Emit the clobber before *GSI. */
222 static void
225 {
229 }
231 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
233 /* Function vect_mark_relevant.
235 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
237 static void
240 {
249 /* If this stmt is an original stmt in a pattern, we might need to mark its
250 related pattern stmt instead of the original stmt. However, such stmts
251 may have their own uses that are not in any pattern, in such cases the
252 stmt itself should be marked. */
254 {
255 /* This is the last stmt in a sequence that was detected as a
256 pattern that can potentially be vectorized. Don't mark the stmt
257 as relevant/live because it's not going to be vectorized.
258 Instead mark the pattern-stmt that replaces it. */
262 "last stmt in pattern. don't mark"
272 {
275 "vec_stmt_relevant_p: forcing live pattern stmt "
278 }
284 }
292 {
297 }
300 }
303 /* Function is_simple_and_all_uses_invariant
305 Return true if STMT_INFO is simple and all uses of it are invariant. */
307 bool
309 loop_vec_info loop_vinfo)
310 {
311 tree op;
312 ssa_op_iter iter;
319 {
323 {
328 }
332 }
334 }
336 /* Function vect_stmt_relevant_p.
338 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
339 is "relevant for vectorization".
341 A stmt is considered "relevant for vectorization" if:
342 - it has uses outside the loop.
343 - it has vdefs (it alters memory).
344 - control stmts in the loop (except for the exit condition).
346 CHECKME: what other side effects would the vectorizer allow? */
348 static bool
351 {
353 ssa_op_iter op_iter;
354 imm_use_iterator imm_iter;
355 use_operand_p use_p;
356 def_operand_p def_p;
361 /* cond stmt other than loop exit cond. */
366 /* changing memory. */
370 {
375 }
377 /* uses outside the loop. */
379 {
381 {
384 {
392 /* We expect all such uses to be in the loop exit phis
393 (because of loop closed form) */
398 }
399 }
400 }
404 {
409 }
412 }
415 /* Function exist_non_indexing_operands_for_use_p
417 USE is one of the uses attached to STMT_INFO. Check if USE is
418 used in STMT_INFO for anything other than indexing an array. */
420 static bool
422 {
423 tree operand;
425 /* USE corresponds to some operand in STMT. If there is no data
426 reference in STMT, then any operand that corresponds to USE
427 is not indexing an array. */
431 /* STMT has a data_ref. FORNOW this means that its of one of
432 the following forms:
433 -1- ARRAY_REF = var
434 -2- var = ARRAY_REF
435 (This should have been verified in analyze_data_refs).
437 'var' in the second case corresponds to a def, not a use,
438 so USE cannot correspond to any operands that are not used
439 for array indexing.
441 Therefore, all we need to check is if STMT falls into the
442 first case, and whether var corresponds to USE. */
446 {
449 {
462 }
464 }
476 }
479 /*
480 Function process_use.
482 Inputs:
483 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
484 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
485 that defined USE. This is done by calling mark_relevant and passing it
486 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
487 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
488 be performed.
490 Outputs:
491 Generally, LIVE_P and RELEVANT are used to define the liveness and
492 relevance info of the DEF_STMT of this USE:
493 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
494 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
495 Exceptions:
496 - case 1: If USE is used only for address computations (e.g. array indexing),
497 which does not need to be directly vectorized, then the liveness/relevance
498 of the respective DEF_STMT is left unchanged.
499 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
500 we skip DEF_STMT cause it had already been processed.
501 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
502 "relevant" will be modified accordingly.
504 Return true if everything is as expected. Return false otherwise. */
506 static opt_result
510 {
511 stmt_vec_info dstmt_vinfo;
514 /* case 1: we are only interested in uses that need to be vectorized. Uses
515 that are used for address computation are not considered relevant. */
521 "not vectorized:"
530 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
531 We have to force the stmt live since the epilogue loop needs it to
532 continue computing the reduction. */
538 {
544 }
546 /* case 3a: outer-loop stmt defining an inner-loop stmt:
547 outer-loop-header-bb:
548 d = dstmt_vinfo
549 inner-loop:
550 stmt # use (d)
551 outer-loop-tail-bb:
552 ... */
554 {
560 {
581 }
582 }
584 /* case 3b: inner-loop stmt defining an outer-loop stmt:
585 outer-loop-header-bb:
586 ...
587 inner-loop:
588 d = dstmt_vinfo
589 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
590 stmt # use (d) */
592 {
598 {
616 }
617 }
618 /* We are also not interested in uses on loop PHI backedges that are
619 inductions. Otherwise we'll needlessly vectorize the IV increment
620 and cause hybrid SLP for SLP inductions. Unless the PHI is live
621 of course. */
628 {
633 }
638 }
641 /* Function vect_mark_stmts_to_be_vectorized.
643 Not all stmts in the loop need to be vectorized. For example:
645 for i...
646 for j...
647 1. T0 = i + j
648 2. T1 = a[T0]
650 3. j = j + 1
652 Stmt 1 and 3 do not need to be vectorized, because loop control and
653 addressing of vectorized data-refs are handled differently.
655 This pass detects such stmts. */
657 opt_result
659 {
663 gimple_stmt_iterator si;
665 basic_block bb;
673 /* 1. Init worklist. */
675 {
678 {
686 }
688 {
698 }
699 }
701 /* 2. Process_worklist */
703 {
704 use_operand_p use_p;
705 ssa_op_iter iter;
712 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
713 (DEF_STMT) as relevant/irrelevant according to the relevance property
714 of STMT. */
717 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
718 propagated as is to the DEF_STMTs of its USEs.
720 One exception is when STMT has been identified as defining a reduction
721 variable; in this case we set the relevance to vect_used_by_reduction.
722 This is because we distinguish between two kinds of relevant stmts -
723 those that are used by a reduction computation, and those that are
724 (also) used by a regular computation. This allows us later on to
725 identify stmts that are used solely by a reduction, and therefore the
726 order of the results that they produce does not have to be kept. */
729 {
758 }
761 {
762 /* Pattern statements are not inserted into the code, so
763 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
764 have to scan the RHS or function arguments instead. */
766 {
772 {
773 opt_result res
783 }
785 {
788 {
789 opt_result res
794 }
795 }
796 }
798 {
800 {
802 opt_result res
807 }
808 }
809 }
810 else
812 {
814 opt_result res
819 }
822 {
823 gather_scatter_info gs_info;
826 opt_result res
830 {
834 }
835 }
839 }
841 /* Function vect_model_simple_cost.
843 Models cost for simple operations, i.e. those that only emit ncopies of a
844 single op. Right now, this does not account for multiple insns that could
845 be generated for the single vector op. We will handle that shortly. */
847 static void
852 slp_tree node,
855 {
860 /* ??? Somehow we need to fix this at the callers. */
865 /* Cost the "broadcast" of a scalar operand in to a vector operand.
866 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
867 cost model. */
873 /* Pass the inside-of-loop statements to the target-specific cost model. */
879 "vect_model_simple_cost: inside_cost = %d, "
881 }
884 /* Model cost for type demotion and promotion operations. PWR is
885 normally zero for single-step promotions and demotions. It will be
886 one if two-step promotion/demotion is required, and so on. NCOPIES
887 is the number of vector results (and thus number of instructions)
888 for the narrowest end of the operation chain. Each additional
889 step doubles the number of instructions required. If WIDEN_ARITH
890 is true the stmt is doing widening arithmetic. */
892 static void
898 {
903 {
905 widen_arith
909 }
911 /* FORNOW: Assuming maximum 2 args per stmts. */
919 "vect_model_promotion_demotion_cost: inside_cost = %d, "
921 }
923 /* Returns true if the current function returns DECL. */
925 static bool
927 {
928 edge_iterator ei;
929 edge e;
931 {
937 /* We often end up with an aggregate copy to the result decl,
938 handle that case as well. First skip intermediate clobbers
939 though. */
941 do
942 {
944 }
950 }
952 }
954 /* Calculate cost of DR's memory access. */
955 void
957 dr_alignment_support alignment_support_scheme,
961 {
963 {
965 {
968 vect_body);
974 }
977 {
978 /* Here, we assign an additional cost for the unaligned store. */
984 "vect_model_store_cost: unaligned supported by "
987 }
990 {
997 }
1001 }
1002 }
1004 /* Calculate cost of DR's memory access. */
1005 void
1007 dr_alignment_support alignment_support_scheme,
1014 {
1016 {
1018 {
1027 }
1029 {
1030 /* Here, we assign an additional cost for the unaligned load. */
1037 "vect_model_load_cost: unaligned supported by "
1041 }
1043 {
1049 /* FIXME: If the misalignment remains fixed across the iterations of
1050 the containing loop, the following cost should be added to the
1051 prologue costs. */
1061 }
1063 {
1066 "vect_model_load_cost: unaligned software "
1069 /* Unaligned software pipeline has a load of an address, an initial
1070 load, and possibly a mask operation to "prime" the loop. However,
1071 if this is an access in a group of loads, which provide grouped
1072 access, then the above cost should only be considered for one
1073 access in the group. Inside the loop, there is a load op
1074 and a realignment op. */
1077 {
1085 }
1094 "vect_model_load_cost: explicit realign optimized"
1098 }
1101 {
1108 }
1112 }
1113 }
1115 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1116 the loop preheader for the vectorized stmt STMT_VINFO. */
1118 static void
1121 {
1124 else
1130 }
1132 /* Function vect_init_vector.
1134 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1135 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1136 vector type a vector with all elements equal to VAL is created first.
1137 Place the initialization at GSI if it is not NULL. Otherwise, place the
1138 initialization at the loop preheader.
1139 Return the DEF of INIT_STMT.
1140 It will be used in the vectorization of STMT_INFO. */
1142 tree
1145 {
1147 tree new_temp;
1149 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1151 {
1154 {
1155 /* Scalar boolean value should be transformed into
1156 all zeros or all ones value before building a vector. */
1158 {
1164 else
1165 {
1171 }
1172 }
1173 else
1174 {
1179 else
1180 /* ??? Condition vectorization expects us to do
1181 promotion of invariant/external defs. */
1185 {
1189 }
1190 }
1191 }
1193 }
1199 }
1202 /* Function vect_get_vec_defs_for_operand.
1204 OP is an operand in STMT_VINFO. This function returns a vector of
1205 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1207 In the case that OP is an SSA_NAME which is defined in the loop, then
1208 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1210 In case OP is an invariant or constant, a new stmt that creates a vector def
1211 needs to be introduced. VECTYPE may be used to specify a required type for
1212 vector invariant. */
1214 void
1218 {
1228 stmt_vec_info def_stmt_info;
1237 {
1239 tree vector_type;
1246 else
1253 }
1254 else
1255 {
1261 }
1262 }
1265 /* Get vectorized definitions for OP0 and OP1. */
1267 void
1274 {
1276 {
1285 }
1286 else
1287 {
1300 }
1301 }
1303 void
1310 {
1316 }
1318 /* Helper function called by vect_finish_replace_stmt and
1319 vect_finish_stmt_generation. Set the location of the new
1320 statement and create and return a stmt_vec_info for it. */
1322 static void
1325 {
1330 {
1333 /* While EH edges will generally prevent vectorization, stmt might
1334 e.g. be in a must-not-throw region. Ensure newly created stmts
1335 that could throw are part of the same region. */
1339 }
1340 else
1342 }
1344 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1345 which sets the same scalar result as STMT_INFO did. Create and return a
1346 stmt_vec_info for VEC_STMT. */
1348 void
1351 {
1359 }
1361 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1362 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1364 void
1368 {
1373 {
1377 {
1381 /* If we have an SSA vuse and insert a store, update virtual
1382 SSA form to avoid triggering the renamer. Do so only
1383 if we can easily see all uses - which is what almost always
1384 happens with the way vectorized stmts are inserted. */
1393 {
1397 }
1398 }
1399 }
1402 }
1404 /* We want to vectorize a call to combined function CFN with function
1405 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1406 as the types of all inputs. Check whether this is possible using
1407 an internal function, returning its code if so or IFN_LAST if not. */
1409 static internal_fn
1412 {
1413 internal_fn ifn;
1416 else
1419 {
1422 {
1426 OPTIMIZE_FOR_SPEED))
1428 }
1429 }
1431 }
1435 gimple_stmt_iterator *);
1437 /* Check whether a load or store statement in the loop described by
1438 LOOP_VINFO is possible in a loop using partial vectors. This is
1439 testing whether the vectorizer pass has the appropriate support,
1440 as well as whether the target does.
1442 VLS_TYPE says whether the statement is a load or store and VECTYPE
1443 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1444 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1445 says how the load or store is going to be implemented and GROUP_SIZE
1446 is the number of load or store statements in the containing group.
1447 If the access is a gather load or scatter store, GS_INFO describes
1448 its arguments. If the load or store is conditional, SCALAR_MASK is the
1449 condition under which it occurs.
1451 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1452 vectors is not supported, otherwise record the required rgroup control
1453 types. */
1455 static void
1457 slp_tree slp_node,
1458 vec_load_store_type vls_type,
1460 vect_memory_access_type
1461 memory_access_type,
1463 tree scalar_mask)
1464 {
1465 /* Invariant loads need no special support. */
1472 else
1480 {
1481 internal_fn ifn
1488 scalar_mask);
1489 else
1490 {
1493 "can't operate on partial vectors because"
1494 " the target doesn't have an appropriate"
1497 }
1499 }
1502 {
1503 internal_fn ifn = (is_load
1504 ? IFN_MASK_GATHER_LOAD
1506 internal_fn len_ifn = (is_load
1507 ? IFN_MASK_LEN_GATHER_LOAD
1519 scalar_mask);
1520 else
1521 {
1524 "can't operate on partial vectors because"
1525 " the target doesn't have an appropriate"
1528 }
1530 }
1534 {
1535 /* Element X of the data must come from iteration i * VF + X of the
1536 scalar loop. We need more work to support other mappings. */
1539 "can't operate on partial vectors because an"
1543 }
1546 {
1549 "can't operate on partial vectors when emulating"
1553 }
1555 /* We might load more scalars than we need for permuting SLP loads.
1556 We checked in get_group_load_store_type that the extra elements
1557 don't leak into a new vector. */
1559 {
1564 };
1568 machine_mode mask_mode;
1569 machine_mode vmode;
1572 {
1577 }
1580 {
1584 }
1587 {
1590 "can't operate on partial vectors because the"
1591 " target doesn't have the appropriate partial"
1594 }
1595 }
1597 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1598 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1599 that needs to be applied to all loads and stores in a vectorized loop.
1600 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1601 otherwise return VEC_MASK & LOOP_MASK.
1603 MASK_TYPE is the type of both masks. If new statements are needed,
1604 insert them before GSI. */
1606 static tree
1609 {
1625 }
1627 /* Determine whether we can use a gather load or scatter store to vectorize
1628 strided load or store STMT_INFO by truncating the current offset to a
1629 smaller width. We need to be able to construct an offset vector:
1631 { 0, X, X*2, X*3, ... }
1633 without loss of precision, where X is STMT_INFO's DR_STEP.
1635 Return true if this is possible, describing the gather load or scatter
1636 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1638 static bool
1642 {
1647 {
1648 /* ??? Perhaps we could use range information here? */
1653 }
1655 /* Get the number of bits in an element. */
1660 /* Set COUNT to the upper limit on the number of elements - 1.
1661 Start with the maximum vectorization factor. */
1664 /* Try lowering COUNT to the number of scalar latch iterations. */
1666 widest_int max_iters;
1671 /* Try scales of 1 and the element size. */
1675 {
1677 widest_int factor;
1681 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1688 /* Find the narrowest viable offset type. */
1693 /* See whether the target supports the operation with an offset
1694 no narrower than OFFSET_TYPE. */
1703 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1704 but we don't need to store that here. */
1712 }
1716 "truncating gather/scatter offset to %d bits"
1720 }
1722 /* Return true if we can use gather/scatter internal functions to
1723 vectorize STMT_INFO, which is a grouped or strided load or store.
1724 MASKED_P is true if load or store is conditional. When returning
1725 true, fill in GS_INFO with the information required to perform the
1726 operation. */
1728 static bool
1732 {
1747 "using gather/scatter for strided/grouped access,"
1751 }
1753 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1754 elements with a known constant step. Return -1 if that step
1755 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1757 static int
1759 {
1762 size_zero_node);
1763 }
1765 /* If the target supports a permute mask that reverses the elements in
1766 a vector of type VECTYPE, return that mask, otherwise return null. */
1768 static tree
1770 {
1773 /* The encoding has a single stepped pattern. */
1780 indices))
1783 }
1785 /* A subroutine of get_load_store_type, with a subset of the same
1786 arguments. Handle the case where STMT_INFO is a load or store that
1787 accesses consecutive elements with a negative step. Sets *POFFSET
1788 to the offset to be applied to the DR for the first access. */
1790 static vect_memory_access_type
1793 vec_load_store_type vls_type,
1795 {
1797 dr_alignment_support alignment_support_scheme;
1800 {
1805 }
1807 /* For backward running DRs the first access in vectype actually is
1808 N-1 elements before the address of the DR. */
1813 alignment_support_scheme
1817 {
1823 }
1826 {
1829 "negative step with invariant source;"
1832 }
1835 {
1841 }
1844 }
1846 /* STMT_INFO is either a masked or unconditional store. Return the value
1847 being stored. */
1849 tree
1851 {
1853 {
1856 }
1858 {
1863 }
1865 }
1867 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1869 This function returns a vector type which can be composed with NETLS pieces,
1870 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1871 same vector size as the return vector. It checks target whether supports
1872 pieces-size vector mode for construction firstly, if target fails to, check
1873 pieces-size scalar mode for construction further. It returns NULL_TREE if
1874 fails to find the available composition.
1876 For example, for (vtype=V16QI, nelts=4), we can probably get:
1877 - V16QI with PTYPE V4QI.
1878 - V4SI with PTYPE SI.
1879 - NULL_TREE. */
1881 static tree
1883 {
1891 /* When we are asked to compose the vector from its components let
1892 that happen directly. */
1894 {
1897 }
1902 {
1903 /* First check if vec_init optab supports construction from
1904 vector pieces directly. */
1907 machine_mode rmode;
1911 {
1914 }
1916 /* Otherwise check if exists an integer type of the same piece size and
1917 if vec_init optab supports construction from it directly. */
1922 {
1925 }
1926 }
1929 }
1931 /* A subroutine of get_load_store_type, with a subset of the same
1932 arguments. Handle the case where STMT_INFO is part of a grouped load
1933 or store.
1935 For stores, the statements in the group are all consecutive
1936 and there is no gap at the end. For loads, the statements in the
1937 group might not be consecutive; there can be gaps between statements
1938 as well as at the end. */
1940 static bool
1950 {
1953 stmt_vec_info first_stmt_info;
1957 {
1961 }
1962 else
1963 {
1967 }
1973 /* True if the vectorized statements would access beyond the last
1974 statement in the group. */
1977 /* True if we can cope with such overrun by peeling for gaps, so that
1978 there is at least one final scalar iteration after the vector loop. */
1981 && loop_vinfo
1984 /* There can only be a gap at the end of the group if the stride is
1985 known at compile time. */
1988 /* Stores can't yet have gaps. */
1992 {
1993 /* For SLP vectorization we directly vectorize a subchain
1994 without permutation. */
1996 first_dr_info
1999 {
2000 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2001 separated by the stride, until we have a complete vector.
2002 Fall back to scalar accesses if that isn't possible. */
2005 else
2007 }
2008 else
2009 {
2012 {
2014 "Grouped store with gaps requires"
2017 }
2018 /* An overrun is fine if the trailing elements are smaller
2019 than the alignment boundary B. Every vector access will
2020 be a multiple of B and so we are guaranteed to access a
2021 non-gap element in the same B-sized block. */
2024 vectype)
2028 /* If the gap splits the vector in half and the target
2029 can do half-vector operations avoid the epilogue peeling
2030 by simply loading half of the vector only. Usually
2031 the construction with an upper zero half will be elided. */
2032 dr_alignment_support alss;
2034 tree half_vtype;
2036 && !masked_p
2039 == dr_aligned
2048 {
2053 }
2056 {
2058 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2059 only correct for single element "interleaving" SLP. */
2062 else
2063 {
2064 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2065 separated by the stride, until we have a complete vector.
2066 Fall back to scalar accesses if that isn't possible. */
2069 else
2071 }
2072 }
2074 {
2077 /* Invariant accesses perform only component accesses, alignment
2078 is irrelevant for them. */
2080 }
2081 else
2084 /* When we have a contiguous access across loop iterations
2085 but the access in the loop doesn't cover the full vector
2086 we can end up with no gap recorded but still excess
2087 elements accessed, see PR103116. Make sure we peel for
2088 gaps if necessary and sufficient and give up if not.
2090 If there is a combination of the access not covering the full
2091 vector and a gap recorded then we may need to peel twice. */
2096 nunits))
2097 {
2102 /* Peeling for gaps assumes that a single scalar iteration
2103 is enough to make sure the last vector iteration doesn't
2104 access excess elements.
2105 ??? Enhancements include peeling multiple iterations
2106 or using masked loads with a static mask. */
2108 {
2111 "peeling for gaps insufficient for "
2114 }
2116 }
2117 }
2118 }
2119 else
2120 {
2121 /* We can always handle this case using elementwise accesses,
2122 but see if something more efficient is available. */
2125 /* If there is a gap at the end of the group then these optimizations
2126 would access excess elements in the last iteration. */
2128 /* An overrun is fine if the trailing elements are smaller than the
2129 alignment boundary B. Every vector access will be a multiple of B
2130 and so we are guaranteed to access a non-gap element in the
2131 same B-sized block. */
2133 && !masked_p
2141 {
2142 /* First cope with the degenerate case of a single-element
2143 vector. */
2145 ;
2147 else
2148 {
2149 /* Otherwise try using LOAD/STORE_LANES. */
2150 *lanes_ifn
2154 masked_p);
2156 {
2159 }
2161 /* If that fails, try using permuting loads. */
2164 single_element_p,
2165 group_size)
2167 {
2170 }
2171 }
2172 }
2174 /* As a last resort, trying using a gather load or scatter store.
2176 ??? Although the code can handle all group sizes correctly,
2177 it probably isn't a win to use separate strided accesses based
2178 on nearby locations. Or, even if it's a win over scalar code,
2179 it might not be a win over vectorizing at a lower VF, if that
2180 allows us to use contiguous accesses. */
2182 && single_element_p
2183 && loop_vinfo
2187 }
2191 {
2194 }
2195 else
2196 {
2198 *alignment_support_scheme
2201 }
2204 {
2205 /* STMT is the leader of the group. Check the operands of all the
2206 stmts of the group. */
2209 {
2213 {
2218 }
2220 }
2221 }
2224 {
2228 "Data access with gaps requires scalar "
2231 }
2234 }
2236 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2237 if there is a memory access type that the vectorized form can use,
2238 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2239 or scatters, fill in GS_INFO accordingly. In addition
2240 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2241 the target does not support the alignment scheme. *MISALIGNMENT
2242 is set according to the alignment of the access (including
2243 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2245 SLP says whether we're performing SLP rather than loop vectorization.
2246 MASKED_P is true if the statement is conditional on a vectorized mask.
2247 VECTYPE is the vector type that the vectorized statements will use.
2248 NCOPIES is the number of vector statements that will be needed. */
2250 static bool
2261 {
2267 {
2271 /* When using internal functions, we rely on pattern recognition
2272 to convert the type of the offset to the type that the target
2273 requires, with the result being a call to an internal function.
2274 If that failed for some reason (e.g. because another pattern
2275 took priority), just handle cases in which the offset already
2276 has the right type. */
2281 {
2287 }
2291 {
2297 }
2299 {
2305 {
2308 "unsupported vector types for emulated "
2311 }
2312 }
2313 /* Gather-scatter accesses perform only component accesses, alignment
2314 is irrelevant for them. */
2316 }
2318 {
2320 masked_p,
2322 alignment_support_scheme,
2325 }
2327 {
2333 else
2335 /* Alignment is irrelevant here. */
2337 }
2338 else
2339 {
2342 {
2345 /* Invariant accesses perform only component accesses, alignment
2346 is irrelevant for them. */
2348 }
2349 else
2350 {
2354 else
2358 *alignment_support_scheme
2362 }
2363 }
2368 {
2371 "Not using elementwise accesses due to variable "
2374 }
2377 {
2382 }
2384 /* FIXME: At the moment the cost model seems to underestimate the
2385 cost of using elementwise accesses. This check preserves the
2386 traditional behavior until that can be fixed. */
2395 {
2400 }
2402 }
2404 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2405 conditional operation STMT_INFO. When returning true, store the mask
2406 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2407 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2408 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2410 static bool
2415 {
2417 tree mask_vectype;
2418 slp_tree mask_node_1;
2421 {
2426 }
2429 {
2434 }
2436 /* If the caller is not prepared for adjusting an external/constant
2437 SLP mask vector type fail. */
2439 && !mask_node
2441 {
2446 }
2453 {
2458 }
2462 {
2465 "vector mask type %T"
2470 }
2477 }
2479 /* Return true if stored value RHS is suitable for vectorizing store
2480 statement STMT_INFO. When returning true, store the type of the
2481 definition in *RHS_DT_OUT, the type of the vectorized store value in
2482 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2484 static bool
2489 {
2490 /* In the case this is a store from a constant make sure
2491 native_encode_expr can handle it. */
2493 {
2498 }
2502 {
2508 }
2511 tree rhs_vectype;
2512 slp_tree slp_op;
2515 {
2520 }
2524 {
2529 }
2535 else
2538 }
2540 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2541 Note that we support masks with floating-point type, in which case the
2542 floats are interpreted as a bitmask. */
2544 static tree
2547 {
2552 {
2556 }
2558 {
2559 REAL_VALUE_TYPE r;
2567 }
2569 }
2571 /* Build an all-zero merge value of type VECTYPE while vectorizing
2572 STMT_INFO as a gather load. */
2574 static tree
2577 {
2578 tree merge;
2582 {
2583 REAL_VALUE_TYPE r;
2589 }
2590 else
2594 }
2596 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2597 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2598 the gather load operation. If the load is conditional, MASK is the
2599 vectorized condition, otherwise MASK is null. PTR is the base
2600 pointer and OFFSET is the vectorized offset. */
2607 {
2616 tree var;
2618 && (!mask
2624 {
2632 }
2637 {
2639 {
2644 else
2648 gassign *new_stmt
2653 {
2660 }
2663 }
2664 else
2665 {
2668 }
2669 }
2670 else
2671 {
2674 }
2681 {
2689 }
2692 }
2694 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2695 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2696 the scatter store operation. If the store is conditional, MASK is the
2697 unvectorized condition, otherwise MASK is null. */
2699 static void
2704 {
2710 poly_uint64 scatter_off_nunits
2713 /* FIXME: Keep the previous costing way in vect_model_store_cost by
2714 costing N scalar stores, but it should be tweaked to use target
2715 specific costs on related scatter store calls. */
2717 {
2731 "vect_model_store_cost: inside_cost = %d, "
2735 }
2741 {
2744 /* Currently gathers and scatters are only supported for
2745 fixed-length vectors. */
2754 }
2756 {
2759 /* Currently gathers and scatters are only supported for
2760 fixed-length vectors. */
2773 }
2774 else
2790 {
2791 gimple_seq seq;
2797 }
2801 {
2804 }
2812 {
2817 }
2830 {
2832 {
2836 else
2841 }
2843 {
2847 else
2852 }
2853 else
2854 {
2859 }
2862 {
2870 }
2873 {
2881 }
2884 {
2885 tree utype;
2888 {
2889 tree var
2891 gassign *new_stmt
2895 mask_op);
2898 }
2902 else
2906 gassign *new_stmt
2911 {
2917 }
2918 }
2920 gcall *new_stmt
2925 }
2927 }
2929 /* Prepare the base and offset in GS_INFO for vectorization.
2930 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2931 to the vectorized offset argument for the first copy of STMT_INFO.
2932 STMT_INFO is the statement described by GS_INFO and LOOP is the
2933 containing loop. */
2935 static void
2940 {
2944 {
2945 basic_block new_bb;
2949 }
2952 else
2953 {
2954 unsigned ncopies
2959 }
2960 }
2962 /* Prepare to implement a grouped or strided load or store using
2963 the gather load or scatter store operation described by GS_INFO.
2964 STMT_INFO is the load or store statement.
2966 Set *DATAREF_BUMP to the amount that should be added to the base
2967 address after each copy of the vectorized statement. Set *VEC_OFFSET
2968 to an invariant offset vector in which element I has the value
2969 I * DR_STEP / SCALE. */
2971 static void
2973 loop_vec_info loop_vinfo,
2978 {
2983 {
2984 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2985 ivtmp_8 = _31 * 16 (step in bytes);
2986 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2987 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2988 tree loop_len
2990 tree tmp
2993 loop_len);
2995 GSI_SAME_STMT);
2996 }
2997 else
2998 {
2999 tree bump
3004 }
3006 /* The offset given in GS_INFO can have pointer type, so use the element
3007 type of the vector instead. */
3010 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
3015 /* Create {0, X, X*2, X*3, ...}. */
3019 }
3021 /* Prepare the pointer IVs which needs to be updated by a variable amount.
3022 Such variable amount is the outcome of .SELECT_VL. In this case, we can
3023 allow each iteration process the flexible number of elements as long as
3024 the number <= vf elments.
3026 Return data reference according to SELECT_VL.
3027 If new statements are needed, insert them before GSI. */
3029 static tree
3033 vect_memory_access_type memory_access_type)
3034 {
3038 /* gather/scatter never reach here. */
3041 /* When we support SELECT_VL pattern, we dynamic adjust
3042 the memory address by .SELECT_VL result.
3044 The result of .SELECT_VL is the number of elements to
3045 be processed of each iteration. So the memory address
3046 adjustment operation should be:
3048 addr = addr + .SELECT_VL (ARG..) * step;
3049 */
3050 tree loop_len
3053 /* Since the outcome of .SELECT_VL is element size, we should adjust
3054 it into bytesize so that it can be used in address pointer variable
3055 amount IVs adjustment. */
3062 }
3064 /* Return the amount that should be added to a vector pointer to move
3065 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3066 being vectorized and MEMORY_ACCESS_TYPE describes the type of
3067 vectorization. */
3069 static tree
3072 vect_memory_access_type memory_access_type,
3074 {
3082 memory_access_type);
3089 }
3091 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3093 static bool
3099 {
3109 /* Multiple types in SLP are handled by creating the appropriate number of
3110 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3111 case of SLP. */
3114 else
3128 /* The encoding uses one stepped pattern for each byte in the word. */
3140 {
3143 {
3148 }
3155 slp_node
3159 }
3163 /* Transform. */
3167 /* Arguments are ready. create the new vector stmt. */
3169 tree vop;
3171 {
3187 else
3189 }
3196 }
3198 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3199 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3200 in a single step. On success, store the binary pack code in
3201 *CONVERT_CODE. */
3203 static bool
3206 {
3211 code_helper code;
3221 }
3223 /* Function vectorizable_call.
3225 Check if STMT_INFO performs a function call that can be vectorized.
3226 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3227 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3228 Return true if STMT_INFO is vectorizable in this way. */
3230 static bool
3235 {
3237 tree vec_dest;
3238 tree scalar_dest;
3239 tree op;
3242 poly_uint64 nunits_in;
3243 poly_uint64 nunits_out;
3249 vect_unknown_def_type };
3257 tree lhs;
3266 /* Is STMT_INFO a vectorizable call? */
3274 /* Handled by vectorizable_load and vectorizable_store. */
3285 /* Process function arguments. */
3290 /* Bail out if the function has more than four arguments, we do not have
3291 interesting builtin functions to vectorize with more than two arguments
3292 except for fma. No arguments is also not good. */
3296 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3299 {
3302 }
3309 {
3311 {
3316 }
3320 {
3325 }
3327 /* We can only handle calls with arguments of the same type. */
3330 {
3335 }
3343 {
3348 }
3349 }
3350 /* If all arguments are external or constant defs, infer the vector type
3351 from the scalar type. */
3357 {
3363 }
3364 /* FORNOW: we don't yet support mixtures of vector sizes for calls,
3365 just mixtures of nunits. E.g. DI->SI versions of __builtin_ctz*
3366 are traditionally vectorized as two VnDI->VnDI IFN_CTZs followed
3367 by a pack of the two vectors into an SI vector. We would need
3368 separate code to handle direct VnDI->VnSI IFN_CTZs. */
3370 {
3376 }
3380 {
3385 }
3388 {
3393 }
3395 /* FORNOW */
3404 else
3407 /* We only handle functions that do not read or clobber memory. */
3409 {
3414 }
3416 /* For now, we only vectorize functions if a target specific builtin
3417 is available. TODO -- in some cases, it might be profitable to
3418 insert the calls for pieces of the vector, in order to be able
3419 to vectorize other operations in the loop. */
3424 /* First try using an internal function. */
3432 vectype_in);
3434 /* If that fails, try asking for a target-specific built-in function. */
3436 {
3443 }
3446 {
3448 && !slp_node
3449 && loop_vinfo
3454 {
3455 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3456 { 0, 1, 2, ... vf - 1 } vector. */
3458 }
3466 else
3467 {
3472 }
3473 }
3479 else
3482 /* Sanity check: make sure that at least one copy of the vectorized stmt
3483 needs to be generated. */
3493 {
3499 {
3504 }
3516 {
3520 OPTIMIZE_FOR_SPEED))
3523 OPTIMIZE_FOR_SPEED)))
3524 {
3527 "can't use a fully-masked loop because no"
3530 }
3531 else
3532 {
3533 unsigned int nvectors
3534 = (slp_node
3542 OPTIMIZE_FOR_SPEED))
3545 else
3547 scalar_mask);
3548 }
3549 }
3551 }
3553 /* Transform. */
3558 /* Handle def. */
3566 {
3568 {
3570 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3572 }
3575 }
3577 {
3580 }
3583 {
3588 {
3589 /* Build argument list for the vectorized call. */
3591 {
3597 /* Arguments are ready. Create the new vector stmt. */
3599 {
3602 {
3604 /* Always true for SLP. */
3609 }
3612 {
3615 }
3620 {
3621 /* We don't define any narrowing conditional functions
3622 at present. */
3625 gcall *call
3631 {
3634 }
3640 }
3641 else
3642 {
3644 {
3646 /* Always true for SLP. */
3648 tree len
3651 signed char biasval
3656 }
3658 {
3660 /* Always true for SLP. */
3668 }
3673 else
3680 }
3682 }
3684 }
3691 {
3694 {
3699 }
3701 }
3706 {
3709 signed char biasval
3714 }
3716 {
3722 }
3726 {
3728 tree new_var
3735 }
3737 {
3738 /* We don't define any narrowing conditional functions at
3739 present. */
3747 {
3750 }
3753 half_res);
3755 }
3756 else
3757 {
3761 else
3768 }
3773 }
3775 {
3778 }
3779 }
3781 {
3783 /* We don't define any narrowing conditional functions at present. */
3786 {
3787 /* Build argument list for the vectorized call. */
3790 else
3794 {
3800 /* Arguments are ready. Create the new vector stmt. */
3802 {
3806 {
3810 }
3814 else
3821 }
3823 }
3826 {
3829 {
3833 }
3839 }
3847 }
3853 {
3856 }
3857 }
3858 else
3859 /* No current target implements this case. */
3864 /* The call in STMT might prevent it from being removed in dce.
3865 We however cannot remove it here, due to the way the ssa name
3866 it defines is mapped to the new definition. So just replace
3867 rhs of the statement with something harmless. */
3875 gassign *new_stmt
3880 }
3883 struct simd_call_arg_info
3884 {
3885 tree vectype;
3886 tree op;
3887 HOST_WIDE_INT linear_step;
3891 };
3893 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3894 is linear within simd lane (but not within whole loop), note it in
3895 *ARGINFO. */
3897 static void
3900 {
3912 {
3913 tree t;
3917 {
3932 CASE_CONVERT:
3944 }
3950 {
3957 }
3958 }
3959 }
3961 /* Function vectorizable_simd_clone_call.
3963 Check if STMT_INFO performs a function call that can be vectorized
3964 by calling a simd clone of the function.
3965 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3966 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3967 Return true if STMT_INFO is vectorizable in this way. */
3969 static bool
3973 stmt_vector_for_cost *)
3974 {
3975 tree vec_dest;
3976 tree scalar_dest;
3979 tree vectype;
3980 poly_uint64 nunits;
3993 /* Is STMT a vectorizable call? */
4001 {
4007 }
4033 /* Process function arguments. */
4036 /* Bail out if the function has zero arguments. */
4047 {
4048 simd_call_arg_info thisarginfo;
4049 affine_iv iv;
4063 {
4068 }
4072 {
4073 /* With SLP we determine the vector type of constants/externals
4074 at analysis time, handling conflicts via
4075 vect_maybe_update_slp_op_vectype. At transform time
4076 we have a vector type recorded for SLP. */
4078 || !slp_node
4083 slp_node);
4084 }
4085 else
4088 /* For linear arguments, the analyze phase should have saved
4089 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
4092 {
4096 thisarginfo.simd_lane_linear
4098 /* If loop has been peeled for alignment, we need to adjust it. */
4102 {
4108 thisarginfo.op
4112 }
4113 }
4117 && loop_vinfo
4122 {
4125 }
4130 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4131 linear too. */
4134 && !vec_stmt
4137 && loop_vinfo
4142 }
4150 else
4153 {
4170 {
4172 {
4203 /* FORNOW */
4207 /* While we can create a traditional data vector from
4208 an incoming integer mode mask we have no good way to
4209 force generate an integer mode mask from a traditional
4210 boolean vector input. */
4218 }
4222 {
4225 }
4229 }
4235 {
4237 SIMD_CLONE_ARG_TYPE_MASK);
4238 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4239 not in a branch, as we'd have to construct an all-true mask. */
4242 }
4244 {
4247 }
4248 }
4257 num_mask_args++;
4260 {
4264 {
4268 slp_node);
4273 }
4277 {
4282 }
4285 {
4288 {
4291 {
4292 /* FORNOW we only have partial support for vector-type masks
4293 that can't hold all of simdlen. */
4296 vect_location,
4297 "in-branch vector clones are not yet"
4300 }
4301 }
4303 {
4306 num_mask_args),
4308 {
4309 /* FORNOW we only have partial support for integer-type masks
4310 that represent the same number of lanes as the
4311 vectorized mask inputs. */
4314 vect_location,
4315 "in-branch vector clones are not yet "
4318 }
4319 }
4320 else
4321 {
4324 vect_location,
4325 "in-branch vector clones not supported"
4328 }
4329 }
4330 }
4336 else
4339 /* If the function isn't const, only allow it in simd loops where user
4340 has asserted that at least nunits consecutive iterations can be
4341 performed using SIMD instructions. */
4346 /* Sanity check: make sure that at least one copy of the vectorized stmt
4347 needs to be generated. */
4351 {
4355 {
4360 }
4361 /* When the original call is pure or const but the SIMD ABI dictates
4362 an aggregate return we will have to use a virtual definition and
4363 in a loop eventually even need to add a virtual PHI. That's
4364 not straight-forward so allow to fix this up via renaming. */
4369 /* ??? For SLP code-gen we end up inserting after the last
4370 vector argument def rather than at the original call position
4371 so automagic virtual operand updating doesn't work. */
4376 {
4378 {
4383 {
4394 }
4404 }
4405 }
4408 {
4412 "can't use a fully-masked loop because a"
4415 }
4419 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4420 dt, slp_node, cost_vec); */
4422 }
4424 /* Transform. */
4429 /* Handle def. */
4435 {
4439 {
4442 }
4443 }
4449 {
4452 }
4453 else
4456 {
4457 poly_uint64 callee_nelements;
4458 poly_uint64 caller_nelements;
4459 /* Build argument list for the vectorized call. */
4462 else
4466 {
4468 tree atype;
4471 {
4478 {
4480 {
4488 {
4495 }
4496 else
4497 {
4501 }
4503 vec_oprnd0
4507 gassign *new_stmt
4509 vec_oprnd0);
4513 }
4514 else
4515 {
4523 else
4526 {
4528 {
4532 op,
4536 }
4537 else
4543 vec_oprnd0);
4544 }
4547 atype))
4548 {
4550 vec_oprnd0);
4551 gassign *new_stmt
4553 vec_oprnd0);
4557 }
4558 else
4560 else
4561 {
4563 gassign *new_stmt
4565 vec_oprnd0);
4569 }
4570 }
4571 }
4575 {
4584 {
4586 {
4587 /* The mask type has fewer elements than simdlen. */
4589 /* FORNOW */
4591 }
4593 {
4594 /* The SIMD clone function has the same number of
4595 elements as the current function. */
4597 {
4601 op,
4604 }
4608 {
4609 vec_loop_masks *loop_masks
4611 tree loop_mask
4615 vec_oprnd0
4619 gsi);
4621 loop_mask });
4623 }
4624 vec_oprnd0
4628 gassign *new_stmt
4630 vec_oprnd0);
4634 }
4635 else
4636 {
4637 /* The mask type has more elements than simdlen. */
4639 /* FORNOW */
4641 }
4642 }
4643 }
4645 {
4647 /* Guess the number of lanes represented by atype. */
4648 poly_uint64 atype_subparts
4650 num_mask_args);
4653 {
4655 {
4659 op,
4662 }
4665 {
4666 /* The mask argument has fewer elements than the
4667 input vector. */
4668 /* FORNOW */
4670 }
4673 {
4674 /* The vector mask argument matches the input
4675 in the number of lanes, but not necessarily
4676 in the mode. */
4681 vec_oprnd0);
4682 gassign *new_stmt
4684 vec_oprnd0);
4688 {
4689 new_stmt
4691 NOP_EXPR,
4692 gimple_assign_lhs
4696 }
4698 }
4699 else
4700 {
4701 /* The mask argument has more elements than the
4702 input vector. */
4703 /* FORNOW */
4705 }
4706 }
4707 }
4708 else
4717 {
4718 gimple_seq stmts;
4723 {
4724 basic_block new_bb;
4728 }
4730 {
4733 }
4738 enum tree_code code
4743 poly_widest_int cst
4748 gassign *new_stmt
4753 UNKNOWN_LOCATION);
4756 }
4757 else
4758 {
4759 enum tree_code code
4764 poly_widest_int cst
4769 gassign *new_stmt
4774 }
4784 }
4785 }
4790 {
4793 tree mask;
4795 SIMD_CLONE_ARG_TYPE_MASK);
4801 {
4803 {
4807 }
4808 else
4812 {
4815 {
4816 /* This means we are dealing with integer mask modes.
4817 First convert to an integer type with the same size as
4818 the current vector type. */
4819 unsigned HOST_WIDE_INT intermediate_size
4821 tree mid_int_type =
4824 new_stmt
4826 mask);
4828 /* Then zero-extend to the mask mode. */
4831 }
4832 else
4836 mask);
4840 }
4842 }
4843 }
4847 {
4854 else
4857 }
4862 {
4864 {
4872 {
4873 tree t;
4875 {
4879 }
4880 else
4891 else
4893 }
4898 }
4900 {
4909 {
4914 {
4918 tem);
4923 }
4925 }
4926 else
4931 new_stmt
4940 else
4943 }
4945 {
4952 }
4954 {
4956 new_stmt
4959 }
4960 }
4966 else
4968 }
4971 {
4974 }
4977 /* Mark the clone as no longer being a candidate for GC. */
4980 /* The call in STMT might prevent it from being removed in dce.
4981 We however cannot remove it here, due to the way the ssa name
4982 it defines is mapped to the new definition. So just replace
4983 rhs of the statement with something harmless. */
4990 {
4994 }
4995 else
5001 }
5004 /* Function vect_gen_widened_results_half
5006 Create a vector stmt whose code, type, number of arguments, and result
5007 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5008 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5009 In the case that CODE is a CALL_EXPR, this means that a call to DECL
5010 needs to be created (DECL is a function-decl of a target-builtin).
5011 STMT_INFO is the original scalar stmt that we are vectorizing. */
5017 stmt_vec_info stmt_info)
5018 {
5020 tree new_temp;
5022 /* Generate half of the widened result: */
5031 }
5034 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5035 For multi-step conversions store the resulting vectors and call the function
5036 recursively. When NARROW_SRC_P is true, there's still a conversion after
5037 narrowing, don't store the vectors in the SLP_NODE or in vector info of
5038 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5040 static void
5043 stmt_vec_info stmt_info,
5048 {
5055 {
5056 /* Create demotion operation. */
5064 /* Store the resulting vector for next recursive call,
5065 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5067 else
5068 {
5069 /* This is the last step of the conversion sequence. Store the
5070 vectors in SLP_NODE or in vector info of the scalar statement
5071 (or in STMT_VINFO_RELATED_STMT chain). */
5074 else
5076 }
5077 }
5079 /* For multi-step demotion operations we first generate demotion operations
5080 from the source type to the intermediate types, and then combine the
5081 results (stored in VEC_OPRNDS) in demotion operation to the destination
5082 type. */
5084 {
5085 /* At each level of recursion we have half of the operands we had at the
5086 previous level. */
5092 narrow_src_p);
5093 }
5096 }
5099 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5100 and VEC_OPRNDS1, for a binary operation associated with scalar statement
5101 STMT_INFO. For multi-step conversions store the resulting vectors and
5102 call the function recursively. */
5104 static void
5110 code_helper ch1,
5112 {
5120 {
5123 else
5126 /* Generate the two halves of promotion operation. */
5129 stmt_info);
5132 stmt_info);
5134 {
5137 }
5138 else
5139 {
5142 }
5144 /* Store the results for the next step. */
5147 }
5151 }
5153 /* Create vectorized promotion stmts for widening stmts using only half the
5154 potential vector size for input. */
5155 static void
5161 code_helper code1,
5163 {
5173 {
5179 /* Widen the first vector input. */
5185 {
5186 /* Widen the second vector input. */
5190 /* Perform the operation. With both vector inputs widened. */
5192 }
5193 else
5194 {
5195 /* Perform the operation. With the single vector input widened. */
5197 }
5203 /* Store the results for the next step. */
5205 }
5209 }
5212 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5213 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5214 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5215 Return true if STMT_INFO is vectorizable in this way. */
5217 static bool
5222 {
5224 tree scalar_dest;
5230 tree new_temp;
5233 poly_uint64 nunits_in;
5234 poly_uint64 nunits_out;
5238 /* For conversions between floating point and integer, there're 2 NARROW
5239 cases. NARROW_SRC is for FLOAT_EXPR, means
5240 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5241 This is safe when the range of the source integer can fit into the lower
5242 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5243 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5244 For other conversions, when there's narrowing, NARROW_DST is used as
5245 default. */
5249 tree vop0;
5257 /* Is STMT a vectorizable conversion? */
5278 {
5281 }
5283 {
5286 }
5287 else
5300 /* Check types of lhs and rhs. */
5305 /* Check the operands of the operation. */
5309 {
5314 }
5329 {
5332 "type conversion to/from bit-precision unsupported."
5335 }
5338 {
5345 tree vectype1_in;
5348 {
5353 }
5354 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5355 OP1. */
5358 }
5360 /* If op0 is an external or constant def, infer the vector type
5361 from the scalar type. */
5367 {
5373 }
5377 {
5380 "can't convert between boolean and non "
5384 }
5391 else
5395 else
5396 {
5399 }
5401 /* Multiple types in SLP are handled by creating the appropriate number of
5402 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5403 case of SLP. */
5408 else
5411 /* Sanity check: make sure that at least one copy of the vectorized stmt
5412 needs to be generated. */
5418 opt_scalar_mode rhs_mode_iter;
5420 /* Supportable by target? */
5422 {
5431 {
5434 }
5436 /* For conversions between float and integer types try whether
5437 we can use intermediate signed integer types to support the
5438 conversion. */
5442 {
5446 scalar_mode intermediate_mode;
5448 {
5451 }
5452 else
5453 {
5458 }
5461 opt_scalar_mode mode_iter;
5463 {
5469 scalar_mode cvt_mode;
5474 cvt_type = build_nonstandard_integer_type
5477 /* Check if the intermediate type can hold OP0's range.
5478 When converting from float to integer this is not necessary
5479 because values that do not fit the (smaller) target type are
5480 unspecified anyway. */
5482 {
5493 }
5496 /* This should only happened for SLP as long as loop vectorizer
5497 only supports same-sized vector. */
5501 vectype_out,
5504 cvt_type,
5510 }
5513 {
5514 multi_step_cvt++;
5520 }
5521 }
5522 /* FALLTHRU */
5523 unsupported:
5531 {
5540 }
5545 {
5546 /* Binary widening operation can only be supported directly by the
5547 architecture. */
5550 }
5558 {
5563 cvt_type
5570 {
5577 }
5584 else
5588 cvt_type,
5592 {
5595 }
5596 }
5603 else
5604 {
5605 multi_step_cvt++;
5608 }
5622 {
5623 cvt_type
5631 else
5637 }
5638 /* If op0 can be represented with low precision integer,
5639 truncate it to cvt_type and the do FLOAT_EXPR. */
5641 {
5646 cvt_type
5664 {
5668 }
5669 }
5675 }
5678 {
5682 {
5687 }
5690 {
5695 }
5697 {
5699 /* The final packing step produces one vector result per copy. */
5700 unsigned int nvectors
5704 widen_arith);
5705 }
5706 else
5707 {
5709 /* The initial unpacking step produces two vector results
5710 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5711 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5712 unsigned int nvectors
5713 = (slp_node
5718 widen_arith);
5719 }
5722 }
5724 /* Transform. */
5730 {
5735 }
5737 /* In case of multi-step conversion, we first generate conversion operations
5738 to the intermediate types, and then from that types to the final one.
5739 We create vector destinations for the intermediate type (TYPES) received
5740 from supportable_*_operation, and store them in the correct order
5741 for future use in vect_create_vectorized_*_stmts (). */
5743 bool widen_or_narrow_float_p
5746 widen_or_narrow_float_p
5751 {
5754 {
5756 intermediate_type);
5758 }
5759 }
5763 widen_or_narrow_float_p
5768 {
5770 ;
5772 {
5776 }
5777 }
5780 {
5784 /* vec_dest is intermediate type operand when multi_step_cvt. */
5786 {
5789 }
5792 {
5793 /* Arguments are ready, create the new vector stmt. */
5796 {
5803 }
5811 else
5813 }
5817 /* In case the vectorization factor (VF) is bigger than the number
5818 of elements that we can fit in a vectype (nunits), we have to
5819 generate more than one vector stmt - i.e - we need to "unroll"
5820 the vector stmt by a factor VF/nunits. */
5826 {
5831 }
5832 /* Arguments are ready. Create the new vector stmts. */
5834 {
5838 {
5841 }
5845 op_type);
5846 else
5851 }
5854 {
5857 {
5861 }
5862 else
5867 else
5869 }
5874 /* In case the vectorization factor (VF) is bigger than the number
5875 of elements that we can fit in a vectype (nunits), we have to
5876 generate more than one vector stmt - i.e - we need to "unroll"
5877 the vector stmt by a factor VF/nunits. */
5880 /* Arguments are ready. Create the new vector stmts. */
5883 {
5888 }
5891 multi_step_cvt,
5895 /* After demoting op0 to cvt_type, convert it to dest. */
5897 {
5899 {
5900 /* Arguments are ready, create the new vector stmt. */
5902 gimple *new_stmt
5908 /* This is the last step of the conversion sequence. Store the
5909 vectors in SLP_NODE or in vector info of the scalar statement
5910 (or in STMT_VINFO_RELATED_STMT chain). */
5913 else
5915 }
5916 }
5918 }
5927 }
5929 /* Return true if we can assume from the scalar form of STMT_INFO that
5930 neither the scalar nor the vector forms will generate code. STMT_INFO
5931 is known not to involve a data reference. */
5933 bool
5935 {
5951 }
5953 /* Function vectorizable_assignment.
5955 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5956 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5957 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5958 Return true if STMT_INFO is vectorizable in this way. */
5960 static bool
5965 {
5966 tree vec_dest;
5967 tree scalar_dest;
5968 tree op;
5970 tree new_temp;
5976 tree vop;
5979 tree vectype_in;
5988 /* Is vectorizable assignment? */
6009 /* Multiple types in SLP are handled by creating the appropriate number of
6010 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6011 case of SLP. */
6014 else
6019 slp_tree slp_op;
6022 {
6027 }
6031 /* We can handle NOP_EXPR conversions that do not change the number
6032 of elements or the vector size. */
6035 && (!vectype_in
6042 {
6045 "can't convert between boolean and non "
6049 }
6051 /* We do not handle bit-precision changes. */
6058 /* But a conversion that does not change the bit-pattern is ok. */
6064 {
6067 "type conversion to/from bit-precision "
6070 }
6073 {
6076 {
6081 }
6086 cost_vec);
6088 }
6090 /* Transform. */
6094 /* Handle def. */
6097 /* Handle use. */
6100 /* Arguments are ready. create the new vector stmt. */
6102 {
6112 else
6114 }
6120 }
6123 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6124 either as shift by a scalar or by a vector. */
6126 bool
6128 {
6130 machine_mode vec_mode;
6131 optab optab;
6133 tree vectype;
6142 {
6148 }
6156 }
6159 /* Function vectorizable_shift.
6161 Check if STMT_INFO performs a shift operation that can be vectorized.
6162 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6163 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6164 Return true if STMT_INFO is vectorizable in this way. */
6166 static bool
6171 {
6172 tree vec_dest;
6173 tree scalar_dest;
6176 tree vectype;
6179 machine_mode vec_mode;
6180 tree new_temp;
6181 optab optab;
6183 machine_mode optab_op2_mode;
6186 poly_uint64 nunits_in;
6187 poly_uint64 nunits_out;
6188 tree vectype_out;
6189 tree op1_vectype;
6208 /* Is STMT a vectorizable binary/unary operation? */
6225 {
6230 }
6232 slp_tree slp_op0;
6235 {
6240 }
6241 /* If op0 is an external or constant def, infer the vector type
6242 from the scalar type. */
6248 {
6253 }
6260 stmt_vec_info op1_def_stmt_info;
6261 slp_tree slp_op1;
6264 {
6269 }
6271 /* Multiple types in SLP are handled by creating the appropriate number of
6272 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6273 case of SLP. */
6276 else
6281 /* Determine whether the shift amount is a vector, or scalar. If the
6282 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6292 {
6293 /* In SLP, need to check whether the shift count is the same,
6294 in loops if it is a constant or invariant, it is always
6295 a scalar shift. */
6297 {
6299 stmt_vec_info slpstmt_info;
6302 {
6306 }
6308 /* For internal SLP defs we have to make sure we see scalar stmts
6309 for all vector elements.
6310 ??? For different vectors we could resort to a different
6311 scalar shift operand but code-generation below simply always
6312 takes the first. */
6317 }
6319 /* If the shift amount is computed by a pattern stmt we cannot
6320 use the scalar amount directly thus give up and use a vector
6321 shift. */
6324 }
6325 else
6326 {
6331 }
6333 /* Vector shifted by vector. */
6336 {
6344 slp_op1);
6345 incompatible_op1_vectype_p
6351 && (!slp_node
6354 {
6357 "unusable type for last operand in"
6360 }
6361 }
6362 /* See if the machine has a vector shifted by scalar insn and if not
6363 then see if it has a vector shifted by vector insn. */
6364 else
6365 {
6369 {
6373 }
6374 else
6375 {
6380 {
6390 slp_op1);
6392 /* Unlike the other binary operators, shifts/rotates have
6393 the rhs being int, instead of the same type as the lhs,
6394 so make sure the scalar is the right type if we are
6395 dealing with vectors of long long/long/short/char. */
6396 incompatible_op1_vectype_p
6397 = (!op1_vectype
6402 {
6405 "unusable type for last operand in"
6408 }
6409 }
6410 }
6411 }
6413 /* Supportable by target? */
6415 {
6420 }
6424 {
6429 }
6430 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6435 {
6439 && (!incompatible_op1_vectype_p
6441 && !vect_maybe_update_slp_op_vectype
6444 {
6449 }
6450 /* Now adjust the constant shift amount in place. */
6452 && incompatible_op1_vectype_p
6454 {
6457 {
6463 }
6464 }
6470 }
6472 /* Transform. */
6479 {
6485 }
6487 /* Handle def. */
6491 {
6492 /* Vector shl and shr insn patterns can be defined with scalar
6493 operand 2 (shift operand). In this case, use constant or loop
6494 invariant op1 directly, without extending it to vector mode
6495 first. */
6498 {
6505 /* Store vec_oprnd1 for every vector stmt to be created.
6506 We check during the analysis that all the shift arguments
6507 are the same.
6508 TODO: Allow different constants for different vector
6509 stmts generated for an SLP instance. */
6513 }
6514 }
6516 {
6518 {
6519 /* If the argument was the same in all lanes create
6520 the correctly typed vector shift amount directly. */
6529 }
6531 /* The constant shift amount has been adjusted in place. */
6532 ;
6533 else
6535 }
6537 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6538 (a special case for certain kind of vector shifts); otherwise,
6539 operand 1 should be of a vector type (the usual case). */
6544 /* Arguments are ready. Create the new vector stmt. */
6546 {
6547 /* For internal defs where we need to use a scalar shift arg
6548 extract the first lane. */
6550 {
6553 gassign *new_stmt
6556 vop1,
6558 bitsize_zero_node));
6561 }
6562 else
6570 else
6572 }
6581 }
6583 /* Function vectorizable_operation.
6585 Check if STMT_INFO performs a binary, unary or ternary operation that can
6586 be vectorized.
6587 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6588 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6589 Return true if STMT_INFO is vectorizable in this way. */
6591 static bool
6596 {
6597 tree vec_dest;
6598 tree scalar_dest;
6600 tree vectype;
6603 machine_mode vec_mode;
6604 tree new_temp;
6606 optab optab;
6611 poly_uint64 nunits_in;
6612 poly_uint64 nunits_out;
6613 tree vectype_out;
6629 /* Is STMT a vectorizable binary/unary operation? */
6634 /* Loads and stores are handled in vectorizable_{load,store}. */
6640 /* Shifts are handled in vectorizable_shift. */
6647 /* Comparisons are handled in vectorizable_comparison. */
6651 /* Conditions are handled in vectorizable_condition. */
6655 /* For pointer addition and subtraction, we should use the normal
6656 plus and minus for the vector operation. */
6662 /* Support only unary or binary operations. */
6665 {
6669 op_type);
6671 }
6676 /* Most operations cannot handle bit-precision types without extra
6677 truncations. */
6681 /* Exception are bitwise binary operations. */
6685 {
6690 }
6692 slp_tree slp_op0;
6695 {
6700 }
6703 /* If op0 is an external or constant def, infer the vector type
6704 from the scalar type. */
6706 {
6707 /* For boolean type we cannot determine vectype by
6708 invariant value (don't know whether it is a vector
6709 of booleans or vector of integers). We use output
6710 vectype because operations on boolean don't change
6711 type. */
6713 {
6715 {
6720 }
6722 }
6723 else
6725 slp_node);
6726 }
6730 {
6737 }
6747 {
6750 {
6755 }
6761 }
6763 {
6766 {
6771 }
6777 }
6779 /* Multiple types in SLP are handled by creating the appropriate number of
6780 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6781 case of SLP. */
6783 {
6786 }
6787 else
6788 {
6791 }
6795 /* Reject attempts to combine mask types with nonmask types, e.g. if
6796 we have an AND between a (nonmask) boolean loaded from memory and
6797 a (mask) boolean result of a comparison.
6799 TODO: We could easily fix these cases up using pattern statements. */
6803 {
6808 }
6810 /* Supportable by target? */
6815 else
6816 {
6819 {
6824 }
6827 }
6831 {
6835 /* When vec_mode is not a vector mode and we verified ops we
6836 do not have to lower like AND are natively supported let
6837 those through even when the mode isn't word_mode. For
6838 ops we have to lower the lowering code assumes we are
6839 dealing with word_mode. */
6843 /* Check only during analysis. */
6845 {
6849 }
6854 }
6862 /* If operating on inactive elements could generate spurious traps,
6863 we need to restrict the operation to active lanes. Note that this
6864 specifically doesn't apply to unhoisted invariants, since they
6865 operate on the same value for every lane.
6867 Similarly, if this operation is part of a reduction, a fully-masked
6868 loop should only change the active lanes of the reduction chain,
6869 keeping the inactive lanes as-is. */
6874 {
6878 {
6881 OPTIMIZE_FOR_SPEED))
6886 OPTIMIZE_FOR_SPEED))
6889 else
6890 {
6893 "can't use a fully-masked loop because no"
6896 }
6897 }
6899 /* Put types on constant and invariant SLP children. */
6904 {
6909 }
6916 {
6917 /* The above vect_model_simple_cost call handles constants
6918 in the prologue and (mis-)costs one of the stmts as
6919 vector stmt. See below for the actual lowering that will
6920 be applied. */
6921 unsigned n
6924 {
6935 /* Bit operations do not have extra cost and are accounted
6936 as vector stmt by vect_model_simple_cost. */
6939 }
6941 {
6942 /* We also need to materialize two large constants. */
6947 }
6948 }
6950 }
6952 /* Transform. */
6961 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6962 vectors with unsigned elements, but the result is signed. So, we
6963 need to compute the MINUS_EXPR into vectype temporary and
6964 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6967 {
6970 }
6971 /* Handle def. */
6972 else
6975 /* In case the vectorization factor (VF) is bigger than the number
6976 of elements that we can fit in a vectype (nunits), we have to generate
6977 more than one vector stmt - i.e - we need to "unroll" the
6978 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6979 from one copy of the vector stmt to the next, in the field
6980 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6981 stages to find the correct vector defs to be used when vectorizing
6982 stmts that use the defs of the current stmt. The example below
6983 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6984 we need to create 4 vectorized stmts):
6986 before vectorization:
6987 RELATED_STMT VEC_STMT
6988 S1: x = memref - -
6989 S2: z = x + 1 - -
6991 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6992 there):
6993 RELATED_STMT VEC_STMT
6994 VS1_0: vx0 = memref0 VS1_1 -
6995 VS1_1: vx1 = memref1 VS1_2 -
6996 VS1_2: vx2 = memref2 VS1_3 -
6997 VS1_3: vx3 = memref3 - -
6998 S1: x = load - VS1_0
6999 S2: z = x + 1 - -
7001 step2: vectorize stmt S2 (done here):
7002 To vectorize stmt S2 we first need to find the relevant vector
7003 def for the first operand 'x'. This is, as usual, obtained from
7004 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
7005 that defines 'x' (S1). This way we find the stmt VS1_0, and the
7006 relevant vector def 'vx0'. Having found 'vx0' we can generate
7007 the vector stmt VS2_0, and as usual, record it in the
7008 STMT_VINFO_VEC_STMT of stmt S2.
7009 When creating the second copy (VS2_1), we obtain the relevant vector
7010 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
7011 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
7012 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
7013 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
7014 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
7015 chain of stmts and pointers:
7016 RELATED_STMT VEC_STMT
7017 VS1_0: vx0 = memref0 VS1_1 -
7018 VS1_1: vx1 = memref1 VS1_2 -
7019 VS1_2: vx2 = memref2 VS1_3 -
7020 VS1_3: vx3 = memref3 - -
7021 S1: x = load - VS1_0
7022 VS2_0: vz0 = vx0 + v1 VS2_1 -
7023 VS2_1: vz1 = vx1 + v1 VS2_2 -
7024 VS2_2: vz2 = vx2 + v1 VS2_3 -
7025 VS2_3: vz3 = vx3 + v1 - -
7026 S2: z = x + 1 - VS2_0 */
7030 /* Arguments are ready. Create the new vector stmt. */
7032 {
7039 {
7040 /* Lower the operation. This follows vector lowering. */
7043 tree word_type
7047 tree high_bits
7056 {
7074 else
7079 {
7085 }
7094 }
7095 else
7096 {
7113 }
7115 signs);
7125 }
7127 {
7128 tree mask;
7132 else
7133 /* Dummy mask. */
7143 {
7144 /* Perform the operation on active elements only and take
7145 inactive elements from the reduction chain input. */
7148 }
7149 else
7150 {
7154 }
7156 {
7159 signed char biasval
7164 }
7165 gcall *call
7168 vops);
7174 }
7175 else
7176 {
7178 /* When combining two masks check if either of them is elsewhere
7179 combined with a loop mask, if that's the case we can mark that the
7180 new combined mask doesn't need to be combined with a loop mask. */
7184 {
7186 ncopies}))
7187 {
7193 }
7196 ncopies }))
7197 {
7203 }
7204 }
7213 /* Enter the combined value into the vector cond hash so we don't
7214 AND it with a loop mask again. */
7217 }
7220 {
7223 new_temp);
7228 }
7232 else
7234 }
7244 }
7246 /* A helper function to ensure data reference DR_INFO's base alignment. */
7248 static void
7250 {
7251 /* Alignment is only analyzed for the first element of a DR group,
7252 use that to look at base alignment we need to enforce. */
7259 {
7262 // We should only be able to increase the alignment of a base object if
7263 // we know what its new alignment should be at compile time.
7270 {
7273 }
7275 }
7276 }
7279 /* Function get_group_alias_ptr_type.
7281 Return the alias type for the group starting at FIRST_STMT_INFO. */
7283 static tree
7285 {
7291 {
7295 {
7300 }
7302 }
7304 }
7307 /* Function scan_operand_equal_p.
7309 Helper function for check_scan_store. Compare two references
7310 with .GOMP_SIMD_LANE bases. */
7312 static bool
7314 {
7319 {
7320 machine_mode mode;
7330 {
7336 {
7341 }
7342 }
7343 }
7350 {
7354 {
7357 {
7360 {
7366 {
7369 }
7370 }
7372 {
7375 }
7378 {
7382 }
7391 }
7395 }
7396 }
7398 }
7402 /* Normal permutation. */
7403 scan_store_kind_perm,
7405 /* Whole vector left shift permutation with zero init. */
7406 scan_store_kind_lshift_zero,
7408 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7409 scan_store_kind_lshift_cond
7410 };
7412 /* Function check_scan_store.
7414 Verify if we can perform the needed permutations or whole vector shifts.
7415 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7416 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7417 to do at each step. */
7419 static int
7422 {
7434 {
7440 {
7443 }
7444 else
7445 {
7450 }
7453 {
7458 {
7462 /* Whole vector shifts shift in zeros, so if init is all zero
7463 constant, there is no need to do anything further. */
7467 {
7472 }
7473 }
7475 }
7477 {
7482 }
7483 }
7486 }
7489 /* Function check_scan_store.
7491 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7493 static bool
7496 vect_memory_access_type memory_access_type)
7497 {
7500 tree ref_type;
7504 || mask
7516 {
7521 }
7523 /* We need to pattern match code built by OpenMP lowering and simplified
7524 by following optimizations into something we can handle.
7525 #pragma omp simd reduction(inscan,+:r)
7526 for (...)
7527 {
7528 r += something ();
7529 #pragma omp scan inclusive (r)
7530 use (r);
7531 }
7532 shall have body with:
7533 // Initialization for input phase, store the reduction initializer:
7534 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7535 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7536 D.2042[_21] = 0;
7537 // Actual input phase:
7538 ...
7539 r.0_5 = D.2042[_20];
7540 _6 = _4 + r.0_5;
7541 D.2042[_20] = _6;
7542 // Initialization for scan phase:
7543 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7544 _26 = D.2043[_25];
7545 _27 = D.2042[_25];
7546 _28 = _26 + _27;
7547 D.2043[_25] = _28;
7548 D.2042[_25] = _28;
7549 // Actual scan phase:
7550 ...
7551 r.1_8 = D.2042[_20];
7552 ...
7553 The "omp simd array" variable D.2042 holds the privatized copy used
7554 inside of the loop and D.2043 is another one that holds copies of
7555 the current original list item. The separate GOMP_SIMD_LANE ifn
7556 kinds are there in order to allow optimizing the initializer store
7557 and combiner sequence, e.g. if it is originally some C++ish user
7558 defined reduction, but allow the vectorizer to pattern recognize it
7559 and turn into the appropriate vectorized scan.
7561 For exclusive scan, this is slightly different:
7562 #pragma omp simd reduction(inscan,+:r)
7563 for (...)
7564 {
7565 use (r);
7566 #pragma omp scan exclusive (r)
7567 r += something ();
7568 }
7569 shall have body with:
7570 // Initialization for input phase, store the reduction initializer:
7571 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7572 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7573 D.2042[_21] = 0;
7574 // Actual input phase:
7575 ...
7576 r.0_5 = D.2042[_20];
7577 _6 = _4 + r.0_5;
7578 D.2042[_20] = _6;
7579 // Initialization for scan phase:
7580 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7581 _26 = D.2043[_25];
7582 D.2044[_25] = _26;
7583 _27 = D.2042[_25];
7584 _28 = _26 + _27;
7585 D.2043[_25] = _28;
7586 // Actual scan phase:
7587 ...
7588 r.1_8 = D.2044[_20];
7589 ... */
7592 {
7593 /* Match the D.2042[_21] = 0; store above. Just require that
7594 it is a constant or external definition store. */
7596 {
7597 fail_init:
7602 }
7612 /* These stores can be vectorized normally. */
7614 }
7617 {
7618 fail:
7623 }
7632 bool inscan_var_store
7636 {
7638 {
7639 use_operand_p use_p;
7640 imm_use_iterator iter;
7642 {
7649 || other_store_stmt
7653 }
7659 }
7660 }
7662 {
7663 use_operand_p use_p;
7664 imm_use_iterator iter;
7666 {
7673 }
7674 }
7675 else
7685 /* For pointer addition, we should use the normal plus for the vector
7686 operation. */
7688 {
7696 }
7724 {
7730 tree lrhs;
7733 else
7735 use_operand_p use_p;
7736 imm_use_iterator iter;
7738 {
7745 }
7746 }
7754 stmt_vec_info other_store_stmt_info
7767 {
7771 }
7782 {
7793 }
7812 {
7817 }
7825 /* The IL is as expected, now check if we can actually vectorize it.
7826 Inclusive scan:
7827 _26 = D.2043[_25];
7828 _27 = D.2042[_25];
7829 _28 = _26 + _27;
7830 D.2043[_25] = _28;
7831 D.2042[_25] = _28;
7832 should be vectorized as (where _40 is the vectorized rhs
7833 from the D.2042[_21] = 0; store):
7834 _30 = MEM <vector(8) int> [(int *)&D.2043];
7835 _31 = MEM <vector(8) int> [(int *)&D.2042];
7836 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7837 _33 = _31 + _32;
7838 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7839 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7840 _35 = _33 + _34;
7841 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7842 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7843 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7844 _37 = _35 + _36;
7845 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7846 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7847 _38 = _30 + _37;
7848 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7849 MEM <vector(8) int> [(int *)&D.2043] = _39;
7850 MEM <vector(8) int> [(int *)&D.2042] = _38;
7851 Exclusive scan:
7852 _26 = D.2043[_25];
7853 D.2044[_25] = _26;
7854 _27 = D.2042[_25];
7855 _28 = _26 + _27;
7856 D.2043[_25] = _28;
7857 should be vectorized as (where _40 is the vectorized rhs
7858 from the D.2042[_21] = 0; store):
7859 _30 = MEM <vector(8) int> [(int *)&D.2043];
7860 _31 = MEM <vector(8) int> [(int *)&D.2042];
7861 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7862 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7863 _34 = _32 + _33;
7864 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7865 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7866 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7867 _36 = _34 + _35;
7868 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7869 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7870 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7871 _38 = _36 + _37;
7872 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7873 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7874 _39 = _30 + _38;
7875 _50 = _31 + _39;
7876 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7877 MEM <vector(8) int> [(int *)&D.2044] = _39;
7878 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7889 }
7892 /* Function vectorizable_scan_store.
7894 Helper of vectorizable_score, arguments like on vectorizable_store.
7895 Handle only the transformation, checking is done in check_scan_store. */
7897 static bool
7901 {
7916 bool inscan_var_store
7920 {
7921 use_operand_p use_p;
7922 imm_use_iterator iter;
7924 {
7930 }
7931 }
7952 {
7956 }
7971 {
7978 else
7979 {
7984 }
7988 {
7995 }
7996 else
7998 }
8019 {
8030 {
8034 dataref_offset);
8040 }
8044 {
8047 (zero_vec
8057 {
8058 /* Whole vector shift shifted in zero bits, but if *init
8059 is not initializer_zerop, we need to replace those elements
8060 with elements from vec_oprnd1. */
8073 }
8075 /* For exclusive scan, perform the perms[i] permutation once
8076 more. */
8080 {
8084 }
8092 }
8100 /* For exclusive scan, new_temp computed above is the exclusive scan
8101 prefix sum. Turn it into inclusive prefix sum for the broadcast
8102 of the last element into orig. */
8104 {
8109 }
8118 {
8121 dataref_offset);
8126 }
8127 }
8131 {
8137 dataref_offset);
8142 }
8144 }
8147 /* Function vectorizable_store.
8149 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8150 that can be vectorized.
8151 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8152 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8153 Return true if STMT_INFO is vectorizable in this way. */
8155 static bool
8160 {
8161 tree data_ref;
8162 tree op;
8164 tree elem_type;
8167 machine_mode vec_mode;
8168 tree dummy;
8176 stmt_vec_info first_stmt_info;
8182 tree aggr_type;
8183 gather_scatter_info gs_info;
8184 poly_uint64 vf;
8185 vec_load_store_type vls_type;
8186 tree ref_type;
8195 /* Is vectorizable store? */
8199 {
8212 }
8213 else
8214 {
8230 }
8234 /* Cannot have hybrid store SLP -- that would mean storing to the
8235 same location twice. */
8242 {
8245 }
8246 else
8249 /* Multiple types in SLP are handled by creating the appropriate number of
8250 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8251 case of SLP. */
8254 else
8259 /* FORNOW. This restriction should be relaxed. */
8261 {
8266 }
8278 vect_memory_access_type memory_access_type;
8281 poly_int64 poffset;
8282 internal_fn lanes_ifn;
8290 {
8292 {
8297 }
8301 {
8306 }
8310 {
8315 }
8316 }
8317 else
8318 {
8319 /* FORNOW. In some cases can vectorize even if data-type not supported
8320 (e.g. - array initialization with 0). */
8323 }
8330 {
8334 }
8335 else
8336 {
8340 }
8343 {
8345 memory_access_type))
8347 }
8351 {
8359 mask);
8363 vectype))
8364 {
8369 }
8380 /* As function vect_transform_stmt shows, for interleaving stores
8381 the whole chain is vectorized when the last store in the chain
8382 is reached, the other stores in the group are skipped. So we
8383 want to only cost the last one here, but it's not trivial to
8384 get the last, as it's equivalent to use the first one for
8385 costing, use the first one instead. */
8387 && !slp
8390 }
8393 /* Transform. */
8398 {
8402 }
8404 {
8408 {
8419 "vect_model_store_cost: inside_cost = %d, "
8424 }
8426 }
8429 {
8430 /* FORNOW */
8434 {
8436 /* VEC_NUM is the number of vect stmts to be created for this
8437 group. */
8444 }
8445 else
8446 /* VEC_NUM is the number of vect stmts to be created for this
8447 group. */
8451 }
8452 else
8457 ncopies);
8459 /* Check if we need to update prologue cost for invariant,
8460 and update it accordingly if so. If it's not for
8461 interleaving store, we can just check vls_type; but if
8462 it's for interleaving store, need to check the def_type
8463 of the stored value since the current vls_type is just
8464 for first_stmt_info. */
8466 {
8471 {
8477 }
8482 };
8486 {
8488 gimple_stmt_iterator incr_gsi;
8491 tree offvar;
8492 tree ivstep;
8493 tree running_off;
8496 tree dr_offset;
8498 /* Checked by get_load_store_type. */
8505 stride_base
8506 = fold_build_pointer_plus
8513 /* For a store with loop-invariant (but other than power-of-2)
8514 stride (i.e. not a grouped access) like so:
8516 for (i = 0; i < n; i += stride)
8517 array[i] = ...;
8519 we generate a new induction variable and new stores from
8520 the components of the (vectorized) rhs:
8522 for (j = 0; ; j += VF*stride)
8523 vectemp = ...;
8524 tmp1 = vectemp[0];
8525 array[j] = tmp1;
8526 tmp2 = vectemp[1];
8527 array[j + stride] = tmp2;
8528 ...
8529 */
8536 {
8539 {
8545 /* First check if vec_extract optab doesn't support extraction
8546 of vector elts directly. */
8548 machine_mode vmode;
8555 {
8556 /* Try to avoid emitting an extract of vector elements
8557 by performing the extracts using an integer type of the
8558 same size, extracting from a vector of those and then
8559 re-interpreting it as the original vector type if
8560 supported. */
8561 unsigned lsize
8564 /* If we can't construct such a vector fall back to
8565 element extracts from the original vector type and
8566 element size stores. */
8574 {
8579 }
8580 /* Else fall back to vector extraction anyway.
8581 Fewer stores are more important than avoiding spilling
8582 of the vector we extract from. Compared to the
8583 construction case in vectorizable_load no store-forwarding
8584 issue exists here for reasonable archs. */
8585 }
8586 }
8589 {
8591 dr_alignment_support dr_align
8593 mis_align);
8596 {
8603 }
8604 }
8607 }
8610 {
8624 }
8630 {
8633 {
8635 {
8637 tree pos
8644 }
8645 }
8651 else
8654 unsigned HOST_WIDE_INT
8657 {
8659 {
8661 /* Pun the vector to extract from if necessary. */
8663 {
8665 tree cvt
8670 }
8671 }
8673 {
8675 {
8676 /* Only need vector extracting when there are more
8677 than one stores. */
8679 inside_cost
8682 /* Take a single lane vector type store as scalar
8683 store to avoid ICE like 110776. */
8687 alignment_support_scheme,
8689 cost_vec);
8690 else
8691 inside_cost
8695 }
8699 /* Extract the i'th component. */
8707 GSI_SAME_STMT);
8715 /* And store it to *running_off. */
8722 {
8730 }
8733 {
8737 }
8738 }
8739 }
8744 }
8748 "vect_model_store_cost: inside_cost = %d, "
8753 }
8756 vec_loop_masks *loop_masks
8760 vec_loop_lens *loop_lens
8765 /* Shouldn't go with length-based approach if fully masked. */
8768 /* Targets with store-lane instructions must not require explicit
8769 realignment. vect_supportable_dr_alignment always returns either
8770 dr_aligned or dr_unaligned_supported for masked operations. */
8772 && !mask
8781 tree bump;
8784 {
8787 }
8789 {
8794 }
8795 else
8796 {
8799 else
8803 }
8808 /* In case the vectorization factor (VF) is bigger than the number
8809 of elements that we can fit in a vectype (nunits), we have to generate
8810 more than one vector stmt - i.e - we need to "unroll" the
8811 vector stmt by a factor VF/nunits. */
8813 /* In case of interleaving (non-unit grouped access):
8815 S1: &base + 2 = x2
8816 S2: &base = x0
8817 S3: &base + 1 = x1
8818 S4: &base + 3 = x3
8820 We create vectorized stores starting from base address (the access of the
8821 first stmt in the chain (S2 in the above example), when the last store stmt
8822 of the chain (S4) is reached:
8824 VS1: &base = vx2
8825 VS2: &base + vec_size*1 = vx0
8826 VS3: &base + vec_size*2 = vx1
8827 VS4: &base + vec_size*3 = vx3
8829 Then permutation statements are generated:
8831 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8832 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8833 ...
8835 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8836 (the order of the data-refs in the output of vect_permute_store_chain
8837 corresponds to the order of scalar stmts in the interleaving chain - see
8838 the documentation of vect_permute_store_chain()).
8840 In case of both multiple types and interleaving, above vector stores and
8841 permutation stmts are created for every copy. The result vector stmts are
8842 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8843 STMT_VINFO_RELATED_STMT for the next copies.
8844 */
8854 {
8858 {
8861 {
8862 /* For interleaved stores we collect vectorized defs for all
8863 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8864 as an input to vect_permute_store_chain(). */
8867 {
8868 /* Since gaps are not supported for interleaved stores,
8869 DR_GROUP_SIZE is the exact number of stmts in the
8870 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8874 else
8875 {
8881 }
8883 }
8886 {
8888 {
8891 mask_vectype);
8893 }
8895 /* We should have catched mismatched types earlier. */
8898 dataref_ptr
8902 }
8903 }
8905 {
8907 /* DR_CHAIN is then used as an input to
8908 vect_permute_store_chain(). */
8910 {
8913 }
8918 }
8921 {
8927 }
8929 /* Get an array into which we can store the individual vectors. */
8932 /* Invalidate the current contents of VEC_ARRAY. This should
8933 become an RTL clobber too, which prevents the vector registers
8934 from being upward-exposed. */
8937 /* Store the individual vectors into the array. */
8939 {
8942 i);
8943 }
8956 {
8960 else
8962 signed char biasval
8966 {
8969 }
8970 }
8974 {
8975 /* Emit:
8976 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8977 LEN, BIAS, VEC_ARRAY). */
8983 vec_array);
8984 }
8986 {
8987 /* Emit:
8988 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8989 VEC_ARRAY). */
8995 }
8996 else
8997 {
8998 /* Emit:
8999 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
9003 }
9008 /* Record that VEC_ARRAY is now dead. */
9013 }
9017 "vect_model_store_cost: inside_cost = %d, "
9022 }
9025 {
9030 {
9033 {
9038 {
9039 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
9040 DR_CHAIN is of size 1. */
9048 {
9051 mask_vectype);
9053 }
9055 /* We should have catched mismatched types earlier. */
9062 else
9063 dataref_ptr
9067 bump);
9068 }
9069 }
9071 {
9080 }
9088 {
9095 }
9098 {
9100 {
9102 inside_cost
9106 }
9113 {
9117 else
9120 signed char biasval
9124 {
9127 }
9128 }
9137 call
9141 else
9148 }
9149 else
9150 {
9151 /* Emulated scatter. */
9154 {
9156 /* For emulated scatter N offset vector element extracts
9157 (we assume the scalar scaling and ptr + offset add is
9158 consumed by the load). */
9159 inside_cost
9162 /* N scalar stores plus extracting the elements. */
9163 inside_cost
9166 inside_cost
9170 }
9173 unsigned HOST_WIDE_INT const_offset_nunits
9179 unsigned HOST_WIDE_INT elt_size
9181 /* We support offset vectors with more elements
9182 than the data vector for now. */
9183 unsigned HOST_WIDE_INT factor
9192 {
9193 /* Compute the offsetted pointer. */
9196 tree idx
9201 tree ptr
9205 /* Extract the element to be stored. */
9206 tree elt
9212 tree ref
9216 }
9217 }
9221 }
9225 "vect_model_store_cost: inside_cost = %d, "
9230 }
9241 {
9244 {
9246 {
9247 /* Get vectorized arguments for SLP_NODE. */
9253 }
9254 else
9255 {
9256 /* For interleaved stores we collect vectorized defs for all the
9257 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9258 input to vect_permute_store_chain().
9260 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9261 is of size 1. */
9264 {
9265 /* Since gaps are not supported for interleaved stores,
9266 DR_GROUP_SIZE is the exact number of stmts in the chain.
9267 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9268 that there is no interleaving, DR_GROUP_SIZE is 1,
9269 and only one iteration of the loop will be executed. */
9273 else
9274 {
9280 }
9282 }
9284 {
9287 mask_vectype);
9289 }
9290 }
9292 /* We should have catched mismatched types earlier. */
9296 bool simd_lane_access_p
9299 && simd_lane_access_p
9300 && !loop_masks
9307 {
9310 }
9312 dataref_ptr
9317 }
9319 {
9321 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9322 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9323 of size 1. */
9325 {
9328 }
9333 else
9336 }
9340 {
9341 /* Permute. */
9344 {
9351 "vect_model_store_cost: "
9353 group_size);
9354 }
9355 else
9358 }
9362 {
9364 {
9368 /* For grouped stores vectorized defs are interleaved in
9369 vect_permute_store_chain(). */
9371 }
9374 {
9378 else
9379 {
9385 /* Generate the permute statement. */
9386 gimple *perm_stmt
9390 gsi);
9394 }
9395 }
9398 {
9404 {
9408 }
9411 }
9427 /* Bump the vector pointer. */
9437 {
9440 }
9441 else
9446 misalign);
9449 /* Compute IFN when LOOP_LENS or final_mask valid. */
9454 {
9455 opt_machine_mode new_ovmode
9458 unsigned factor
9463 }
9465 {
9470 }
9473 {
9475 {
9476 /* Pass VF value to 'len' argument of
9477 MASK_LEN_STORE if LOOP_LENS is invalid. */
9479 }
9481 {
9482 /* Pass all ones value to 'mask' argument of
9483 MASK_LEN_STORE if final_mask is invalid. */
9486 }
9487 }
9489 {
9490 signed char biasval
9494 }
9496 /* Arguments are ready. Create the new vector stmt. */
9498 {
9501 /* Need conversion if it's wrapped with VnQI. */
9503 {
9504 tree new_vtype
9506 new_vmode);
9509 gassign *new_stmt
9513 }
9519 else
9526 }
9528 {
9530 gcall *call
9536 }
9537 else
9538 {
9539 data_ref
9541 dataref_offset ? dataref_offset
9544 ;
9545 else
9552 }
9560 }
9562 {
9566 }
9567 }
9570 {
9571 /* When vectorizing a store into the function result assign
9572 a penalty if the function returns in a multi-register location.
9573 In this case we assume we'll end up with having to spill the
9574 vector result and do piecewise loads as a conservative estimate. */
9580 {
9582 /* ??? Handle PARALLEL in some way. */
9584 {
9586 /* Assume that a single reg-reg move is possible and cheap,
9587 do not account for vector to gp register move cost. */
9589 {
9590 /* Spill. */
9591 prologue_cost
9594 /* Loads. */
9595 prologue_cost
9598 }
9599 }
9600 }
9603 "vect_model_store_cost: inside_cost = %d, "
9606 }
9609 }
9611 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9612 VECTOR_CST mask. No checks are made that the target platform supports the
9613 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9614 vect_gen_perm_mask_checked. */
9616 tree
9618 {
9619 tree mask_type;
9626 }
9628 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9629 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9631 tree
9633 {
9637 }
9639 /* Given a vector variable X and Y, that was generated for the scalar
9640 STMT_INFO, generate instructions to permute the vector elements of X and Y
9641 using permutation mask MASK_VEC, insert them at *GSI and return the
9642 permuted vector variable. */
9644 static tree
9648 {
9656 else
9660 /* Generate the permute statement. */
9665 }
9667 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9668 inserting them on the loops preheader edge. Returns true if we
9669 were successful in doing so (and thus STMT_INFO can be moved then),
9670 otherwise returns false. HOIST_P indicates if we want to hoist the
9671 definitions of all SSA uses, it would be false when we are costing. */
9673 static bool
9675 {
9676 ssa_op_iter i;
9677 tree op;
9681 {
9685 {
9686 /* Make sure we don't need to recurse. While we could do
9687 so in simple cases when there are more complex use webs
9688 we don't have an easy way to preserve stmt order to fulfil
9689 dependencies within them. */
9690 tree op2;
9691 ssa_op_iter i2;
9695 {
9700 }
9702 }
9703 }
9712 {
9716 {
9720 }
9721 }
9724 }
9726 /* vectorizable_load.
9728 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9729 that can be vectorized.
9730 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9731 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9732 Return true if STMT_INFO is vectorizable in this way. */
9734 static bool
9739 {
9740 tree scalar_dest;
9747 tree elem_type;
9748 /* Avoid false positive uninitialized warning, see PR110652. */
9750 machine_mode mode;
9751 tree dummy;
9758 poly_uint64 group_gap_adj;
9764 stmt_vec_info first_stmt_info;
9772 poly_uint64 vf;
9773 tree aggr_type;
9774 gather_scatter_info gs_info;
9775 tree ref_type;
9791 {
9806 }
9807 else
9808 {
9828 }
9834 {
9838 }
9839 else
9842 /* Multiple types in SLP are handled by creating the appropriate number of
9843 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
9844 case of SLP. */
9847 else
9852 /* FORNOW. This restriction should be relaxed. */
9854 {
9859 }
9861 /* Invalidate assumptions made by dependence analysis when vectorization
9862 on the unrolled body effectively re-orders stmts. */
9867 {
9870 "cannot perform implicit CSE when unrolling "
9873 }
9878 /* FORNOW. In some cases can vectorize even if data-type not supported
9879 (e.g. - data copies). */
9881 {
9886 }
9888 /* Check if the load is a part of an interleaving chain. */
9890 {
9892 /* FORNOW */
9899 /* Refuse non-SLP vectorization of SLP-only groups. */
9901 {
9906 }
9908 /* Invalidate assumptions made by dependence analysis when vectorization
9909 on the unrolled body effectively re-orders stmts. */
9914 {
9917 "cannot perform implicit CSE when performing "
9920 }
9921 }
9922 else
9926 {
9930 {
9931 /* In BB vectorization we may not actually use a loaded vector
9932 accessing elements in excess of DR_GROUP_SIZE. */
9943 {
9946 "BB vectorization with gaps at the end of "
9949 }
9950 }
9956 {
9959 vect_location,
9962 }
9963 }
9965 vect_memory_access_type memory_access_type;
9968 poly_int64 poffset;
9969 internal_fn lanes_ifn;
9977 {
9979 {
9985 }
9988 {
9993 }
9997 {
10002 }
10003 }
10008 {
10010 && mask
10012 mask_vectype))
10013 {
10018 }
10028 mask);
10041 }
10051 /* Transform. */
10057 {
10059 /* If we have versioned for aliasing or the loop doesn't
10060 have any data dependencies that would preclude this,
10061 then we are sure this is a loop invariant load and
10062 thus we can insert it on the preheader edge.
10063 TODO: hoist_defs_of_uses should ideally be computed
10064 once at analysis time, remembered and used in the
10065 transform time. */
10067 && !nested_in_vect_loop
10070 {
10071 enum vect_cost_model_location cost_loc
10076 cost_loc);
10081 "vect_model_load_cost: inside_cost = %d, "
10085 }
10087 {
10097 tree vuse;
10100 else
10105 }
10106 /* These copies are all equivalent. */
10110 else
10111 {
10116 }
10121 else
10122 {
10126 }
10128 }
10132 {
10133 gimple_stmt_iterator incr_gsi;
10135 tree offvar;
10136 tree ivstep;
10137 tree running_off;
10140 /* Checked by get_load_store_type. */
10143 tree dr_offset;
10150 {
10153 }
10154 else
10155 {
10158 }
10161 {
10164 }
10165 else
10166 {
10168 cst_offset
10171 first_stmt_info));
10174 }
10177 {
10185 /* For a load with loop-invariant (but other than power-of-2)
10186 stride (i.e. not a grouped access) like so:
10188 for (i = 0; i < n; i += stride)
10189 ... = array[i];
10191 we generate a new induction variable and new accesses to
10192 form a new vector (or vectors, depending on ncopies):
10194 for (j = 0; ; j += VF*stride)
10195 tmp1 = array[j];
10196 tmp2 = array[j + stride];
10197 ...
10198 vectemp = {tmp1, tmp2, ...}
10199 */
10213 }
10223 {
10225 {
10226 /* First check if vec_init optab supports construction from vector
10227 elts directly. Otherwise avoid emitting a constructor of
10228 vector elements by performing the loads using an integer type
10229 of the same size, constructing a vector of those and then
10230 re-interpreting it as the original vector type. This avoids a
10231 huge runtime penalty due to the general inability to perform
10232 store forwarding from smaller stores to a larger load. */
10233 tree ptype;
10234 tree vtype
10239 {
10244 }
10245 }
10246 else
10247 {
10251 }
10253 }
10254 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10259 {
10260 /* For SLP permutation support we need to load the whole group,
10261 not only the number of vector stmts the permutation result
10262 fits in. */
10264 {
10265 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10266 variable VF. */
10270 }
10271 else
10273 }
10275 unsigned HOST_WIDE_INT
10279 {
10284 {
10286 {
10287 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10288 avoid ICE, see PR110776. */
10295 else
10299 }
10313 {
10314 n_groups++;
10315 /* When doing SLP make sure to not load elements from
10316 the next vector iteration, those will not be accessed
10317 so just use the last element again. See PR107451. */
10319 {
10321 gimple *incr
10326 }
10328 }
10329 }
10332 {
10336 else
10337 {
10343 {
10344 new_stmt
10346 VIEW_CONVERT_EXPR,
10350 gsi);
10351 }
10352 }
10353 }
10356 {
10358 {
10361 else
10363 }
10364 else
10365 {
10369 }
10370 }
10371 }
10373 {
10376 {
10382 }
10383 else
10386 }
10390 "vect_model_load_cost: inside_cost = %u, "
10392 inside_cost);
10395 }
10403 {
10405 {
10408 }
10409 else
10410 {
10413 }
10414 /* For SLP vectorization we directly vectorize a subchain
10415 without permutation. */
10418 /* For BB vectorization always use the first stmt to base
10419 the data ref pointer on. */
10421 first_stmt_info_for_drptr
10424 /* Check if the chain of loads is already vectorized. */
10426 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10427 ??? But we can only do so if there is exactly one
10428 as we have no way to get at the rest. Leave the CSE
10429 opportunity alone.
10430 ??? With the group load eventually participating
10431 in multiple different permutations (having multiple
10432 slp nodes which refer to the same group) the CSE
10433 is even wrong code. See PR56270. */
10435 {
10438 }
10442 /* VEC_NUM is the number of vect stmts to be created for this group. */
10444 {
10446 /* If an SLP permutation is from N elements to N elements,
10447 and if one vector holds a whole number of N, we can load
10448 the inputs to the permutation in the same way as an
10449 unpermuted sequence. In other cases we need to load the
10450 whole group, not only the number of vector stmts the
10451 permutation result fits in. */
10456 {
10457 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10458 variable VF; see vect_transform_slp_perm_load. */
10463 }
10464 else
10465 {
10467 group_gap_adj
10469 }
10470 }
10471 else
10475 }
10476 else
10477 {
10485 }
10488 vec_loop_masks *loop_masks
10492 vec_loop_lens *loop_lens
10497 /* Shouldn't go with length-based approach if fully masked. */
10500 /* Targets with store-lane instructions must not require explicit
10501 realignment. vect_supportable_dr_alignment always returns either
10502 dr_aligned or dr_unaligned_supported for masked operations. */
10504 && !mask
10509 /* In case the vectorization factor (VF) is bigger than the number
10510 of elements that we can fit in a vectype (nunits), we have to generate
10511 more than one vector stmt - i.e - we need to "unroll" the
10512 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10513 from one copy of the vector stmt to the next, in the field
10514 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10515 stages to find the correct vector defs to be used when vectorizing
10516 stmts that use the defs of the current stmt. The example below
10517 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10518 need to create 4 vectorized stmts):
10520 before vectorization:
10521 RELATED_STMT VEC_STMT
10522 S1: x = memref - -
10523 S2: z = x + 1 - -
10525 step 1: vectorize stmt S1:
10526 We first create the vector stmt VS1_0, and, as usual, record a
10527 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10528 Next, we create the vector stmt VS1_1, and record a pointer to
10529 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10530 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10531 stmts and pointers:
10532 RELATED_STMT VEC_STMT
10533 VS1_0: vx0 = memref0 VS1_1 -
10534 VS1_1: vx1 = memref1 VS1_2 -
10535 VS1_2: vx2 = memref2 VS1_3 -
10536 VS1_3: vx3 = memref3 - -
10537 S1: x = load - VS1_0
10538 S2: z = x + 1 - -
10539 */
10541 /* In case of interleaving (non-unit grouped access):
10543 S1: x2 = &base + 2
10544 S2: x0 = &base
10545 S3: x1 = &base + 1
10546 S4: x3 = &base + 3
10548 Vectorized loads are created in the order of memory accesses
10549 starting from the access of the first stmt of the chain:
10551 VS1: vx0 = &base
10552 VS2: vx1 = &base + vec_size*1
10553 VS3: vx3 = &base + vec_size*2
10554 VS4: vx4 = &base + vec_size*3
10556 Then permutation statements are generated:
10558 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10559 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10560 ...
10562 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10563 (the order of the data-refs in the output of vect_permute_load_chain
10564 corresponds to the order of scalar stmts in the interleaving chain - see
10565 the documentation of vect_permute_load_chain()).
10566 The generation of permutation stmts and recording them in
10567 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10569 In case of both multiple types and interleaving, the vector loads and
10570 permutation stmts above are created for every copy. The result vector
10571 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10572 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10574 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10575 on a target that supports unaligned accesses (dr_unaligned_supported)
10576 we generate the following code:
10577 p = initial_addr;
10578 indx = 0;
10579 loop {
10580 p = p + indx * vectype_size;
10581 vec_dest = *(p);
10582 indx = indx + 1;
10583 }
10585 Otherwise, the data reference is potentially unaligned on a target that
10586 does not support unaligned accesses (dr_explicit_realign_optimized) -
10587 then generate the following code, in which the data in each iteration is
10588 obtained by two vector loads, one from the previous iteration, and one
10589 from the current iteration:
10590 p1 = initial_addr;
10591 msq_init = *(floor(p1))
10592 p2 = initial_addr + VS - 1;
10593 realignment_token = call target_builtin;
10594 indx = 0;
10595 loop {
10596 p2 = p2 + indx * vectype_size
10597 lsq = *(floor(p2))
10598 vec_dest = realign_load (msq, lsq, realignment_token)
10599 indx = indx + 1;
10600 msq = lsq;
10601 } */
10603 /* If the misalignment remains the same throughout the execution of the
10604 loop, we can create the init_addr and permutation mask at the loop
10605 preheader. Otherwise, it needs to be created inside the loop.
10606 This can only occur when vectorizing memory accesses in the inner-loop
10607 nested within an outer-loop that is being vectorized. */
10612 {
10615 }
10617 bool diff_first_stmt_info
10624 {
10625 /* If we have different first_stmt_info, we can't set up realignment
10626 here, since we can't guarantee first_stmt_info DR has been
10627 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10628 distance from first_stmt_info DR instead as below. */
10630 {
10637 {
10640 size_one_node);
10642 }
10643 }
10644 }
10645 else
10649 offset = (offset
10653 tree bump;
10656 {
10659 }
10661 {
10666 }
10667 else
10668 {
10671 else
10675 }
10680 {
10684 else
10687 }
10691 {
10698 {
10700 {
10701 /* An IFN_LOAD_LANES will load all its vector results,
10702 regardless of which ones we actually need. Account
10703 for the cost of unused results. */
10705 {
10708 do
10709 {
10712 }
10715 {
10718 "vect_model_load_cost: %d "
10720 gaps);
10722 alignment_support_scheme,
10726 }
10727 }
10732 }
10734 /* 1. Create the vector or array pointer update chain. */
10736 dataref_ptr
10740 else
10741 {
10745 }
10762 {
10766 else
10768 signed char biasval
10772 {
10775 }
10776 }
10780 {
10781 /* Emit:
10782 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10783 VEC_MASK, LEN, BIAS). */
10789 }
10791 {
10792 /* Emit:
10793 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10794 VEC_MASK). */
10799 final_mask);
10800 }
10801 else
10802 {
10803 /* Emit:
10804 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10807 }
10813 /* Extract each vector into an SSA_NAME. */
10815 {
10819 }
10821 /* Record the mapping between SSA_NAMEs and statements. */
10824 /* Record that VEC_ARRAY is now dead. */
10830 }
10834 "vect_model_load_cost: inside_cost = %u, "
10839 }
10842 {
10849 {
10850 /* 1. Create the vector or array pointer update chain. */
10852 {
10857 else
10858 dataref_ptr
10862 }
10864 {
10869 }
10876 {
10881 {
10883 final_mask
10894 }
10896 /* 2. Create the vector-load in the loop. */
10899 {
10901 {
10903 inside_cost
10907 }
10914 {
10916 final_len
10920 else
10921 final_len
10924 signed char biasval
10928 {
10931 }
10932 }
10936 call
10945 else
10952 }
10954 {
10955 /* The builtin decls path for gather is legacy, x86 only. */
10958 {
10960 inside_cost
10964 }
10965 poly_uint64 offset_nunits
10968 {
10969 new_stmt = vect_build_one_gather_load_call
10972 final_mask);
10974 }
10976 {
10977 /* We have a offset vector with half the number of
10978 lanes but the builtins will produce full vectype
10979 data with just the lower lanes filled. */
10980 new_stmt = vect_build_one_gather_load_call
10983 final_mask);
10989 /* now put upper half of final_mask in final_mask low. */
10991 && !SCALAR_INT_MODE_P
10993 {
11000 tree perm_mask = vect_gen_perm_mask_checked
11003 VEC_PERM_EXPR,
11004 final_mask,
11005 final_mask,
11006 perm_mask);
11011 }
11013 {
11015 VEC_UNPACK_HI_EXPR,
11016 final_mask);
11017 final_mask = make_ssa_name
11022 }
11024 new_stmt = vect_build_one_gather_load_call
11026 dataref_ptr,
11028 final_mask);
11034 /* compose low + high. */
11041 tree perm_mask
11044 VEC_PERM_EXPR,
11047 }
11049 {
11050 /* We have a offset vector with double the number of
11051 lanes. Select the low/high part accordingly. */
11054 {
11061 tree perm_mask = vect_gen_perm_mask_checked
11064 VEC_PERM_EXPR,
11065 vec_offset,
11066 vec_offset,
11067 perm_mask);
11072 }
11073 new_stmt = vect_build_one_gather_load_call
11077 }
11078 else
11080 }
11081 else
11082 {
11083 /* Emulated gather-scatter. */
11087 {
11088 /* For emulated gathers N offset vector element
11089 offset add is consumed by the load). */
11093 /* N scalar loads plus gathering them into a
11094 vector. */
11095 inside_cost
11098 inside_cost
11102 }
11103 unsigned HOST_WIDE_INT const_offset_nunits
11109 /* We support offset vectors with more elements
11110 than the data vector for now. */
11111 unsigned HOST_WIDE_INT factor
11120 {
11123 tree idx
11128 scale);
11140 }
11145 }
11148 /* DATA_REF is null if we've already built the statement. */
11150 {
11153 }
11158 /* Store vector loads in the corresponding SLP_NODE. */
11161 }
11165 }
11172 "vect_model_load_cost: inside_cost = %u, "
11176 }
11181 {
11182 /* 1. Create the vector or array pointer update chain. */
11184 {
11185 bool simd_lane_access_p
11196 {
11199 }
11201 {
11202 dataref_ptr
11206 bump);
11207 /* Adjust the pointer by the difference to first_stmt. */
11208 data_reference_p ptrdr
11210 tree diff
11218 {
11222 alignment_support_scheme,
11225 }
11226 }
11227 else
11228 dataref_ptr
11230 at_loop,
11235 }
11237 {
11241 bump);
11242 else
11247 }
11254 {
11259 {
11271 }
11273 /* 2. Create the vector-load in the loop. */
11275 {
11278 {
11288 {
11289 align
11292 }
11293 else
11298 misalign);
11301 /* Compute IFN when LOOP_LENS or final_mask valid. */
11306 {
11307 opt_machine_mode new_ovmode
11310 unsigned factor
11315 }
11317 {
11322 }
11325 {
11327 {
11328 /* Pass VF value to 'len' argument of
11329 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11331 }
11333 {
11334 /* Pass all ones value to 'mask' argument of
11335 MASK_LEN_LOAD if final_mask is invalid. */
11338 }
11339 }
11341 {
11342 signed char biasval
11346 }
11349 {
11356 bias);
11357 else
11365 /* Need conversion if it's wrapped with VnQI. */
11367 {
11370 tree var
11374 gsi);
11378 }
11379 }
11381 {
11385 final_mask);
11389 }
11390 else
11391 {
11395 unsigned int vect_align
11397 unsigned int scalar_dr_size
11399 /* If there's no peeling for gaps but we have a gap
11400 with slp loads then load the lower half of the
11401 vector only. See get_group_load_store_type for
11402 when we apply this optimization. */
11404 && loop_vinfo
11409 {
11410 tree half_vtype;
11411 new_vtype
11416 }
11417 tree offset
11422 {
11423 unsigned HOST_WIDE_INT gap_offset
11427 }
11428 data_ref
11431 ;
11432 else
11437 {
11443 gsi);
11448 {
11452 }
11453 else
11454 {
11458 }
11463 else
11464 {
11471 vec_dest,
11473 }
11474 }
11475 }
11477 }
11479 {
11489 dr_explicit_realign,
11494 else
11496 // For explicit realign the target alignment should be
11497 // known at compile time.
11498 unsigned HOST_WIDE_INT align
11505 data_ref
11519 bump);
11525 else
11529 data_ref
11532 }
11534 {
11539 else
11541 // We should only be doing this if we know the target
11542 // alignment at compile time.
11543 unsigned HOST_WIDE_INT align
11553 }
11556 }
11558 /* One common place to cost the above vect load for different
11559 alignment support schemes. */
11561 {
11562 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11563 only need to take care of the first stmt, whose
11564 stmt_info is first_stmt_info, vec_num iterating on it
11565 will cover the cost for the remaining, it's consistent
11566 with transforming. For the prologue cost for realign,
11567 we only need to count it once for the whole group. */
11578 }
11579 else
11580 {
11582 /* DATA_REF is null if we've already built the statement. */
11584 {
11587 }
11591 }
11593 /* 3. Handle explicit realignment if necessary/supported.
11594 Create in loop:
11595 vec_dest = realign_load (msq, lsq, realignment_token) */
11599 {
11611 {
11615 UNKNOWN_LOCATION);
11617 }
11618 }
11621 {
11625 else
11626 {
11631 }
11632 }
11634 /* Collect vector loads and later create their permutation in
11635 vect_transform_grouped_load (). */
11639 /* Store vector loads in the corresponding SLP_NODE. */
11643 /* With SLP permutation we load the gaps as well, without
11644 we need to skip the gaps after we manage to fully load
11645 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11649 && !slp_perm
11651 {
11652 poly_wide_int bump_val
11661 }
11662 }
11663 /* Bump the vector pointer to account for a gap or for excess
11664 elements loaded for a permuted SLP load. */
11668 {
11669 poly_wide_int bump_val
11676 }
11682 {
11684 /* For SLP we know we've seen all possible uses of dr_chain so
11685 direct vect_transform_slp_perm_load to DCE the unused parts.
11686 ??? This is a hack to prevent compile-time issues as seen
11687 in PR101120 and friends. */
11689 {
11694 }
11695 else
11696 {
11701 }
11702 }
11703 else
11704 {
11706 {
11708 /* We assume that the cost of a single load-lanes instruction
11709 is equivalent to the cost of DR_GROUP_SIZE separate loads.
11710 If a grouped access is instead being provided by a
11711 load-and-permute operation, include the cost of the
11712 permutes. */
11714 {
11715 /* Uses an even and odd extract operations or shuffle
11716 operations for each needed permute. */
11724 "vect_model_load_cost:"
11726 group_size);
11727 }
11729 {
11733 }
11734 }
11737 }
11739 }
11744 {
11750 "vect_model_load_cost: inside_cost = %u, "
11753 }
11756 }
11758 /* Function vect_is_simple_cond.
11760 Input:
11761 LOOP - the loop that is being vectorized.
11762 COND - Condition that is checked for simple use.
11764 Output:
11765 *COMP_VECTYPE - the vector type for the comparison.
11766 *DTS - The def types for the arguments of the comparison
11768 Returns whether a COND can be vectorized. Checks whether
11769 condition operands are supportable using vec_is_simple_use. */
11771 static bool
11775 {
11778 slp_tree slp_op;
11780 /* Mask case. */
11783 {
11786 || !*comp_vectype
11790 }
11799 {
11803 }
11807 else
11811 {
11815 }
11819 else
11828 /* Invariant comparison. */
11830 {
11834 else
11835 {
11836 /* If we can widen the comparison to match vectype do so. */
11838 && !slp_node
11841 scalar_type = build_nonstandard_integer_type
11844 slp_node);
11845 }
11846 }
11849 }
11851 /* vectorizable_condition.
11853 Check if STMT_INFO is conditional modify expression that can be vectorized.
11854 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
11855 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
11856 at GSI.
11858 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
11860 Return true if STMT_INFO is vectorizable in this way. */
11862 static bool
11867 {
11875 tree vec_compare;
11876 tree new_temp;
11891 tree vec_cmp_type;
11897 /* Is vectorizable conditional operation? */
11909 bool for_reduction
11912 {
11920 }
11921 else
11922 {
11925 }
11931 {
11934 }
11935 else
11936 {
11939 }
11975 {
11978 }
11980 /* For conditional reductions, the "then" value needs to be the candidate
11981 value calculated by this iteration while the "else" value needs to be
11982 the result carried over from previous iterations. If the COND_EXPR
11983 is the other way around, we need to swap it. */
11986 {
11989 else
11990 {
11995 else
11996 {
11998 /* Make sure we don't accidentally use the old condition. */
12000 }
12001 }
12003 }
12006 {
12007 /* Boolean values may have another representation in vectors
12008 and therefore we prefer bit operations over comparison for
12009 them (which also works for scalar masks). We store opcodes
12010 to use in bitop1 and bitop2. Statement is vectorized as
12011 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12012 depending on bitop1 and bitop2 arity. */
12014 {
12042 }
12044 }
12049 {
12054 }
12057 {
12059 {
12061 optab optab;
12068 {
12070 optab_default);
12073 }
12074 }
12078 /* Count one reduction-like operation per vector. */
12081 && (masked
12083 cond_code)
12085 ERROR_MARK))))
12089 && (!vect_maybe_update_slp_op_vectype
12092 && !vect_maybe_update_slp_op_vectype
12096 {
12101 }
12105 {
12107 {
12113 else
12117 }
12118 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12120 {
12123 "conditional reduction prevents the use"
12126 }
12127 }
12133 }
12135 /* Transform. */
12137 /* Handle def. */
12144 /* See whether another part of the vectorized code applies a loop
12145 mask to the condition, or to its inverse. */
12150 {
12153 }
12155 {
12158 else
12159 {
12163 else
12164 {
12169 {
12173 }
12174 else
12175 {
12176 /* Try the inverse of the current mask. We check if the
12177 inverse mask is live and if so we generate a negate of
12178 the current mask such that we still honor NaNs. */
12182 {
12187 }
12188 }
12189 }
12190 }
12191 }
12193 /* Handle cond expr. */
12198 reduction_type != EXTRACT_LAST_REDUCTION
12200 else
12205 reduction_type != EXTRACT_LAST_REDUCTION
12208 /* Arguments are ready. Create the new vector stmt. */
12210 {
12220 else
12221 {
12224 {
12229 }
12230 else
12231 {
12236 vec_cond_rhs);
12237 else
12238 new_stmt
12240 vec_cond_rhs);
12246 {
12247 /* Instead of doing ~x ? y : z do x ? z : y. */
12250 }
12251 else
12252 {
12255 new_stmt
12257 else
12258 new_stmt
12263 }
12264 }
12265 }
12267 /* If we decided to apply a loop mask to the result of the vector
12268 comparison, AND the comparison with the mask now. Later passes
12269 should then be able to reuse the AND results between mulitple
12270 vector statements.
12272 For example:
12273 for (int i = 0; i < 100; ++i)
12274 x[i] = y[i] ? z[i] : 10;
12276 results in following optimized GIMPLE:
12278 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12279 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12280 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12281 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12282 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12283 vect_iftmp.11_47, { 10, ... }>;
12285 instead of using a masked and unmasked forms of
12286 vec != { 0, ... } (masked in the MASK_LOAD,
12287 unmasked in the VEC_COND_EXPR). */
12289 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12290 in cases where that's necessary. */
12294 {
12296 {
12299 vec_compare);
12302 }
12305 {
12308 BIT_NOT_EXPR,
12309 vec_compare);
12312 }
12316 {
12318 {
12321 signed char biasval
12324 }
12325 else
12326 {
12329 }
12330 }
12332 {
12333 tree loop_mask
12337 gassign *g
12339 loop_mask);
12342 }
12343 }
12347 {
12351 new_stmt = gimple_build_call_internal
12354 else
12355 new_stmt = gimple_build_call_internal
12357 vec_then_clause);
12362 else
12363 {
12364 /* In this case we're moving the definition to later in the
12365 block. That doesn't matter because the only uses of the
12366 lhs are in phi statements. */
12370 }
12371 }
12372 else
12373 {
12378 }
12381 else
12383 }
12394 }
12396 /* Helper of vectorizable_comparison.
12398 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12399 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12400 comparison, put it in VEC_STMT, and insert it at GSI.
12402 Return true if STMT_INFO is vectorizable in this way. */
12404 static bool
12409 {
12413 tree new_temp;
12417 poly_uint64 nunits;
12424 tree mask_type;
12425 tree mask;
12438 else
12462 /* Invariant comparison. */
12464 {
12467 else
12469 slp_node);
12472 }
12476 /* Can't compare mask and non-mask types. */
12481 /* Boolean values may have another representation in vectors
12482 and therefore we prefer bit operations over comparison for
12483 them (which also works for scalar masks). We store opcodes
12484 to use in bitop1 and bitop2. Statement is vectorized as
12485 BITOP2 (rhs1 BITOP1 rhs2) or
12486 rhs1 BITOP2 (BITOP1 rhs2)
12487 depending on bitop1 and bitop2 arity. */
12490 {
12492 {
12495 }
12497 {
12500 }
12502 {
12506 }
12508 {
12512 }
12513 else
12514 {
12518 }
12519 }
12522 {
12524 {
12527 }
12528 else
12529 {
12531 optab optab;
12538 {
12542 }
12543 }
12545 /* Put types on constant and invariant SLP children. */
12549 {
12554 }
12560 }
12562 /* Transform. */
12564 /* Handle def. */
12574 /* Arguments are ready. Create the new vector stmt. */
12576 {
12582 {
12586 }
12587 else
12588 {
12591 else
12593 vec_rhs2);
12596 {
12600 else
12602 new_temp);
12604 }
12605 }
12608 else
12610 }
12619 }
12621 /* vectorizable_comparison.
12623 Check if STMT_INFO is comparison expression that can be vectorized.
12624 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12625 comparison, put it in VEC_STMT, and insert it at GSI.
12627 Return true if STMT_INFO is vectorizable in this way. */
12629 static bool
12634 {
12657 }
12659 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
12660 can handle all live statements in the node. Otherwise return true
12661 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
12662 VEC_STMT_P is as for vectorizable_live_operation. */
12664 static bool
12669 {
12671 {
12672 stmt_vec_info slp_stmt_info;
12675 {
12681 }
12682 }
12690 }
12692 /* Make sure the statement is vectorizable. */
12694 opt_result
12699 {
12703 gimple_seq pattern_def_seq;
12711 "not vectorized:"
12718 {
12719 gimple_stmt_iterator si;
12722 {
12723 stmt_vec_info pattern_def_stmt_info
12727 {
12728 /* Analyze def stmt of STMT if it's a pattern stmt. */
12734 opt_result res
12737 cost_vec);
12740 }
12741 }
12742 }
12744 /* Skip stmts that do not need to be vectorized. In loops this is expected
12745 to include:
12746 - the COND_EXPR which is the loop exit condition
12747 - any LABEL_EXPRs in the loop
12748 - computations that are used only for array indexing or loop control.
12749 In basic blocks we only analyze statements that are a part of some SLP
12750 instance, therefore, all the statements are relevant.
12752 Pattern statement needs to be analyzed instead of the original statement
12753 if the original statement is not relevant. Otherwise, we analyze both
12754 statements. In basic blocks we are called from some SLP instance
12755 traversal, don't analyze pattern stmts instead, the pattern stmts
12756 already will be part of SLP instance. */
12761 {
12763 && pattern_stmt_info
12766 {
12767 /* Analyze PATTERN_STMT instead of the original stmt. */
12773 }
12774 else
12775 {
12780 }
12781 }
12784 && pattern_stmt_info
12787 {
12788 /* Analyze PATTERN_STMT too. */
12794 opt_result res
12799 }
12802 {
12826 }
12833 {
12838 }
12841 {
12846 }
12852 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
12853 -mveclibabi= takes preference over library functions with
12854 the simd attribute. */
12857 cost_vec)
12874 cost_vec)
12879 else
12880 {
12886 cost_vec)
12892 cost_vec)
12900 cost_vec)
12902 }
12909 "not vectorized:"
12913 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
12914 need extra handling, except for vectorizable reductions. */
12922 "not vectorized:"
12927 }
12930 /* Function vect_transform_stmt.
12932 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
12934 bool
12938 {
12950 {
12962 NULL);
12974 NULL);
12986 NULL);
12992 && !slp_node
12995 /* In case of interleaving, the whole chain is vectorized when the
12996 last store in the chain is reached. Store stmts before the last
12997 one are skipped, and there vec_stmt_info shouldn't be freed
12998 meanwhile. */
12999 ;
13000 else
13001 {
13006 }
13062 {
13067 }
13069 }
13075 {
13076 /* Handle stmts whose DEF is used outside the loop-nest that is
13077 being vectorized. */
13081 }
13087 }
13090 /* Remove a group of stores (for SLP or interleaving), free their
13091 stmt_vec_info. */
13093 void
13095 {
13099 {
13102 /* Free the attached stmt_vec_info and remove the stmt. */
13105 }
13106 }
13108 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13109 elements of type SCALAR_TYPE, or null if the target doesn't support
13110 such a type.
13112 If NUNITS is zero, return a vector type that contains elements of
13113 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13115 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13116 for this vectorization region and want to "autodetect" the best choice.
13117 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13118 and we want the new type to be interoperable with it. PREVAILING_MODE
13119 in this case can be a scalar integer mode or a vector mode; when it
13120 is a vector mode, the function acts like a tree-level version of
13121 related_vector_mode. */
13123 tree
13126 {
13128 scalar_mode inner_mode;
13129 machine_mode simd_mode;
13130 tree vectype;
13141 /* Interoperability between modes requires one to be a constant multiple
13142 of the other, so that the number of vectors required for each operation
13143 is a compile-time constant. */
13151 /* For vector types of elements whose mode precision doesn't
13152 match their types precision we use a element type of mode
13153 precision. The vectorization routines will have to make sure
13154 they support the proper result truncation/extension.
13155 We also make sure to build vector types with INTEGER_TYPE
13156 component type only. */
13163 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13164 When the component mode passes the above test simply use a type
13165 corresponding to that mode. The theory is that any use that
13166 would cause problems with this will disable vectorization anyway. */
13171 /* We can't build a vector type of elements with alignment bigger than
13172 their size. */
13177 /* If we felt back to using the mode fail if there was
13178 no scalar type for it. */
13182 /* If no prevailing mode was supplied, use the mode the target prefers.
13183 Otherwise lookup a vector mode based on the prevailing mode. */
13185 {
13189 {
13190 /* Traditional behavior is not to take the integer mode
13191 literally, but simply to use it as a way of determining
13192 the vector size. It is up to mode_for_vector to decide
13193 what the TYPE_MODE should be.
13195 Note that nunits == 1 is allowed in order to support single
13196 element vector types. */
13200 }
13201 }
13205 {
13206 /* Fall back to using mode_for_vector, mostly in the hope of being
13207 able to use an integer mode. */
13214 }
13218 /* In cases where the mode was chosen by mode_for_vector, check that
13219 the target actually supports the chosen mode, or that it at least
13220 allows the vector mode to be replaced by a like-sized integer. */
13225 /* Re-attach the address-space qualifier if we canonicalized the scalar
13226 type. */
13228 return build_qualified_type
13232 }
13234 /* Function get_vectype_for_scalar_type.
13236 Returns the vector type corresponding to SCALAR_TYPE as supported
13237 by the target. If GROUP_SIZE is nonzero and we're performing BB
13238 vectorization, make sure that the number of elements in the vector
13239 is no bigger than GROUP_SIZE. */
13241 tree
13244 {
13245 /* For BB vectorization, we should always have a group size once we've
13246 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13247 are tentative requests during things like early data reference
13248 analysis and pattern recognition. */
13251 else
13255 scalar_type);
13259 /* Register the natural choice of vector type, before the group size
13260 has been applied. */
13264 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13265 try again with an explicit number of elements. */
13267 && group_size
13269 {
13270 /* Start with the biggest number of units that fits within
13271 GROUP_SIZE and halve it until we find a valid vector type.
13272 Usually either the first attempt will succeed or all will
13273 fail (in the latter case because GROUP_SIZE is too small
13274 for the target), but it's possible that a target could have
13275 a hole between supported vector types.
13277 If GROUP_SIZE is not a power of 2, this has the effect of
13278 trying the largest power of 2 that fits within the group,
13279 even though the group is not a multiple of that vector size.
13280 The BB vectorizer will then try to carve up the group into
13281 smaller pieces. */
13283 do
13284 {
13288 }
13290 }
13293 }
13295 /* Return the vector type corresponding to SCALAR_TYPE as supported
13296 by the target. NODE, if nonnull, is the SLP tree node that will
13297 use the returned vector type. */
13299 tree
13301 {
13306 }
13308 /* Function get_mask_type_for_scalar_type.
13310 Returns the mask type corresponding to a result of comparison
13311 of vectors of specified SCALAR_TYPE as supported by target.
13312 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13313 make sure that the number of elements in the vector is no bigger
13314 than GROUP_SIZE. */
13316 tree
13319 {
13326 }
13328 /* Function get_same_sized_vectype
13330 Returns a vector type corresponding to SCALAR_TYPE of size
13331 VECTOR_TYPE if supported by the target. */
13333 tree
13335 {
13339 poly_uint64 nunits;
13346 }
13348 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13349 would not change the chosen vector modes. */
13351 bool
13353 {
13360 }
13362 /* Function vect_is_simple_use.
13364 Input:
13365 VINFO - the vect info of the loop or basic block that is being vectorized.
13366 OPERAND - operand in the loop or bb.
13367 Output:
13368 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13369 case OPERAND is an SSA_NAME that is defined in the vectorizable region
13370 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13371 the definition could be anywhere in the function
13372 DT - the type of definition
13374 Returns whether a stmt with OPERAND can be vectorized.
13375 For loops, supportable operands are constants, loop invariants, and operands
13376 that are defined by the current iteration of the loop. Unsupportable
13377 operands are those that are defined by a previous iteration of the loop (as
13378 is the case in reduction/induction computations).
13379 For basic blocks, supportable operands are constants and bb invariants.
13380 For now, operands defined outside the basic block are not supported. */
13382 bool
13385 {
13393 {
13399 else
13401 }
13411 else
13412 {
13417 else
13418 {
13424 }
13427 }
13430 {
13433 {
13464 }
13465 }
13468 {
13473 }
13476 }
13478 /* Function vect_is_simple_use.
13480 Same as vect_is_simple_use but also determines the vector operand
13481 type of OPERAND and stores it to *VECTYPE. If the definition of
13482 OPERAND is vect_uninitialized_def, vect_constant_def or
13483 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
13484 is responsible to compute the best suited vector type for the
13485 scalar operand. */
13487 bool
13491 {
13492 stmt_vec_info def_stmt_info;
13502 /* Now get a vector type if the def is internal, otherwise supply
13503 NULL_TREE and leave it up to the caller to figure out a proper
13504 type for the use stmt. */
13511 {
13517 }
13522 else
13526 }
13528 /* Function vect_is_simple_use.
13530 Same as vect_is_simple_use but determines the operand by operand
13531 position OPERAND from either STMT or SLP_NODE, filling in *OP
13532 and *SLP_DEF (when SLP_NODE is not NULL). */
13534 bool
13539 {
13541 {
13546 {
13549 }
13550 else
13551 {
13557 }
13558 }
13559 else
13560 {
13563 {
13566 {
13569 else
13571 }
13574 else
13576 }
13579 else
13582 }
13583 }
13585 /* If OP is not NULL and is external or constant update its vector
13586 type with VECTYPE. Returns true if successful or false if not,
13587 for example when conflicting vector types are present. */
13589 bool
13591 {
13596 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13597 should be handled by patters. Allow vect_constant_def for now. */
13603 }
13605 /* Function supportable_widening_operation
13607 Check whether an operation represented by the code CODE is a
13608 widening operation that is supported by the target platform in
13609 vector form (i.e., when operating on arguments of type VECTYPE_IN
13610 producing a result of type VECTYPE_OUT).
13612 Widening operations we currently support are NOP (CONVERT), FLOAT,
13613 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13614 are supported by the target platform either directly (via vector
13615 tree-codes), or via target builtins.
13617 Output:
13618 - CODE1 and CODE2 are codes of vector operations to be used when
13619 vectorizing the operation, if available.
13620 - MULTI_STEP_CVT determines the number of required intermediate steps in
13621 case of multi-step conversion (like char->short->int - in that case
13622 MULTI_STEP_CVT will be 1).
13623 - INTERM_TYPES contains the intermediate type required to perform the
13624 widening operation (short in the above example). */
13626 bool
13628 code_helper code,
13629 stmt_vec_info stmt_info,
13635 {
13638 machine_mode vec_mode;
13654 {
13656 /* Don't set c1 and c2 if code is not a tree_code. */
13660 /* The result of a vectorized widening operation usually requires
13661 two vectors (because the widened results do not fit into one vector).
13662 The generated vector results would normally be expected to be
13663 generated in the same order as in the original scalar computation,
13664 i.e. if 8 results are generated in each vector iteration, they are
13665 to be organized as follows:
13666 vect1: [res1,res2,res3,res4],
13667 vect2: [res5,res6,res7,res8].
13669 However, in the special case that the result of the widening
13670 operation is used in a reduction computation only, the order doesn't
13671 matter (because when vectorizing a reduction we change the order of
13672 the computation). Some targets can take advantage of this and
13673 generate more efficient code. For example, targets like Altivec,
13674 that support widen_mult using a sequence of {mult_even,mult_odd}
13675 generate the following vectors:
13676 vect1: [res1,res3,res5,res7],
13677 vect2: [res2,res4,res6,res8].
13679 When vectorizing outer-loops, we execute the inner-loop sequentially
13680 (each vectorized inner-loop iteration contributes to VF outer-loop
13681 iterations in parallel). We therefore don't allow to change the
13682 order of the computation in the inner-loop during outer-loop
13683 vectorization. */
13684 /* TODO: Another case in which order doesn't *really* matter is when we
13685 widen and then contract again, e.g. (short)((int)x * y >> 8).
13686 Normally, pack_trunc performs an even/odd permute, whereas the
13687 repack from an even/odd expansion would be an interleave, which
13688 would be significantly simpler for e.g. AVX2. */
13689 /* In any case, in order to avoid duplicating the code below, recurse
13690 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13691 are properly set up for the caller. If we fail, we'll continue with
13692 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13700 interm_types))
13701 {
13702 /* Elements in a vector with vect_used_by_reduction property cannot
13703 be reordered if the use chain with this property does not have the
13704 same operation. One such an example is s += a * b, where elements
13705 in a and b cannot be reordered. Here we check if the vector defined
13706 by STMT is only directly used in the reduction statement. */
13712 }
13728 /* Support the recursion induced just above. */
13738 CASE_CONVERT:
13755 }
13761 {
13762 /* The signedness is determined from output operand. */
13765 }
13771 {
13772 /* If the input and result modes are the same, a different optab
13773 is needed where we pass in the number of units in vectype. */
13776 }
13780 {
13781 /* If this is an internal fn then we must check whether the target
13782 supports either a low-high split or an even-odd split. */
13792 /* If we don't support low-high, then check for even-odd. */
13795 || !optab2
13797 {
13803 }
13804 }
13806 {
13808 {
13809 /* The signedness is determined from output operand. */
13812 }
13818 {
13819 /* If the input and result modes are the same, a different optab
13820 is needed where we pass in the number of units in vectype. */
13823 }
13824 else
13825 {
13828 }
13831 }
13843 {
13846 /* For scalar masks we may have different boolean
13847 vector types having the same QImode. Thus we
13848 add additional check for elements number. */
13852 }
13854 /* Check if it's a multi-step conversion that can be done using intermediate
13855 types. */
13863 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
13864 intermediate steps in promotion sequence. We try
13865 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
13866 not. */
13869 {
13872 intermediate_type
13875 {
13876 tree intermediate_element_type
13879 intermediate_type
13881 intermediate_mode);
13882 }
13883 else
13884 intermediate_type
13892 {
13893 /* If the input and result modes are the same, a different optab
13894 is needed where we pass in the number of units in vectype. */
13897 }
13898 else
13899 {
13902 }
13920 {
13926 }
13930 }
13934 }
13937 /* Function supportable_narrowing_operation
13939 Check whether an operation represented by the code CODE is a
13940 narrowing operation that is supported by the target platform in
13941 vector form (i.e., when operating on arguments of type VECTYPE_IN
13942 and producing a result of type VECTYPE_OUT).
13944 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
13945 and FLOAT. This function checks if these operations are supported by
13946 the target platform directly via vector tree-codes.
13948 Output:
13949 - CODE1 is the code of a vector operation to be used when
13950 vectorizing the operation, if available.
13951 - MULTI_STEP_CVT determines the number of required intermediate steps in
13952 case of multi-step conversion (like int->short->char - in that case
13953 MULTI_STEP_CVT will be 1).
13954 - INTERM_TYPES contains the intermediate type required to perform the
13955 narrowing operation (short in the above example). */
13957 bool
13962 {
13963 machine_mode vec_mode;
13980 {
13981 CASE_CONVERT:
13989 else
13995 /* The signedness is determined from output operand. */
14006 }
14018 {
14021 /* For scalar masks we may have different boolean
14022 vector types having the same QImode. Thus we
14023 add additional check for elements number. */
14027 }
14032 /* Check if it's a multi-step conversion that can be done using intermediate
14033 types. */
14038 else
14041 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14042 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14043 costly than signed. */
14045 {
14048 intermediate_type
14050 interm_optab
14056 {
14060 }
14061 }
14063 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14064 intermediate steps in promotion sequence. We try
14065 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14068 {
14071 intermediate_type
14073 else
14074 intermediate_type
14082 else
14083 interm_optab
14085 optab_default);
14097 {
14103 }
14108 }
14112 }
14114 /* Generate and return a vector mask of MASK_TYPE such that
14115 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14116 Add the statements to SEQ. */
14118 tree
14121 {
14125 OPTIMIZE_FOR_SPEED));
14129 tree tmp;
14132 else
14137 }
14139 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14140 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14142 tree
14144 tree end_index)
14145 {
14148 }
14150 /* Try to compute the vector types required to vectorize STMT_INFO,
14151 returning true on success and false if vectorization isn't possible.
14152 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14153 take sure that the number of elements in the vectors is no bigger
14154 than GROUP_SIZE.
14156 On success:
14158 - Set *STMT_VECTYPE_OUT to:
14159 - NULL_TREE if the statement doesn't need to be vectorized;
14160 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14162 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14163 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14164 statement does not help to determine the overall number of units. */
14166 opt_result
14171 {
14174 /* For BB vectorization, we should always have a group size once we've
14175 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14176 are tentative requests during things like early data reference
14177 analysis and pattern recognition. */
14180 else
14187 /* MASK_STORE has no lhs, but is ok. */
14189 {
14191 {
14192 /* Ignore calls with no lhs. These must be calls to
14193 #pragma omp simd functions, and what vectorization factor
14194 it really needs can't be determined until
14195 vectorizable_simd_clone_call. */
14200 }
14204 }
14206 tree vectype;
14209 {
14214 }
14216 {
14225 }
14226 else
14227 {
14232 else
14236 {
14239 "get vectype for scalar type (group size %d):"
14241 else
14244 }
14248 "not vectorized:"
14250 scalar_type);
14254 }
14259 stmt);
14263 /* Don't try to compute scalar types if the stmt produces a boolean
14264 vector; use the existing vector type instead. */
14267 {
14268 /* The number of units is set according to the smallest scalar
14269 type (or the largest vector size, but we only support one
14270 vector size per vectorization). */
14274 {
14278 scalar_type);
14280 group_size);
14284 scalar_type);
14287 nunits_vectype);
14288 }
14289 }
14294 "Not vectorized: Incompatible number "
14299 {
14303 }
14307 }
14309 /* Generate and return statement sequence that sets vector length LEN that is:
14311 min_of_start_and_end = min (START_INDEX, END_INDEX);
14312 left_len = END_INDEX - min_of_start_and_end;
14313 rhs = min (left_len, LEN_LIMIT);
14314 LEN = rhs;
14316 Note: the cost of the code generated by this function is modeled
14317 by vect_estimate_min_profitable_iters, so changes here may need
14318 corresponding changes there. */
14320 gimple_seq
14322 {
14334 }