| blob | 390c8472fd6c48259dffc4d9cf3ba1a0f0cf30d7 |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2023 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
61 /* For lang_hooks.types.type_for_mode. */
64 /* Return the vectorized type for the given statement. */
66 tree
68 {
70 }
72 /* Return TRUE iff the given statement is in an inner loop relative to
73 the loop being vectorized. */
74 bool
76 {
88 }
90 /* Record the cost of a statement, either by directly informing the
91 target model or by saving it in a vector for later processing.
92 Return a preliminary estimate of the statement's cost. */
94 static unsigned
100 {
108 stmt_info_for_cost si
114 }
116 unsigned
121 {
124 }
126 unsigned
131 {
134 }
136 unsigned
140 {
145 }
147 /* Return a variable of type ELEM_TYPE[NELEMS]. */
149 static tree
151 {
154 }
156 /* ARRAY is an array of vectors created by create_vector_array.
157 Return an SSA_NAME for the vector in index N. The reference
158 is part of the vectorization of STMT_INFO and the vector is associated
159 with scalar destination SCALAR_DEST. */
161 static tree
165 {
182 }
184 /* ARRAY is an array of vectors created by create_vector_array.
185 Emit code to store SSA_NAME VECT in index N of the array.
186 The store is part of the vectorization of STMT_INFO. */
188 static void
192 {
193 tree array_ref;
202 }
204 /* PTR is a pointer to an array of type TYPE. Return a representation
205 of *PTR. The memory reference replaces those in FIRST_DR
206 (and its group). */
208 static tree
210 {
211 tree mem_ref;
214 /* Arrays have the same alignment as their type. */
217 }
219 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
220 Emit the clobber before *GSI. */
222 static void
225 {
229 }
231 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
233 /* Function vect_mark_relevant.
235 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
237 static void
240 {
249 /* If this stmt is an original stmt in a pattern, we might need to mark its
250 related pattern stmt instead of the original stmt. However, such stmts
251 may have their own uses that are not in any pattern, in such cases the
252 stmt itself should be marked. */
254 {
255 /* This is the last stmt in a sequence that was detected as a
256 pattern that can potentially be vectorized. Don't mark the stmt
257 as relevant/live because it's not going to be vectorized.
258 Instead mark the pattern-stmt that replaces it. */
262 "last stmt in pattern. don't mark"
272 {
275 "vec_stmt_relevant_p: forcing live pattern stmt "
278 }
284 }
292 {
297 }
300 }
303 /* Function is_simple_and_all_uses_invariant
305 Return true if STMT_INFO is simple and all uses of it are invariant. */
307 bool
309 loop_vec_info loop_vinfo)
310 {
311 tree op;
312 ssa_op_iter iter;
319 {
323 {
328 }
332 }
334 }
336 /* Function vect_stmt_relevant_p.
338 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
339 is "relevant for vectorization".
341 A stmt is considered "relevant for vectorization" if:
342 - it has uses outside the loop.
343 - it has vdefs (it alters memory).
344 - control stmts in the loop (except for the exit condition).
346 CHECKME: what other side effects would the vectorizer allow? */
348 static bool
351 {
353 ssa_op_iter op_iter;
354 imm_use_iterator imm_iter;
355 use_operand_p use_p;
356 def_operand_p def_p;
361 /* cond stmt other than loop exit cond. */
366 /* changing memory. */
370 {
375 }
377 /* uses outside the loop. */
379 {
381 {
384 {
392 /* We expect all such uses to be in the loop exit phis
393 (because of loop closed form) */
398 }
399 }
400 }
404 {
409 }
412 }
415 /* Function exist_non_indexing_operands_for_use_p
417 USE is one of the uses attached to STMT_INFO. Check if USE is
418 used in STMT_INFO for anything other than indexing an array. */
420 static bool
422 {
423 tree operand;
425 /* USE corresponds to some operand in STMT. If there is no data
426 reference in STMT, then any operand that corresponds to USE
427 is not indexing an array. */
431 /* STMT has a data_ref. FORNOW this means that its of one of
432 the following forms:
433 -1- ARRAY_REF = var
434 -2- var = ARRAY_REF
435 (This should have been verified in analyze_data_refs).
437 'var' in the second case corresponds to a def, not a use,
438 so USE cannot correspond to any operands that are not used
439 for array indexing.
441 Therefore, all we need to check is if STMT falls into the
442 first case, and whether var corresponds to USE. */
446 {
449 {
462 }
464 }
476 }
479 /*
480 Function process_use.
482 Inputs:
483 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
484 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
485 that defined USE. This is done by calling mark_relevant and passing it
486 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
487 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
488 be performed.
490 Outputs:
491 Generally, LIVE_P and RELEVANT are used to define the liveness and
492 relevance info of the DEF_STMT of this USE:
493 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
494 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
495 Exceptions:
496 - case 1: If USE is used only for address computations (e.g. array indexing),
497 which does not need to be directly vectorized, then the liveness/relevance
498 of the respective DEF_STMT is left unchanged.
499 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
500 we skip DEF_STMT cause it had already been processed.
501 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
502 "relevant" will be modified accordingly.
504 Return true if everything is as expected. Return false otherwise. */
506 static opt_result
510 {
511 stmt_vec_info dstmt_vinfo;
514 /* case 1: we are only interested in uses that need to be vectorized. Uses
515 that are used for address computation are not considered relevant. */
521 "not vectorized:"
530 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
531 We have to force the stmt live since the epilogue loop needs it to
532 continue computing the reduction. */
538 {
544 }
546 /* case 3a: outer-loop stmt defining an inner-loop stmt:
547 outer-loop-header-bb:
548 d = dstmt_vinfo
549 inner-loop:
550 stmt # use (d)
551 outer-loop-tail-bb:
552 ... */
554 {
560 {
581 }
582 }
584 /* case 3b: inner-loop stmt defining an outer-loop stmt:
585 outer-loop-header-bb:
586 ...
587 inner-loop:
588 d = dstmt_vinfo
589 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
590 stmt # use (d) */
592 {
598 {
616 }
617 }
618 /* We are also not interested in uses on loop PHI backedges that are
619 inductions. Otherwise we'll needlessly vectorize the IV increment
620 and cause hybrid SLP for SLP inductions. Unless the PHI is live
621 of course. */
628 {
633 }
638 }
641 /* Function vect_mark_stmts_to_be_vectorized.
643 Not all stmts in the loop need to be vectorized. For example:
645 for i...
646 for j...
647 1. T0 = i + j
648 2. T1 = a[T0]
650 3. j = j + 1
652 Stmt 1 and 3 do not need to be vectorized, because loop control and
653 addressing of vectorized data-refs are handled differently.
655 This pass detects such stmts. */
657 opt_result
659 {
663 gimple_stmt_iterator si;
665 basic_block bb;
673 /* 1. Init worklist. */
675 {
678 {
686 }
688 {
698 }
699 }
701 /* 2. Process_worklist */
703 {
704 use_operand_p use_p;
705 ssa_op_iter iter;
712 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
713 (DEF_STMT) as relevant/irrelevant according to the relevance property
714 of STMT. */
717 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
718 propagated as is to the DEF_STMTs of its USEs.
720 One exception is when STMT has been identified as defining a reduction
721 variable; in this case we set the relevance to vect_used_by_reduction.
722 This is because we distinguish between two kinds of relevant stmts -
723 those that are used by a reduction computation, and those that are
724 (also) used by a regular computation. This allows us later on to
725 identify stmts that are used solely by a reduction, and therefore the
726 order of the results that they produce does not have to be kept. */
729 {
758 }
761 {
762 /* Pattern statements are not inserted into the code, so
763 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
764 have to scan the RHS or function arguments instead. */
766 {
772 {
773 opt_result res
783 }
785 {
788 {
789 opt_result res
794 }
795 }
796 }
798 {
800 {
802 opt_result res
807 }
808 }
809 }
810 else
812 {
814 opt_result res
819 }
822 {
823 gather_scatter_info gs_info;
826 opt_result res
830 {
834 }
835 }
839 }
841 /* Function vect_model_simple_cost.
843 Models cost for simple operations, i.e. those that only emit ncopies of a
844 single op. Right now, this does not account for multiple insns that could
845 be generated for the single vector op. We will handle that shortly. */
847 static void
852 slp_tree node,
855 {
860 /* ??? Somehow we need to fix this at the callers. */
865 /* Cost the "broadcast" of a scalar operand in to a vector operand.
866 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
867 cost model. */
873 /* Pass the inside-of-loop statements to the target-specific cost model. */
879 "vect_model_simple_cost: inside_cost = %d, "
881 }
884 /* Model cost for type demotion and promotion operations. PWR is
885 normally zero for single-step promotions and demotions. It will be
886 one if two-step promotion/demotion is required, and so on. NCOPIES
887 is the number of vector results (and thus number of instructions)
888 for the narrowest end of the operation chain. Each additional
889 step doubles the number of instructions required. If WIDEN_ARITH
890 is true the stmt is doing widening arithmetic. */
892 static void
898 {
903 {
905 widen_arith
909 }
911 /* FORNOW: Assuming maximum 2 args per stmts. */
919 "vect_model_promotion_demotion_cost: inside_cost = %d, "
921 }
923 /* Returns true if the current function returns DECL. */
925 static bool
927 {
928 edge_iterator ei;
929 edge e;
931 {
937 /* We often end up with an aggregate copy to the result decl,
938 handle that case as well. First skip intermediate clobbers
939 though. */
941 do
942 {
944 }
950 }
952 }
954 /* Calculate cost of DR's memory access. */
955 void
957 dr_alignment_support alignment_support_scheme,
961 {
963 {
965 {
968 vect_body);
974 }
977 {
978 /* Here, we assign an additional cost for the unaligned store. */
984 "vect_model_store_cost: unaligned supported by "
987 }
990 {
997 }
1001 }
1002 }
1004 /* Calculate cost of DR's memory access. */
1005 void
1007 dr_alignment_support alignment_support_scheme,
1014 {
1016 {
1018 {
1027 }
1029 {
1030 /* Here, we assign an additional cost for the unaligned load. */
1037 "vect_model_load_cost: unaligned supported by "
1041 }
1043 {
1049 /* FIXME: If the misalignment remains fixed across the iterations of
1050 the containing loop, the following cost should be added to the
1051 prologue costs. */
1061 }
1063 {
1066 "vect_model_load_cost: unaligned software "
1069 /* Unaligned software pipeline has a load of an address, an initial
1070 load, and possibly a mask operation to "prime" the loop. However,
1071 if this is an access in a group of loads, which provide grouped
1072 access, then the above cost should only be considered for one
1073 access in the group. Inside the loop, there is a load op
1074 and a realignment op. */
1077 {
1085 }
1094 "vect_model_load_cost: explicit realign optimized"
1098 }
1101 {
1108 }
1112 }
1113 }
1115 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1116 the loop preheader for the vectorized stmt STMT_VINFO. */
1118 static void
1121 {
1124 else
1130 }
1132 /* Function vect_init_vector.
1134 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1135 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1136 vector type a vector with all elements equal to VAL is created first.
1137 Place the initialization at GSI if it is not NULL. Otherwise, place the
1138 initialization at the loop preheader.
1139 Return the DEF of INIT_STMT.
1140 It will be used in the vectorization of STMT_INFO. */
1142 tree
1145 {
1147 tree new_temp;
1149 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1151 {
1154 {
1155 /* Scalar boolean value should be transformed into
1156 all zeros or all ones value before building a vector. */
1158 {
1164 else
1165 {
1171 }
1172 }
1173 else
1174 {
1179 else
1180 /* ??? Condition vectorization expects us to do
1181 promotion of invariant/external defs. */
1185 {
1189 }
1190 }
1191 }
1193 }
1199 }
1202 /* Function vect_get_vec_defs_for_operand.
1204 OP is an operand in STMT_VINFO. This function returns a vector of
1205 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1207 In the case that OP is an SSA_NAME which is defined in the loop, then
1208 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1210 In case OP is an invariant or constant, a new stmt that creates a vector def
1211 needs to be introduced. VECTYPE may be used to specify a required type for
1212 vector invariant. */
1214 void
1218 {
1228 stmt_vec_info def_stmt_info;
1237 {
1239 tree vector_type;
1246 else
1253 }
1254 else
1255 {
1261 }
1262 }
1265 /* Get vectorized definitions for OP0 and OP1. */
1267 void
1274 {
1276 {
1285 }
1286 else
1287 {
1300 }
1301 }
1303 void
1310 {
1316 }
1318 /* Helper function called by vect_finish_replace_stmt and
1319 vect_finish_stmt_generation. Set the location of the new
1320 statement and create and return a stmt_vec_info for it. */
1322 static void
1325 {
1330 {
1333 /* While EH edges will generally prevent vectorization, stmt might
1334 e.g. be in a must-not-throw region. Ensure newly created stmts
1335 that could throw are part of the same region. */
1339 }
1340 else
1342 }
1344 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1345 which sets the same scalar result as STMT_INFO did. Create and return a
1346 stmt_vec_info for VEC_STMT. */
1348 void
1351 {
1359 }
1361 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1362 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1364 void
1368 {
1373 {
1377 {
1381 /* If we have an SSA vuse and insert a store, update virtual
1382 SSA form to avoid triggering the renamer. Do so only
1383 if we can easily see all uses - which is what almost always
1384 happens with the way vectorized stmts are inserted. */
1393 {
1397 }
1398 }
1399 }
1402 }
1404 /* We want to vectorize a call to combined function CFN with function
1405 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1406 as the types of all inputs. Check whether this is possible using
1407 an internal function, returning its code if so or IFN_LAST if not. */
1409 static internal_fn
1412 {
1413 internal_fn ifn;
1416 else
1419 {
1422 {
1427 /* The type size of both the vectype_in and vectype_out should be
1428 exactly the same when vectype_out isn't participating the optab.
1429 While there is no restriction for type size when vectype_out
1430 is part of the optab query. */
1435 OPTIMIZE_FOR_SPEED))
1437 }
1438 }
1440 }
1444 gimple_stmt_iterator *);
1446 /* Check whether a load or store statement in the loop described by
1447 LOOP_VINFO is possible in a loop using partial vectors. This is
1448 testing whether the vectorizer pass has the appropriate support,
1449 as well as whether the target does.
1451 VLS_TYPE says whether the statement is a load or store and VECTYPE
1452 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1453 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1454 says how the load or store is going to be implemented and GROUP_SIZE
1455 is the number of load or store statements in the containing group.
1456 If the access is a gather load or scatter store, GS_INFO describes
1457 its arguments. If the load or store is conditional, SCALAR_MASK is the
1458 condition under which it occurs.
1460 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1461 vectors is not supported, otherwise record the required rgroup control
1462 types. */
1464 static void
1466 slp_tree slp_node,
1467 vec_load_store_type vls_type,
1469 vect_memory_access_type
1470 memory_access_type,
1472 tree scalar_mask)
1473 {
1474 /* Invariant loads need no special support. */
1481 else
1489 {
1490 internal_fn ifn
1497 scalar_mask);
1498 else
1499 {
1502 "can't operate on partial vectors because"
1503 " the target doesn't have an appropriate"
1506 }
1508 }
1511 {
1512 internal_fn ifn = (is_load
1513 ? IFN_MASK_GATHER_LOAD
1515 internal_fn len_ifn = (is_load
1516 ? IFN_MASK_LEN_GATHER_LOAD
1528 scalar_mask);
1529 else
1530 {
1533 "can't operate on partial vectors because"
1534 " the target doesn't have an appropriate"
1537 }
1539 }
1543 {
1544 /* Element X of the data must come from iteration i * VF + X of the
1545 scalar loop. We need more work to support other mappings. */
1548 "can't operate on partial vectors because an"
1552 }
1555 {
1558 "can't operate on partial vectors when emulating"
1562 }
1564 /* We might load more scalars than we need for permuting SLP loads.
1565 We checked in get_group_load_store_type that the extra elements
1566 don't leak into a new vector. */
1568 {
1573 };
1577 machine_mode mask_mode;
1578 machine_mode vmode;
1581 {
1586 }
1589 {
1593 }
1596 {
1599 "can't operate on partial vectors because the"
1600 " target doesn't have the appropriate partial"
1603 }
1604 }
1606 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1607 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1608 that needs to be applied to all loads and stores in a vectorized loop.
1609 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1610 otherwise return VEC_MASK & LOOP_MASK.
1612 MASK_TYPE is the type of both masks. If new statements are needed,
1613 insert them before GSI. */
1615 static tree
1618 {
1634 }
1636 /* Determine whether we can use a gather load or scatter store to vectorize
1637 strided load or store STMT_INFO by truncating the current offset to a
1638 smaller width. We need to be able to construct an offset vector:
1640 { 0, X, X*2, X*3, ... }
1642 without loss of precision, where X is STMT_INFO's DR_STEP.
1644 Return true if this is possible, describing the gather load or scatter
1645 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1647 static bool
1651 {
1656 {
1657 /* ??? Perhaps we could use range information here? */
1662 }
1664 /* Get the number of bits in an element. */
1669 /* Set COUNT to the upper limit on the number of elements - 1.
1670 Start with the maximum vectorization factor. */
1673 /* Try lowering COUNT to the number of scalar latch iterations. */
1675 widest_int max_iters;
1680 /* Try scales of 1 and the element size. */
1684 {
1686 widest_int factor;
1690 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1697 /* Find the narrowest viable offset type. */
1702 /* See whether the target supports the operation with an offset
1703 no narrower than OFFSET_TYPE. */
1712 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1713 but we don't need to store that here. */
1721 }
1725 "truncating gather/scatter offset to %d bits"
1729 }
1731 /* Return true if we can use gather/scatter internal functions to
1732 vectorize STMT_INFO, which is a grouped or strided load or store.
1733 MASKED_P is true if load or store is conditional. When returning
1734 true, fill in GS_INFO with the information required to perform the
1735 operation. */
1737 static bool
1741 {
1756 "using gather/scatter for strided/grouped access,"
1760 }
1762 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1763 elements with a known constant step. Return -1 if that step
1764 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1766 static int
1768 {
1771 size_zero_node);
1772 }
1774 /* If the target supports a permute mask that reverses the elements in
1775 a vector of type VECTYPE, return that mask, otherwise return null. */
1777 static tree
1779 {
1782 /* The encoding has a single stepped pattern. */
1789 indices))
1792 }
1794 /* A subroutine of get_load_store_type, with a subset of the same
1795 arguments. Handle the case where STMT_INFO is a load or store that
1796 accesses consecutive elements with a negative step. Sets *POFFSET
1797 to the offset to be applied to the DR for the first access. */
1799 static vect_memory_access_type
1802 vec_load_store_type vls_type,
1804 {
1806 dr_alignment_support alignment_support_scheme;
1809 {
1814 }
1816 /* For backward running DRs the first access in vectype actually is
1817 N-1 elements before the address of the DR. */
1822 alignment_support_scheme
1826 {
1832 }
1835 {
1838 "negative step with invariant source;"
1841 }
1844 {
1850 }
1853 }
1855 /* STMT_INFO is either a masked or unconditional store. Return the value
1856 being stored. */
1858 tree
1860 {
1862 {
1865 }
1867 {
1872 }
1874 }
1876 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1878 This function returns a vector type which can be composed with NETLS pieces,
1879 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1880 same vector size as the return vector. It checks target whether supports
1881 pieces-size vector mode for construction firstly, if target fails to, check
1882 pieces-size scalar mode for construction further. It returns NULL_TREE if
1883 fails to find the available composition.
1885 For example, for (vtype=V16QI, nelts=4), we can probably get:
1886 - V16QI with PTYPE V4QI.
1887 - V4SI with PTYPE SI.
1888 - NULL_TREE. */
1890 static tree
1892 {
1900 /* When we are asked to compose the vector from its components let
1901 that happen directly. */
1903 {
1906 }
1911 {
1912 /* First check if vec_init optab supports construction from
1913 vector pieces directly. */
1916 machine_mode rmode;
1920 {
1923 }
1925 /* Otherwise check if exists an integer type of the same piece size and
1926 if vec_init optab supports construction from it directly. */
1931 {
1934 }
1935 }
1938 }
1940 /* A subroutine of get_load_store_type, with a subset of the same
1941 arguments. Handle the case where STMT_INFO is part of a grouped load
1942 or store.
1944 For stores, the statements in the group are all consecutive
1945 and there is no gap at the end. For loads, the statements in the
1946 group might not be consecutive; there can be gaps between statements
1947 as well as at the end. */
1949 static bool
1959 {
1962 stmt_vec_info first_stmt_info;
1966 {
1970 }
1971 else
1972 {
1976 }
1982 /* True if the vectorized statements would access beyond the last
1983 statement in the group. */
1986 /* True if we can cope with such overrun by peeling for gaps, so that
1987 there is at least one final scalar iteration after the vector loop. */
1990 && loop_vinfo
1993 /* There can only be a gap at the end of the group if the stride is
1994 known at compile time. */
1997 /* Stores can't yet have gaps. */
2001 {
2002 /* For SLP vectorization we directly vectorize a subchain
2003 without permutation. */
2005 first_dr_info
2008 {
2009 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2010 separated by the stride, until we have a complete vector.
2011 Fall back to scalar accesses if that isn't possible. */
2014 else
2016 }
2017 else
2018 {
2021 {
2023 "Grouped store with gaps requires"
2026 }
2027 /* An overrun is fine if the trailing elements are smaller
2028 than the alignment boundary B. Every vector access will
2029 be a multiple of B and so we are guaranteed to access a
2030 non-gap element in the same B-sized block. */
2033 vectype)
2037 /* If the gap splits the vector in half and the target
2038 can do half-vector operations avoid the epilogue peeling
2039 by simply loading half of the vector only. Usually
2040 the construction with an upper zero half will be elided. */
2041 dr_alignment_support alss;
2043 tree half_vtype;
2045 && !masked_p
2048 == dr_aligned
2057 {
2062 }
2065 {
2067 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2068 only correct for single element "interleaving" SLP. */
2071 else
2072 {
2073 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2074 separated by the stride, until we have a complete vector.
2075 Fall back to scalar accesses if that isn't possible. */
2078 else
2080 }
2081 }
2083 {
2086 /* Invariant accesses perform only component accesses, alignment
2087 is irrelevant for them. */
2089 }
2090 else
2093 /* When we have a contiguous access across loop iterations
2094 but the access in the loop doesn't cover the full vector
2095 we can end up with no gap recorded but still excess
2096 elements accessed, see PR103116. Make sure we peel for
2097 gaps if necessary and sufficient and give up if not.
2099 If there is a combination of the access not covering the full
2100 vector and a gap recorded then we may need to peel twice. */
2105 nunits))
2106 {
2111 /* Peeling for gaps assumes that a single scalar iteration
2112 is enough to make sure the last vector iteration doesn't
2113 access excess elements.
2114 ??? Enhancements include peeling multiple iterations
2115 or using masked loads with a static mask. */
2117 {
2120 "peeling for gaps insufficient for "
2123 }
2125 }
2126 }
2127 }
2128 else
2129 {
2130 /* We can always handle this case using elementwise accesses,
2131 but see if something more efficient is available. */
2134 /* If there is a gap at the end of the group then these optimizations
2135 would access excess elements in the last iteration. */
2137 /* An overrun is fine if the trailing elements are smaller than the
2138 alignment boundary B. Every vector access will be a multiple of B
2139 and so we are guaranteed to access a non-gap element in the
2140 same B-sized block. */
2142 && !masked_p
2150 {
2151 /* First cope with the degenerate case of a single-element
2152 vector. */
2154 ;
2156 else
2157 {
2158 /* Otherwise try using LOAD/STORE_LANES. */
2159 *lanes_ifn
2163 masked_p);
2165 {
2168 }
2170 /* If that fails, try using permuting loads. */
2173 single_element_p,
2174 group_size)
2176 {
2179 }
2180 }
2181 }
2183 /* As a last resort, trying using a gather load or scatter store.
2185 ??? Although the code can handle all group sizes correctly,
2186 it probably isn't a win to use separate strided accesses based
2187 on nearby locations. Or, even if it's a win over scalar code,
2188 it might not be a win over vectorizing at a lower VF, if that
2189 allows us to use contiguous accesses. */
2191 && single_element_p
2192 && loop_vinfo
2196 }
2200 {
2203 }
2204 else
2205 {
2207 *alignment_support_scheme
2210 }
2213 {
2214 /* STMT is the leader of the group. Check the operands of all the
2215 stmts of the group. */
2218 {
2222 {
2227 }
2229 }
2230 }
2233 {
2237 "Data access with gaps requires scalar "
2240 }
2243 }
2245 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2246 if there is a memory access type that the vectorized form can use,
2247 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2248 or scatters, fill in GS_INFO accordingly. In addition
2249 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2250 the target does not support the alignment scheme. *MISALIGNMENT
2251 is set according to the alignment of the access (including
2252 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2254 SLP says whether we're performing SLP rather than loop vectorization.
2255 MASKED_P is true if the statement is conditional on a vectorized mask.
2256 VECTYPE is the vector type that the vectorized statements will use.
2257 NCOPIES is the number of vector statements that will be needed. */
2259 static bool
2270 {
2276 {
2280 /* When using internal functions, we rely on pattern recognition
2281 to convert the type of the offset to the type that the target
2282 requires, with the result being a call to an internal function.
2283 If that failed for some reason (e.g. because another pattern
2284 took priority), just handle cases in which the offset already
2285 has the right type. */
2290 {
2296 }
2300 {
2306 }
2308 {
2314 {
2317 "unsupported vector types for emulated "
2320 }
2321 }
2322 /* Gather-scatter accesses perform only component accesses, alignment
2323 is irrelevant for them. */
2325 }
2327 {
2329 masked_p,
2331 alignment_support_scheme,
2334 }
2336 {
2342 else
2344 /* Alignment is irrelevant here. */
2346 }
2347 else
2348 {
2351 {
2354 /* Invariant accesses perform only component accesses, alignment
2355 is irrelevant for them. */
2357 }
2358 else
2359 {
2363 else
2367 *alignment_support_scheme
2371 }
2372 }
2377 {
2380 "Not using elementwise accesses due to variable "
2383 }
2386 {
2391 }
2393 /* FIXME: At the moment the cost model seems to underestimate the
2394 cost of using elementwise accesses. This check preserves the
2395 traditional behavior until that can be fixed. */
2404 {
2409 }
2411 }
2413 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2414 conditional operation STMT_INFO. When returning true, store the mask
2415 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2416 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2417 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2419 static bool
2424 {
2426 tree mask_vectype;
2427 slp_tree mask_node_1;
2430 {
2435 }
2438 {
2443 }
2445 /* If the caller is not prepared for adjusting an external/constant
2446 SLP mask vector type fail. */
2448 && !mask_node
2450 {
2455 }
2460 mask_node_1);
2463 {
2468 }
2472 {
2475 "vector mask type %T"
2480 }
2487 }
2489 /* Return true if stored value is suitable for vectorizing store
2490 statement STMT_INFO. When returning true, store the scalar stored
2491 in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2492 the type of the vectorized store value in
2493 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2495 static bool
2500 {
2503 {
2507 }
2509 op_no = vect_slp_child_index_for_operand
2513 tree rhs_vectype;
2516 {
2521 }
2523 /* In the case this is a store from a constant make sure
2524 native_encode_expr can handle it. */
2526 {
2531 }
2535 {
2540 }
2546 else
2549 }
2551 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2552 Note that we support masks with floating-point type, in which case the
2553 floats are interpreted as a bitmask. */
2555 static tree
2558 {
2563 {
2567 }
2569 {
2570 REAL_VALUE_TYPE r;
2578 }
2580 }
2582 /* Build an all-zero merge value of type VECTYPE while vectorizing
2583 STMT_INFO as a gather load. */
2585 static tree
2588 {
2589 tree merge;
2593 {
2594 REAL_VALUE_TYPE r;
2600 }
2601 else
2605 }
2607 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2608 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2609 the gather load operation. If the load is conditional, MASK is the
2610 vectorized condition, otherwise MASK is null. PTR is the base
2611 pointer and OFFSET is the vectorized offset. */
2618 {
2627 tree var;
2629 && (!mask
2635 {
2643 }
2648 {
2650 {
2655 else
2659 gassign *new_stmt
2664 {
2671 }
2674 }
2675 else
2676 {
2679 }
2680 }
2681 else
2682 {
2685 }
2692 {
2700 }
2703 }
2705 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2706 instructions before GSI. GS_INFO describes the scatter store operation.
2707 PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
2708 vectorized data to store.
2709 If the store is conditional, MASK is the vectorized condition, otherwise
2710 MASK is null. */
2717 {
2730 {
2733 tree utype;
2736 else
2740 gassign *new_stmt
2745 {
2751 }
2752 }
2753 else
2754 {
2757 }
2761 {
2769 }
2773 {
2781 }
2784 gcall *new_stmt
2787 }
2789 /* Prepare the base and offset in GS_INFO for vectorization.
2790 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2791 to the vectorized offset argument for the first copy of STMT_INFO.
2792 STMT_INFO is the statement described by GS_INFO and LOOP is the
2793 containing loop. */
2795 static void
2800 {
2804 {
2805 basic_block new_bb;
2809 }
2812 else
2813 {
2814 unsigned ncopies
2819 }
2820 }
2822 /* Prepare to implement a grouped or strided load or store using
2823 the gather load or scatter store operation described by GS_INFO.
2824 STMT_INFO is the load or store statement.
2826 Set *DATAREF_BUMP to the amount that should be added to the base
2827 address after each copy of the vectorized statement. Set *VEC_OFFSET
2828 to an invariant offset vector in which element I has the value
2829 I * DR_STEP / SCALE. */
2831 static void
2833 loop_vec_info loop_vinfo,
2838 {
2843 {
2844 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2845 ivtmp_8 = _31 * 16 (step in bytes);
2846 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2847 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2848 tree loop_len
2850 tree tmp
2853 loop_len);
2855 GSI_SAME_STMT);
2856 }
2857 else
2858 {
2859 tree bump
2864 }
2866 /* The offset given in GS_INFO can have pointer type, so use the element
2867 type of the vector instead. */
2870 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2875 /* Create {0, X, X*2, X*3, ...}. */
2879 }
2881 /* Prepare the pointer IVs which needs to be updated by a variable amount.
2882 Such variable amount is the outcome of .SELECT_VL. In this case, we can
2883 allow each iteration process the flexible number of elements as long as
2884 the number <= vf elments.
2886 Return data reference according to SELECT_VL.
2887 If new statements are needed, insert them before GSI. */
2889 static tree
2893 vect_memory_access_type memory_access_type)
2894 {
2898 /* gather/scatter never reach here. */
2901 /* When we support SELECT_VL pattern, we dynamic adjust
2902 the memory address by .SELECT_VL result.
2904 The result of .SELECT_VL is the number of elements to
2905 be processed of each iteration. So the memory address
2906 adjustment operation should be:
2908 addr = addr + .SELECT_VL (ARG..) * step;
2909 */
2910 tree loop_len
2913 /* Since the outcome of .SELECT_VL is element size, we should adjust
2914 it into bytesize so that it can be used in address pointer variable
2915 amount IVs adjustment. */
2922 }
2924 /* Return the amount that should be added to a vector pointer to move
2925 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
2926 being vectorized and MEMORY_ACCESS_TYPE describes the type of
2927 vectorization. */
2929 static tree
2932 vect_memory_access_type memory_access_type,
2934 {
2942 memory_access_type);
2949 }
2951 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
2953 static bool
2959 {
2969 /* Multiple types in SLP are handled by creating the appropriate number of
2970 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2971 case of SLP. */
2974 else
2980 {
2986 }
2997 /* The encoding uses one stepped pattern for each byte in the word. */
3009 {
3012 {
3017 }
3024 slp_node
3028 }
3032 /* Transform. */
3036 /* Arguments are ready. create the new vector stmt. */
3038 tree vop;
3040 {
3056 else
3058 }
3065 }
3067 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3068 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3069 in a single step. On success, store the binary pack code in
3070 *CONVERT_CODE. */
3072 static bool
3075 {
3080 code_helper code;
3090 }
3092 /* Function vectorizable_call.
3094 Check if STMT_INFO performs a function call that can be vectorized.
3095 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3096 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3097 Return true if STMT_INFO is vectorizable in this way. */
3099 static bool
3104 {
3106 tree vec_dest;
3107 tree scalar_dest;
3108 tree op;
3111 poly_uint64 nunits_in;
3112 poly_uint64 nunits_out;
3118 vect_unknown_def_type };
3126 tree lhs;
3136 /* Is STMT_INFO a vectorizable call? */
3144 /* Handled by vectorizable_load and vectorizable_store. */
3155 /* Process function arguments. */
3160 /* Bail out if the function has more than four arguments, we do not have
3161 interesting builtin functions to vectorize with more than two arguments
3162 except for fma. No arguments is also not good. */
3166 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3169 {
3172 }
3173 /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3174 argument just says whether it is well-defined at zero or not and what
3175 value should be returned for it. */
3177 {
3180 }
3187 {
3189 {
3194 }
3198 {
3203 }
3205 /* We can only handle calls with arguments of the same type. */
3208 {
3213 }
3221 {
3226 }
3227 }
3228 /* If all arguments are external or constant defs, infer the vector type
3229 from the scalar type. */
3235 {
3241 }
3245 {
3250 }
3253 {
3258 }
3260 /* FORNOW */
3269 else
3272 /* We only handle functions that do not read or clobber memory. */
3274 {
3279 }
3281 /* For now, we only vectorize functions if a target specific builtin
3282 is available. TODO -- in some cases, it might be profitable to
3283 insert the calls for pieces of the vector, in order to be able
3284 to vectorize other operations in the loop. */
3289 /* First try using an internal function. */
3297 vectype_in);
3299 /* If that fails, try asking for a target-specific built-in function. */
3301 {
3308 }
3311 {
3313 && !slp_node
3314 && loop_vinfo
3319 {
3320 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3321 { 0, 1, 2, ... vf - 1 } vector. */
3323 }
3331 else
3332 {
3337 }
3338 }
3344 else
3347 /* Sanity check: make sure that at least one copy of the vectorized stmt
3348 needs to be generated. */
3358 {
3364 {
3369 }
3381 {
3385 OPTIMIZE_FOR_SPEED))
3388 OPTIMIZE_FOR_SPEED)))
3389 {
3392 "can't use a fully-masked loop because no"
3395 }
3396 else
3397 {
3398 unsigned int nvectors
3399 = (slp_node
3407 OPTIMIZE_FOR_SPEED))
3410 else
3412 scalar_mask);
3413 }
3414 }
3416 }
3418 /* Transform. */
3423 /* Handle def. */
3431 {
3433 {
3435 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3437 }
3440 }
3442 {
3445 }
3450 {
3455 {
3456 /* Build argument list for the vectorized call. */
3458 {
3464 /* Arguments are ready. Create the new vector stmt. */
3466 {
3469 {
3471 /* Always true for SLP. */
3476 }
3479 {
3482 }
3490 {
3491 /* We don't define any narrowing conditional functions
3492 at present. */
3495 gcall *call
3501 {
3504 }
3510 }
3511 else
3512 {
3514 {
3516 /* Always true for SLP. */
3518 tree len
3521 signed char biasval
3526 }
3528 {
3530 /* Always true for SLP. */
3538 }
3543 else
3550 }
3552 }
3554 }
3561 {
3564 {
3569 }
3571 }
3578 {
3581 signed char biasval
3586 }
3588 {
3594 }
3598 {
3600 tree new_var
3607 }
3609 {
3610 /* We don't define any narrowing conditional functions at
3611 present. */
3619 {
3622 }
3625 half_res);
3627 }
3628 else
3629 {
3633 else
3640 }
3645 }
3647 {
3650 }
3651 }
3653 {
3655 /* We don't define any narrowing conditional functions at present. */
3658 {
3659 /* Build argument list for the vectorized call. */
3662 else
3666 {
3672 /* Arguments are ready. Create the new vector stmt. */
3674 {
3678 {
3682 }
3686 else
3693 }
3695 }
3698 {
3701 {
3705 }
3711 }
3719 }
3725 {
3728 }
3729 }
3730 else
3731 /* No current target implements this case. */
3736 /* The call in STMT might prevent it from being removed in dce.
3737 We however cannot remove it here, due to the way the ssa name
3738 it defines is mapped to the new definition. So just replace
3739 rhs of the statement with something harmless. */
3747 gassign *new_stmt
3752 }
3755 struct simd_call_arg_info
3756 {
3757 tree vectype;
3758 tree op;
3759 HOST_WIDE_INT linear_step;
3763 };
3765 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3766 is linear within simd lane (but not within whole loop), note it in
3767 *ARGINFO. */
3769 static void
3772 {
3784 {
3785 tree t;
3789 {
3804 CASE_CONVERT:
3816 }
3822 {
3829 }
3830 }
3831 }
3833 /* Function vectorizable_simd_clone_call.
3835 Check if STMT_INFO performs a function call that can be vectorized
3836 by calling a simd clone of the function.
3837 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3838 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3839 Return true if STMT_INFO is vectorizable in this way. */
3841 static bool
3845 stmt_vector_for_cost *)
3846 {
3847 tree vec_dest;
3848 tree scalar_dest;
3851 tree vectype;
3852 poly_uint64 nunits;
3865 /* Is STMT a vectorizable call? */
3873 {
3879 }
3905 /* Process function arguments. */
3908 /* Bail out if the function has zero arguments. */
3919 {
3920 simd_call_arg_info thisarginfo;
3921 affine_iv iv;
3935 {
3940 }
3944 {
3945 /* With SLP we determine the vector type of constants/externals
3946 at analysis time, handling conflicts via
3947 vect_maybe_update_slp_op_vectype. At transform time
3948 we have a vector type recorded for SLP. */
3950 || !slp_node
3955 slp_node);
3956 }
3957 else
3960 /* For linear arguments, the analyze phase should have saved
3961 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
3964 {
3968 thisarginfo.simd_lane_linear
3970 /* If loop has been peeled for alignment, we need to adjust it. */
3974 {
3980 thisarginfo.op
3984 }
3985 }
3989 && loop_vinfo
3994 {
3997 }
4002 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4003 linear too. */
4006 && !vec_stmt
4009 && loop_vinfo
4014 }
4022 else
4025 {
4028 /* The number of arguments in the call and the number of parameters in
4029 the simdclone should match. However, when the simdclone is
4030 'inbranch', it could have one more paramater than nargs when using
4031 an inbranch simdclone to call a non-inbranch call, either in a
4032 non-masked loop using a all true constant mask, or inside a masked
4033 loop using it's mask. */
4036 simd_nargs--;
4051 {
4053 {
4084 /* FORNOW */
4088 /* While we can create a traditional data vector from
4089 an incoming integer mode mask we have no good way to
4090 force generate an integer mode mask from a traditional
4091 boolean vector input. */
4099 }
4103 {
4106 }
4110 }
4116 {
4118 SIMD_CLONE_ARG_TYPE_MASK);
4119 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4120 not in a branch, as we'd have to construct an all-true mask. */
4123 }
4125 {
4128 }
4129 }
4138 num_mask_args++;
4141 {
4145 {
4149 slp_node);
4154 }
4158 {
4163 }
4166 {
4169 {
4172 {
4173 /* FORNOW we only have partial support for vector-type masks
4174 that can't hold all of simdlen. */
4177 vect_location,
4178 "in-branch vector clones are not yet"
4181 }
4182 }
4184 {
4187 num_mask_args),
4189 {
4190 /* FORNOW we only have partial support for integer-type masks
4191 that represent the same number of lanes as the
4192 vectorized mask inputs. */
4195 vect_location,
4196 "in-branch vector clones are not yet "
4199 }
4200 }
4201 else
4202 {
4205 vect_location,
4206 "in-branch vector clones not supported"
4209 }
4210 }
4211 }
4217 else
4220 /* If the function isn't const, only allow it in simd loops where user
4221 has asserted that at least nunits consecutive iterations can be
4222 performed using SIMD instructions. */
4227 /* Sanity check: make sure that at least one copy of the vectorized stmt
4228 needs to be generated. */
4232 {
4236 {
4241 }
4242 /* When the original call is pure or const but the SIMD ABI dictates
4243 an aggregate return we will have to use a virtual definition and
4244 in a loop eventually even need to add a virtual PHI. That's
4245 not straight-forward so allow to fix this up via renaming. */
4250 /* ??? For SLP code-gen we end up inserting after the last
4251 vector argument def rather than at the original call position
4252 so automagic virtual operand updating doesn't work. */
4257 {
4259 {
4264 {
4274 }
4284 }
4285 }
4288 {
4292 "can't use a fully-masked loop because a"
4295 }
4299 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4300 dt, slp_node, cost_vec); */
4302 }
4304 /* Transform. */
4309 /* Handle def. */
4315 {
4319 {
4322 }
4323 }
4329 {
4332 }
4333 else
4336 {
4337 poly_uint64 callee_nelements;
4338 poly_uint64 caller_nelements;
4339 /* Build argument list for the vectorized call. */
4342 else
4346 {
4348 tree atype;
4351 {
4358 {
4360 {
4368 {
4375 }
4376 else
4377 {
4381 }
4383 vec_oprnd0
4387 gassign *new_stmt
4389 vec_oprnd0);
4393 }
4394 else
4395 {
4403 else
4406 {
4408 {
4412 op,
4416 }
4417 else
4423 vec_oprnd0);
4424 }
4427 atype))
4428 {
4430 vec_oprnd0);
4431 gassign *new_stmt
4433 vec_oprnd0);
4437 }
4438 else
4440 else
4441 {
4443 gassign *new_stmt
4445 vec_oprnd0);
4449 }
4450 }
4451 }
4455 {
4464 {
4466 {
4467 /* The mask type has fewer elements than simdlen. */
4469 /* FORNOW */
4471 }
4473 {
4474 /* The SIMD clone function has the same number of
4475 elements as the current function. */
4477 {
4481 op,
4484 }
4488 {
4489 vec_loop_masks *loop_masks
4491 tree loop_mask
4495 vec_oprnd0
4499 gsi);
4501 loop_mask });
4503 }
4504 vec_oprnd0
4508 gassign *new_stmt
4510 vec_oprnd0);
4514 }
4515 else
4516 {
4517 /* The mask type has more elements than simdlen. */
4519 /* FORNOW */
4521 }
4522 }
4523 }
4525 {
4527 /* Guess the number of lanes represented by atype. */
4528 poly_uint64 atype_subparts
4530 num_mask_args);
4533 {
4535 {
4539 op,
4542 }
4545 {
4546 /* The mask argument has fewer elements than the
4547 input vector. */
4548 /* FORNOW */
4550 }
4553 {
4554 /* The vector mask argument matches the input
4555 in the number of lanes, but not necessarily
4556 in the mode. */
4561 vec_oprnd0);
4562 gassign *new_stmt
4564 vec_oprnd0);
4568 {
4569 new_stmt
4571 NOP_EXPR,
4572 gimple_assign_lhs
4576 }
4578 }
4579 else
4580 {
4581 /* The mask argument has more elements than the
4582 input vector. */
4583 /* FORNOW */
4585 }
4586 }
4587 }
4588 else
4597 {
4598 gimple_seq stmts;
4603 {
4604 basic_block new_bb;
4608 }
4610 {
4613 }
4618 enum tree_code code
4623 poly_widest_int cst
4628 gassign *new_stmt
4633 UNKNOWN_LOCATION);
4636 }
4637 else
4638 {
4639 enum tree_code code
4644 poly_widest_int cst
4649 gassign *new_stmt
4654 }
4664 }
4665 }
4670 {
4673 tree mask;
4675 SIMD_CLONE_ARG_TYPE_MASK);
4681 {
4683 {
4687 }
4688 else
4693 {
4694 /* This means we are dealing with integer mask modes.
4695 First convert to an integer type with the same size as
4696 the current vector type. */
4697 unsigned HOST_WIDE_INT intermediate_size
4699 tree mid_int_type =
4702 new_stmt
4704 mask);
4706 /* Then zero-extend to the mask mode. */
4709 }
4711 {
4713 integer_one_node);
4715 integer_zero_node);
4719 }
4720 else
4728 }
4729 }
4733 {
4740 else
4743 }
4748 {
4750 {
4758 {
4759 tree t;
4761 {
4765 }
4766 else
4777 else
4779 }
4784 }
4786 {
4795 {
4800 {
4804 tem);
4809 }
4811 }
4812 else
4817 new_stmt
4826 else
4829 }
4831 {
4838 }
4840 {
4842 new_stmt
4845 }
4846 }
4852 else
4854 }
4857 {
4860 }
4863 /* Mark the clone as no longer being a candidate for GC. */
4866 /* The call in STMT might prevent it from being removed in dce.
4867 We however cannot remove it here, due to the way the ssa name
4868 it defines is mapped to the new definition. So just replace
4869 rhs of the statement with something harmless. */
4876 {
4880 }
4881 else
4887 }
4890 /* Function vect_gen_widened_results_half
4892 Create a vector stmt whose code, type, number of arguments, and result
4893 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
4894 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
4895 In the case that CODE is a CALL_EXPR, this means that a call to DECL
4896 needs to be created (DECL is a function-decl of a target-builtin).
4897 STMT_INFO is the original scalar stmt that we are vectorizing. */
4903 stmt_vec_info stmt_info)
4904 {
4906 tree new_temp;
4908 /* Generate half of the widened result: */
4917 }
4920 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
4921 For multi-step conversions store the resulting vectors and call the function
4922 recursively. When NARROW_SRC_P is true, there's still a conversion after
4923 narrowing, don't store the vectors in the SLP_NODE or in vector info of
4924 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
4926 static void
4929 stmt_vec_info stmt_info,
4934 {
4941 {
4942 /* Create demotion operation. */
4950 /* Store the resulting vector for next recursive call,
4951 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
4953 else
4954 {
4955 /* This is the last step of the conversion sequence. Store the
4956 vectors in SLP_NODE or in vector info of the scalar statement
4957 (or in STMT_VINFO_RELATED_STMT chain). */
4960 else
4962 }
4963 }
4965 /* For multi-step demotion operations we first generate demotion operations
4966 from the source type to the intermediate types, and then combine the
4967 results (stored in VEC_OPRNDS) in demotion operation to the destination
4968 type. */
4970 {
4971 /* At each level of recursion we have half of the operands we had at the
4972 previous level. */
4978 narrow_src_p);
4979 }
4982 }
4985 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
4986 and VEC_OPRNDS1, for a binary operation associated with scalar statement
4987 STMT_INFO. For multi-step conversions store the resulting vectors and
4988 call the function recursively. */
4990 static void
4996 code_helper ch1,
4998 {
5006 {
5009 else
5012 /* Generate the two halves of promotion operation. */
5015 stmt_info);
5018 stmt_info);
5020 {
5023 }
5024 else
5025 {
5028 }
5030 /* Store the results for the next step. */
5033 }
5037 }
5039 /* Create vectorized promotion stmts for widening stmts using only half the
5040 potential vector size for input. */
5041 static void
5047 code_helper code1,
5049 {
5059 {
5065 /* Widen the first vector input. */
5071 {
5072 /* Widen the second vector input. */
5076 /* Perform the operation. With both vector inputs widened. */
5078 }
5079 else
5080 {
5081 /* Perform the operation. With the single vector input widened. */
5083 }
5089 /* Store the results for the next step. */
5091 }
5095 }
5098 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5099 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5100 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5101 Return true if STMT_INFO is vectorizable in this way. */
5103 static bool
5108 {
5110 tree scalar_dest;
5116 tree new_temp;
5119 poly_uint64 nunits_in;
5120 poly_uint64 nunits_out;
5124 /* For conversions between floating point and integer, there're 2 NARROW
5125 cases. NARROW_SRC is for FLOAT_EXPR, means
5126 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5127 This is safe when the range of the source integer can fit into the lower
5128 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5129 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5130 For other conversions, when there's narrowing, NARROW_DST is used as
5131 default. */
5135 tree vop0;
5143 /* Is STMT a vectorizable conversion? */
5164 {
5167 }
5169 {
5172 }
5173 else
5186 /* Check types of lhs and rhs. */
5191 /* Check the operands of the operation. */
5195 {
5200 }
5215 {
5218 "type conversion to/from bit-precision unsupported."
5221 }
5224 {
5231 tree vectype1_in;
5234 {
5239 }
5240 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5241 OP1. */
5244 }
5246 /* If op0 is an external or constant def, infer the vector type
5247 from the scalar type. */
5253 {
5259 }
5263 {
5266 "can't convert between boolean and non "
5270 }
5277 else
5281 else
5282 {
5285 }
5287 /* Multiple types in SLP are handled by creating the appropriate number of
5288 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5289 case of SLP. */
5294 else
5297 /* Sanity check: make sure that at least one copy of the vectorized stmt
5298 needs to be generated. */
5304 opt_scalar_mode rhs_mode_iter;
5306 /* Supportable by target? */
5308 {
5317 {
5320 }
5322 /* For conversions between float and integer types try whether
5323 we can use intermediate signed integer types to support the
5324 conversion. */
5328 {
5332 scalar_mode intermediate_mode;
5334 {
5337 }
5338 else
5339 {
5344 }
5347 opt_scalar_mode mode_iter;
5349 {
5355 scalar_mode cvt_mode;
5360 cvt_type = build_nonstandard_integer_type
5363 /* Check if the intermediate type can hold OP0's range.
5364 When converting from float to integer this is not necessary
5365 because values that do not fit the (smaller) target type are
5366 unspecified anyway. */
5368 {
5379 }
5382 /* This should only happened for SLP as long as loop vectorizer
5383 only supports same-sized vector. */
5387 vectype_out,
5390 cvt_type,
5396 }
5399 {
5400 multi_step_cvt++;
5406 }
5407 }
5408 /* FALLTHRU */
5409 unsupported:
5417 {
5426 }
5431 {
5432 /* Binary widening operation can only be supported directly by the
5433 architecture. */
5436 }
5444 {
5449 cvt_type
5456 {
5463 }
5470 else
5474 cvt_type,
5478 {
5481 }
5482 }
5489 else
5490 {
5491 multi_step_cvt++;
5494 }
5508 {
5509 cvt_type
5517 else
5523 }
5524 /* If op0 can be represented with low precision integer,
5525 truncate it to cvt_type and the do FLOAT_EXPR. */
5527 {
5532 cvt_type
5550 {
5554 }
5555 }
5561 }
5564 {
5568 {
5573 }
5576 {
5581 }
5583 {
5585 /* The final packing step produces one vector result per copy. */
5586 unsigned int nvectors
5590 widen_arith);
5591 }
5592 else
5593 {
5595 /* The initial unpacking step produces two vector results
5596 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5597 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5598 unsigned int nvectors
5599 = (slp_node
5604 widen_arith);
5605 }
5608 }
5610 /* Transform. */
5616 {
5621 }
5623 /* In case of multi-step conversion, we first generate conversion operations
5624 to the intermediate types, and then from that types to the final one.
5625 We create vector destinations for the intermediate type (TYPES) received
5626 from supportable_*_operation, and store them in the correct order
5627 for future use in vect_create_vectorized_*_stmts (). */
5629 bool widen_or_narrow_float_p
5632 widen_or_narrow_float_p
5637 {
5640 {
5642 intermediate_type);
5644 }
5645 }
5649 widen_or_narrow_float_p
5654 {
5656 ;
5658 {
5662 }
5663 }
5666 {
5670 /* vec_dest is intermediate type operand when multi_step_cvt. */
5672 {
5675 }
5678 {
5679 /* Arguments are ready, create the new vector stmt. */
5682 {
5689 }
5697 else
5699 }
5703 /* In case the vectorization factor (VF) is bigger than the number
5704 of elements that we can fit in a vectype (nunits), we have to
5705 generate more than one vector stmt - i.e - we need to "unroll"
5706 the vector stmt by a factor VF/nunits. */
5712 {
5717 }
5718 /* Arguments are ready. Create the new vector stmts. */
5720 {
5724 {
5727 }
5731 op_type);
5732 else
5737 }
5740 {
5743 {
5747 }
5748 else
5753 else
5755 }
5760 /* In case the vectorization factor (VF) is bigger than the number
5761 of elements that we can fit in a vectype (nunits), we have to
5762 generate more than one vector stmt - i.e - we need to "unroll"
5763 the vector stmt by a factor VF/nunits. */
5766 /* Arguments are ready. Create the new vector stmts. */
5769 {
5774 }
5777 multi_step_cvt,
5781 /* After demoting op0 to cvt_type, convert it to dest. */
5783 {
5785 {
5786 /* Arguments are ready, create the new vector stmt. */
5788 gimple *new_stmt
5794 /* This is the last step of the conversion sequence. Store the
5795 vectors in SLP_NODE or in vector info of the scalar statement
5796 (or in STMT_VINFO_RELATED_STMT chain). */
5799 else
5801 }
5802 }
5804 }
5813 }
5815 /* Return true if we can assume from the scalar form of STMT_INFO that
5816 neither the scalar nor the vector forms will generate code. STMT_INFO
5817 is known not to involve a data reference. */
5819 bool
5821 {
5837 }
5839 /* Function vectorizable_assignment.
5841 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5842 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5843 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5844 Return true if STMT_INFO is vectorizable in this way. */
5846 static bool
5851 {
5852 tree vec_dest;
5853 tree scalar_dest;
5854 tree op;
5856 tree new_temp;
5862 tree vop;
5865 tree vectype_in;
5874 /* Is vectorizable assignment? */
5895 /* Multiple types in SLP are handled by creating the appropriate number of
5896 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5897 case of SLP. */
5900 else
5905 slp_tree slp_op;
5908 {
5913 }
5917 /* We can handle NOP_EXPR conversions that do not change the number
5918 of elements or the vector size. */
5921 && (!vectype_in
5928 {
5931 "can't convert between boolean and non "
5935 }
5937 /* We do not handle bit-precision changes. */
5944 /* But a conversion that does not change the bit-pattern is ok. */
5952 {
5955 "type conversion to/from bit-precision "
5958 }
5961 {
5964 {
5969 }
5974 cost_vec);
5976 }
5978 /* Transform. */
5982 /* Handle def. */
5985 /* Handle use. */
5988 /* Arguments are ready. create the new vector stmt. */
5990 {
6000 else
6002 }
6008 }
6011 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6012 either as shift by a scalar or by a vector. */
6014 bool
6016 {
6018 machine_mode vec_mode;
6019 optab optab;
6021 tree vectype;
6030 {
6036 }
6044 }
6047 /* Function vectorizable_shift.
6049 Check if STMT_INFO performs a shift operation that can be vectorized.
6050 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6051 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6052 Return true if STMT_INFO is vectorizable in this way. */
6054 static bool
6059 {
6060 tree vec_dest;
6061 tree scalar_dest;
6064 tree vectype;
6067 machine_mode vec_mode;
6068 tree new_temp;
6069 optab optab;
6071 machine_mode optab_op2_mode;
6074 poly_uint64 nunits_in;
6075 poly_uint64 nunits_out;
6076 tree vectype_out;
6077 tree op1_vectype;
6096 /* Is STMT a vectorizable binary/unary operation? */
6113 {
6118 }
6120 slp_tree slp_op0;
6123 {
6128 }
6129 /* If op0 is an external or constant def, infer the vector type
6130 from the scalar type. */
6136 {
6141 }
6148 stmt_vec_info op1_def_stmt_info;
6149 slp_tree slp_op1;
6152 {
6157 }
6159 /* Multiple types in SLP are handled by creating the appropriate number of
6160 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6161 case of SLP. */
6164 else
6169 /* Determine whether the shift amount is a vector, or scalar. If the
6170 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6180 {
6181 /* In SLP, need to check whether the shift count is the same,
6182 in loops if it is a constant or invariant, it is always
6183 a scalar shift. */
6185 {
6187 stmt_vec_info slpstmt_info;
6190 {
6194 }
6196 /* For internal SLP defs we have to make sure we see scalar stmts
6197 for all vector elements.
6198 ??? For different vectors we could resort to a different
6199 scalar shift operand but code-generation below simply always
6200 takes the first. */
6205 }
6207 /* If the shift amount is computed by a pattern stmt we cannot
6208 use the scalar amount directly thus give up and use a vector
6209 shift. */
6212 }
6213 else
6214 {
6219 }
6221 /* Vector shifted by vector. */
6224 {
6232 slp_op1);
6233 incompatible_op1_vectype_p
6239 && (!slp_node
6242 {
6245 "unusable type for last operand in"
6248 }
6249 }
6250 /* See if the machine has a vector shifted by scalar insn and if not
6251 then see if it has a vector shifted by vector insn. */
6252 else
6253 {
6257 {
6261 }
6262 else
6263 {
6268 {
6278 slp_op1);
6280 /* Unlike the other binary operators, shifts/rotates have
6281 the rhs being int, instead of the same type as the lhs,
6282 so make sure the scalar is the right type if we are
6283 dealing with vectors of long long/long/short/char. */
6284 incompatible_op1_vectype_p
6285 = (!op1_vectype
6290 {
6293 "unusable type for last operand in"
6296 }
6297 }
6298 }
6299 }
6301 /* Supportable by target? */
6303 {
6308 }
6312 {
6317 }
6318 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6323 {
6327 && (!incompatible_op1_vectype_p
6329 && !vect_maybe_update_slp_op_vectype
6332 {
6337 }
6338 /* Now adjust the constant shift amount in place. */
6340 && incompatible_op1_vectype_p
6342 {
6345 {
6351 }
6352 }
6358 }
6360 /* Transform. */
6367 {
6373 }
6375 /* Handle def. */
6379 {
6380 /* Vector shl and shr insn patterns can be defined with scalar
6381 operand 2 (shift operand). In this case, use constant or loop
6382 invariant op1 directly, without extending it to vector mode
6383 first. */
6386 {
6393 /* Store vec_oprnd1 for every vector stmt to be created.
6394 We check during the analysis that all the shift arguments
6395 are the same.
6396 TODO: Allow different constants for different vector
6397 stmts generated for an SLP instance. */
6401 }
6402 }
6404 {
6406 {
6407 /* If the argument was the same in all lanes create
6408 the correctly typed vector shift amount directly. */
6417 }
6419 /* The constant shift amount has been adjusted in place. */
6420 ;
6421 else
6423 }
6425 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6426 (a special case for certain kind of vector shifts); otherwise,
6427 operand 1 should be of a vector type (the usual case). */
6432 /* Arguments are ready. Create the new vector stmt. */
6434 {
6435 /* For internal defs where we need to use a scalar shift arg
6436 extract the first lane. */
6438 {
6441 gassign *new_stmt
6444 vop1,
6446 bitsize_zero_node));
6449 }
6450 else
6458 else
6460 }
6469 }
6471 /* Function vectorizable_operation.
6473 Check if STMT_INFO performs a binary, unary or ternary operation that can
6474 be vectorized.
6475 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6476 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6477 Return true if STMT_INFO is vectorizable in this way. */
6479 static bool
6484 {
6485 tree vec_dest;
6486 tree scalar_dest;
6488 tree vectype;
6491 machine_mode vec_mode;
6492 tree new_temp;
6494 optab optab;
6499 poly_uint64 nunits_in;
6500 poly_uint64 nunits_out;
6501 tree vectype_out;
6517 /* Is STMT a vectorizable binary/unary operation? */
6522 /* Loads and stores are handled in vectorizable_{load,store}. */
6528 /* Shifts are handled in vectorizable_shift. */
6535 /* Comparisons are handled in vectorizable_comparison. */
6539 /* Conditions are handled in vectorizable_condition. */
6543 /* For pointer addition and subtraction, we should use the normal
6544 plus and minus for the vector operation. */
6550 /* Support only unary or binary operations. */
6553 {
6557 op_type);
6559 }
6564 /* Most operations cannot handle bit-precision types without extra
6565 truncations. */
6569 /* Exception are bitwise binary operations. */
6573 {
6578 }
6580 slp_tree slp_op0;
6583 {
6588 }
6591 /* If op0 is an external or constant def, infer the vector type
6592 from the scalar type. */
6594 {
6595 /* For boolean type we cannot determine vectype by
6596 invariant value (don't know whether it is a vector
6597 of booleans or vector of integers). We use output
6598 vectype because operations on boolean don't change
6599 type. */
6601 {
6603 {
6608 }
6610 }
6611 else
6613 slp_node);
6614 }
6618 {
6625 }
6635 {
6638 {
6643 }
6649 }
6651 {
6654 {
6659 }
6665 }
6667 /* Multiple types in SLP are handled by creating the appropriate number of
6668 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6669 case of SLP. */
6671 {
6674 }
6675 else
6676 {
6679 }
6683 /* Reject attempts to combine mask types with nonmask types, e.g. if
6684 we have an AND between a (nonmask) boolean loaded from memory and
6685 a (mask) boolean result of a comparison.
6687 TODO: We could easily fix these cases up using pattern statements. */
6691 {
6696 }
6698 /* Supportable by target? */
6703 else
6704 {
6707 {
6712 }
6715 }
6719 {
6723 /* When vec_mode is not a vector mode and we verified ops we
6724 do not have to lower like AND are natively supported let
6725 those through even when the mode isn't word_mode. For
6726 ops we have to lower the lowering code assumes we are
6727 dealing with word_mode. */
6731 /* Check only during analysis. */
6733 {
6737 }
6742 }
6750 /* If operating on inactive elements could generate spurious traps,
6751 we need to restrict the operation to active lanes. Note that this
6752 specifically doesn't apply to unhoisted invariants, since they
6753 operate on the same value for every lane.
6755 Similarly, if this operation is part of a reduction, a fully-masked
6756 loop should only change the active lanes of the reduction chain,
6757 keeping the inactive lanes as-is. */
6762 {
6766 {
6769 OPTIMIZE_FOR_SPEED))
6774 OPTIMIZE_FOR_SPEED))
6777 else
6778 {
6781 "can't use a fully-masked loop because no"
6784 }
6785 }
6787 /* Put types on constant and invariant SLP children. */
6792 {
6797 }
6804 {
6805 /* The above vect_model_simple_cost call handles constants
6806 in the prologue and (mis-)costs one of the stmts as
6807 vector stmt. See below for the actual lowering that will
6808 be applied. */
6809 unsigned n
6812 {
6823 /* Bit operations do not have extra cost and are accounted
6824 as vector stmt by vect_model_simple_cost. */
6827 }
6829 {
6830 /* We also need to materialize two large constants. */
6835 }
6836 }
6838 }
6840 /* Transform. */
6849 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6850 vectors with unsigned elements, but the result is signed. So, we
6851 need to compute the MINUS_EXPR into vectype temporary and
6852 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6855 {
6858 }
6859 /* Handle def. */
6860 else
6863 /* In case the vectorization factor (VF) is bigger than the number
6864 of elements that we can fit in a vectype (nunits), we have to generate
6865 more than one vector stmt - i.e - we need to "unroll" the
6866 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6867 from one copy of the vector stmt to the next, in the field
6868 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6869 stages to find the correct vector defs to be used when vectorizing
6870 stmts that use the defs of the current stmt. The example below
6871 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6872 we need to create 4 vectorized stmts):
6874 before vectorization:
6875 RELATED_STMT VEC_STMT
6876 S1: x = memref - -
6877 S2: z = x + 1 - -
6879 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6880 there):
6881 RELATED_STMT VEC_STMT
6882 VS1_0: vx0 = memref0 VS1_1 -
6883 VS1_1: vx1 = memref1 VS1_2 -
6884 VS1_2: vx2 = memref2 VS1_3 -
6885 VS1_3: vx3 = memref3 - -
6886 S1: x = load - VS1_0
6887 S2: z = x + 1 - -
6889 step2: vectorize stmt S2 (done here):
6890 To vectorize stmt S2 we first need to find the relevant vector
6891 def for the first operand 'x'. This is, as usual, obtained from
6892 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
6893 that defines 'x' (S1). This way we find the stmt VS1_0, and the
6894 relevant vector def 'vx0'. Having found 'vx0' we can generate
6895 the vector stmt VS2_0, and as usual, record it in the
6896 STMT_VINFO_VEC_STMT of stmt S2.
6897 When creating the second copy (VS2_1), we obtain the relevant vector
6898 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
6899 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
6900 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
6901 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
6902 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
6903 chain of stmts and pointers:
6904 RELATED_STMT VEC_STMT
6905 VS1_0: vx0 = memref0 VS1_1 -
6906 VS1_1: vx1 = memref1 VS1_2 -
6907 VS1_2: vx2 = memref2 VS1_3 -
6908 VS1_3: vx3 = memref3 - -
6909 S1: x = load - VS1_0
6910 VS2_0: vz0 = vx0 + v1 VS2_1 -
6911 VS2_1: vz1 = vx1 + v1 VS2_2 -
6912 VS2_2: vz2 = vx2 + v1 VS2_3 -
6913 VS2_3: vz3 = vx3 + v1 - -
6914 S2: z = x + 1 - VS2_0 */
6918 /* Arguments are ready. Create the new vector stmt. */
6920 {
6927 {
6928 /* Lower the operation. This follows vector lowering. */
6931 tree word_type
6935 tree high_bits
6944 {
6962 else
6967 {
6973 }
6982 }
6983 else
6984 {
7001 }
7003 signs);
7013 }
7015 {
7016 tree mask;
7020 else
7021 /* Dummy mask. */
7031 {
7032 /* Perform the operation on active elements only and take
7033 inactive elements from the reduction chain input. */
7036 }
7037 else
7038 {
7042 }
7044 {
7047 signed char biasval
7052 }
7053 gcall *call
7056 vops);
7062 }
7063 else
7064 {
7066 /* When combining two masks check if either of them is elsewhere
7067 combined with a loop mask, if that's the case we can mark that the
7068 new combined mask doesn't need to be combined with a loop mask. */
7072 {
7074 ncopies}))
7075 {
7081 }
7084 ncopies }))
7085 {
7091 }
7092 }
7101 /* Enter the combined value into the vector cond hash so we don't
7102 AND it with a loop mask again. */
7105 }
7108 {
7111 new_temp);
7116 }
7120 else
7122 }
7132 }
7134 /* A helper function to ensure data reference DR_INFO's base alignment. */
7136 static void
7138 {
7139 /* Alignment is only analyzed for the first element of a DR group,
7140 use that to look at base alignment we need to enforce. */
7147 {
7150 // We should only be able to increase the alignment of a base object if
7151 // we know what its new alignment should be at compile time.
7158 {
7161 }
7163 }
7164 }
7167 /* Function get_group_alias_ptr_type.
7169 Return the alias type for the group starting at FIRST_STMT_INFO. */
7171 static tree
7173 {
7179 {
7183 {
7188 }
7190 }
7192 }
7195 /* Function scan_operand_equal_p.
7197 Helper function for check_scan_store. Compare two references
7198 with .GOMP_SIMD_LANE bases. */
7200 static bool
7202 {
7207 {
7208 machine_mode mode;
7218 {
7224 {
7229 }
7230 }
7231 }
7238 {
7242 {
7245 {
7248 {
7254 {
7257 }
7258 }
7260 {
7263 }
7266 {
7270 }
7279 }
7283 }
7284 }
7286 }
7290 /* Normal permutation. */
7291 scan_store_kind_perm,
7293 /* Whole vector left shift permutation with zero init. */
7294 scan_store_kind_lshift_zero,
7296 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7297 scan_store_kind_lshift_cond
7298 };
7300 /* Function check_scan_store.
7302 Verify if we can perform the needed permutations or whole vector shifts.
7303 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7304 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7305 to do at each step. */
7307 static int
7310 {
7322 {
7328 {
7331 }
7332 else
7333 {
7338 }
7341 {
7346 {
7350 /* Whole vector shifts shift in zeros, so if init is all zero
7351 constant, there is no need to do anything further. */
7355 {
7360 }
7361 }
7363 }
7365 {
7370 }
7371 }
7374 }
7377 /* Function check_scan_store.
7379 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7381 static bool
7384 vect_memory_access_type memory_access_type)
7385 {
7388 tree ref_type;
7392 || mask
7404 {
7409 }
7411 /* We need to pattern match code built by OpenMP lowering and simplified
7412 by following optimizations into something we can handle.
7413 #pragma omp simd reduction(inscan,+:r)
7414 for (...)
7415 {
7416 r += something ();
7417 #pragma omp scan inclusive (r)
7418 use (r);
7419 }
7420 shall have body with:
7421 // Initialization for input phase, store the reduction initializer:
7422 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7423 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7424 D.2042[_21] = 0;
7425 // Actual input phase:
7426 ...
7427 r.0_5 = D.2042[_20];
7428 _6 = _4 + r.0_5;
7429 D.2042[_20] = _6;
7430 // Initialization for scan phase:
7431 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7432 _26 = D.2043[_25];
7433 _27 = D.2042[_25];
7434 _28 = _26 + _27;
7435 D.2043[_25] = _28;
7436 D.2042[_25] = _28;
7437 // Actual scan phase:
7438 ...
7439 r.1_8 = D.2042[_20];
7440 ...
7441 The "omp simd array" variable D.2042 holds the privatized copy used
7442 inside of the loop and D.2043 is another one that holds copies of
7443 the current original list item. The separate GOMP_SIMD_LANE ifn
7444 kinds are there in order to allow optimizing the initializer store
7445 and combiner sequence, e.g. if it is originally some C++ish user
7446 defined reduction, but allow the vectorizer to pattern recognize it
7447 and turn into the appropriate vectorized scan.
7449 For exclusive scan, this is slightly different:
7450 #pragma omp simd reduction(inscan,+:r)
7451 for (...)
7452 {
7453 use (r);
7454 #pragma omp scan exclusive (r)
7455 r += something ();
7456 }
7457 shall have body with:
7458 // Initialization for input phase, store the reduction initializer:
7459 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7460 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7461 D.2042[_21] = 0;
7462 // Actual input phase:
7463 ...
7464 r.0_5 = D.2042[_20];
7465 _6 = _4 + r.0_5;
7466 D.2042[_20] = _6;
7467 // Initialization for scan phase:
7468 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7469 _26 = D.2043[_25];
7470 D.2044[_25] = _26;
7471 _27 = D.2042[_25];
7472 _28 = _26 + _27;
7473 D.2043[_25] = _28;
7474 // Actual scan phase:
7475 ...
7476 r.1_8 = D.2044[_20];
7477 ... */
7480 {
7481 /* Match the D.2042[_21] = 0; store above. Just require that
7482 it is a constant or external definition store. */
7484 {
7485 fail_init:
7490 }
7500 /* These stores can be vectorized normally. */
7502 }
7505 {
7506 fail:
7511 }
7520 bool inscan_var_store
7524 {
7526 {
7527 use_operand_p use_p;
7528 imm_use_iterator iter;
7530 {
7537 || other_store_stmt
7541 }
7547 }
7548 }
7550 {
7551 use_operand_p use_p;
7552 imm_use_iterator iter;
7554 {
7561 }
7562 }
7563 else
7573 /* For pointer addition, we should use the normal plus for the vector
7574 operation. */
7576 {
7584 }
7612 {
7618 tree lrhs;
7621 else
7623 use_operand_p use_p;
7624 imm_use_iterator iter;
7626 {
7633 }
7634 }
7642 stmt_vec_info other_store_stmt_info
7655 {
7659 }
7670 {
7681 }
7700 {
7705 }
7713 /* The IL is as expected, now check if we can actually vectorize it.
7714 Inclusive scan:
7715 _26 = D.2043[_25];
7716 _27 = D.2042[_25];
7717 _28 = _26 + _27;
7718 D.2043[_25] = _28;
7719 D.2042[_25] = _28;
7720 should be vectorized as (where _40 is the vectorized rhs
7721 from the D.2042[_21] = 0; store):
7722 _30 = MEM <vector(8) int> [(int *)&D.2043];
7723 _31 = MEM <vector(8) int> [(int *)&D.2042];
7724 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7725 _33 = _31 + _32;
7726 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7727 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7728 _35 = _33 + _34;
7729 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7730 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7731 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7732 _37 = _35 + _36;
7733 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7734 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7735 _38 = _30 + _37;
7736 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7737 MEM <vector(8) int> [(int *)&D.2043] = _39;
7738 MEM <vector(8) int> [(int *)&D.2042] = _38;
7739 Exclusive scan:
7740 _26 = D.2043[_25];
7741 D.2044[_25] = _26;
7742 _27 = D.2042[_25];
7743 _28 = _26 + _27;
7744 D.2043[_25] = _28;
7745 should be vectorized as (where _40 is the vectorized rhs
7746 from the D.2042[_21] = 0; store):
7747 _30 = MEM <vector(8) int> [(int *)&D.2043];
7748 _31 = MEM <vector(8) int> [(int *)&D.2042];
7749 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7750 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7751 _34 = _32 + _33;
7752 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7753 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7754 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7755 _36 = _34 + _35;
7756 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7757 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7758 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7759 _38 = _36 + _37;
7760 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7761 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7762 _39 = _30 + _38;
7763 _50 = _31 + _39;
7764 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7765 MEM <vector(8) int> [(int *)&D.2044] = _39;
7766 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7777 }
7780 /* Function vectorizable_scan_store.
7782 Helper of vectorizable_score, arguments like on vectorizable_store.
7783 Handle only the transformation, checking is done in check_scan_store. */
7785 static bool
7789 {
7804 bool inscan_var_store
7808 {
7809 use_operand_p use_p;
7810 imm_use_iterator iter;
7812 {
7818 }
7819 }
7840 {
7844 }
7859 {
7866 else
7867 {
7872 }
7876 {
7883 }
7884 else
7886 }
7907 {
7918 {
7922 dataref_offset);
7928 }
7932 {
7935 (zero_vec
7945 {
7946 /* Whole vector shift shifted in zero bits, but if *init
7947 is not initializer_zerop, we need to replace those elements
7948 with elements from vec_oprnd1. */
7961 }
7963 /* For exclusive scan, perform the perms[i] permutation once
7964 more. */
7968 {
7972 }
7980 }
7988 /* For exclusive scan, new_temp computed above is the exclusive scan
7989 prefix sum. Turn it into inclusive prefix sum for the broadcast
7990 of the last element into orig. */
7992 {
7997 }
8006 {
8009 dataref_offset);
8014 }
8015 }
8019 {
8025 dataref_offset);
8030 }
8032 }
8035 /* Function vectorizable_store.
8037 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8038 that can be vectorized.
8039 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8040 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8041 Return true if STMT_INFO is vectorizable in this way. */
8043 static bool
8048 {
8049 tree data_ref;
8051 tree elem_type;
8054 machine_mode vec_mode;
8055 tree dummy;
8063 stmt_vec_info first_stmt_info;
8069 tree aggr_type;
8070 gather_scatter_info gs_info;
8071 poly_uint64 vf;
8072 vec_load_store_type vls_type;
8073 tree ref_type;
8082 /* Is vectorizable store? */
8087 {
8100 }
8101 else
8102 {
8113 mask_index = vect_slp_child_index_for_operand
8120 }
8122 /* Cannot have hybrid store SLP -- that would mean storing to the
8123 same location twice. */
8130 {
8133 }
8134 else
8137 /* Multiple types in SLP are handled by creating the appropriate number of
8138 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8139 case of SLP. */
8142 else
8147 /* FORNOW. This restriction should be relaxed. */
8149 {
8154 }
8156 tree op;
8157 slp_tree op_node;
8168 vect_memory_access_type memory_access_type;
8171 poly_int64 poffset;
8172 internal_fn lanes_ifn;
8180 {
8182 {
8187 }
8191 {
8196 }
8200 {
8205 }
8206 }
8207 else
8208 {
8209 /* FORNOW. In some cases can vectorize even if data-type not supported
8210 (e.g. - array initialization with 0). */
8213 }
8220 {
8224 }
8225 else
8226 {
8230 }
8233 {
8235 memory_access_type))
8237 }
8241 {
8249 mask);
8253 || (mask
8255 mask_vectype))))
8256 {
8261 }
8272 /* As function vect_transform_stmt shows, for interleaving stores
8273 the whole chain is vectorized when the last store in the chain
8274 is reached, the other stores in the group are skipped. So we
8275 want to only cost the last one here, but it's not trivial to
8276 get the last, as it's equivalent to use the first one for
8277 costing, use the first one instead. */
8279 && !slp
8282 }
8285 /* Transform. */
8290 {
8294 {
8305 "vect_model_store_cost: inside_cost = %d, "
8310 }
8312 }
8315 {
8316 /* FORNOW */
8320 {
8322 /* VEC_NUM is the number of vect stmts to be created for this
8323 group. */
8330 }
8331 else
8332 /* VEC_NUM is the number of vect stmts to be created for this
8333 group. */
8337 }
8338 else
8343 ncopies);
8345 /* Check if we need to update prologue cost for invariant,
8346 and update it accordingly if so. If it's not for
8347 interleaving store, we can just check vls_type; but if
8348 it's for interleaving store, need to check the def_type
8349 of the stored value since the current vls_type is just
8350 for first_stmt_info. */
8352 {
8357 {
8363 }
8368 };
8372 {
8374 gimple_stmt_iterator incr_gsi;
8377 tree offvar;
8378 tree ivstep;
8379 tree running_off;
8382 tree dr_offset;
8384 /* Checked by get_load_store_type. */
8391 stride_base
8392 = fold_build_pointer_plus
8399 /* For a store with loop-invariant (but other than power-of-2)
8400 stride (i.e. not a grouped access) like so:
8402 for (i = 0; i < n; i += stride)
8403 array[i] = ...;
8405 we generate a new induction variable and new stores from
8406 the components of the (vectorized) rhs:
8408 for (j = 0; ; j += VF*stride)
8409 vectemp = ...;
8410 tmp1 = vectemp[0];
8411 array[j] = tmp1;
8412 tmp2 = vectemp[1];
8413 array[j + stride] = tmp2;
8414 ...
8415 */
8422 {
8425 {
8431 /* First check if vec_extract optab doesn't support extraction
8432 of vector elts directly. */
8434 machine_mode vmode;
8441 {
8442 /* Try to avoid emitting an extract of vector elements
8443 by performing the extracts using an integer type of the
8444 same size, extracting from a vector of those and then
8445 re-interpreting it as the original vector type if
8446 supported. */
8447 unsigned lsize
8450 /* If we can't construct such a vector fall back to
8451 element extracts from the original vector type and
8452 element size stores. */
8460 {
8465 }
8466 /* Else fall back to vector extraction anyway.
8467 Fewer stores are more important than avoiding spilling
8468 of the vector we extract from. Compared to the
8469 construction case in vectorizable_load no store-forwarding
8470 issue exists here for reasonable archs. */
8471 }
8472 }
8475 {
8477 dr_alignment_support dr_align
8479 mis_align);
8482 {
8489 }
8490 }
8493 }
8496 {
8510 }
8515 /* For costing some adjacent vector stores, we'd like to cost with
8516 the total number of them once instead of cost each one by one. */
8519 {
8522 {
8524 {
8526 tree pos
8533 }
8534 }
8540 else
8543 unsigned HOST_WIDE_INT
8546 {
8548 {
8550 /* Pun the vector to extract from if necessary. */
8552 {
8554 tree cvt
8559 }
8560 }
8562 {
8564 {
8565 /* Only need vector extracting when there are more
8566 than one stores. */
8568 inside_cost
8571 /* Take a single lane vector type store as scalar
8572 store to avoid ICE like 110776. */
8575 n_adjacent_stores++;
8576 else
8577 inside_cost
8581 }
8585 /* Extract the i'th component. */
8593 GSI_SAME_STMT);
8601 /* And store it to *running_off. */
8608 {
8616 }
8619 {
8623 }
8624 }
8625 }
8630 }
8633 {
8640 "vect_model_store_cost: inside_cost = %d, "
8643 }
8646 }
8649 vec_loop_masks *loop_masks
8653 vec_loop_lens *loop_lens
8658 /* Shouldn't go with length-based approach if fully masked. */
8661 /* Targets with store-lane instructions must not require explicit
8662 realignment. vect_supportable_dr_alignment always returns either
8663 dr_aligned or dr_unaligned_supported for masked operations. */
8665 && !mask
8674 tree bump;
8677 {
8680 }
8682 {
8687 }
8688 else
8689 {
8692 else
8696 }
8701 /* In case the vectorization factor (VF) is bigger than the number
8702 of elements that we can fit in a vectype (nunits), we have to generate
8703 more than one vector stmt - i.e - we need to "unroll" the
8704 vector stmt by a factor VF/nunits. */
8706 /* In case of interleaving (non-unit grouped access):
8708 S1: &base + 2 = x2
8709 S2: &base = x0
8710 S3: &base + 1 = x1
8711 S4: &base + 3 = x3
8713 We create vectorized stores starting from base address (the access of the
8714 first stmt in the chain (S2 in the above example), when the last store stmt
8715 of the chain (S4) is reached:
8717 VS1: &base = vx2
8718 VS2: &base + vec_size*1 = vx0
8719 VS3: &base + vec_size*2 = vx1
8720 VS4: &base + vec_size*3 = vx3
8722 Then permutation statements are generated:
8724 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8725 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8726 ...
8728 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8729 (the order of the data-refs in the output of vect_permute_store_chain
8730 corresponds to the order of scalar stmts in the interleaving chain - see
8731 the documentation of vect_permute_store_chain()).
8733 In case of both multiple types and interleaving, above vector stores and
8734 permutation stmts are created for every copy. The result vector stmts are
8735 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8736 STMT_VINFO_RELATED_STMT for the next copies.
8737 */
8747 {
8750 /* For costing some adjacent vector stores, we'd like to cost with
8751 the total number of them once instead of cost each one by one. */
8754 {
8757 {
8758 /* For interleaved stores we collect vectorized defs for all
8759 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8760 as an input to vect_permute_store_chain(). */
8763 {
8764 /* Since gaps are not supported for interleaved stores,
8765 DR_GROUP_SIZE is the exact number of stmts in the
8766 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8770 else
8771 {
8777 }
8779 }
8782 {
8784 {
8787 mask_vectype);
8789 }
8791 /* We should have catched mismatched types earlier. */
8794 dataref_ptr
8798 }
8799 }
8801 {
8803 /* DR_CHAIN is then used as an input to
8804 vect_permute_store_chain(). */
8806 {
8809 }
8814 }
8817 {
8820 }
8822 /* Get an array into which we can store the individual vectors. */
8825 /* Invalidate the current contents of VEC_ARRAY. This should
8826 become an RTL clobber too, which prevents the vector registers
8827 from being upward-exposed. */
8830 /* Store the individual vectors into the array. */
8832 {
8835 i);
8836 }
8849 {
8853 else
8855 signed char biasval
8859 {
8862 }
8863 }
8867 {
8868 /* Emit:
8869 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8870 LEN, BIAS, VEC_ARRAY). */
8876 vec_array);
8877 }
8879 {
8880 /* Emit:
8881 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
8882 VEC_ARRAY). */
8888 }
8889 else
8890 {
8891 /* Emit:
8892 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
8896 }
8901 /* Record that VEC_ARRAY is now dead. */
8906 }
8909 {
8916 "vect_model_store_cost: inside_cost = %d, "
8919 }
8922 }
8925 {
8930 {
8933 {
8938 {
8939 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
8940 DR_CHAIN is of size 1. */
8944 else
8948 {
8951 else
8953 ncopies,
8955 mask_vectype);
8956 }
8962 else
8963 dataref_ptr
8967 bump);
8968 }
8969 }
8971 {
8976 }
8980 {
8982 {
8986 /* We should have catched mismatched types earlier. */
8989 }
8995 {
9003 }
9006 {
9008 {
9010 inside_cost
9014 }
9021 {
9026 else
9028 signed char biasval
9032 {
9035 }
9036 }
9040 call = gimple_build_call_internal
9045 call = gimple_build_call_internal
9048 else
9055 }
9057 {
9058 /* The builtin decls path for scatter is legacy, x86 only. */
9060 && (!final_mask
9061 || SCALAR_INT_MODE_P
9064 {
9066 inside_cost
9070 }
9071 poly_uint64 offset_nunits
9074 {
9075 new_stmt = vect_build_one_scatter_store_call
9081 }
9083 {
9084 /* We have a offset vector with half the number of
9085 lanes but the builtins will store full vectype
9086 data from the lower lanes. */
9087 new_stmt = vect_build_one_scatter_store_call
9089 dataref_ptr,
9100 tree perm_mask
9104 perm_mask);
9110 {
9112 VEC_UNPACK_HI_EXPR,
9113 final_mask);
9114 final_mask = make_ssa_name
9119 }
9120 new_stmt = vect_build_one_scatter_store_call
9122 dataref_ptr,
9127 }
9129 {
9130 /* We have a offset vector with double the number of
9131 lanes. Select the low/high part accordingly. */
9134 {
9141 tree perm_mask = vect_gen_perm_mask_checked
9144 VEC_PERM_EXPR,
9145 vec_offset,
9146 vec_offset,
9147 perm_mask);
9152 }
9153 new_stmt = vect_build_one_scatter_store_call
9159 }
9160 else
9162 }
9163 else
9164 {
9165 /* Emulated scatter. */
9168 {
9170 /* For emulated scatter N offset vector element extracts
9171 (we assume the scalar scaling and ptr + offset add is
9172 consumed by the load). */
9173 inside_cost
9176 /* N scalar stores plus extracting the elements. */
9177 inside_cost
9180 inside_cost
9184 }
9187 unsigned HOST_WIDE_INT const_offset_nunits
9193 unsigned HOST_WIDE_INT elt_size
9195 /* We support offset vectors with more elements
9196 than the data vector for now. */
9197 unsigned HOST_WIDE_INT factor
9200 unsigned elt_offset
9207 {
9208 /* Compute the offsetted pointer. */
9211 tree idx
9217 tree ptr
9222 /* Extract the element to be stored. */
9223 tree elt
9230 tree ref
9236 }
9239 }
9240 }
9243 }
9250 "vect_model_store_cost: inside_cost = %d, "
9255 }
9263 /* For costing some adjacent vector stores, we'd like to cost with
9264 the total number of them once instead of cost each one by one. */
9269 {
9272 {
9274 {
9275 /* Get vectorized arguments for SLP_NODE. */
9281 }
9282 else
9283 {
9284 /* For interleaved stores we collect vectorized defs for all the
9285 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9286 input to vect_permute_store_chain().
9288 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9289 is of size 1. */
9292 {
9293 /* Since gaps are not supported for interleaved stores,
9294 DR_GROUP_SIZE is the exact number of stmts in the chain.
9295 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9296 that there is no interleaving, DR_GROUP_SIZE is 1,
9297 and only one iteration of the loop will be executed. */
9301 else
9302 {
9308 }
9310 }
9312 {
9315 mask_vectype);
9317 }
9318 }
9320 /* We should have catched mismatched types earlier. */
9324 bool simd_lane_access_p
9327 && simd_lane_access_p
9328 && !loop_masks
9335 {
9338 }
9340 dataref_ptr
9345 }
9347 {
9349 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9350 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9351 of size 1. */
9353 {
9356 }
9361 else
9364 }
9368 {
9369 /* Permute. */
9372 {
9379 "vect_model_store_cost: "
9381 group_size);
9382 }
9383 else
9386 }
9390 {
9392 {
9396 /* For grouped stores vectorized defs are interleaved in
9397 vect_permute_store_chain(). */
9399 }
9402 {
9406 else
9407 {
9413 /* Generate the permute statement. */
9414 gimple *perm_stmt
9418 gsi);
9422 }
9423 }
9426 {
9427 n_adjacent_stores++;
9430 {
9434 }
9437 }
9453 /* Bump the vector pointer. */
9463 {
9466 }
9467 else
9472 misalign);
9475 /* Compute IFN when LOOP_LENS or final_mask valid. */
9480 {
9481 opt_machine_mode new_ovmode
9484 unsigned factor
9489 }
9491 {
9496 }
9499 {
9501 {
9502 /* Pass VF value to 'len' argument of
9503 MASK_LEN_STORE if LOOP_LENS is invalid. */
9505 }
9507 {
9508 /* Pass all ones value to 'mask' argument of
9509 MASK_LEN_STORE if final_mask is invalid. */
9512 }
9513 }
9515 {
9516 signed char biasval
9520 }
9522 /* Arguments are ready. Create the new vector stmt. */
9524 {
9527 /* Need conversion if it's wrapped with VnQI. */
9529 {
9530 tree new_vtype
9532 new_vmode);
9535 gassign *new_stmt
9539 }
9545 else
9552 }
9554 {
9556 gcall *call
9562 }
9563 else
9564 {
9565 data_ref
9567 dataref_offset ? dataref_offset
9570 ;
9571 else
9578 }
9586 }
9588 {
9592 }
9593 }
9596 {
9602 /* When vectorizing a store into the function result assign
9603 a penalty if the function returns in a multi-register location.
9604 In this case we assume we'll end up with having to spill the
9605 vector result and do piecewise loads as a conservative estimate. */
9611 {
9613 /* ??? Handle PARALLEL in some way. */
9615 {
9617 /* Assume that a single reg-reg move is possible and cheap,
9618 do not account for vector to gp register move cost. */
9620 {
9621 /* Spill. */
9622 prologue_cost
9625 /* Loads. */
9626 prologue_cost
9629 }
9630 }
9631 }
9634 "vect_model_store_cost: inside_cost = %d, "
9637 }
9640 }
9642 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9643 VECTOR_CST mask. No checks are made that the target platform supports the
9644 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9645 vect_gen_perm_mask_checked. */
9647 tree
9649 {
9650 tree mask_type;
9657 }
9659 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9660 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9662 tree
9664 {
9668 }
9670 /* Given a vector variable X and Y, that was generated for the scalar
9671 STMT_INFO, generate instructions to permute the vector elements of X and Y
9672 using permutation mask MASK_VEC, insert them at *GSI and return the
9673 permuted vector variable. */
9675 static tree
9679 {
9687 else
9691 /* Generate the permute statement. */
9696 }
9698 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9699 inserting them on the loops preheader edge. Returns true if we
9700 were successful in doing so (and thus STMT_INFO can be moved then),
9701 otherwise returns false. HOIST_P indicates if we want to hoist the
9702 definitions of all SSA uses, it would be false when we are costing. */
9704 static bool
9706 {
9707 ssa_op_iter i;
9708 tree op;
9712 {
9716 {
9717 /* Make sure we don't need to recurse. While we could do
9718 so in simple cases when there are more complex use webs
9719 we don't have an easy way to preserve stmt order to fulfil
9720 dependencies within them. */
9721 tree op2;
9722 ssa_op_iter i2;
9726 {
9731 }
9733 }
9734 }
9743 {
9747 {
9751 }
9752 }
9755 }
9757 /* vectorizable_load.
9759 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9760 that can be vectorized.
9761 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9762 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9763 Return true if STMT_INFO is vectorizable in this way. */
9765 static bool
9770 {
9771 tree scalar_dest;
9778 tree elem_type;
9779 /* Avoid false positive uninitialized warning, see PR110652. */
9781 machine_mode mode;
9782 tree dummy;
9789 poly_uint64 group_gap_adj;
9795 stmt_vec_info first_stmt_info;
9803 poly_uint64 vf;
9804 tree aggr_type;
9805 gather_scatter_info gs_info;
9806 tree ref_type;
9823 {
9838 }
9839 else
9840 {
9855 mask_index = vect_slp_child_index_for_operand
9861 }
9867 {
9871 }
9872 else
9875 /* Multiple types in SLP are handled by creating the appropriate number of
9876 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
9877 case of SLP. */
9880 else
9885 /* FORNOW. This restriction should be relaxed. */
9887 {
9892 }
9894 /* Invalidate assumptions made by dependence analysis when vectorization
9895 on the unrolled body effectively re-orders stmts. */
9900 {
9903 "cannot perform implicit CSE when unrolling "
9906 }
9911 /* FORNOW. In some cases can vectorize even if data-type not supported
9912 (e.g. - data copies). */
9914 {
9919 }
9921 /* Check if the load is a part of an interleaving chain. */
9923 {
9925 /* FORNOW */
9932 /* Refuse non-SLP vectorization of SLP-only groups. */
9934 {
9939 }
9941 /* Invalidate assumptions made by dependence analysis when vectorization
9942 on the unrolled body effectively re-orders stmts. */
9947 {
9950 "cannot perform implicit CSE when performing "
9953 }
9954 }
9955 else
9959 {
9963 {
9964 /* In BB vectorization we may not actually use a loaded vector
9965 accessing elements in excess of DR_GROUP_SIZE. */
9976 {
9979 "BB vectorization with gaps at the end of "
9982 }
9983 }
9989 {
9992 vect_location,
9995 }
9996 }
9998 vect_memory_access_type memory_access_type;
10001 poly_int64 poffset;
10002 internal_fn lanes_ifn;
10010 {
10012 {
10018 }
10021 {
10026 }
10030 {
10035 }
10036 }
10041 {
10043 && mask
10045 mask_vectype))
10046 {
10051 }
10061 mask);
10074 }
10084 /* Transform. */
10090 {
10092 /* If we have versioned for aliasing or the loop doesn't
10093 have any data dependencies that would preclude this,
10094 then we are sure this is a loop invariant load and
10095 thus we can insert it on the preheader edge.
10096 TODO: hoist_defs_of_uses should ideally be computed
10097 once at analysis time, remembered and used in the
10098 transform time. */
10100 && !nested_in_vect_loop
10103 {
10104 enum vect_cost_model_location cost_loc
10109 cost_loc);
10114 "vect_model_load_cost: inside_cost = %d, "
10118 }
10120 {
10130 tree vuse;
10133 else
10138 }
10139 /* These copies are all equivalent. */
10143 else
10144 {
10149 }
10154 else
10155 {
10159 }
10161 }
10165 {
10166 gimple_stmt_iterator incr_gsi;
10168 tree offvar;
10169 tree ivstep;
10170 tree running_off;
10173 /* Checked by get_load_store_type. */
10176 tree dr_offset;
10183 {
10186 }
10187 else
10188 {
10191 }
10194 {
10197 }
10198 else
10199 {
10201 cst_offset
10204 first_stmt_info));
10207 }
10210 {
10218 /* For a load with loop-invariant (but other than power-of-2)
10219 stride (i.e. not a grouped access) like so:
10221 for (i = 0; i < n; i += stride)
10222 ... = array[i];
10224 we generate a new induction variable and new accesses to
10225 form a new vector (or vectors, depending on ncopies):
10227 for (j = 0; ; j += VF*stride)
10228 tmp1 = array[j];
10229 tmp2 = array[j + stride];
10230 ...
10231 vectemp = {tmp1, tmp2, ...}
10232 */
10246 }
10256 {
10258 {
10259 /* First check if vec_init optab supports construction from vector
10260 elts directly. Otherwise avoid emitting a constructor of
10261 vector elements by performing the loads using an integer type
10262 of the same size, constructing a vector of those and then
10263 re-interpreting it as the original vector type. This avoids a
10264 huge runtime penalty due to the general inability to perform
10265 store forwarding from smaller stores to a larger load. */
10266 tree ptype;
10267 tree vtype
10272 {
10277 }
10278 }
10279 else
10280 {
10284 }
10286 }
10287 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10292 {
10293 /* For SLP permutation support we need to load the whole group,
10294 not only the number of vector stmts the permutation result
10295 fits in. */
10297 {
10298 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10299 variable VF. */
10303 }
10304 else
10306 }
10308 unsigned HOST_WIDE_INT
10311 /* For costing some adjacent vector loads, we'd like to cost with
10312 the total number of them once instead of cost each one by one. */
10315 {
10320 {
10322 {
10323 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10324 avoid ICE, see PR110776. */
10327 n_adjacent_loads++;
10328 else
10332 }
10346 {
10347 n_groups++;
10348 /* When doing SLP make sure to not load elements from
10349 the next vector iteration, those will not be accessed
10350 so just use the last element again. See PR107451. */
10352 {
10354 gimple *incr
10359 }
10361 }
10362 }
10365 {
10369 else
10370 {
10376 {
10377 new_stmt
10379 VIEW_CONVERT_EXPR,
10383 gsi);
10384 }
10385 }
10386 }
10389 {
10391 {
10394 else
10396 }
10397 else
10398 {
10402 }
10403 }
10404 }
10406 {
10409 {
10415 }
10416 else
10419 }
10422 {
10430 "vect_model_load_cost: inside_cost = %u, "
10432 inside_cost);
10433 }
10436 }
10444 {
10446 {
10449 }
10450 else
10451 {
10454 }
10455 /* For SLP vectorization we directly vectorize a subchain
10456 without permutation. */
10459 /* For BB vectorization always use the first stmt to base
10460 the data ref pointer on. */
10462 first_stmt_info_for_drptr
10465 /* Check if the chain of loads is already vectorized. */
10467 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10468 ??? But we can only do so if there is exactly one
10469 as we have no way to get at the rest. Leave the CSE
10470 opportunity alone.
10471 ??? With the group load eventually participating
10472 in multiple different permutations (having multiple
10473 slp nodes which refer to the same group) the CSE
10474 is even wrong code. See PR56270. */
10476 {
10479 }
10483 /* VEC_NUM is the number of vect stmts to be created for this group. */
10485 {
10487 /* If an SLP permutation is from N elements to N elements,
10488 and if one vector holds a whole number of N, we can load
10489 the inputs to the permutation in the same way as an
10490 unpermuted sequence. In other cases we need to load the
10491 whole group, not only the number of vector stmts the
10492 permutation result fits in. */
10497 {
10498 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10499 variable VF; see vect_transform_slp_perm_load. */
10504 }
10505 else
10506 {
10508 group_gap_adj
10510 }
10511 }
10512 else
10516 }
10517 else
10518 {
10526 }
10529 vec_loop_masks *loop_masks
10533 vec_loop_lens *loop_lens
10538 /* Shouldn't go with length-based approach if fully masked. */
10541 /* Targets with store-lane instructions must not require explicit
10542 realignment. vect_supportable_dr_alignment always returns either
10543 dr_aligned or dr_unaligned_supported for masked operations. */
10545 && !mask
10550 /* In case the vectorization factor (VF) is bigger than the number
10551 of elements that we can fit in a vectype (nunits), we have to generate
10552 more than one vector stmt - i.e - we need to "unroll" the
10553 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10554 from one copy of the vector stmt to the next, in the field
10555 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10556 stages to find the correct vector defs to be used when vectorizing
10557 stmts that use the defs of the current stmt. The example below
10558 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10559 need to create 4 vectorized stmts):
10561 before vectorization:
10562 RELATED_STMT VEC_STMT
10563 S1: x = memref - -
10564 S2: z = x + 1 - -
10566 step 1: vectorize stmt S1:
10567 We first create the vector stmt VS1_0, and, as usual, record a
10568 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10569 Next, we create the vector stmt VS1_1, and record a pointer to
10570 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10571 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10572 stmts and pointers:
10573 RELATED_STMT VEC_STMT
10574 VS1_0: vx0 = memref0 VS1_1 -
10575 VS1_1: vx1 = memref1 VS1_2 -
10576 VS1_2: vx2 = memref2 VS1_3 -
10577 VS1_3: vx3 = memref3 - -
10578 S1: x = load - VS1_0
10579 S2: z = x + 1 - -
10580 */
10582 /* In case of interleaving (non-unit grouped access):
10584 S1: x2 = &base + 2
10585 S2: x0 = &base
10586 S3: x1 = &base + 1
10587 S4: x3 = &base + 3
10589 Vectorized loads are created in the order of memory accesses
10590 starting from the access of the first stmt of the chain:
10592 VS1: vx0 = &base
10593 VS2: vx1 = &base + vec_size*1
10594 VS3: vx3 = &base + vec_size*2
10595 VS4: vx4 = &base + vec_size*3
10597 Then permutation statements are generated:
10599 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10600 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10601 ...
10603 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10604 (the order of the data-refs in the output of vect_permute_load_chain
10605 corresponds to the order of scalar stmts in the interleaving chain - see
10606 the documentation of vect_permute_load_chain()).
10607 The generation of permutation stmts and recording them in
10608 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10610 In case of both multiple types and interleaving, the vector loads and
10611 permutation stmts above are created for every copy. The result vector
10612 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10613 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10615 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10616 on a target that supports unaligned accesses (dr_unaligned_supported)
10617 we generate the following code:
10618 p = initial_addr;
10619 indx = 0;
10620 loop {
10621 p = p + indx * vectype_size;
10622 vec_dest = *(p);
10623 indx = indx + 1;
10624 }
10626 Otherwise, the data reference is potentially unaligned on a target that
10627 does not support unaligned accesses (dr_explicit_realign_optimized) -
10628 then generate the following code, in which the data in each iteration is
10629 obtained by two vector loads, one from the previous iteration, and one
10630 from the current iteration:
10631 p1 = initial_addr;
10632 msq_init = *(floor(p1))
10633 p2 = initial_addr + VS - 1;
10634 realignment_token = call target_builtin;
10635 indx = 0;
10636 loop {
10637 p2 = p2 + indx * vectype_size
10638 lsq = *(floor(p2))
10639 vec_dest = realign_load (msq, lsq, realignment_token)
10640 indx = indx + 1;
10641 msq = lsq;
10642 } */
10644 /* If the misalignment remains the same throughout the execution of the
10645 loop, we can create the init_addr and permutation mask at the loop
10646 preheader. Otherwise, it needs to be created inside the loop.
10647 This can only occur when vectorizing memory accesses in the inner-loop
10648 nested within an outer-loop that is being vectorized. */
10653 {
10656 }
10658 bool diff_first_stmt_info
10665 {
10666 /* If we have different first_stmt_info, we can't set up realignment
10667 here, since we can't guarantee first_stmt_info DR has been
10668 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10669 distance from first_stmt_info DR instead as below. */
10671 {
10678 {
10681 size_one_node);
10683 }
10684 }
10685 }
10686 else
10690 offset = (offset
10694 tree bump;
10697 {
10700 }
10702 {
10707 }
10708 else
10709 {
10712 else
10716 }
10721 {
10725 else
10728 }
10732 {
10738 /* For costing some adjacent vector loads, we'd like to cost with
10739 the total number of them once instead of cost each one by one. */
10742 {
10744 {
10745 /* An IFN_LOAD_LANES will load all its vector results,
10746 regardless of which ones we actually need. Account
10747 for the cost of unused results. */
10749 {
10752 do
10753 {
10756 }
10759 {
10762 "vect_model_load_cost: %d "
10764 gaps);
10766 alignment_support_scheme,
10770 }
10771 }
10772 n_adjacent_loads++;
10774 }
10776 /* 1. Create the vector or array pointer update chain. */
10778 dataref_ptr
10782 else
10783 {
10787 }
10804 {
10808 else
10810 signed char biasval
10814 {
10817 }
10818 }
10822 {
10823 /* Emit:
10824 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10825 VEC_MASK, LEN, BIAS). */
10831 }
10833 {
10834 /* Emit:
10835 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
10836 VEC_MASK). */
10841 final_mask);
10842 }
10843 else
10844 {
10845 /* Emit:
10846 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
10849 }
10855 /* Extract each vector into an SSA_NAME. */
10857 {
10861 }
10863 /* Record the mapping between SSA_NAMEs and statements. */
10866 /* Record that VEC_ARRAY is now dead. */
10872 }
10875 {
10883 "vect_model_load_cost: inside_cost = %u, "
10886 }
10889 }
10892 {
10899 {
10900 /* 1. Create the vector or array pointer update chain. */
10902 {
10907 else
10908 dataref_ptr
10912 }
10914 {
10919 }
10923 {
10928 {
10932 final_mask
10943 }
10945 /* 2. Create the vector-load in the loop. */
10948 {
10950 {
10952 inside_cost
10956 }
10963 {
10965 final_len
10969 else
10970 final_len
10973 signed char biasval
10977 {
10980 }
10981 }
10985 call
10994 else
11001 }
11003 {
11004 /* The builtin decls path for gather is legacy, x86 only. */
11007 {
11009 inside_cost
11013 }
11014 poly_uint64 offset_nunits
11017 {
11018 new_stmt = vect_build_one_gather_load_call
11021 final_mask);
11023 }
11025 {
11026 /* We have a offset vector with half the number of
11027 lanes but the builtins will produce full vectype
11028 data with just the lower lanes filled. */
11029 new_stmt = vect_build_one_gather_load_call
11032 final_mask);
11038 /* now put upper half of final_mask in final_mask low. */
11040 && !SCALAR_INT_MODE_P
11042 {
11049 tree perm_mask = vect_gen_perm_mask_checked
11052 VEC_PERM_EXPR,
11053 final_mask,
11054 final_mask,
11055 perm_mask);
11060 }
11062 {
11064 VEC_UNPACK_HI_EXPR,
11065 final_mask);
11066 final_mask = make_ssa_name
11071 }
11073 new_stmt = vect_build_one_gather_load_call
11075 dataref_ptr,
11077 final_mask);
11083 /* compose low + high. */
11090 tree perm_mask
11093 VEC_PERM_EXPR,
11096 }
11098 {
11099 /* We have a offset vector with double the number of
11100 lanes. Select the low/high part accordingly. */
11103 {
11110 tree perm_mask = vect_gen_perm_mask_checked
11113 VEC_PERM_EXPR,
11114 vec_offset,
11115 vec_offset,
11116 perm_mask);
11121 }
11122 new_stmt = vect_build_one_gather_load_call
11126 }
11127 else
11129 }
11130 else
11131 {
11132 /* Emulated gather-scatter. */
11136 {
11137 /* For emulated gathers N offset vector element
11138 offset add is consumed by the load). */
11142 /* N scalar loads plus gathering them into a
11143 vector. */
11144 inside_cost
11147 inside_cost
11151 }
11152 unsigned HOST_WIDE_INT const_offset_nunits
11158 /* We support offset vectors with more elements
11159 than the data vector for now. */
11160 unsigned HOST_WIDE_INT factor
11163 unsigned elt_offset
11170 {
11173 tree idx
11178 scale);
11190 }
11195 }
11198 /* DATA_REF is null if we've already built the statement. */
11200 {
11203 }
11208 /* Store vector loads in the corresponding SLP_NODE. */
11211 }
11215 }
11222 "vect_model_load_cost: inside_cost = %u, "
11226 }
11230 /* For costing some adjacent vector loads, we'd like to cost with
11231 the total number of them once instead of cost each one by one. */
11234 {
11235 /* 1. Create the vector or array pointer update chain. */
11237 {
11238 bool simd_lane_access_p
11249 {
11252 }
11254 {
11255 dataref_ptr
11259 bump);
11260 /* Adjust the pointer by the difference to first_stmt. */
11261 data_reference_p ptrdr
11263 tree diff
11271 {
11275 alignment_support_scheme,
11278 }
11279 }
11280 else
11281 dataref_ptr
11283 at_loop,
11286 }
11288 {
11292 bump);
11293 else
11296 }
11303 {
11308 {
11322 }
11324 /* 2. Create the vector-load in the loop. */
11326 {
11329 {
11339 {
11340 align
11343 }
11344 else
11349 misalign);
11352 /* Compute IFN when LOOP_LENS or final_mask valid. */
11357 {
11358 opt_machine_mode new_ovmode
11361 unsigned factor
11366 }
11368 {
11373 }
11376 {
11378 {
11379 /* Pass VF value to 'len' argument of
11380 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11382 }
11384 {
11385 /* Pass all ones value to 'mask' argument of
11386 MASK_LEN_LOAD if final_mask is invalid. */
11389 }
11390 }
11392 {
11393 signed char biasval
11397 }
11400 {
11407 bias);
11408 else
11416 /* Need conversion if it's wrapped with VnQI. */
11418 {
11421 tree var
11425 gsi);
11429 }
11430 }
11432 {
11436 final_mask);
11440 }
11441 else
11442 {
11446 unsigned int vect_align
11448 unsigned int scalar_dr_size
11450 /* If there's no peeling for gaps but we have a gap
11451 with slp loads then load the lower half of the
11452 vector only. See get_group_load_store_type for
11453 when we apply this optimization. */
11455 && loop_vinfo
11460 {
11461 tree half_vtype;
11462 new_vtype
11467 }
11468 tree offset
11473 {
11474 unsigned HOST_WIDE_INT gap_offset
11478 }
11479 data_ref
11482 ;
11483 else
11488 {
11494 gsi);
11499 {
11503 }
11504 else
11505 {
11509 }
11514 else
11515 {
11522 vec_dest,
11524 }
11525 }
11526 }
11528 }
11530 {
11540 dr_explicit_realign,
11545 else
11547 // For explicit realign the target alignment should be
11548 // known at compile time.
11549 unsigned HOST_WIDE_INT align
11556 data_ref
11570 bump);
11576 else
11580 data_ref
11583 }
11585 {
11590 else
11592 // We should only be doing this if we know the target
11593 // alignment at compile time.
11594 unsigned HOST_WIDE_INT align
11604 }
11607 }
11609 /* One common place to cost the above vect load for different
11610 alignment support schemes. */
11612 {
11613 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11614 only need to take care of the first stmt, whose
11615 stmt_info is first_stmt_info, vec_num iterating on it
11616 will cover the cost for the remaining, it's consistent
11617 with transforming. For the prologue cost for realign,
11618 we only need to count it once for the whole group. */
11625 {
11626 /* Leave realign cases alone to keep them simple. */
11634 else
11635 n_adjacent_loads++;
11636 }
11637 }
11638 else
11639 {
11641 /* DATA_REF is null if we've already built the statement. */
11643 {
11646 }
11650 }
11652 /* 3. Handle explicit realignment if necessary/supported.
11653 Create in loop:
11654 vec_dest = realign_load (msq, lsq, realignment_token) */
11658 {
11670 {
11674 UNKNOWN_LOCATION);
11676 }
11677 }
11680 {
11684 else
11685 {
11690 }
11691 }
11693 /* Collect vector loads and later create their permutation in
11694 vect_transform_grouped_load (). */
11698 /* Store vector loads in the corresponding SLP_NODE. */
11702 /* With SLP permutation we load the gaps as well, without
11703 we need to skip the gaps after we manage to fully load
11704 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11708 && !slp_perm
11710 {
11711 poly_wide_int bump_val
11720 }
11721 }
11722 /* Bump the vector pointer to account for a gap or for excess
11723 elements loaded for a permuted SLP load. */
11727 {
11728 poly_wide_int bump_val
11735 }
11741 {
11743 /* For SLP we know we've seen all possible uses of dr_chain so
11744 direct vect_transform_slp_perm_load to DCE the unused parts.
11745 ??? This is a hack to prevent compile-time issues as seen
11746 in PR101120 and friends. */
11748 {
11753 }
11754 else
11755 {
11760 }
11761 }
11762 else
11763 {
11765 {
11767 /* We assume that the cost of a single load-lanes instruction
11768 is equivalent to the cost of DR_GROUP_SIZE separate loads.
11769 If a grouped access is instead being provided by a
11770 load-and-permute operation, include the cost of the
11771 permutes. */
11773 {
11774 /* Uses an even and odd extract operations or shuffle
11775 operations for each needed permute. */
11783 "vect_model_load_cost:"
11785 group_size);
11786 }
11788 {
11792 }
11793 }
11796 }
11798 }
11803 {
11814 "vect_model_load_cost: inside_cost = %u, "
11817 }
11820 }
11822 /* Function vect_is_simple_cond.
11824 Input:
11825 LOOP - the loop that is being vectorized.
11826 COND - Condition that is checked for simple use.
11828 Output:
11829 *COMP_VECTYPE - the vector type for the comparison.
11830 *DTS - The def types for the arguments of the comparison
11832 Returns whether a COND can be vectorized. Checks whether
11833 condition operands are supportable using vec_is_simple_use. */
11835 static bool
11839 {
11842 slp_tree slp_op;
11844 /* Mask case. */
11847 {
11850 || !*comp_vectype
11854 }
11863 {
11867 }
11871 else
11875 {
11879 }
11883 else
11892 /* Invariant comparison. */
11894 {
11898 else
11899 {
11900 /* If we can widen the comparison to match vectype do so. */
11902 && !slp_node
11905 scalar_type = build_nonstandard_integer_type
11908 slp_node);
11909 }
11910 }
11913 }
11915 /* vectorizable_condition.
11917 Check if STMT_INFO is conditional modify expression that can be vectorized.
11918 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
11919 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
11920 at GSI.
11922 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
11924 Return true if STMT_INFO is vectorizable in this way. */
11926 static bool
11931 {
11939 tree vec_compare;
11940 tree new_temp;
11955 tree vec_cmp_type;
11961 /* Is vectorizable conditional operation? */
11973 bool for_reduction
11976 {
11984 }
11985 else
11986 {
11989 }
11995 {
11998 }
11999 else
12000 {
12003 }
12039 {
12042 }
12044 /* For conditional reductions, the "then" value needs to be the candidate
12045 value calculated by this iteration while the "else" value needs to be
12046 the result carried over from previous iterations. If the COND_EXPR
12047 is the other way around, we need to swap it. */
12050 {
12053 else
12054 {
12059 else
12060 {
12062 /* Make sure we don't accidentally use the old condition. */
12064 }
12065 }
12067 }
12070 {
12071 /* Boolean values may have another representation in vectors
12072 and therefore we prefer bit operations over comparison for
12073 them (which also works for scalar masks). We store opcodes
12074 to use in bitop1 and bitop2. Statement is vectorized as
12075 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12076 depending on bitop1 and bitop2 arity. */
12078 {
12106 }
12108 }
12113 {
12118 }
12121 {
12123 {
12125 optab optab;
12132 {
12134 optab_default);
12137 }
12138 }
12142 /* Count one reduction-like operation per vector. */
12145 && (masked
12147 cond_code)
12149 ERROR_MARK))))
12153 && (!vect_maybe_update_slp_op_vectype
12156 && !vect_maybe_update_slp_op_vectype
12160 {
12165 }
12169 {
12171 {
12177 else
12181 }
12182 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12184 {
12187 "conditional reduction prevents the use"
12190 }
12191 }
12197 }
12199 /* Transform. */
12201 /* Handle def. */
12208 /* See whether another part of the vectorized code applies a loop
12209 mask to the condition, or to its inverse. */
12214 {
12217 }
12219 {
12222 else
12223 {
12227 else
12228 {
12233 {
12237 }
12238 else
12239 {
12240 /* Try the inverse of the current mask. We check if the
12241 inverse mask is live and if so we generate a negate of
12242 the current mask such that we still honor NaNs. */
12246 {
12251 }
12252 }
12253 }
12254 }
12255 }
12257 /* Handle cond expr. */
12262 reduction_type != EXTRACT_LAST_REDUCTION
12264 else
12269 reduction_type != EXTRACT_LAST_REDUCTION
12272 /* Arguments are ready. Create the new vector stmt. */
12274 {
12284 else
12285 {
12288 {
12293 }
12294 else
12295 {
12300 vec_cond_rhs);
12301 else
12302 new_stmt
12304 vec_cond_rhs);
12310 {
12311 /* Instead of doing ~x ? y : z do x ? z : y. */
12314 }
12315 else
12316 {
12319 new_stmt
12321 else
12322 new_stmt
12327 }
12328 }
12329 }
12331 /* If we decided to apply a loop mask to the result of the vector
12332 comparison, AND the comparison with the mask now. Later passes
12333 should then be able to reuse the AND results between mulitple
12334 vector statements.
12336 For example:
12337 for (int i = 0; i < 100; ++i)
12338 x[i] = y[i] ? z[i] : 10;
12340 results in following optimized GIMPLE:
12342 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12343 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12344 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12345 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12346 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12347 vect_iftmp.11_47, { 10, ... }>;
12349 instead of using a masked and unmasked forms of
12350 vec != { 0, ... } (masked in the MASK_LOAD,
12351 unmasked in the VEC_COND_EXPR). */
12353 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12354 in cases where that's necessary. */
12358 {
12360 {
12363 vec_compare);
12366 }
12369 {
12372 BIT_NOT_EXPR,
12373 vec_compare);
12376 }
12380 {
12382 {
12385 signed char biasval
12388 }
12389 else
12390 {
12393 }
12394 }
12396 {
12397 tree loop_mask
12401 gassign *g
12403 loop_mask);
12406 }
12407 }
12411 {
12415 new_stmt = gimple_build_call_internal
12418 else
12419 new_stmt = gimple_build_call_internal
12421 vec_then_clause);
12426 else
12427 {
12428 /* In this case we're moving the definition to later in the
12429 block. That doesn't matter because the only uses of the
12430 lhs are in phi statements. */
12434 }
12435 }
12436 else
12437 {
12442 }
12445 else
12447 }
12458 }
12460 /* Helper of vectorizable_comparison.
12462 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12463 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12464 comparison, put it in VEC_STMT, and insert it at GSI.
12466 Return true if STMT_INFO is vectorizable in this way. */
12468 static bool
12473 {
12477 tree new_temp;
12481 poly_uint64 nunits;
12488 tree mask_type;
12489 tree mask;
12502 else
12526 /* Invariant comparison. */
12528 {
12531 else
12533 slp_node);
12536 }
12540 /* Can't compare mask and non-mask types. */
12545 /* Boolean values may have another representation in vectors
12546 and therefore we prefer bit operations over comparison for
12547 them (which also works for scalar masks). We store opcodes
12548 to use in bitop1 and bitop2. Statement is vectorized as
12549 BITOP2 (rhs1 BITOP1 rhs2) or
12550 rhs1 BITOP2 (BITOP1 rhs2)
12551 depending on bitop1 and bitop2 arity. */
12554 {
12556 {
12559 }
12561 {
12564 }
12566 {
12570 }
12572 {
12576 }
12577 else
12578 {
12582 }
12583 }
12586 {
12588 {
12591 }
12592 else
12593 {
12595 optab optab;
12602 {
12606 }
12607 }
12609 /* Put types on constant and invariant SLP children. */
12613 {
12618 }
12624 }
12626 /* Transform. */
12628 /* Handle def. */
12638 /* Arguments are ready. Create the new vector stmt. */
12640 {
12646 {
12650 }
12651 else
12652 {
12655 else
12657 vec_rhs2);
12660 {
12664 else
12666 new_temp);
12668 }
12669 }
12672 else
12674 }
12683 }
12685 /* vectorizable_comparison.
12687 Check if STMT_INFO is comparison expression that can be vectorized.
12688 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12689 comparison, put it in VEC_STMT, and insert it at GSI.
12691 Return true if STMT_INFO is vectorizable in this way. */
12693 static bool
12698 {
12721 }
12723 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
12724 can handle all live statements in the node. Otherwise return true
12725 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
12726 VEC_STMT_P is as for vectorizable_live_operation. */
12728 static bool
12733 {
12735 {
12736 stmt_vec_info slp_stmt_info;
12739 {
12745 }
12746 }
12754 }
12756 /* Make sure the statement is vectorizable. */
12758 opt_result
12763 {
12767 gimple_seq pattern_def_seq;
12775 "not vectorized:"
12782 {
12783 gimple_stmt_iterator si;
12786 {
12787 stmt_vec_info pattern_def_stmt_info
12791 {
12792 /* Analyze def stmt of STMT if it's a pattern stmt. */
12798 opt_result res
12801 cost_vec);
12804 }
12805 }
12806 }
12808 /* Skip stmts that do not need to be vectorized. In loops this is expected
12809 to include:
12810 - the COND_EXPR which is the loop exit condition
12811 - any LABEL_EXPRs in the loop
12812 - computations that are used only for array indexing or loop control.
12813 In basic blocks we only analyze statements that are a part of some SLP
12814 instance, therefore, all the statements are relevant.
12816 Pattern statement needs to be analyzed instead of the original statement
12817 if the original statement is not relevant. Otherwise, we analyze both
12818 statements. In basic blocks we are called from some SLP instance
12819 traversal, don't analyze pattern stmts instead, the pattern stmts
12820 already will be part of SLP instance. */
12825 {
12827 && pattern_stmt_info
12830 {
12831 /* Analyze PATTERN_STMT instead of the original stmt. */
12837 }
12838 else
12839 {
12844 }
12845 }
12848 && pattern_stmt_info
12851 {
12852 /* Analyze PATTERN_STMT too. */
12858 opt_result res
12863 }
12866 {
12890 }
12897 {
12902 }
12905 {
12910 }
12916 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
12917 -mveclibabi= takes preference over library functions with
12918 the simd attribute. */
12921 cost_vec)
12938 cost_vec)
12943 else
12944 {
12950 cost_vec)
12956 cost_vec)
12964 cost_vec)
12966 }
12973 "not vectorized:"
12977 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
12978 need extra handling, except for vectorizable reductions. */
12986 "not vectorized:"
12991 }
12994 /* Function vect_transform_stmt.
12996 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
12998 bool
13002 {
13014 {
13026 NULL);
13038 NULL);
13050 NULL);
13056 && !slp_node
13059 /* In case of interleaving, the whole chain is vectorized when the
13060 last store in the chain is reached. Store stmts before the last
13061 one are skipped, and there vec_stmt_info shouldn't be freed
13062 meanwhile. */
13063 ;
13064 else
13065 {
13070 }
13126 {
13131 }
13133 }
13139 {
13140 /* Handle stmts whose DEF is used outside the loop-nest that is
13141 being vectorized. */
13145 }
13151 }
13154 /* Remove a group of stores (for SLP or interleaving), free their
13155 stmt_vec_info. */
13157 void
13159 {
13163 {
13166 /* Free the attached stmt_vec_info and remove the stmt. */
13169 }
13170 }
13172 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13173 elements of type SCALAR_TYPE, or null if the target doesn't support
13174 such a type.
13176 If NUNITS is zero, return a vector type that contains elements of
13177 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13179 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13180 for this vectorization region and want to "autodetect" the best choice.
13181 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13182 and we want the new type to be interoperable with it. PREVAILING_MODE
13183 in this case can be a scalar integer mode or a vector mode; when it
13184 is a vector mode, the function acts like a tree-level version of
13185 related_vector_mode. */
13187 tree
13190 {
13192 scalar_mode inner_mode;
13193 machine_mode simd_mode;
13194 tree vectype;
13205 /* Interoperability between modes requires one to be a constant multiple
13206 of the other, so that the number of vectors required for each operation
13207 is a compile-time constant. */
13215 /* For vector types of elements whose mode precision doesn't
13216 match their types precision we use a element type of mode
13217 precision. The vectorization routines will have to make sure
13218 they support the proper result truncation/extension.
13219 We also make sure to build vector types with INTEGER_TYPE
13220 component type only. */
13227 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13228 When the component mode passes the above test simply use a type
13229 corresponding to that mode. The theory is that any use that
13230 would cause problems with this will disable vectorization anyway. */
13235 /* We can't build a vector type of elements with alignment bigger than
13236 their size. */
13241 /* If we felt back to using the mode fail if there was
13242 no scalar type for it. */
13246 /* If no prevailing mode was supplied, use the mode the target prefers.
13247 Otherwise lookup a vector mode based on the prevailing mode. */
13249 {
13253 {
13254 /* Traditional behavior is not to take the integer mode
13255 literally, but simply to use it as a way of determining
13256 the vector size. It is up to mode_for_vector to decide
13257 what the TYPE_MODE should be.
13259 Note that nunits == 1 is allowed in order to support single
13260 element vector types. */
13264 }
13265 }
13269 {
13270 /* Fall back to using mode_for_vector, mostly in the hope of being
13271 able to use an integer mode. */
13278 }
13282 /* In cases where the mode was chosen by mode_for_vector, check that
13283 the target actually supports the chosen mode, or that it at least
13284 allows the vector mode to be replaced by a like-sized integer. */
13289 /* Re-attach the address-space qualifier if we canonicalized the scalar
13290 type. */
13292 return build_qualified_type
13296 }
13298 /* Function get_vectype_for_scalar_type.
13300 Returns the vector type corresponding to SCALAR_TYPE as supported
13301 by the target. If GROUP_SIZE is nonzero and we're performing BB
13302 vectorization, make sure that the number of elements in the vector
13303 is no bigger than GROUP_SIZE. */
13305 tree
13308 {
13309 /* For BB vectorization, we should always have a group size once we've
13310 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13311 are tentative requests during things like early data reference
13312 analysis and pattern recognition. */
13315 else
13319 scalar_type);
13323 /* Register the natural choice of vector type, before the group size
13324 has been applied. */
13328 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13329 try again with an explicit number of elements. */
13331 && group_size
13333 {
13334 /* Start with the biggest number of units that fits within
13335 GROUP_SIZE and halve it until we find a valid vector type.
13336 Usually either the first attempt will succeed or all will
13337 fail (in the latter case because GROUP_SIZE is too small
13338 for the target), but it's possible that a target could have
13339 a hole between supported vector types.
13341 If GROUP_SIZE is not a power of 2, this has the effect of
13342 trying the largest power of 2 that fits within the group,
13343 even though the group is not a multiple of that vector size.
13344 The BB vectorizer will then try to carve up the group into
13345 smaller pieces. */
13347 do
13348 {
13352 }
13354 }
13357 }
13359 /* Return the vector type corresponding to SCALAR_TYPE as supported
13360 by the target. NODE, if nonnull, is the SLP tree node that will
13361 use the returned vector type. */
13363 tree
13365 {
13370 }
13372 /* Function get_mask_type_for_scalar_type.
13374 Returns the mask type corresponding to a result of comparison
13375 of vectors of specified SCALAR_TYPE as supported by target.
13376 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13377 make sure that the number of elements in the vector is no bigger
13378 than GROUP_SIZE. */
13380 tree
13383 {
13390 }
13392 /* Function get_mask_type_for_scalar_type.
13394 Returns the mask type corresponding to a result of comparison
13395 of vectors of specified SCALAR_TYPE as supported by target.
13396 NODE, if nonnull, is the SLP tree node that will use the returned
13397 vector type. */
13399 tree
13401 slp_tree node)
13402 {
13409 }
13411 /* Function get_same_sized_vectype
13413 Returns a vector type corresponding to SCALAR_TYPE of size
13414 VECTOR_TYPE if supported by the target. */
13416 tree
13418 {
13422 poly_uint64 nunits;
13429 }
13431 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13432 would not change the chosen vector modes. */
13434 bool
13436 {
13443 }
13445 /* Function vect_is_simple_use.
13447 Input:
13448 VINFO - the vect info of the loop or basic block that is being vectorized.
13449 OPERAND - operand in the loop or bb.
13450 Output:
13451 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13452 case OPERAND is an SSA_NAME that is defined in the vectorizable region
13453 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13454 the definition could be anywhere in the function
13455 DT - the type of definition
13457 Returns whether a stmt with OPERAND can be vectorized.
13458 For loops, supportable operands are constants, loop invariants, and operands
13459 that are defined by the current iteration of the loop. Unsupportable
13460 operands are those that are defined by a previous iteration of the loop (as
13461 is the case in reduction/induction computations).
13462 For basic blocks, supportable operands are constants and bb invariants.
13463 For now, operands defined outside the basic block are not supported. */
13465 bool
13468 {
13476 {
13482 else
13484 }
13494 else
13495 {
13500 else
13501 {
13507 }
13510 }
13513 {
13516 {
13547 }
13548 }
13551 {
13556 }
13559 }
13561 /* Function vect_is_simple_use.
13563 Same as vect_is_simple_use but also determines the vector operand
13564 type of OPERAND and stores it to *VECTYPE. If the definition of
13565 OPERAND is vect_uninitialized_def, vect_constant_def or
13566 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
13567 is responsible to compute the best suited vector type for the
13568 scalar operand. */
13570 bool
13574 {
13575 stmt_vec_info def_stmt_info;
13585 /* Now get a vector type if the def is internal, otherwise supply
13586 NULL_TREE and leave it up to the caller to figure out a proper
13587 type for the use stmt. */
13594 {
13600 }
13605 else
13609 }
13611 /* Function vect_is_simple_use.
13613 Same as vect_is_simple_use but determines the operand by operand
13614 position OPERAND from either STMT or SLP_NODE, filling in *OP
13615 and *SLP_DEF (when SLP_NODE is not NULL). */
13617 bool
13622 {
13624 {
13629 {
13632 }
13633 else
13634 {
13640 }
13641 }
13642 else
13643 {
13646 {
13649 {
13652 else
13654 }
13657 else
13659 }
13662 else
13665 }
13666 }
13668 /* If OP is not NULL and is external or constant update its vector
13669 type with VECTYPE. Returns true if successful or false if not,
13670 for example when conflicting vector types are present. */
13672 bool
13674 {
13679 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
13680 should be handled by patters. Allow vect_constant_def for now. */
13686 }
13688 /* Function supportable_widening_operation
13690 Check whether an operation represented by the code CODE is a
13691 widening operation that is supported by the target platform in
13692 vector form (i.e., when operating on arguments of type VECTYPE_IN
13693 producing a result of type VECTYPE_OUT).
13695 Widening operations we currently support are NOP (CONVERT), FLOAT,
13696 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
13697 are supported by the target platform either directly (via vector
13698 tree-codes), or via target builtins.
13700 Output:
13701 - CODE1 and CODE2 are codes of vector operations to be used when
13702 vectorizing the operation, if available.
13703 - MULTI_STEP_CVT determines the number of required intermediate steps in
13704 case of multi-step conversion (like char->short->int - in that case
13705 MULTI_STEP_CVT will be 1).
13706 - INTERM_TYPES contains the intermediate type required to perform the
13707 widening operation (short in the above example). */
13709 bool
13711 code_helper code,
13712 stmt_vec_info stmt_info,
13718 {
13721 machine_mode vec_mode;
13737 {
13739 /* Don't set c1 and c2 if code is not a tree_code. */
13743 /* The result of a vectorized widening operation usually requires
13744 two vectors (because the widened results do not fit into one vector).
13745 The generated vector results would normally be expected to be
13746 generated in the same order as in the original scalar computation,
13747 i.e. if 8 results are generated in each vector iteration, they are
13748 to be organized as follows:
13749 vect1: [res1,res2,res3,res4],
13750 vect2: [res5,res6,res7,res8].
13752 However, in the special case that the result of the widening
13753 operation is used in a reduction computation only, the order doesn't
13754 matter (because when vectorizing a reduction we change the order of
13755 the computation). Some targets can take advantage of this and
13756 generate more efficient code. For example, targets like Altivec,
13757 that support widen_mult using a sequence of {mult_even,mult_odd}
13758 generate the following vectors:
13759 vect1: [res1,res3,res5,res7],
13760 vect2: [res2,res4,res6,res8].
13762 When vectorizing outer-loops, we execute the inner-loop sequentially
13763 (each vectorized inner-loop iteration contributes to VF outer-loop
13764 iterations in parallel). We therefore don't allow to change the
13765 order of the computation in the inner-loop during outer-loop
13766 vectorization. */
13767 /* TODO: Another case in which order doesn't *really* matter is when we
13768 widen and then contract again, e.g. (short)((int)x * y >> 8).
13769 Normally, pack_trunc performs an even/odd permute, whereas the
13770 repack from an even/odd expansion would be an interleave, which
13771 would be significantly simpler for e.g. AVX2. */
13772 /* In any case, in order to avoid duplicating the code below, recurse
13773 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
13774 are properly set up for the caller. If we fail, we'll continue with
13775 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
13783 interm_types))
13784 {
13785 /* Elements in a vector with vect_used_by_reduction property cannot
13786 be reordered if the use chain with this property does not have the
13787 same operation. One such an example is s += a * b, where elements
13788 in a and b cannot be reordered. Here we check if the vector defined
13789 by STMT is only directly used in the reduction statement. */
13795 }
13811 /* Support the recursion induced just above. */
13821 CASE_CONVERT:
13838 }
13844 {
13845 /* The signedness is determined from output operand. */
13848 }
13854 {
13855 /* If the input and result modes are the same, a different optab
13856 is needed where we pass in the number of units in vectype. */
13859 }
13863 {
13864 /* If this is an internal fn then we must check whether the target
13865 supports either a low-high split or an even-odd split. */
13875 /* If we don't support low-high, then check for even-odd. */
13878 || !optab2
13880 {
13886 }
13887 }
13889 {
13891 {
13892 /* The signedness is determined from output operand. */
13895 }
13901 {
13902 /* If the input and result modes are the same, a different optab
13903 is needed where we pass in the number of units in vectype. */
13906 }
13907 else
13908 {
13911 }
13914 }
13926 {
13929 /* For scalar masks we may have different boolean
13930 vector types having the same QImode. Thus we
13931 add additional check for elements number. */
13935 }
13937 /* Check if it's a multi-step conversion that can be done using intermediate
13938 types. */
13946 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
13947 intermediate steps in promotion sequence. We try
13948 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
13949 not. */
13952 {
13955 intermediate_type
13958 {
13959 tree intermediate_element_type
13962 intermediate_type
13964 intermediate_mode);
13965 }
13966 else
13967 intermediate_type
13975 {
13976 /* If the input and result modes are the same, a different optab
13977 is needed where we pass in the number of units in vectype. */
13980 }
13981 else
13982 {
13985 }
14003 {
14009 }
14013 }
14017 }
14020 /* Function supportable_narrowing_operation
14022 Check whether an operation represented by the code CODE is a
14023 narrowing operation that is supported by the target platform in
14024 vector form (i.e., when operating on arguments of type VECTYPE_IN
14025 and producing a result of type VECTYPE_OUT).
14027 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14028 and FLOAT. This function checks if these operations are supported by
14029 the target platform directly via vector tree-codes.
14031 Output:
14032 - CODE1 is the code of a vector operation to be used when
14033 vectorizing the operation, if available.
14034 - MULTI_STEP_CVT determines the number of required intermediate steps in
14035 case of multi-step conversion (like int->short->char - in that case
14036 MULTI_STEP_CVT will be 1).
14037 - INTERM_TYPES contains the intermediate type required to perform the
14038 narrowing operation (short in the above example). */
14040 bool
14045 {
14046 machine_mode vec_mode;
14063 {
14064 CASE_CONVERT:
14072 else
14078 /* The signedness is determined from output operand. */
14089 }
14101 {
14104 /* For scalar masks we may have different boolean
14105 vector types having the same QImode. Thus we
14106 add additional check for elements number. */
14110 }
14115 /* Check if it's a multi-step conversion that can be done using intermediate
14116 types. */
14121 else
14124 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14125 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14126 costly than signed. */
14128 {
14131 intermediate_type
14133 interm_optab
14139 {
14143 }
14144 }
14146 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14147 intermediate steps in promotion sequence. We try
14148 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14151 {
14154 intermediate_type
14156 else
14157 intermediate_type
14165 else
14166 interm_optab
14168 optab_default);
14180 {
14186 }
14191 }
14195 }
14197 /* Generate and return a vector mask of MASK_TYPE such that
14198 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14199 Add the statements to SEQ. */
14201 tree
14204 {
14208 OPTIMIZE_FOR_SPEED));
14212 tree tmp;
14215 else
14220 }
14222 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14223 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14225 tree
14227 tree end_index)
14228 {
14231 }
14233 /* Try to compute the vector types required to vectorize STMT_INFO,
14234 returning true on success and false if vectorization isn't possible.
14235 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14236 take sure that the number of elements in the vectors is no bigger
14237 than GROUP_SIZE.
14239 On success:
14241 - Set *STMT_VECTYPE_OUT to:
14242 - NULL_TREE if the statement doesn't need to be vectorized;
14243 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14245 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14246 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14247 statement does not help to determine the overall number of units. */
14249 opt_result
14254 {
14257 /* For BB vectorization, we should always have a group size once we've
14258 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14259 are tentative requests during things like early data reference
14260 analysis and pattern recognition. */
14263 else
14270 /* MASK_STORE has no lhs, but is ok. */
14272 {
14274 {
14275 /* Ignore calls with no lhs. These must be calls to
14276 #pragma omp simd functions, and what vectorization factor
14277 it really needs can't be determined until
14278 vectorizable_simd_clone_call. */
14283 }
14287 }
14289 tree vectype;
14292 {
14297 }
14299 {
14308 }
14309 else
14310 {
14315 else
14319 {
14322 "get vectype for scalar type (group size %d):"
14324 else
14327 }
14331 "not vectorized:"
14333 scalar_type);
14337 }
14342 stmt);
14346 /* Don't try to compute scalar types if the stmt produces a boolean
14347 vector; use the existing vector type instead. */
14350 {
14351 /* The number of units is set according to the smallest scalar
14352 type (or the largest vector size, but we only support one
14353 vector size per vectorization). */
14357 {
14361 scalar_type);
14363 group_size);
14367 scalar_type);
14370 nunits_vectype);
14371 }
14372 }
14377 "Not vectorized: Incompatible number "
14382 {
14386 }
14390 }
14392 /* Generate and return statement sequence that sets vector length LEN that is:
14394 min_of_start_and_end = min (START_INDEX, END_INDEX);
14395 left_len = END_INDEX - min_of_start_and_end;
14396 rhs = min (left_len, LEN_LIMIT);
14397 LEN = rhs;
14399 Note: the cost of the code generated by this function is modeled
14400 by vect_estimate_min_profitable_iters, so changes here may need
14401 corresponding changes there. */
14403 gimple_seq
14405 {
14417 }