1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
21 //===----------------------------------------------------------------------===//
23 #include "LegalizeTypes.h"
24 #include "llvm/CodeGen/PseudoSourceValue.h"
25 #include "llvm/Target/TargetData.h"
26 #include "llvm/Support/ErrorHandling.h"
27 #include "llvm/Support/raw_ostream.h"
30 //===----------------------------------------------------------------------===//
31 // Result Vector Scalarization: <1 x ty> -> ty.
32 //===----------------------------------------------------------------------===//
34 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode
*N
, unsigned ResNo
) {
35 DEBUG(dbgs() << "Scalarize node result " << ResNo
<< ": ";
38 SDValue R
= SDValue();
40 switch (N
->getOpcode()) {
43 dbgs() << "ScalarizeVectorResult #" << ResNo
<< ": ";
47 llvm_unreachable("Do not know how to scalarize the result of this operator!");
49 case ISD::BIT_CONVERT
: R
= ScalarizeVecRes_BIT_CONVERT(N
); break;
50 case ISD::BUILD_VECTOR
: R
= N
->getOperand(0); break;
51 case ISD::CONVERT_RNDSAT
: R
= ScalarizeVecRes_CONVERT_RNDSAT(N
); break;
52 case ISD::EXTRACT_SUBVECTOR
: R
= ScalarizeVecRes_EXTRACT_SUBVECTOR(N
); break;
53 case ISD::FP_ROUND_INREG
: R
= ScalarizeVecRes_InregOp(N
); break;
54 case ISD::FPOWI
: R
= ScalarizeVecRes_FPOWI(N
); break;
55 case ISD::INSERT_VECTOR_ELT
: R
= ScalarizeVecRes_INSERT_VECTOR_ELT(N
); break;
56 case ISD::LOAD
: R
= ScalarizeVecRes_LOAD(cast
<LoadSDNode
>(N
));break;
57 case ISD::SCALAR_TO_VECTOR
: R
= ScalarizeVecRes_SCALAR_TO_VECTOR(N
); break;
58 case ISD::SIGN_EXTEND_INREG
: R
= ScalarizeVecRes_InregOp(N
); break;
59 case ISD::SELECT
: R
= ScalarizeVecRes_SELECT(N
); break;
60 case ISD::SELECT_CC
: R
= ScalarizeVecRes_SELECT_CC(N
); break;
61 case ISD::SETCC
: R
= ScalarizeVecRes_SETCC(N
); break;
62 case ISD::UNDEF
: R
= ScalarizeVecRes_UNDEF(N
); break;
63 case ISD::VECTOR_SHUFFLE
: R
= ScalarizeVecRes_VECTOR_SHUFFLE(N
); break;
64 case ISD::VSETCC
: R
= ScalarizeVecRes_VSETCC(N
); break;
84 case ISD::SIGN_EXTEND
:
85 case ISD::ZERO_EXTEND
:
87 R
= ScalarizeVecRes_UnaryOp(N
);
109 R
= ScalarizeVecRes_BinOp(N
);
113 // If R is null, the sub-method took care of registering the result.
115 SetScalarizedVector(SDValue(N
, ResNo
), R
);
118 SDValue
DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode
*N
) {
119 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
120 SDValue RHS
= GetScalarizedVector(N
->getOperand(1));
121 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(),
122 LHS
.getValueType(), LHS
, RHS
);
125 SDValue
DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode
*N
) {
126 EVT NewVT
= N
->getValueType(0).getVectorElementType();
127 return DAG
.getNode(ISD::BIT_CONVERT
, N
->getDebugLoc(),
128 NewVT
, N
->getOperand(0));
131 SDValue
DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode
*N
) {
132 EVT NewVT
= N
->getValueType(0).getVectorElementType();
133 SDValue Op0
= GetScalarizedVector(N
->getOperand(0));
134 return DAG
.getConvertRndSat(NewVT
, N
->getDebugLoc(),
135 Op0
, DAG
.getValueType(NewVT
),
136 DAG
.getValueType(Op0
.getValueType()),
139 cast
<CvtRndSatSDNode
>(N
)->getCvtCode());
142 SDValue
DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode
*N
) {
143 return DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, N
->getDebugLoc(),
144 N
->getValueType(0).getVectorElementType(),
145 N
->getOperand(0), N
->getOperand(1));
148 SDValue
DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode
*N
) {
149 SDValue Op
= GetScalarizedVector(N
->getOperand(0));
150 return DAG
.getNode(ISD::FPOWI
, N
->getDebugLoc(),
151 Op
.getValueType(), Op
, N
->getOperand(1));
154 SDValue
DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode
*N
) {
155 // The value to insert may have a wider type than the vector element type,
156 // so be sure to truncate it to the element type if necessary.
157 SDValue Op
= N
->getOperand(1);
158 EVT EltVT
= N
->getValueType(0).getVectorElementType();
159 if (Op
.getValueType() != EltVT
)
160 // FIXME: Can this happen for floating point types?
161 Op
= DAG
.getNode(ISD::TRUNCATE
, N
->getDebugLoc(), EltVT
, Op
);
165 SDValue
DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode
*N
) {
166 assert(N
->isUnindexed() && "Indexed vector load?");
168 SDValue Result
= DAG
.getLoad(ISD::UNINDEXED
,
169 N
->getExtensionType(),
170 N
->getValueType(0).getVectorElementType(),
172 N
->getChain(), N
->getBasePtr(),
173 DAG
.getUNDEF(N
->getBasePtr().getValueType()),
174 N
->getSrcValue(), N
->getSrcValueOffset(),
175 N
->getMemoryVT().getVectorElementType(),
176 N
->isVolatile(), N
->isNonTemporal(),
177 N
->getOriginalAlignment());
179 // Legalized the chain result - switch anything that used the old chain to
181 ReplaceValueWith(SDValue(N
, 1), Result
.getValue(1));
185 SDValue
DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode
*N
) {
186 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
187 EVT DestVT
= N
->getValueType(0).getVectorElementType();
188 SDValue Op
= GetScalarizedVector(N
->getOperand(0));
189 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), DestVT
, Op
);
192 SDValue
DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode
*N
) {
193 EVT EltVT
= N
->getValueType(0).getVectorElementType();
194 EVT ExtVT
= cast
<VTSDNode
>(N
->getOperand(1))->getVT().getVectorElementType();
195 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
196 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), EltVT
,
197 LHS
, DAG
.getValueType(ExtVT
));
200 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode
*N
) {
201 // If the operand is wider than the vector element type then it is implicitly
202 // truncated. Make that explicit here.
203 EVT EltVT
= N
->getValueType(0).getVectorElementType();
204 SDValue InOp
= N
->getOperand(0);
205 if (InOp
.getValueType() != EltVT
)
206 return DAG
.getNode(ISD::TRUNCATE
, N
->getDebugLoc(), EltVT
, InOp
);
210 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode
*N
) {
211 SDValue LHS
= GetScalarizedVector(N
->getOperand(1));
212 return DAG
.getNode(ISD::SELECT
, N
->getDebugLoc(),
213 LHS
.getValueType(), N
->getOperand(0), LHS
,
214 GetScalarizedVector(N
->getOperand(2)));
217 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode
*N
) {
218 SDValue LHS
= GetScalarizedVector(N
->getOperand(2));
219 return DAG
.getNode(ISD::SELECT_CC
, N
->getDebugLoc(), LHS
.getValueType(),
220 N
->getOperand(0), N
->getOperand(1),
221 LHS
, GetScalarizedVector(N
->getOperand(3)),
225 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode
*N
) {
226 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
227 SDValue RHS
= GetScalarizedVector(N
->getOperand(1));
228 DebugLoc DL
= N
->getDebugLoc();
230 // Turn it into a scalar SETCC.
231 return DAG
.getNode(ISD::SETCC
, DL
, MVT::i1
, LHS
, RHS
, N
->getOperand(2));
234 SDValue
DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode
*N
) {
235 return DAG
.getUNDEF(N
->getValueType(0).getVectorElementType());
238 SDValue
DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode
*N
) {
239 // Figure out if the scalar is the LHS or RHS and return it.
240 SDValue Arg
= N
->getOperand(2).getOperand(0);
241 if (Arg
.getOpcode() == ISD::UNDEF
)
242 return DAG
.getUNDEF(N
->getValueType(0).getVectorElementType());
243 unsigned Op
= !cast
<ConstantSDNode
>(Arg
)->isNullValue();
244 return GetScalarizedVector(N
->getOperand(Op
));
247 SDValue
DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode
*N
) {
248 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
249 SDValue RHS
= GetScalarizedVector(N
->getOperand(1));
250 EVT NVT
= N
->getValueType(0).getVectorElementType();
251 EVT SVT
= TLI
.getSetCCResultType(LHS
.getValueType());
252 DebugLoc DL
= N
->getDebugLoc();
254 // Turn it into a scalar SETCC.
255 SDValue Res
= DAG
.getNode(ISD::SETCC
, DL
, SVT
, LHS
, RHS
, N
->getOperand(2));
257 // VSETCC always returns a sign-extended value, while SETCC may not. The
258 // SETCC result type may not match the vector element type. Correct these.
259 if (NVT
.bitsLE(SVT
)) {
260 // The SETCC result type is bigger than the vector element type.
261 // Ensure the SETCC result is sign-extended.
262 if (TLI
.getBooleanContents() !=
263 TargetLowering::ZeroOrNegativeOneBooleanContent
)
264 Res
= DAG
.getNode(ISD::SIGN_EXTEND_INREG
, DL
, SVT
, Res
,
265 DAG
.getValueType(MVT::i1
));
266 // Truncate to the final type.
267 return DAG
.getNode(ISD::TRUNCATE
, DL
, NVT
, Res
);
270 // The SETCC result type is smaller than the vector element type.
271 // If the SetCC result is not sign-extended, chop it down to MVT::i1.
272 if (TLI
.getBooleanContents() !=
273 TargetLowering::ZeroOrNegativeOneBooleanContent
)
274 Res
= DAG
.getNode(ISD::TRUNCATE
, DL
, MVT::i1
, Res
);
275 // Sign extend to the final type.
276 return DAG
.getNode(ISD::SIGN_EXTEND
, DL
, NVT
, Res
);
280 //===----------------------------------------------------------------------===//
281 // Operand Vector Scalarization <1 x ty> -> ty.
282 //===----------------------------------------------------------------------===//
284 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode
*N
, unsigned OpNo
) {
285 DEBUG(dbgs() << "Scalarize node operand " << OpNo
<< ": ";
288 SDValue Res
= SDValue();
290 if (Res
.getNode() == 0) {
291 switch (N
->getOpcode()) {
294 dbgs() << "ScalarizeVectorOperand Op #" << OpNo
<< ": ";
298 llvm_unreachable("Do not know how to scalarize this operator's operand!");
299 case ISD::BIT_CONVERT
:
300 Res
= ScalarizeVecOp_BIT_CONVERT(N
);
302 case ISD::CONCAT_VECTORS
:
303 Res
= ScalarizeVecOp_CONCAT_VECTORS(N
);
305 case ISD::EXTRACT_VECTOR_ELT
:
306 Res
= ScalarizeVecOp_EXTRACT_VECTOR_ELT(N
);
309 Res
= ScalarizeVecOp_STORE(cast
<StoreSDNode
>(N
), OpNo
);
314 // If the result is null, the sub-method took care of registering results etc.
315 if (!Res
.getNode()) return false;
317 // If the result is N, the sub-method updated N in place. Tell the legalizer
319 if (Res
.getNode() == N
)
322 assert(Res
.getValueType() == N
->getValueType(0) && N
->getNumValues() == 1 &&
323 "Invalid operand expansion");
325 ReplaceValueWith(SDValue(N
, 0), Res
);
329 /// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs
330 /// to be scalarized, it must be <1 x ty>. Convert the element instead.
331 SDValue
DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode
*N
) {
332 SDValue Elt
= GetScalarizedVector(N
->getOperand(0));
333 return DAG
.getNode(ISD::BIT_CONVERT
, N
->getDebugLoc(),
334 N
->getValueType(0), Elt
);
337 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
338 /// use a BUILD_VECTOR instead.
339 SDValue
DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode
*N
) {
340 SmallVector
<SDValue
, 8> Ops(N
->getNumOperands());
341 for (unsigned i
= 0, e
= N
->getNumOperands(); i
< e
; ++i
)
342 Ops
[i
] = GetScalarizedVector(N
->getOperand(i
));
343 return DAG
.getNode(ISD::BUILD_VECTOR
, N
->getDebugLoc(), N
->getValueType(0),
344 &Ops
[0], Ops
.size());
347 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
348 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
350 SDValue
DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode
*N
) {
351 SDValue Res
= GetScalarizedVector(N
->getOperand(0));
352 if (Res
.getValueType() != N
->getValueType(0))
353 Res
= DAG
.getNode(ISD::ANY_EXTEND
, N
->getDebugLoc(), N
->getValueType(0),
358 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
359 /// scalarized, it must be <1 x ty>. Just store the element.
360 SDValue
DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode
*N
, unsigned OpNo
){
361 assert(N
->isUnindexed() && "Indexed store of one-element vector?");
362 assert(OpNo
== 1 && "Do not know how to scalarize this operand!");
363 DebugLoc dl
= N
->getDebugLoc();
365 if (N
->isTruncatingStore())
366 return DAG
.getTruncStore(N
->getChain(), dl
,
367 GetScalarizedVector(N
->getOperand(1)),
369 N
->getSrcValue(), N
->getSrcValueOffset(),
370 N
->getMemoryVT().getVectorElementType(),
371 N
->isVolatile(), N
->isNonTemporal(),
374 return DAG
.getStore(N
->getChain(), dl
, GetScalarizedVector(N
->getOperand(1)),
375 N
->getBasePtr(), N
->getSrcValue(), N
->getSrcValueOffset(),
376 N
->isVolatile(), N
->isNonTemporal(),
377 N
->getOriginalAlignment());
381 //===----------------------------------------------------------------------===//
382 // Result Vector Splitting
383 //===----------------------------------------------------------------------===//
385 /// SplitVectorResult - This method is called when the specified result of the
386 /// specified node is found to need vector splitting. At this point, the node
387 /// may also have invalid operands or may have other results that need
388 /// legalization, we just know that (at least) one result needs vector
390 void DAGTypeLegalizer::SplitVectorResult(SDNode
*N
, unsigned ResNo
) {
391 DEBUG(dbgs() << "Split node result: ";
396 switch (N
->getOpcode()) {
399 dbgs() << "SplitVectorResult #" << ResNo
<< ": ";
403 llvm_unreachable("Do not know how to split the result of this operator!");
405 case ISD::MERGE_VALUES
: SplitRes_MERGE_VALUES(N
, Lo
, Hi
); break;
406 case ISD::SELECT
: SplitRes_SELECT(N
, Lo
, Hi
); break;
407 case ISD::SELECT_CC
: SplitRes_SELECT_CC(N
, Lo
, Hi
); break;
408 case ISD::UNDEF
: SplitRes_UNDEF(N
, Lo
, Hi
); break;
410 case ISD::BIT_CONVERT
: SplitVecRes_BIT_CONVERT(N
, Lo
, Hi
); break;
411 case ISD::BUILD_VECTOR
: SplitVecRes_BUILD_VECTOR(N
, Lo
, Hi
); break;
412 case ISD::CONCAT_VECTORS
: SplitVecRes_CONCAT_VECTORS(N
, Lo
, Hi
); break;
413 case ISD::CONVERT_RNDSAT
: SplitVecRes_CONVERT_RNDSAT(N
, Lo
, Hi
); break;
414 case ISD::EXTRACT_SUBVECTOR
: SplitVecRes_EXTRACT_SUBVECTOR(N
, Lo
, Hi
); break;
415 case ISD::FP_ROUND_INREG
: SplitVecRes_InregOp(N
, Lo
, Hi
); break;
416 case ISD::FPOWI
: SplitVecRes_FPOWI(N
, Lo
, Hi
); break;
417 case ISD::INSERT_VECTOR_ELT
: SplitVecRes_INSERT_VECTOR_ELT(N
, Lo
, Hi
); break;
418 case ISD::SCALAR_TO_VECTOR
: SplitVecRes_SCALAR_TO_VECTOR(N
, Lo
, Hi
); break;
419 case ISD::SIGN_EXTEND_INREG
: SplitVecRes_InregOp(N
, Lo
, Hi
); break;
421 SplitVecRes_LOAD(cast
<LoadSDNode
>(N
), Lo
, Hi
);
425 SplitVecRes_SETCC(N
, Lo
, Hi
);
427 case ISD::VECTOR_SHUFFLE
:
428 SplitVecRes_VECTOR_SHUFFLE(cast
<ShuffleVectorSDNode
>(N
), Lo
, Hi
);
443 case ISD::FNEARBYINT
:
444 case ISD::FP_TO_SINT
:
445 case ISD::FP_TO_UINT
:
446 case ISD::SINT_TO_FP
:
447 case ISD::UINT_TO_FP
:
449 case ISD::SIGN_EXTEND
:
450 case ISD::ZERO_EXTEND
:
451 case ISD::ANY_EXTEND
:
457 SplitVecRes_UnaryOp(N
, Lo
, Hi
);
479 SplitVecRes_BinOp(N
, Lo
, Hi
);
483 // If Lo/Hi is null, the sub-method took care of registering results etc.
485 SetSplitVector(SDValue(N
, ResNo
), Lo
, Hi
);
488 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode
*N
, SDValue
&Lo
,
490 SDValue LHSLo
, LHSHi
;
491 GetSplitVector(N
->getOperand(0), LHSLo
, LHSHi
);
492 SDValue RHSLo
, RHSHi
;
493 GetSplitVector(N
->getOperand(1), RHSLo
, RHSHi
);
494 DebugLoc dl
= N
->getDebugLoc();
496 Lo
= DAG
.getNode(N
->getOpcode(), dl
, LHSLo
.getValueType(), LHSLo
, RHSLo
);
497 Hi
= DAG
.getNode(N
->getOpcode(), dl
, LHSHi
.getValueType(), LHSHi
, RHSHi
);
500 void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode
*N
, SDValue
&Lo
,
502 // We know the result is a vector. The input may be either a vector or a
505 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
506 DebugLoc dl
= N
->getDebugLoc();
508 SDValue InOp
= N
->getOperand(0);
509 EVT InVT
= InOp
.getValueType();
511 // Handle some special cases efficiently.
512 switch (getTypeAction(InVT
)) {
514 assert(false && "Unknown type action!");
518 case ScalarizeVector
:
522 // A scalar to vector conversion, where the scalar needs expansion.
523 // If the vector is being split in two then we can just convert the
526 GetExpandedOp(InOp
, Lo
, Hi
);
527 if (TLI
.isBigEndian())
529 Lo
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, LoVT
, Lo
);
530 Hi
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, HiVT
, Hi
);
535 // If the input is a vector that needs to be split, convert each split
536 // piece of the input now.
537 GetSplitVector(InOp
, Lo
, Hi
);
538 Lo
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, LoVT
, Lo
);
539 Hi
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, HiVT
, Hi
);
543 // In the general case, convert the input to an integer and split it by hand.
544 EVT LoIntVT
= EVT::getIntegerVT(*DAG
.getContext(), LoVT
.getSizeInBits());
545 EVT HiIntVT
= EVT::getIntegerVT(*DAG
.getContext(), HiVT
.getSizeInBits());
546 if (TLI
.isBigEndian())
547 std::swap(LoIntVT
, HiIntVT
);
549 SplitInteger(BitConvertToInteger(InOp
), LoIntVT
, HiIntVT
, Lo
, Hi
);
551 if (TLI
.isBigEndian())
553 Lo
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, LoVT
, Lo
);
554 Hi
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, HiVT
, Hi
);
557 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode
*N
, SDValue
&Lo
,
560 DebugLoc dl
= N
->getDebugLoc();
561 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
562 unsigned LoNumElts
= LoVT
.getVectorNumElements();
563 SmallVector
<SDValue
, 8> LoOps(N
->op_begin(), N
->op_begin()+LoNumElts
);
564 Lo
= DAG
.getNode(ISD::BUILD_VECTOR
, dl
, LoVT
, &LoOps
[0], LoOps
.size());
566 SmallVector
<SDValue
, 8> HiOps(N
->op_begin()+LoNumElts
, N
->op_end());
567 Hi
= DAG
.getNode(ISD::BUILD_VECTOR
, dl
, HiVT
, &HiOps
[0], HiOps
.size());
570 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode
*N
, SDValue
&Lo
,
572 assert(!(N
->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
573 DebugLoc dl
= N
->getDebugLoc();
574 unsigned NumSubvectors
= N
->getNumOperands() / 2;
575 if (NumSubvectors
== 1) {
576 Lo
= N
->getOperand(0);
577 Hi
= N
->getOperand(1);
582 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
584 SmallVector
<SDValue
, 8> LoOps(N
->op_begin(), N
->op_begin()+NumSubvectors
);
585 Lo
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, LoVT
, &LoOps
[0], LoOps
.size());
587 SmallVector
<SDValue
, 8> HiOps(N
->op_begin()+NumSubvectors
, N
->op_end());
588 Hi
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, HiVT
, &HiOps
[0], HiOps
.size());
591 void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode
*N
, SDValue
&Lo
,
594 DebugLoc dl
= N
->getDebugLoc();
595 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
597 SDValue DTyOpLo
= DAG
.getValueType(LoVT
);
598 SDValue DTyOpHi
= DAG
.getValueType(HiVT
);
600 SDValue RndOp
= N
->getOperand(3);
601 SDValue SatOp
= N
->getOperand(4);
602 ISD::CvtCode CvtCode
= cast
<CvtRndSatSDNode
>(N
)->getCvtCode();
606 EVT InVT
= N
->getOperand(0).getValueType();
607 switch (getTypeAction(InVT
)) {
608 default: llvm_unreachable("Unexpected type action!");
610 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
611 LoVT
.getVectorNumElements());
612 VLo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
613 DAG
.getIntPtrConstant(0));
614 VHi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
615 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
619 GetSplitVector(N
->getOperand(0), VLo
, VHi
);
622 // If the result needs to be split and the input needs to be widened,
623 // the two types must have different lengths. Use the widened result
624 // and extract from it to do the split.
625 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
626 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
627 LoVT
.getVectorNumElements());
628 VLo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
629 DAG
.getIntPtrConstant(0));
630 VHi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
631 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
636 SDValue STyOpLo
= DAG
.getValueType(VLo
.getValueType());
637 SDValue STyOpHi
= DAG
.getValueType(VHi
.getValueType());
639 Lo
= DAG
.getConvertRndSat(LoVT
, dl
, VLo
, DTyOpLo
, STyOpLo
, RndOp
, SatOp
,
641 Hi
= DAG
.getConvertRndSat(HiVT
, dl
, VHi
, DTyOpHi
, STyOpHi
, RndOp
, SatOp
,
645 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode
*N
, SDValue
&Lo
,
647 SDValue Vec
= N
->getOperand(0);
648 SDValue Idx
= N
->getOperand(1);
649 EVT IdxVT
= Idx
.getValueType();
650 DebugLoc dl
= N
->getDebugLoc();
653 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
655 Lo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, LoVT
, Vec
, Idx
);
656 Idx
= DAG
.getNode(ISD::ADD
, dl
, IdxVT
, Idx
,
657 DAG
.getConstant(LoVT
.getVectorNumElements(), IdxVT
));
658 Hi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, HiVT
, Vec
, Idx
);
661 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode
*N
, SDValue
&Lo
,
663 DebugLoc dl
= N
->getDebugLoc();
664 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
665 Lo
= DAG
.getNode(ISD::FPOWI
, dl
, Lo
.getValueType(), Lo
, N
->getOperand(1));
666 Hi
= DAG
.getNode(ISD::FPOWI
, dl
, Hi
.getValueType(), Hi
, N
->getOperand(1));
669 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode
*N
, SDValue
&Lo
,
671 SDValue LHSLo
, LHSHi
;
672 GetSplitVector(N
->getOperand(0), LHSLo
, LHSHi
);
673 DebugLoc dl
= N
->getDebugLoc();
676 GetSplitDestVTs(cast
<VTSDNode
>(N
->getOperand(1))->getVT(), LoVT
, HiVT
);
678 Lo
= DAG
.getNode(N
->getOpcode(), dl
, LHSLo
.getValueType(), LHSLo
,
679 DAG
.getValueType(LoVT
));
680 Hi
= DAG
.getNode(N
->getOpcode(), dl
, LHSHi
.getValueType(), LHSHi
,
681 DAG
.getValueType(HiVT
));
684 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode
*N
, SDValue
&Lo
,
686 SDValue Vec
= N
->getOperand(0);
687 SDValue Elt
= N
->getOperand(1);
688 SDValue Idx
= N
->getOperand(2);
689 DebugLoc dl
= N
->getDebugLoc();
690 GetSplitVector(Vec
, Lo
, Hi
);
692 if (ConstantSDNode
*CIdx
= dyn_cast
<ConstantSDNode
>(Idx
)) {
693 unsigned IdxVal
= CIdx
->getZExtValue();
694 unsigned LoNumElts
= Lo
.getValueType().getVectorNumElements();
695 if (IdxVal
< LoNumElts
)
696 Lo
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
,
697 Lo
.getValueType(), Lo
, Elt
, Idx
);
699 Hi
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
, Hi
.getValueType(), Hi
, Elt
,
700 DAG
.getIntPtrConstant(IdxVal
- LoNumElts
));
704 // Spill the vector to the stack.
705 EVT VecVT
= Vec
.getValueType();
706 EVT EltVT
= VecVT
.getVectorElementType();
707 SDValue StackPtr
= DAG
.CreateStackTemporary(VecVT
);
708 SDValue Store
= DAG
.getStore(DAG
.getEntryNode(), dl
, Vec
, StackPtr
, NULL
, 0,
711 // Store the new element. This may be larger than the vector element type,
712 // so use a truncating store.
713 SDValue EltPtr
= GetVectorElementPointer(StackPtr
, EltVT
, Idx
);
714 const Type
*VecType
= VecVT
.getTypeForEVT(*DAG
.getContext());
716 TLI
.getTargetData()->getPrefTypeAlignment(VecType
);
717 Store
= DAG
.getTruncStore(Store
, dl
, Elt
, EltPtr
, NULL
, 0, EltVT
,
720 // Load the Lo part from the stack slot.
721 Lo
= DAG
.getLoad(Lo
.getValueType(), dl
, Store
, StackPtr
, NULL
, 0,
724 // Increment the pointer to the other part.
725 unsigned IncrementSize
= Lo
.getValueType().getSizeInBits() / 8;
726 StackPtr
= DAG
.getNode(ISD::ADD
, dl
, StackPtr
.getValueType(), StackPtr
,
727 DAG
.getIntPtrConstant(IncrementSize
));
729 // Load the Hi part from the stack slot.
730 Hi
= DAG
.getLoad(Hi
.getValueType(), dl
, Store
, StackPtr
, NULL
, 0, false,
731 false, MinAlign(Alignment
, IncrementSize
));
734 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode
*N
, SDValue
&Lo
,
737 DebugLoc dl
= N
->getDebugLoc();
738 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
739 Lo
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, LoVT
, N
->getOperand(0));
740 Hi
= DAG
.getUNDEF(HiVT
);
743 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode
*LD
, SDValue
&Lo
,
745 assert(ISD::isUNINDEXEDLoad(LD
) && "Indexed load during type legalization!");
747 DebugLoc dl
= LD
->getDebugLoc();
748 GetSplitDestVTs(LD
->getValueType(0), LoVT
, HiVT
);
750 ISD::LoadExtType ExtType
= LD
->getExtensionType();
751 SDValue Ch
= LD
->getChain();
752 SDValue Ptr
= LD
->getBasePtr();
753 SDValue Offset
= DAG
.getUNDEF(Ptr
.getValueType());
754 const Value
*SV
= LD
->getSrcValue();
755 int SVOffset
= LD
->getSrcValueOffset();
756 EVT MemoryVT
= LD
->getMemoryVT();
757 unsigned Alignment
= LD
->getOriginalAlignment();
758 bool isVolatile
= LD
->isVolatile();
759 bool isNonTemporal
= LD
->isNonTemporal();
761 EVT LoMemVT
, HiMemVT
;
762 GetSplitDestVTs(MemoryVT
, LoMemVT
, HiMemVT
);
764 Lo
= DAG
.getLoad(ISD::UNINDEXED
, ExtType
, LoVT
, dl
, Ch
, Ptr
, Offset
,
765 SV
, SVOffset
, LoMemVT
, isVolatile
, isNonTemporal
, Alignment
);
767 unsigned IncrementSize
= LoMemVT
.getSizeInBits()/8;
768 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
769 DAG
.getIntPtrConstant(IncrementSize
));
770 SVOffset
+= IncrementSize
;
771 Hi
= DAG
.getLoad(ISD::UNINDEXED
, ExtType
, HiVT
, dl
, Ch
, Ptr
, Offset
,
772 SV
, SVOffset
, HiMemVT
, isVolatile
, isNonTemporal
, Alignment
);
774 // Build a factor node to remember that this load is independent of the
776 Ch
= DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
.getValue(1),
779 // Legalized the chain result - switch anything that used the old chain to
781 ReplaceValueWith(SDValue(LD
, 1), Ch
);
784 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode
*N
, SDValue
&Lo
, SDValue
&Hi
) {
786 DebugLoc DL
= N
->getDebugLoc();
787 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
790 EVT InVT
= N
->getOperand(0).getValueType();
791 SDValue LL
, LH
, RL
, RH
;
792 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
793 LoVT
.getVectorNumElements());
794 LL
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(0),
795 DAG
.getIntPtrConstant(0));
796 LH
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(0),
797 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
799 RL
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(1),
800 DAG
.getIntPtrConstant(0));
801 RH
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(1),
802 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
804 Lo
= DAG
.getNode(N
->getOpcode(), DL
, LoVT
, LL
, RL
, N
->getOperand(2));
805 Hi
= DAG
.getNode(N
->getOpcode(), DL
, HiVT
, LH
, RH
, N
->getOperand(2));
808 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode
*N
, SDValue
&Lo
,
810 // Get the dest types - they may not match the input types, e.g. int_to_fp.
812 DebugLoc dl
= N
->getDebugLoc();
813 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
816 EVT InVT
= N
->getOperand(0).getValueType();
817 switch (getTypeAction(InVT
)) {
818 default: llvm_unreachable("Unexpected type action!");
820 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
821 LoVT
.getVectorNumElements());
822 Lo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
823 DAG
.getIntPtrConstant(0));
824 Hi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
825 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
829 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
832 // If the result needs to be split and the input needs to be widened,
833 // the two types must have different lengths. Use the widened result
834 // and extract from it to do the split.
835 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
836 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
837 LoVT
.getVectorNumElements());
838 Lo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
839 DAG
.getIntPtrConstant(0));
840 Hi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
841 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
846 Lo
= DAG
.getNode(N
->getOpcode(), dl
, LoVT
, Lo
);
847 Hi
= DAG
.getNode(N
->getOpcode(), dl
, HiVT
, Hi
);
850 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode
*N
,
851 SDValue
&Lo
, SDValue
&Hi
) {
852 // The low and high parts of the original input give four input vectors.
854 DebugLoc dl
= N
->getDebugLoc();
855 GetSplitVector(N
->getOperand(0), Inputs
[0], Inputs
[1]);
856 GetSplitVector(N
->getOperand(1), Inputs
[2], Inputs
[3]);
857 EVT NewVT
= Inputs
[0].getValueType();
858 unsigned NewElts
= NewVT
.getVectorNumElements();
860 // If Lo or Hi uses elements from at most two of the four input vectors, then
861 // express it as a vector shuffle of those two inputs. Otherwise extract the
862 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
863 SmallVector
<int, 16> Ops
;
864 for (unsigned High
= 0; High
< 2; ++High
) {
865 SDValue
&Output
= High
? Hi
: Lo
;
867 // Build a shuffle mask for the output, discovering on the fly which
868 // input vectors to use as shuffle operands (recorded in InputUsed).
869 // If building a suitable shuffle vector proves too hard, then bail
870 // out with useBuildVector set.
871 unsigned InputUsed
[2] = { -1U, -1U }; // Not yet discovered.
872 unsigned FirstMaskIdx
= High
* NewElts
;
873 bool useBuildVector
= false;
874 for (unsigned MaskOffset
= 0; MaskOffset
< NewElts
; ++MaskOffset
) {
875 // The mask element. This indexes into the input.
876 int Idx
= N
->getMaskElt(FirstMaskIdx
+ MaskOffset
);
878 // The input vector this mask element indexes into.
879 unsigned Input
= (unsigned)Idx
/ NewElts
;
881 if (Input
>= array_lengthof(Inputs
)) {
882 // The mask element does not index into any input vector.
887 // Turn the index into an offset from the start of the input vector.
888 Idx
-= Input
* NewElts
;
890 // Find or create a shuffle vector operand to hold this input.
892 for (OpNo
= 0; OpNo
< array_lengthof(InputUsed
); ++OpNo
) {
893 if (InputUsed
[OpNo
] == Input
) {
894 // This input vector is already an operand.
896 } else if (InputUsed
[OpNo
] == -1U) {
897 // Create a new operand for this input vector.
898 InputUsed
[OpNo
] = Input
;
903 if (OpNo
>= array_lengthof(InputUsed
)) {
904 // More than two input vectors used! Give up on trying to create a
905 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
906 useBuildVector
= true;
910 // Add the mask index for the new shuffle vector.
911 Ops
.push_back(Idx
+ OpNo
* NewElts
);
914 if (useBuildVector
) {
915 EVT EltVT
= NewVT
.getVectorElementType();
916 SmallVector
<SDValue
, 16> SVOps
;
918 // Extract the input elements by hand.
919 for (unsigned MaskOffset
= 0; MaskOffset
< NewElts
; ++MaskOffset
) {
920 // The mask element. This indexes into the input.
921 int Idx
= N
->getMaskElt(FirstMaskIdx
+ MaskOffset
);
923 // The input vector this mask element indexes into.
924 unsigned Input
= (unsigned)Idx
/ NewElts
;
926 if (Input
>= array_lengthof(Inputs
)) {
927 // The mask element is "undef" or indexes off the end of the input.
928 SVOps
.push_back(DAG
.getUNDEF(EltVT
));
932 // Turn the index into an offset from the start of the input vector.
933 Idx
-= Input
* NewElts
;
935 // Extract the vector element by hand.
936 SVOps
.push_back(DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
,
937 Inputs
[Input
], DAG
.getIntPtrConstant(Idx
)));
940 // Construct the Lo/Hi output using a BUILD_VECTOR.
941 Output
= DAG
.getNode(ISD::BUILD_VECTOR
,dl
,NewVT
, &SVOps
[0], SVOps
.size());
942 } else if (InputUsed
[0] == -1U) {
943 // No input vectors were used! The result is undefined.
944 Output
= DAG
.getUNDEF(NewVT
);
946 SDValue Op0
= Inputs
[InputUsed
[0]];
947 // If only one input was used, use an undefined vector for the other.
948 SDValue Op1
= InputUsed
[1] == -1U ?
949 DAG
.getUNDEF(NewVT
) : Inputs
[InputUsed
[1]];
950 // At least one input vector was used. Create a new shuffle vector.
951 Output
= DAG
.getVectorShuffle(NewVT
, dl
, Op0
, Op1
, &Ops
[0]);
959 //===----------------------------------------------------------------------===//
960 // Operand Vector Splitting
961 //===----------------------------------------------------------------------===//
963 /// SplitVectorOperand - This method is called when the specified operand of the
964 /// specified node is found to need vector splitting. At this point, all of the
965 /// result types of the node are known to be legal, but other operands of the
966 /// node may need legalization as well as the specified one.
967 bool DAGTypeLegalizer::SplitVectorOperand(SDNode
*N
, unsigned OpNo
) {
968 DEBUG(dbgs() << "Split node operand: ";
971 SDValue Res
= SDValue();
973 if (Res
.getNode() == 0) {
974 switch (N
->getOpcode()) {
977 dbgs() << "SplitVectorOperand Op #" << OpNo
<< ": ";
981 llvm_unreachable("Do not know how to split this operator's operand!");
983 case ISD::BIT_CONVERT
: Res
= SplitVecOp_BIT_CONVERT(N
); break;
984 case ISD::EXTRACT_SUBVECTOR
: Res
= SplitVecOp_EXTRACT_SUBVECTOR(N
); break;
985 case ISD::EXTRACT_VECTOR_ELT
:Res
= SplitVecOp_EXTRACT_VECTOR_ELT(N
); break;
987 Res
= SplitVecOp_STORE(cast
<StoreSDNode
>(N
), OpNo
);
993 case ISD::FP_TO_SINT
:
994 case ISD::FP_TO_UINT
:
995 case ISD::SINT_TO_FP
:
996 case ISD::UINT_TO_FP
:
998 case ISD::SIGN_EXTEND
:
999 case ISD::ZERO_EXTEND
:
1000 case ISD::ANY_EXTEND
:
1001 Res
= SplitVecOp_UnaryOp(N
);
1006 // If the result is null, the sub-method took care of registering results etc.
1007 if (!Res
.getNode()) return false;
1009 // If the result is N, the sub-method updated N in place. Tell the legalizer
1011 if (Res
.getNode() == N
)
1014 assert(Res
.getValueType() == N
->getValueType(0) && N
->getNumValues() == 1 &&
1015 "Invalid operand expansion");
1017 ReplaceValueWith(SDValue(N
, 0), Res
);
1021 SDValue
DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode
*N
) {
1022 // The result has a legal vector type, but the input needs splitting.
1023 EVT ResVT
= N
->getValueType(0);
1025 DebugLoc dl
= N
->getDebugLoc();
1026 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
1027 EVT InVT
= Lo
.getValueType();
1029 EVT OutVT
= EVT::getVectorVT(*DAG
.getContext(), ResVT
.getVectorElementType(),
1030 InVT
.getVectorNumElements());
1032 Lo
= DAG
.getNode(N
->getOpcode(), dl
, OutVT
, Lo
);
1033 Hi
= DAG
.getNode(N
->getOpcode(), dl
, OutVT
, Hi
);
1035 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, ResVT
, Lo
, Hi
);
1038 SDValue
DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode
*N
) {
1039 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will
1040 // end up being split all the way down to individual components. Convert the
1041 // split pieces into integers and reassemble.
1043 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
1044 Lo
= BitConvertToInteger(Lo
);
1045 Hi
= BitConvertToInteger(Hi
);
1047 if (TLI
.isBigEndian())
1050 return DAG
.getNode(ISD::BIT_CONVERT
, N
->getDebugLoc(), N
->getValueType(0),
1051 JoinIntegers(Lo
, Hi
));
1054 SDValue
DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode
*N
) {
1055 // We know that the extracted result type is legal. For now, assume the index
1057 EVT SubVT
= N
->getValueType(0);
1058 SDValue Idx
= N
->getOperand(1);
1059 DebugLoc dl
= N
->getDebugLoc();
1061 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
1063 uint64_t LoElts
= Lo
.getValueType().getVectorNumElements();
1064 uint64_t IdxVal
= cast
<ConstantSDNode
>(Idx
)->getZExtValue();
1066 if (IdxVal
< LoElts
) {
1067 assert(IdxVal
+ SubVT
.getVectorNumElements() <= LoElts
&&
1068 "Extracted subvector crosses vector split!");
1069 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, SubVT
, Lo
, Idx
);
1071 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, SubVT
, Hi
,
1072 DAG
.getConstant(IdxVal
- LoElts
, Idx
.getValueType()));
1076 SDValue
DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode
*N
) {
1077 SDValue Vec
= N
->getOperand(0);
1078 SDValue Idx
= N
->getOperand(1);
1079 EVT VecVT
= Vec
.getValueType();
1081 if (isa
<ConstantSDNode
>(Idx
)) {
1082 uint64_t IdxVal
= cast
<ConstantSDNode
>(Idx
)->getZExtValue();
1083 assert(IdxVal
< VecVT
.getVectorNumElements() && "Invalid vector index!");
1086 GetSplitVector(Vec
, Lo
, Hi
);
1088 uint64_t LoElts
= Lo
.getValueType().getVectorNumElements();
1090 if (IdxVal
< LoElts
)
1091 return SDValue(DAG
.UpdateNodeOperands(N
, Lo
, Idx
), 0);
1092 return SDValue(DAG
.UpdateNodeOperands(N
, Hi
,
1093 DAG
.getConstant(IdxVal
- LoElts
,
1094 Idx
.getValueType())),
1098 // Store the vector to the stack.
1099 EVT EltVT
= VecVT
.getVectorElementType();
1100 DebugLoc dl
= N
->getDebugLoc();
1101 SDValue StackPtr
= DAG
.CreateStackTemporary(VecVT
);
1102 int SPFI
= cast
<FrameIndexSDNode
>(StackPtr
.getNode())->getIndex();
1103 const Value
*SV
= PseudoSourceValue::getFixedStack(SPFI
);
1104 SDValue Store
= DAG
.getStore(DAG
.getEntryNode(), dl
, Vec
, StackPtr
, SV
, 0,
1107 // Load back the required element.
1108 StackPtr
= GetVectorElementPointer(StackPtr
, EltVT
, Idx
);
1109 return DAG
.getExtLoad(ISD::EXTLOAD
, N
->getValueType(0), dl
, Store
, StackPtr
,
1110 SV
, 0, EltVT
, false, false, 0);
1113 SDValue
DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode
*N
, unsigned OpNo
) {
1114 assert(N
->isUnindexed() && "Indexed store of vector?");
1115 assert(OpNo
== 1 && "Can only split the stored value");
1116 DebugLoc dl
= N
->getDebugLoc();
1118 bool isTruncating
= N
->isTruncatingStore();
1119 SDValue Ch
= N
->getChain();
1120 SDValue Ptr
= N
->getBasePtr();
1121 int SVOffset
= N
->getSrcValueOffset();
1122 EVT MemoryVT
= N
->getMemoryVT();
1123 unsigned Alignment
= N
->getOriginalAlignment();
1124 bool isVol
= N
->isVolatile();
1125 bool isNT
= N
->isNonTemporal();
1127 GetSplitVector(N
->getOperand(1), Lo
, Hi
);
1129 EVT LoMemVT
, HiMemVT
;
1130 GetSplitDestVTs(MemoryVT
, LoMemVT
, HiMemVT
);
1132 unsigned IncrementSize
= LoMemVT
.getSizeInBits()/8;
1135 Lo
= DAG
.getTruncStore(Ch
, dl
, Lo
, Ptr
, N
->getSrcValue(), SVOffset
,
1136 LoMemVT
, isVol
, isNT
, Alignment
);
1138 Lo
= DAG
.getStore(Ch
, dl
, Lo
, Ptr
, N
->getSrcValue(), SVOffset
,
1139 isVol
, isNT
, Alignment
);
1141 // Increment the pointer to the other half.
1142 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
1143 DAG
.getIntPtrConstant(IncrementSize
));
1144 SVOffset
+= IncrementSize
;
1147 Hi
= DAG
.getTruncStore(Ch
, dl
, Hi
, Ptr
, N
->getSrcValue(), SVOffset
,
1148 HiMemVT
, isVol
, isNT
, Alignment
);
1150 Hi
= DAG
.getStore(Ch
, dl
, Hi
, Ptr
, N
->getSrcValue(), SVOffset
,
1151 isVol
, isNT
, Alignment
);
1153 return DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
, Hi
);
1157 //===----------------------------------------------------------------------===//
1158 // Result Vector Widening
1159 //===----------------------------------------------------------------------===//
1161 void DAGTypeLegalizer::WidenVectorResult(SDNode
*N
, unsigned ResNo
) {
1162 DEBUG(dbgs() << "Widen node result " << ResNo
<< ": ";
1166 // See if the target wants to custom widen this node.
1167 if (CustomWidenLowerNode(N
, N
->getValueType(ResNo
)))
1170 SDValue Res
= SDValue();
1171 switch (N
->getOpcode()) {
1174 dbgs() << "WidenVectorResult #" << ResNo
<< ": ";
1178 llvm_unreachable("Do not know how to widen the result of this operator!");
1180 case ISD::BIT_CONVERT
: Res
= WidenVecRes_BIT_CONVERT(N
); break;
1181 case ISD::BUILD_VECTOR
: Res
= WidenVecRes_BUILD_VECTOR(N
); break;
1182 case ISD::CONCAT_VECTORS
: Res
= WidenVecRes_CONCAT_VECTORS(N
); break;
1183 case ISD::CONVERT_RNDSAT
: Res
= WidenVecRes_CONVERT_RNDSAT(N
); break;
1184 case ISD::EXTRACT_SUBVECTOR
: Res
= WidenVecRes_EXTRACT_SUBVECTOR(N
); break;
1185 case ISD::FP_ROUND_INREG
: Res
= WidenVecRes_InregOp(N
); break;
1186 case ISD::INSERT_VECTOR_ELT
: Res
= WidenVecRes_INSERT_VECTOR_ELT(N
); break;
1187 case ISD::LOAD
: Res
= WidenVecRes_LOAD(N
); break;
1188 case ISD::SCALAR_TO_VECTOR
: Res
= WidenVecRes_SCALAR_TO_VECTOR(N
); break;
1189 case ISD::SIGN_EXTEND_INREG
: Res
= WidenVecRes_InregOp(N
); break;
1190 case ISD::SELECT
: Res
= WidenVecRes_SELECT(N
); break;
1191 case ISD::SELECT_CC
: Res
= WidenVecRes_SELECT_CC(N
); break;
1192 case ISD::SETCC
: Res
= WidenVecRes_SETCC(N
); break;
1193 case ISD::UNDEF
: Res
= WidenVecRes_UNDEF(N
); break;
1194 case ISD::VECTOR_SHUFFLE
:
1195 Res
= WidenVecRes_VECTOR_SHUFFLE(cast
<ShuffleVectorSDNode
>(N
));
1198 Res
= WidenVecRes_VSETCC(N
);
1205 case ISD::FCOPYSIGN
:
1221 Res
= WidenVecRes_Binary(N
);
1225 Res
= WidenVecRes_POWI(N
);
1231 Res
= WidenVecRes_Shift(N
);
1235 case ISD::FP_TO_SINT
:
1236 case ISD::FP_TO_UINT
:
1237 case ISD::SINT_TO_FP
:
1238 case ISD::UINT_TO_FP
:
1240 case ISD::SIGN_EXTEND
:
1241 case ISD::ZERO_EXTEND
:
1242 case ISD::ANY_EXTEND
:
1243 Res
= WidenVecRes_Convert(N
);
1259 Res
= WidenVecRes_Unary(N
);
1263 // If Res is null, the sub-method took care of registering the result.
1265 SetWidenedVector(SDValue(N
, ResNo
), Res
);
1268 SDValue
DAGTypeLegalizer::WidenVecRes_Binary(SDNode
*N
) {
1269 // Binary op widening.
1270 unsigned Opcode
= N
->getOpcode();
1271 DebugLoc dl
= N
->getDebugLoc();
1272 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1273 EVT WidenEltVT
= WidenVT
.getVectorElementType();
1275 unsigned NumElts
= VT
.getVectorNumElements();
1276 while (!TLI
.isTypeSynthesizable(VT
) && NumElts
!= 1) {
1277 NumElts
= NumElts
/ 2;
1278 VT
= EVT::getVectorVT(*DAG
.getContext(), WidenEltVT
, NumElts
);
1281 if (NumElts
!= 1 && !TLI
.canOpTrap(N
->getOpcode(), VT
)) {
1282 // Operation doesn't trap so just widen as normal.
1283 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1284 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1285 return DAG
.getNode(N
->getOpcode(), dl
, WidenVT
, InOp1
, InOp2
);
1286 } else if (NumElts
== 1) {
1287 // No legal vector version so unroll the vector operation and then widen.
1288 return DAG
.UnrollVectorOp(N
, WidenVT
.getVectorNumElements());
1290 // Since the operation can trap, apply operation on the original vector.
1292 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1293 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1294 unsigned CurNumElts
= N
->getValueType(0).getVectorNumElements();
1296 SmallVector
<SDValue
, 16> ConcatOps(CurNumElts
);
1297 unsigned ConcatEnd
= 0; // Current ConcatOps index.
1298 int Idx
= 0; // Current Idx into input vectors.
1300 // NumElts := greatest synthesizable vector size (at most WidenVT)
1301 // while (orig. vector has unhandled elements) {
1302 // take munches of size NumElts from the beginning and add to ConcatOps
1303 // NumElts := next smaller supported vector size or 1
1305 while (CurNumElts
!= 0) {
1306 while (CurNumElts
>= NumElts
) {
1307 SDValue EOp1
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, VT
, InOp1
,
1308 DAG
.getIntPtrConstant(Idx
));
1309 SDValue EOp2
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, VT
, InOp2
,
1310 DAG
.getIntPtrConstant(Idx
));
1311 ConcatOps
[ConcatEnd
++] = DAG
.getNode(Opcode
, dl
, VT
, EOp1
, EOp2
);
1313 CurNumElts
-= NumElts
;
1316 NumElts
= NumElts
/ 2;
1317 VT
= EVT::getVectorVT(*DAG
.getContext(), WidenEltVT
, NumElts
);
1318 } while (!TLI
.isTypeSynthesizable(VT
) && NumElts
!= 1);
1321 for (unsigned i
= 0; i
!= CurNumElts
; ++i
, ++Idx
) {
1322 SDValue EOp1
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, WidenEltVT
,
1323 InOp1
, DAG
.getIntPtrConstant(Idx
));
1324 SDValue EOp2
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, WidenEltVT
,
1325 InOp2
, DAG
.getIntPtrConstant(Idx
));
1326 ConcatOps
[ConcatEnd
++] = DAG
.getNode(Opcode
, dl
, WidenEltVT
,
1333 // Check to see if we have a single operation with the widen type.
1334 if (ConcatEnd
== 1) {
1335 VT
= ConcatOps
[0].getValueType();
1337 return ConcatOps
[0];
1340 // while (Some element of ConcatOps is not of type MaxVT) {
1341 // From the end of ConcatOps, collect elements of the same type and put
1342 // them into an op of the next larger supported type
1344 while (ConcatOps
[ConcatEnd
-1].getValueType() != MaxVT
) {
1345 Idx
= ConcatEnd
- 1;
1346 VT
= ConcatOps
[Idx
--].getValueType();
1347 while (Idx
>= 0 && ConcatOps
[Idx
].getValueType() == VT
)
1350 int NextSize
= VT
.isVector() ? VT
.getVectorNumElements() : 1;
1354 NextVT
= EVT::getVectorVT(*DAG
.getContext(), WidenEltVT
, NextSize
);
1355 } while (!TLI
.isTypeSynthesizable(NextVT
));
1357 if (!VT
.isVector()) {
1358 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
1359 SDValue VecOp
= DAG
.getUNDEF(NextVT
);
1360 unsigned NumToInsert
= ConcatEnd
- Idx
- 1;
1361 for (unsigned i
= 0, OpIdx
= Idx
+1; i
< NumToInsert
; i
++, OpIdx
++) {
1362 VecOp
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
, NextVT
, VecOp
,
1363 ConcatOps
[OpIdx
], DAG
.getIntPtrConstant(i
));
1365 ConcatOps
[Idx
+1] = VecOp
;
1366 ConcatEnd
= Idx
+ 2;
1369 // Vector type, create a CONCAT_VECTORS of type NextVT
1370 SDValue undefVec
= DAG
.getUNDEF(VT
);
1371 unsigned OpsToConcat
= NextSize
/VT
.getVectorNumElements();
1372 SmallVector
<SDValue
, 16> SubConcatOps(OpsToConcat
);
1373 unsigned RealVals
= ConcatEnd
- Idx
- 1;
1374 unsigned SubConcatEnd
= 0;
1375 unsigned SubConcatIdx
= Idx
+ 1;
1376 while (SubConcatEnd
< RealVals
)
1377 SubConcatOps
[SubConcatEnd
++] = ConcatOps
[++Idx
];
1378 while (SubConcatEnd
< OpsToConcat
)
1379 SubConcatOps
[SubConcatEnd
++] = undefVec
;
1380 ConcatOps
[SubConcatIdx
] = DAG
.getNode(ISD::CONCAT_VECTORS
, dl
,
1381 NextVT
, &SubConcatOps
[0],
1383 ConcatEnd
= SubConcatIdx
+ 1;
1387 // Check to see if we have a single operation with the widen type.
1388 if (ConcatEnd
== 1) {
1389 VT
= ConcatOps
[0].getValueType();
1391 return ConcatOps
[0];
1394 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
1396 WidenVT
.getVectorNumElements()/MaxVT
.getVectorNumElements();
1397 if (NumOps
!= ConcatEnd
) {
1398 SDValue UndefVal
= DAG
.getUNDEF(MaxVT
);
1399 for (unsigned j
= ConcatEnd
; j
< NumOps
; ++j
)
1400 ConcatOps
[j
] = UndefVal
;
1402 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &ConcatOps
[0], NumOps
);
1406 SDValue
DAGTypeLegalizer::WidenVecRes_Convert(SDNode
*N
) {
1407 SDValue InOp
= N
->getOperand(0);
1408 DebugLoc dl
= N
->getDebugLoc();
1410 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1411 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1413 EVT InVT
= InOp
.getValueType();
1414 EVT InEltVT
= InVT
.getVectorElementType();
1415 EVT InWidenVT
= EVT::getVectorVT(*DAG
.getContext(), InEltVT
, WidenNumElts
);
1417 unsigned Opcode
= N
->getOpcode();
1418 unsigned InVTNumElts
= InVT
.getVectorNumElements();
1420 if (getTypeAction(InVT
) == WidenVector
) {
1421 InOp
= GetWidenedVector(N
->getOperand(0));
1422 InVT
= InOp
.getValueType();
1423 InVTNumElts
= InVT
.getVectorNumElements();
1424 if (InVTNumElts
== WidenNumElts
)
1425 return DAG
.getNode(Opcode
, dl
, WidenVT
, InOp
);
1428 if (TLI
.isTypeSynthesizable(InWidenVT
)) {
1429 // Because the result and the input are different vector types, widening
1430 // the result could create a legal type but widening the input might make
1431 // it an illegal type that might lead to repeatedly splitting the input
1432 // and then widening it. To avoid this, we widen the input only if
1433 // it results in a legal type.
1434 if (WidenNumElts
% InVTNumElts
== 0) {
1435 // Widen the input and call convert on the widened input vector.
1436 unsigned NumConcat
= WidenNumElts
/InVTNumElts
;
1437 SmallVector
<SDValue
, 16> Ops(NumConcat
);
1439 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1440 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
1442 return DAG
.getNode(Opcode
, dl
, WidenVT
,
1443 DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, InWidenVT
,
1444 &Ops
[0], NumConcat
));
1447 if (InVTNumElts
% WidenNumElts
== 0) {
1448 // Extract the input and convert the shorten input vector.
1449 return DAG
.getNode(Opcode
, dl
, WidenVT
,
1450 DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InWidenVT
,
1451 InOp
, DAG
.getIntPtrConstant(0)));
1455 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1456 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1457 EVT EltVT
= WidenVT
.getVectorElementType();
1458 unsigned MinElts
= std::min(InVTNumElts
, WidenNumElts
);
1460 for (i
=0; i
< MinElts
; ++i
)
1461 Ops
[i
] = DAG
.getNode(Opcode
, dl
, EltVT
,
1462 DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, InEltVT
, InOp
,
1463 DAG
.getIntPtrConstant(i
)));
1465 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1466 for (; i
< WidenNumElts
; ++i
)
1469 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1472 SDValue
DAGTypeLegalizer::WidenVecRes_POWI(SDNode
*N
) {
1473 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1474 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1475 SDValue ShOp
= N
->getOperand(1);
1476 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), WidenVT
, InOp
, ShOp
);
1479 SDValue
DAGTypeLegalizer::WidenVecRes_Shift(SDNode
*N
) {
1480 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1481 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1482 SDValue ShOp
= N
->getOperand(1);
1484 EVT ShVT
= ShOp
.getValueType();
1485 if (getTypeAction(ShVT
) == WidenVector
) {
1486 ShOp
= GetWidenedVector(ShOp
);
1487 ShVT
= ShOp
.getValueType();
1489 EVT ShWidenVT
= EVT::getVectorVT(*DAG
.getContext(),
1490 ShVT
.getVectorElementType(),
1491 WidenVT
.getVectorNumElements());
1492 if (ShVT
!= ShWidenVT
)
1493 ShOp
= ModifyToType(ShOp
, ShWidenVT
);
1495 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), WidenVT
, InOp
, ShOp
);
1498 SDValue
DAGTypeLegalizer::WidenVecRes_Unary(SDNode
*N
) {
1499 // Unary op widening.
1500 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1501 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1502 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), WidenVT
, InOp
);
1505 SDValue
DAGTypeLegalizer::WidenVecRes_InregOp(SDNode
*N
) {
1506 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1507 EVT ExtVT
= EVT::getVectorVT(*DAG
.getContext(),
1508 cast
<VTSDNode
>(N
->getOperand(1))->getVT()
1509 .getVectorElementType(),
1510 WidenVT
.getVectorNumElements());
1511 SDValue WidenLHS
= GetWidenedVector(N
->getOperand(0));
1512 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(),
1513 WidenVT
, WidenLHS
, DAG
.getValueType(ExtVT
));
1516 SDValue
DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode
*N
) {
1517 SDValue InOp
= N
->getOperand(0);
1518 EVT InVT
= InOp
.getValueType();
1519 EVT VT
= N
->getValueType(0);
1520 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1521 DebugLoc dl
= N
->getDebugLoc();
1523 switch (getTypeAction(InVT
)) {
1525 assert(false && "Unknown type action!");
1529 case PromoteInteger
:
1530 // If the InOp is promoted to the same size, convert it. Otherwise,
1531 // fall out of the switch and widen the promoted input.
1532 InOp
= GetPromotedInteger(InOp
);
1533 InVT
= InOp
.getValueType();
1534 if (WidenVT
.bitsEq(InVT
))
1535 return DAG
.getNode(ISD::BIT_CONVERT
, dl
, WidenVT
, InOp
);
1540 case ScalarizeVector
:
1544 // If the InOp is widened to the same size, convert it. Otherwise, fall
1545 // out of the switch and widen the widened input.
1546 InOp
= GetWidenedVector(InOp
);
1547 InVT
= InOp
.getValueType();
1548 if (WidenVT
.bitsEq(InVT
))
1549 // The input widens to the same size. Convert to the widen value.
1550 return DAG
.getNode(ISD::BIT_CONVERT
, dl
, WidenVT
, InOp
);
1554 unsigned WidenSize
= WidenVT
.getSizeInBits();
1555 unsigned InSize
= InVT
.getSizeInBits();
1556 if (WidenSize
% InSize
== 0) {
1557 // Determine new input vector type. The new input vector type will use
1558 // the same element type (if its a vector) or use the input type as a
1559 // vector. It is the same size as the type to widen to.
1561 unsigned NewNumElts
= WidenSize
/ InSize
;
1562 if (InVT
.isVector()) {
1563 EVT InEltVT
= InVT
.getVectorElementType();
1564 NewInVT
= EVT::getVectorVT(*DAG
.getContext(), InEltVT
,
1565 WidenSize
/ InEltVT
.getSizeInBits());
1567 NewInVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
, NewNumElts
);
1570 if (TLI
.isTypeSynthesizable(NewInVT
)) {
1571 // Because the result and the input are different vector types, widening
1572 // the result could create a legal type but widening the input might make
1573 // it an illegal type that might lead to repeatedly splitting the input
1574 // and then widening it. To avoid this, we widen the input only if
1575 // it results in a legal type.
1576 SmallVector
<SDValue
, 16> Ops(NewNumElts
);
1577 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1579 for (unsigned i
= 1; i
< NewNumElts
; ++i
)
1583 if (InVT
.isVector())
1584 NewVec
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
,
1585 NewInVT
, &Ops
[0], NewNumElts
);
1587 NewVec
= DAG
.getNode(ISD::BUILD_VECTOR
, dl
,
1588 NewInVT
, &Ops
[0], NewNumElts
);
1589 return DAG
.getNode(ISD::BIT_CONVERT
, dl
, WidenVT
, NewVec
);
1593 return CreateStackStoreLoad(InOp
, WidenVT
);
1596 SDValue
DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode
*N
) {
1597 DebugLoc dl
= N
->getDebugLoc();
1598 // Build a vector with undefined for the new nodes.
1599 EVT VT
= N
->getValueType(0);
1600 EVT EltVT
= VT
.getVectorElementType();
1601 unsigned NumElts
= VT
.getVectorNumElements();
1603 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1604 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1606 SmallVector
<SDValue
, 16> NewOps(N
->op_begin(), N
->op_end());
1607 NewOps
.reserve(WidenNumElts
);
1608 for (unsigned i
= NumElts
; i
< WidenNumElts
; ++i
)
1609 NewOps
.push_back(DAG
.getUNDEF(EltVT
));
1611 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &NewOps
[0], NewOps
.size());
1614 SDValue
DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode
*N
) {
1615 EVT InVT
= N
->getOperand(0).getValueType();
1616 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1617 DebugLoc dl
= N
->getDebugLoc();
1618 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1619 unsigned NumOperands
= N
->getNumOperands();
1621 bool InputWidened
= false; // Indicates we need to widen the input.
1622 if (getTypeAction(InVT
) != WidenVector
) {
1623 if (WidenVT
.getVectorNumElements() % InVT
.getVectorNumElements() == 0) {
1624 // Add undef vectors to widen to correct length.
1625 unsigned NumConcat
= WidenVT
.getVectorNumElements() /
1626 InVT
.getVectorNumElements();
1627 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1628 SmallVector
<SDValue
, 16> Ops(NumConcat
);
1629 for (unsigned i
=0; i
< NumOperands
; ++i
)
1630 Ops
[i
] = N
->getOperand(i
);
1631 for (unsigned i
= NumOperands
; i
!= NumConcat
; ++i
)
1633 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &Ops
[0], NumConcat
);
1636 InputWidened
= true;
1637 if (WidenVT
== TLI
.getTypeToTransformTo(*DAG
.getContext(), InVT
)) {
1638 // The inputs and the result are widen to the same value.
1640 for (i
=1; i
< NumOperands
; ++i
)
1641 if (N
->getOperand(i
).getOpcode() != ISD::UNDEF
)
1644 if (i
> NumOperands
)
1645 // Everything but the first operand is an UNDEF so just return the
1646 // widened first operand.
1647 return GetWidenedVector(N
->getOperand(0));
1649 if (NumOperands
== 2) {
1650 // Replace concat of two operands with a shuffle.
1651 SmallVector
<int, 16> MaskOps(WidenNumElts
);
1652 for (unsigned i
=0; i
< WidenNumElts
/2; ++i
) {
1654 MaskOps
[i
+WidenNumElts
/2] = i
+WidenNumElts
;
1656 return DAG
.getVectorShuffle(WidenVT
, dl
,
1657 GetWidenedVector(N
->getOperand(0)),
1658 GetWidenedVector(N
->getOperand(1)),
1664 // Fall back to use extracts and build vector.
1665 EVT EltVT
= WidenVT
.getVectorElementType();
1666 unsigned NumInElts
= InVT
.getVectorNumElements();
1667 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1669 for (unsigned i
=0; i
< NumOperands
; ++i
) {
1670 SDValue InOp
= N
->getOperand(i
);
1672 InOp
= GetWidenedVector(InOp
);
1673 for (unsigned j
=0; j
< NumInElts
; ++j
)
1674 Ops
[Idx
++] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
1675 DAG
.getIntPtrConstant(j
));
1677 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1678 for (; Idx
< WidenNumElts
; ++Idx
)
1679 Ops
[Idx
] = UndefVal
;
1680 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1683 SDValue
DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode
*N
) {
1684 DebugLoc dl
= N
->getDebugLoc();
1685 SDValue InOp
= N
->getOperand(0);
1686 SDValue RndOp
= N
->getOperand(3);
1687 SDValue SatOp
= N
->getOperand(4);
1689 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(),
1690 N
->getValueType(0));
1691 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1693 EVT InVT
= InOp
.getValueType();
1694 EVT InEltVT
= InVT
.getVectorElementType();
1695 EVT InWidenVT
= EVT::getVectorVT(*DAG
.getContext(), InEltVT
, WidenNumElts
);
1697 SDValue DTyOp
= DAG
.getValueType(WidenVT
);
1698 SDValue STyOp
= DAG
.getValueType(InWidenVT
);
1699 ISD::CvtCode CvtCode
= cast
<CvtRndSatSDNode
>(N
)->getCvtCode();
1701 unsigned InVTNumElts
= InVT
.getVectorNumElements();
1702 if (getTypeAction(InVT
) == WidenVector
) {
1703 InOp
= GetWidenedVector(InOp
);
1704 InVT
= InOp
.getValueType();
1705 InVTNumElts
= InVT
.getVectorNumElements();
1706 if (InVTNumElts
== WidenNumElts
)
1707 return DAG
.getConvertRndSat(WidenVT
, dl
, InOp
, DTyOp
, STyOp
, RndOp
,
1711 if (TLI
.isTypeSynthesizable(InWidenVT
)) {
1712 // Because the result and the input are different vector types, widening
1713 // the result could create a legal type but widening the input might make
1714 // it an illegal type that might lead to repeatedly splitting the input
1715 // and then widening it. To avoid this, we widen the input only if
1716 // it results in a legal type.
1717 if (WidenNumElts
% InVTNumElts
== 0) {
1718 // Widen the input and call convert on the widened input vector.
1719 unsigned NumConcat
= WidenNumElts
/InVTNumElts
;
1720 SmallVector
<SDValue
, 16> Ops(NumConcat
);
1722 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1723 for (unsigned i
= 1; i
!= NumConcat
; ++i
) {
1726 InOp
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, InWidenVT
, &Ops
[0],NumConcat
);
1727 return DAG
.getConvertRndSat(WidenVT
, dl
, InOp
, DTyOp
, STyOp
, RndOp
,
1731 if (InVTNumElts
% WidenNumElts
== 0) {
1732 // Extract the input and convert the shorten input vector.
1733 InOp
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InWidenVT
, InOp
,
1734 DAG
.getIntPtrConstant(0));
1735 return DAG
.getConvertRndSat(WidenVT
, dl
, InOp
, DTyOp
, STyOp
, RndOp
,
1740 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1741 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1742 EVT EltVT
= WidenVT
.getVectorElementType();
1743 DTyOp
= DAG
.getValueType(EltVT
);
1744 STyOp
= DAG
.getValueType(InEltVT
);
1746 unsigned MinElts
= std::min(InVTNumElts
, WidenNumElts
);
1748 for (i
=0; i
< MinElts
; ++i
) {
1749 SDValue ExtVal
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, InEltVT
, InOp
,
1750 DAG
.getIntPtrConstant(i
));
1751 Ops
[i
] = DAG
.getConvertRndSat(WidenVT
, dl
, ExtVal
, DTyOp
, STyOp
, RndOp
,
1755 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1756 for (; i
< WidenNumElts
; ++i
)
1759 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1762 SDValue
DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode
*N
) {
1763 EVT VT
= N
->getValueType(0);
1764 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1765 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1766 SDValue InOp
= N
->getOperand(0);
1767 SDValue Idx
= N
->getOperand(1);
1768 DebugLoc dl
= N
->getDebugLoc();
1770 if (getTypeAction(InOp
.getValueType()) == WidenVector
)
1771 InOp
= GetWidenedVector(InOp
);
1773 EVT InVT
= InOp
.getValueType();
1775 ConstantSDNode
*CIdx
= dyn_cast
<ConstantSDNode
>(Idx
);
1777 unsigned IdxVal
= CIdx
->getZExtValue();
1778 // Check if we can just return the input vector after widening.
1779 if (IdxVal
== 0 && InVT
== WidenVT
)
1782 // Check if we can extract from the vector.
1783 unsigned InNumElts
= InVT
.getVectorNumElements();
1784 if (IdxVal
% WidenNumElts
== 0 && IdxVal
+ WidenNumElts
< InNumElts
)
1785 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, WidenVT
, InOp
, Idx
);
1788 // We could try widening the input to the right length but for now, extract
1789 // the original elements, fill the rest with undefs and build a vector.
1790 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1791 EVT EltVT
= VT
.getVectorElementType();
1792 EVT IdxVT
= Idx
.getValueType();
1793 unsigned NumElts
= VT
.getVectorNumElements();
1796 unsigned IdxVal
= CIdx
->getZExtValue();
1797 for (i
=0; i
< NumElts
; ++i
)
1798 Ops
[i
] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
1799 DAG
.getConstant(IdxVal
+i
, IdxVT
));
1801 Ops
[0] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
, Idx
);
1802 for (i
=1; i
< NumElts
; ++i
) {
1803 SDValue NewIdx
= DAG
.getNode(ISD::ADD
, dl
, Idx
.getValueType(), Idx
,
1804 DAG
.getConstant(i
, IdxVT
));
1805 Ops
[i
] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
, NewIdx
);
1809 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1810 for (; i
< WidenNumElts
; ++i
)
1812 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1815 SDValue
DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode
*N
) {
1816 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1817 return DAG
.getNode(ISD::INSERT_VECTOR_ELT
, N
->getDebugLoc(),
1818 InOp
.getValueType(), InOp
,
1819 N
->getOperand(1), N
->getOperand(2));
1822 SDValue
DAGTypeLegalizer::WidenVecRes_LOAD(SDNode
*N
) {
1823 LoadSDNode
*LD
= cast
<LoadSDNode
>(N
);
1824 ISD::LoadExtType ExtType
= LD
->getExtensionType();
1827 SmallVector
<SDValue
, 16> LdChain
; // Chain for the series of load
1828 if (ExtType
!= ISD::NON_EXTLOAD
)
1829 Result
= GenWidenVectorExtLoads(LdChain
, LD
, ExtType
);
1831 Result
= GenWidenVectorLoads(LdChain
, LD
);
1833 // If we generate a single load, we can use that for the chain. Otherwise,
1834 // build a factor node to remember the multiple loads are independent and
1837 if (LdChain
.size() == 1)
1838 NewChain
= LdChain
[0];
1840 NewChain
= DAG
.getNode(ISD::TokenFactor
, LD
->getDebugLoc(), MVT::Other
,
1841 &LdChain
[0], LdChain
.size());
1843 // Modified the chain - switch anything that used the old chain to use
1845 ReplaceValueWith(SDValue(N
, 1), NewChain
);
1850 SDValue
DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode
*N
) {
1851 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1852 return DAG
.getNode(ISD::SCALAR_TO_VECTOR
, N
->getDebugLoc(),
1853 WidenVT
, N
->getOperand(0));
1856 SDValue
DAGTypeLegalizer::WidenVecRes_SELECT(SDNode
*N
) {
1857 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1858 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1860 SDValue Cond1
= N
->getOperand(0);
1861 EVT CondVT
= Cond1
.getValueType();
1862 if (CondVT
.isVector()) {
1863 EVT CondEltVT
= CondVT
.getVectorElementType();
1864 EVT CondWidenVT
= EVT::getVectorVT(*DAG
.getContext(),
1865 CondEltVT
, WidenNumElts
);
1866 if (getTypeAction(CondVT
) == WidenVector
)
1867 Cond1
= GetWidenedVector(Cond1
);
1869 if (Cond1
.getValueType() != CondWidenVT
)
1870 Cond1
= ModifyToType(Cond1
, CondWidenVT
);
1873 SDValue InOp1
= GetWidenedVector(N
->getOperand(1));
1874 SDValue InOp2
= GetWidenedVector(N
->getOperand(2));
1875 assert(InOp1
.getValueType() == WidenVT
&& InOp2
.getValueType() == WidenVT
);
1876 return DAG
.getNode(ISD::SELECT
, N
->getDebugLoc(),
1877 WidenVT
, Cond1
, InOp1
, InOp2
);
1880 SDValue
DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode
*N
) {
1881 SDValue InOp1
= GetWidenedVector(N
->getOperand(2));
1882 SDValue InOp2
= GetWidenedVector(N
->getOperand(3));
1883 return DAG
.getNode(ISD::SELECT_CC
, N
->getDebugLoc(),
1884 InOp1
.getValueType(), N
->getOperand(0),
1885 N
->getOperand(1), InOp1
, InOp2
, N
->getOperand(4));
1888 SDValue
DAGTypeLegalizer::WidenVecRes_SETCC(SDNode
*N
) {
1889 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1890 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1891 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1892 return DAG
.getNode(ISD::SETCC
, N
->getDebugLoc(), WidenVT
,
1893 InOp1
, InOp2
, N
->getOperand(2));
1896 SDValue
DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode
*N
) {
1897 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1898 return DAG
.getUNDEF(WidenVT
);
1901 SDValue
DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode
*N
) {
1902 EVT VT
= N
->getValueType(0);
1903 DebugLoc dl
= N
->getDebugLoc();
1905 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1906 unsigned NumElts
= VT
.getVectorNumElements();
1907 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1909 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1910 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1912 // Adjust mask based on new input vector length.
1913 SmallVector
<int, 16> NewMask
;
1914 for (unsigned i
= 0; i
!= NumElts
; ++i
) {
1915 int Idx
= N
->getMaskElt(i
);
1916 if (Idx
< (int)NumElts
)
1917 NewMask
.push_back(Idx
);
1919 NewMask
.push_back(Idx
- NumElts
+ WidenNumElts
);
1921 for (unsigned i
= NumElts
; i
!= WidenNumElts
; ++i
)
1922 NewMask
.push_back(-1);
1923 return DAG
.getVectorShuffle(WidenVT
, dl
, InOp1
, InOp2
, &NewMask
[0]);
1926 SDValue
DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode
*N
) {
1927 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1928 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1930 SDValue InOp1
= N
->getOperand(0);
1931 EVT InVT
= InOp1
.getValueType();
1932 assert(InVT
.isVector() && "can not widen non vector type");
1933 EVT WidenInVT
= EVT::getVectorVT(*DAG
.getContext(),
1934 InVT
.getVectorElementType(), WidenNumElts
);
1935 InOp1
= GetWidenedVector(InOp1
);
1936 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1938 // Assume that the input and output will be widen appropriately. If not,
1939 // we will have to unroll it at some point.
1940 assert(InOp1
.getValueType() == WidenInVT
&&
1941 InOp2
.getValueType() == WidenInVT
&&
1942 "Input not widened to expected type!");
1943 return DAG
.getNode(ISD::VSETCC
, N
->getDebugLoc(),
1944 WidenVT
, InOp1
, InOp2
, N
->getOperand(2));
1948 //===----------------------------------------------------------------------===//
1949 // Widen Vector Operand
1950 //===----------------------------------------------------------------------===//
1951 bool DAGTypeLegalizer::WidenVectorOperand(SDNode
*N
, unsigned ResNo
) {
1952 DEBUG(dbgs() << "Widen node operand " << ResNo
<< ": ";
1955 SDValue Res
= SDValue();
1957 switch (N
->getOpcode()) {
1960 dbgs() << "WidenVectorOperand op #" << ResNo
<< ": ";
1964 llvm_unreachable("Do not know how to widen this operator's operand!");
1966 case ISD::BIT_CONVERT
: Res
= WidenVecOp_BIT_CONVERT(N
); break;
1967 case ISD::CONCAT_VECTORS
: Res
= WidenVecOp_CONCAT_VECTORS(N
); break;
1968 case ISD::EXTRACT_SUBVECTOR
: Res
= WidenVecOp_EXTRACT_SUBVECTOR(N
); break;
1969 case ISD::EXTRACT_VECTOR_ELT
: Res
= WidenVecOp_EXTRACT_VECTOR_ELT(N
); break;
1970 case ISD::STORE
: Res
= WidenVecOp_STORE(N
); break;
1973 case ISD::FP_TO_SINT
:
1974 case ISD::FP_TO_UINT
:
1975 case ISD::SINT_TO_FP
:
1976 case ISD::UINT_TO_FP
:
1978 case ISD::SIGN_EXTEND
:
1979 case ISD::ZERO_EXTEND
:
1980 case ISD::ANY_EXTEND
:
1981 Res
= WidenVecOp_Convert(N
);
1985 // If Res is null, the sub-method took care of registering the result.
1986 if (!Res
.getNode()) return false;
1988 // If the result is N, the sub-method updated N in place. Tell the legalizer
1990 if (Res
.getNode() == N
)
1994 assert(Res
.getValueType() == N
->getValueType(0) && N
->getNumValues() == 1 &&
1995 "Invalid operand expansion");
1997 ReplaceValueWith(SDValue(N
, 0), Res
);
2001 SDValue
DAGTypeLegalizer::WidenVecOp_Convert(SDNode
*N
) {
2002 // Since the result is legal and the input is illegal, it is unlikely
2003 // that we can fix the input to a legal type so unroll the convert
2004 // into some scalar code and create a nasty build vector.
2005 EVT VT
= N
->getValueType(0);
2006 EVT EltVT
= VT
.getVectorElementType();
2007 DebugLoc dl
= N
->getDebugLoc();
2008 unsigned NumElts
= VT
.getVectorNumElements();
2009 SDValue InOp
= N
->getOperand(0);
2010 if (getTypeAction(InOp
.getValueType()) == WidenVector
)
2011 InOp
= GetWidenedVector(InOp
);
2012 EVT InVT
= InOp
.getValueType();
2013 EVT InEltVT
= InVT
.getVectorElementType();
2015 unsigned Opcode
= N
->getOpcode();
2016 SmallVector
<SDValue
, 16> Ops(NumElts
);
2017 for (unsigned i
=0; i
< NumElts
; ++i
)
2018 Ops
[i
] = DAG
.getNode(Opcode
, dl
, EltVT
,
2019 DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, InEltVT
, InOp
,
2020 DAG
.getIntPtrConstant(i
)));
2022 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, VT
, &Ops
[0], NumElts
);
2025 SDValue
DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode
*N
) {
2026 EVT VT
= N
->getValueType(0);
2027 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
2028 EVT InWidenVT
= InOp
.getValueType();
2029 DebugLoc dl
= N
->getDebugLoc();
2031 // Check if we can convert between two legal vector types and extract.
2032 unsigned InWidenSize
= InWidenVT
.getSizeInBits();
2033 unsigned Size
= VT
.getSizeInBits();
2034 if (InWidenSize
% Size
== 0 && !VT
.isVector()) {
2035 unsigned NewNumElts
= InWidenSize
/ Size
;
2036 EVT NewVT
= EVT::getVectorVT(*DAG
.getContext(), VT
, NewNumElts
);
2037 if (TLI
.isTypeSynthesizable(NewVT
)) {
2038 SDValue BitOp
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, NewVT
, InOp
);
2039 return DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, VT
, BitOp
,
2040 DAG
.getIntPtrConstant(0));
2044 return CreateStackStoreLoad(InOp
, VT
);
2047 SDValue
DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode
*N
) {
2048 // If the input vector is not legal, it is likely that we will not find a
2049 // legal vector of the same size. Replace the concatenate vector with a
2050 // nasty build vector.
2051 EVT VT
= N
->getValueType(0);
2052 EVT EltVT
= VT
.getVectorElementType();
2053 DebugLoc dl
= N
->getDebugLoc();
2054 unsigned NumElts
= VT
.getVectorNumElements();
2055 SmallVector
<SDValue
, 16> Ops(NumElts
);
2057 EVT InVT
= N
->getOperand(0).getValueType();
2058 unsigned NumInElts
= InVT
.getVectorNumElements();
2061 unsigned NumOperands
= N
->getNumOperands();
2062 for (unsigned i
=0; i
< NumOperands
; ++i
) {
2063 SDValue InOp
= N
->getOperand(i
);
2064 if (getTypeAction(InOp
.getValueType()) == WidenVector
)
2065 InOp
= GetWidenedVector(InOp
);
2066 for (unsigned j
=0; j
< NumInElts
; ++j
)
2067 Ops
[Idx
++] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
2068 DAG
.getIntPtrConstant(j
));
2070 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, VT
, &Ops
[0], NumElts
);
2073 SDValue
DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode
*N
) {
2074 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
2075 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, N
->getDebugLoc(),
2076 N
->getValueType(0), InOp
, N
->getOperand(1));
2079 SDValue
DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode
*N
) {
2080 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
2081 return DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, N
->getDebugLoc(),
2082 N
->getValueType(0), InOp
, N
->getOperand(1));
2085 SDValue
DAGTypeLegalizer::WidenVecOp_STORE(SDNode
*N
) {
2086 // We have to widen the value but we want only to store the original
2088 StoreSDNode
*ST
= cast
<StoreSDNode
>(N
);
2090 SmallVector
<SDValue
, 16> StChain
;
2091 if (ST
->isTruncatingStore())
2092 GenWidenVectorTruncStores(StChain
, ST
);
2094 GenWidenVectorStores(StChain
, ST
);
2096 if (StChain
.size() == 1)
2099 return DAG
.getNode(ISD::TokenFactor
, ST
->getDebugLoc(),
2100 MVT::Other
,&StChain
[0],StChain
.size());
2103 //===----------------------------------------------------------------------===//
2104 // Vector Widening Utilities
2105 //===----------------------------------------------------------------------===//
2107 // Utility function to find the type to chop up a widen vector for load/store
2108 // TLI: Target lowering used to determine legal types.
2109 // Width: Width left need to load/store.
2110 // WidenVT: The widen vector type to load to/store from
2111 // Align: If 0, don't allow use of a wider type
2112 // WidenEx: If Align is not 0, the amount additional we can load/store from.
2114 static EVT
FindMemType(SelectionDAG
& DAG
, const TargetLowering
&TLI
,
2115 unsigned Width
, EVT WidenVT
,
2116 unsigned Align
= 0, unsigned WidenEx
= 0) {
2117 EVT WidenEltVT
= WidenVT
.getVectorElementType();
2118 unsigned WidenWidth
= WidenVT
.getSizeInBits();
2119 unsigned WidenEltWidth
= WidenEltVT
.getSizeInBits();
2120 unsigned AlignInBits
= Align
*8;
2122 // If we have one element to load/store, return it.
2123 EVT RetVT
= WidenEltVT
;
2124 if (Width
== WidenEltWidth
)
2127 // See if there is larger legal integer than the element type to load/store
2129 for (VT
= (unsigned)MVT::LAST_INTEGER_VALUETYPE
;
2130 VT
>= (unsigned)MVT::FIRST_INTEGER_VALUETYPE
; --VT
) {
2131 EVT
MemVT((MVT::SimpleValueType
) VT
);
2132 unsigned MemVTWidth
= MemVT
.getSizeInBits();
2133 if (MemVT
.getSizeInBits() <= WidenEltWidth
)
2135 if (TLI
.isTypeSynthesizable(MemVT
) && (WidenWidth
% MemVTWidth
) == 0 &&
2136 (MemVTWidth
<= Width
||
2137 (Align
!=0 && MemVTWidth
<=AlignInBits
&& MemVTWidth
<=Width
+WidenEx
))) {
2143 // See if there is a larger vector type to load/store that has the same vector
2144 // element type and is evenly divisible with the WidenVT.
2145 for (VT
= (unsigned)MVT::LAST_VECTOR_VALUETYPE
;
2146 VT
>= (unsigned)MVT::FIRST_VECTOR_VALUETYPE
; --VT
) {
2147 EVT MemVT
= (MVT::SimpleValueType
) VT
;
2148 unsigned MemVTWidth
= MemVT
.getSizeInBits();
2149 if (TLI
.isTypeSynthesizable(MemVT
) && WidenEltVT
== MemVT
.getVectorElementType() &&
2150 (WidenWidth
% MemVTWidth
) == 0 &&
2151 (MemVTWidth
<= Width
||
2152 (Align
!=0 && MemVTWidth
<=AlignInBits
&& MemVTWidth
<=Width
+WidenEx
))) {
2153 if (RetVT
.getSizeInBits() < MemVTWidth
|| MemVT
== WidenVT
)
2161 // Builds a vector type from scalar loads
2162 // VecTy: Resulting Vector type
2163 // LDOps: Load operators to build a vector type
2164 // [Start,End) the list of loads to use.
2165 static SDValue
BuildVectorFromScalar(SelectionDAG
& DAG
, EVT VecTy
,
2166 SmallVector
<SDValue
, 16>& LdOps
,
2167 unsigned Start
, unsigned End
) {
2168 DebugLoc dl
= LdOps
[Start
].getDebugLoc();
2169 EVT LdTy
= LdOps
[Start
].getValueType();
2170 unsigned Width
= VecTy
.getSizeInBits();
2171 unsigned NumElts
= Width
/ LdTy
.getSizeInBits();
2172 EVT NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), LdTy
, NumElts
);
2175 SDValue VecOp
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, NewVecVT
,LdOps
[Start
]);
2177 for (unsigned i
= Start
+ 1; i
!= End
; ++i
) {
2178 EVT NewLdTy
= LdOps
[i
].getValueType();
2179 if (NewLdTy
!= LdTy
) {
2180 NumElts
= Width
/ NewLdTy
.getSizeInBits();
2181 NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), NewLdTy
, NumElts
);
2182 VecOp
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, NewVecVT
, VecOp
);
2183 // Readjust position and vector position based on new load type
2184 Idx
= Idx
* LdTy
.getSizeInBits() / NewLdTy
.getSizeInBits();
2187 VecOp
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
, NewVecVT
, VecOp
, LdOps
[i
],
2188 DAG
.getIntPtrConstant(Idx
++));
2190 return DAG
.getNode(ISD::BIT_CONVERT
, dl
, VecTy
, VecOp
);
2193 SDValue
DAGTypeLegalizer::GenWidenVectorLoads(SmallVector
<SDValue
, 16>& LdChain
,
2195 // The strategy assumes that we can efficiently load powers of two widths.
2196 // The routines chops the vector into the largest vector loads with the same
2197 // element type or scalar loads and then recombines it to the widen vector
2199 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(),LD
->getValueType(0));
2200 unsigned WidenWidth
= WidenVT
.getSizeInBits();
2201 EVT LdVT
= LD
->getMemoryVT();
2202 DebugLoc dl
= LD
->getDebugLoc();
2203 assert(LdVT
.isVector() && WidenVT
.isVector());
2204 assert(LdVT
.getVectorElementType() == WidenVT
.getVectorElementType());
2207 SDValue Chain
= LD
->getChain();
2208 SDValue BasePtr
= LD
->getBasePtr();
2209 int SVOffset
= LD
->getSrcValueOffset();
2210 unsigned Align
= LD
->getAlignment();
2211 bool isVolatile
= LD
->isVolatile();
2212 bool isNonTemporal
= LD
->isNonTemporal();
2213 const Value
*SV
= LD
->getSrcValue();
2215 int LdWidth
= LdVT
.getSizeInBits();
2216 int WidthDiff
= WidenWidth
- LdWidth
; // Difference
2217 unsigned LdAlign
= (isVolatile
) ? 0 : Align
; // Allow wider loads
2219 // Find the vector type that can load from.
2220 EVT NewVT
= FindMemType(DAG
, TLI
, LdWidth
, WidenVT
, LdAlign
, WidthDiff
);
2221 int NewVTWidth
= NewVT
.getSizeInBits();
2222 SDValue LdOp
= DAG
.getLoad(NewVT
, dl
, Chain
, BasePtr
, SV
, SVOffset
,
2223 isVolatile
, isNonTemporal
, Align
);
2224 LdChain
.push_back(LdOp
.getValue(1));
2226 // Check if we can load the element with one instruction
2227 if (LdWidth
<= NewVTWidth
) {
2228 if (NewVT
.isVector()) {
2229 if (NewVT
!= WidenVT
) {
2230 assert(WidenWidth
% NewVTWidth
== 0);
2231 unsigned NumConcat
= WidenWidth
/ NewVTWidth
;
2232 SmallVector
<SDValue
, 16> ConcatOps(NumConcat
);
2233 SDValue UndefVal
= DAG
.getUNDEF(NewVT
);
2234 ConcatOps
[0] = LdOp
;
2235 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
2236 ConcatOps
[i
] = UndefVal
;
2237 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &ConcatOps
[0],
2242 unsigned NumElts
= WidenWidth
/ NewVTWidth
;
2243 EVT NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), NewVT
, NumElts
);
2244 SDValue VecOp
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, NewVecVT
, LdOp
);
2245 return DAG
.getNode(ISD::BIT_CONVERT
, dl
, WidenVT
, VecOp
);
2249 // Load vector by using multiple loads from largest vector to scalar
2250 SmallVector
<SDValue
, 16> LdOps
;
2251 LdOps
.push_back(LdOp
);
2253 LdWidth
-= NewVTWidth
;
2254 unsigned Offset
= 0;
2256 while (LdWidth
> 0) {
2257 unsigned Increment
= NewVTWidth
/ 8;
2258 Offset
+= Increment
;
2259 BasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(), BasePtr
,
2260 DAG
.getIntPtrConstant(Increment
));
2262 if (LdWidth
< NewVTWidth
) {
2263 // Our current type we are using is too large, find a better size
2264 NewVT
= FindMemType(DAG
, TLI
, LdWidth
, WidenVT
, LdAlign
, WidthDiff
);
2265 NewVTWidth
= NewVT
.getSizeInBits();
2268 SDValue LdOp
= DAG
.getLoad(NewVT
, dl
, Chain
, BasePtr
, SV
,
2269 SVOffset
+Offset
, isVolatile
,
2270 isNonTemporal
, MinAlign(Align
, Increment
));
2271 LdChain
.push_back(LdOp
.getValue(1));
2272 LdOps
.push_back(LdOp
);
2274 LdWidth
-= NewVTWidth
;
2277 // Build the vector from the loads operations
2278 unsigned End
= LdOps
.size();
2279 if (LdOps
[0].getValueType().isVector()) {
2280 // If the load contains vectors, build the vector using concat vector.
2281 // All of the vectors used to loads are power of 2 and the scalars load
2282 // can be combined to make a power of 2 vector.
2283 SmallVector
<SDValue
, 16> ConcatOps(End
);
2286 EVT LdTy
= LdOps
[i
].getValueType();
2287 // First combine the scalar loads to a vector
2288 if (!LdTy
.isVector()) {
2289 for (--i
; i
>= 0; --i
) {
2290 LdTy
= LdOps
[i
].getValueType();
2291 if (LdTy
.isVector())
2294 ConcatOps
[--Idx
] = BuildVectorFromScalar(DAG
, LdTy
, LdOps
, i
+1, End
);
2296 ConcatOps
[--Idx
] = LdOps
[i
];
2297 for (--i
; i
>= 0; --i
) {
2298 EVT NewLdTy
= LdOps
[i
].getValueType();
2299 if (NewLdTy
!= LdTy
) {
2300 // Create a larger vector
2301 ConcatOps
[End
-1] = DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, NewLdTy
,
2302 &ConcatOps
[Idx
], End
- Idx
);
2306 ConcatOps
[--Idx
] = LdOps
[i
];
2309 if (WidenWidth
!= LdTy
.getSizeInBits()*(End
- Idx
)) {
2310 // We need to fill the rest with undefs to build the vector
2311 unsigned NumOps
= WidenWidth
/ LdTy
.getSizeInBits();
2312 SmallVector
<SDValue
, 16> WidenOps(NumOps
);
2313 SDValue UndefVal
= DAG
.getUNDEF(LdTy
);
2315 for (; i
!= End
-Idx
; ++i
)
2316 WidenOps
[i
] = ConcatOps
[Idx
+i
];
2317 for (; i
!= NumOps
; ++i
)
2318 WidenOps
[i
] = UndefVal
;
2319 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &WidenOps
[0],NumOps
);
2321 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
,
2322 &ConcatOps
[Idx
], End
- Idx
);
2323 } else // All the loads are scalar loads.
2324 return BuildVectorFromScalar(DAG
, WidenVT
, LdOps
, 0, End
);
2328 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVector
<SDValue
, 16>& LdChain
,
2330 ISD::LoadExtType ExtType
) {
2331 // For extension loads, it may not be more efficient to chop up the vector
2332 // and then extended it. Instead, we unroll the load and build a new vector.
2333 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(),LD
->getValueType(0));
2334 EVT LdVT
= LD
->getMemoryVT();
2335 DebugLoc dl
= LD
->getDebugLoc();
2336 assert(LdVT
.isVector() && WidenVT
.isVector());
2339 SDValue Chain
= LD
->getChain();
2340 SDValue BasePtr
= LD
->getBasePtr();
2341 int SVOffset
= LD
->getSrcValueOffset();
2342 unsigned Align
= LD
->getAlignment();
2343 bool isVolatile
= LD
->isVolatile();
2344 bool isNonTemporal
= LD
->isNonTemporal();
2345 const Value
*SV
= LD
->getSrcValue();
2347 EVT EltVT
= WidenVT
.getVectorElementType();
2348 EVT LdEltVT
= LdVT
.getVectorElementType();
2349 unsigned NumElts
= LdVT
.getVectorNumElements();
2351 // Load each element and widen
2352 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
2353 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
2354 unsigned Increment
= LdEltVT
.getSizeInBits() / 8;
2355 Ops
[0] = DAG
.getExtLoad(ExtType
, EltVT
, dl
, Chain
, BasePtr
, SV
, SVOffset
,
2356 LdEltVT
, isVolatile
, isNonTemporal
, Align
);
2357 LdChain
.push_back(Ops
[0].getValue(1));
2358 unsigned i
= 0, Offset
= Increment
;
2359 for (i
=1; i
< NumElts
; ++i
, Offset
+= Increment
) {
2360 SDValue NewBasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(),
2361 BasePtr
, DAG
.getIntPtrConstant(Offset
));
2362 Ops
[i
] = DAG
.getExtLoad(ExtType
, EltVT
, dl
, Chain
, NewBasePtr
, SV
,
2363 SVOffset
+ Offset
, LdEltVT
, isVolatile
,
2364 isNonTemporal
, Align
);
2365 LdChain
.push_back(Ops
[i
].getValue(1));
2368 // Fill the rest with undefs
2369 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
2370 for (; i
!= WidenNumElts
; ++i
)
2373 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], Ops
.size());
2377 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector
<SDValue
, 16>& StChain
,
2379 // The strategy assumes that we can efficiently store powers of two widths.
2380 // The routines chops the vector into the largest vector stores with the same
2381 // element type or scalar stores.
2382 SDValue Chain
= ST
->getChain();
2383 SDValue BasePtr
= ST
->getBasePtr();
2384 const Value
*SV
= ST
->getSrcValue();
2385 int SVOffset
= ST
->getSrcValueOffset();
2386 unsigned Align
= ST
->getAlignment();
2387 bool isVolatile
= ST
->isVolatile();
2388 bool isNonTemporal
= ST
->isNonTemporal();
2389 SDValue ValOp
= GetWidenedVector(ST
->getValue());
2390 DebugLoc dl
= ST
->getDebugLoc();
2392 EVT StVT
= ST
->getMemoryVT();
2393 unsigned StWidth
= StVT
.getSizeInBits();
2394 EVT ValVT
= ValOp
.getValueType();
2395 unsigned ValWidth
= ValVT
.getSizeInBits();
2396 EVT ValEltVT
= ValVT
.getVectorElementType();
2397 unsigned ValEltWidth
= ValEltVT
.getSizeInBits();
2398 assert(StVT
.getVectorElementType() == ValEltVT
);
2400 int Idx
= 0; // current index to store
2401 unsigned Offset
= 0; // offset from base to store
2402 while (StWidth
!= 0) {
2403 // Find the largest vector type we can store with
2404 EVT NewVT
= FindMemType(DAG
, TLI
, StWidth
, ValVT
);
2405 unsigned NewVTWidth
= NewVT
.getSizeInBits();
2406 unsigned Increment
= NewVTWidth
/ 8;
2407 if (NewVT
.isVector()) {
2408 unsigned NumVTElts
= NewVT
.getVectorNumElements();
2410 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, NewVT
, ValOp
,
2411 DAG
.getIntPtrConstant(Idx
));
2412 StChain
.push_back(DAG
.getStore(Chain
, dl
, EOp
, BasePtr
, SV
,
2413 SVOffset
+ Offset
, isVolatile
,
2415 MinAlign(Align
, Offset
)));
2416 StWidth
-= NewVTWidth
;
2417 Offset
+= Increment
;
2419 BasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(), BasePtr
,
2420 DAG
.getIntPtrConstant(Increment
));
2421 } while (StWidth
!= 0 && StWidth
>= NewVTWidth
);
2423 // Cast the vector to the scalar type we can store
2424 unsigned NumElts
= ValWidth
/ NewVTWidth
;
2425 EVT NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), NewVT
, NumElts
);
2426 SDValue VecOp
= DAG
.getNode(ISD::BIT_CONVERT
, dl
, NewVecVT
, ValOp
);
2427 // Readjust index position based on new vector type
2428 Idx
= Idx
* ValEltWidth
/ NewVTWidth
;
2430 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, NewVT
, VecOp
,
2431 DAG
.getIntPtrConstant(Idx
++));
2432 StChain
.push_back(DAG
.getStore(Chain
, dl
, EOp
, BasePtr
, SV
,
2433 SVOffset
+ Offset
, isVolatile
,
2434 isNonTemporal
, MinAlign(Align
, Offset
)));
2435 StWidth
-= NewVTWidth
;
2436 Offset
+= Increment
;
2437 BasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(), BasePtr
,
2438 DAG
.getIntPtrConstant(Increment
));
2439 } while (StWidth
!= 0 && StWidth
>= NewVTWidth
);
2440 // Restore index back to be relative to the original widen element type
2441 Idx
= Idx
* NewVTWidth
/ ValEltWidth
;
2447 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVector
<SDValue
, 16>& StChain
,
2449 // For extension loads, it may not be more efficient to truncate the vector
2450 // and then store it. Instead, we extract each element and then store it.
2451 SDValue Chain
= ST
->getChain();
2452 SDValue BasePtr
= ST
->getBasePtr();
2453 const Value
*SV
= ST
->getSrcValue();
2454 int SVOffset
= ST
->getSrcValueOffset();
2455 unsigned Align
= ST
->getAlignment();
2456 bool isVolatile
= ST
->isVolatile();
2457 bool isNonTemporal
= ST
->isNonTemporal();
2458 SDValue ValOp
= GetWidenedVector(ST
->getValue());
2459 DebugLoc dl
= ST
->getDebugLoc();
2461 EVT StVT
= ST
->getMemoryVT();
2462 EVT ValVT
= ValOp
.getValueType();
2464 // It must be true that we the widen vector type is bigger than where
2465 // we need to store.
2466 assert(StVT
.isVector() && ValOp
.getValueType().isVector());
2467 assert(StVT
.bitsLT(ValOp
.getValueType()));
2469 // For truncating stores, we can not play the tricks of chopping legal
2470 // vector types and bit cast it to the right type. Instead, we unroll
2472 EVT StEltVT
= StVT
.getVectorElementType();
2473 EVT ValEltVT
= ValVT
.getVectorElementType();
2474 unsigned Increment
= ValEltVT
.getSizeInBits() / 8;
2475 unsigned NumElts
= StVT
.getVectorNumElements();
2476 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, ValEltVT
, ValOp
,
2477 DAG
.getIntPtrConstant(0));
2478 StChain
.push_back(DAG
.getTruncStore(Chain
, dl
, EOp
, BasePtr
, SV
,
2480 isVolatile
, isNonTemporal
, Align
));
2481 unsigned Offset
= Increment
;
2482 for (unsigned i
=1; i
< NumElts
; ++i
, Offset
+= Increment
) {
2483 SDValue NewBasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(),
2484 BasePtr
, DAG
.getIntPtrConstant(Offset
));
2485 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, ValEltVT
, ValOp
,
2486 DAG
.getIntPtrConstant(0));
2487 StChain
.push_back(DAG
.getTruncStore(Chain
, dl
, EOp
, NewBasePtr
, SV
,
2488 SVOffset
+ Offset
, StEltVT
,
2489 isVolatile
, isNonTemporal
,
2490 MinAlign(Align
, Offset
)));
2494 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
2495 /// input vector must have the same element type as NVT.
2496 SDValue
DAGTypeLegalizer::ModifyToType(SDValue InOp
, EVT NVT
) {
2497 // Note that InOp might have been widened so it might already have
2498 // the right width or it might need be narrowed.
2499 EVT InVT
= InOp
.getValueType();
2500 assert(InVT
.getVectorElementType() == NVT
.getVectorElementType() &&
2501 "input and widen element type must match");
2502 DebugLoc dl
= InOp
.getDebugLoc();
2504 // Check if InOp already has the right width.
2508 unsigned InNumElts
= InVT
.getVectorNumElements();
2509 unsigned WidenNumElts
= NVT
.getVectorNumElements();
2510 if (WidenNumElts
> InNumElts
&& WidenNumElts
% InNumElts
== 0) {
2511 unsigned NumConcat
= WidenNumElts
/ InNumElts
;
2512 SmallVector
<SDValue
, 16> Ops(NumConcat
);
2513 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
2515 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
2518 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, NVT
, &Ops
[0], NumConcat
);
2521 if (WidenNumElts
< InNumElts
&& InNumElts
% WidenNumElts
)
2522 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, NVT
, InOp
,
2523 DAG
.getIntPtrConstant(0));
2525 // Fall back to extract and build.
2526 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
2527 EVT EltVT
= NVT
.getVectorElementType();
2528 unsigned MinNumElts
= std::min(WidenNumElts
, InNumElts
);
2530 for (Idx
= 0; Idx
< MinNumElts
; ++Idx
)
2531 Ops
[Idx
] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
2532 DAG
.getIntPtrConstant(Idx
));
2534 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
2535 for ( ; Idx
< WidenNumElts
; ++Idx
)
2536 Ops
[Idx
] = UndefVal
;
2537 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, NVT
, &Ops
[0], WidenNumElts
);