1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
21 //===----------------------------------------------------------------------===//
23 #include "LegalizeTypes.h"
24 #include "llvm/CodeGen/PseudoSourceValue.h"
25 #include "llvm/Target/TargetData.h"
26 #include "llvm/Support/ErrorHandling.h"
27 #include "llvm/Support/raw_ostream.h"
30 //===----------------------------------------------------------------------===//
31 // Result Vector Scalarization: <1 x ty> -> ty.
32 //===----------------------------------------------------------------------===//
34 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode
*N
, unsigned ResNo
) {
35 DEBUG(dbgs() << "Scalarize node result " << ResNo
<< ": ";
38 SDValue R
= SDValue();
40 switch (N
->getOpcode()) {
43 dbgs() << "ScalarizeVectorResult #" << ResNo
<< ": ";
47 llvm_unreachable("Do not know how to scalarize the result of this operator!");
49 case ISD::BITCAST
: R
= ScalarizeVecRes_BITCAST(N
); break;
50 case ISD::BUILD_VECTOR
: R
= N
->getOperand(0); break;
51 case ISD::CONVERT_RNDSAT
: R
= ScalarizeVecRes_CONVERT_RNDSAT(N
); break;
52 case ISD::EXTRACT_SUBVECTOR
: R
= ScalarizeVecRes_EXTRACT_SUBVECTOR(N
); break;
53 case ISD::FP_ROUND_INREG
: R
= ScalarizeVecRes_InregOp(N
); break;
54 case ISD::FPOWI
: R
= ScalarizeVecRes_FPOWI(N
); break;
55 case ISD::INSERT_VECTOR_ELT
: R
= ScalarizeVecRes_INSERT_VECTOR_ELT(N
); break;
56 case ISD::LOAD
: R
= ScalarizeVecRes_LOAD(cast
<LoadSDNode
>(N
));break;
57 case ISD::SCALAR_TO_VECTOR
: R
= ScalarizeVecRes_SCALAR_TO_VECTOR(N
); break;
58 case ISD::SIGN_EXTEND_INREG
: R
= ScalarizeVecRes_InregOp(N
); break;
59 case ISD::SELECT
: R
= ScalarizeVecRes_SELECT(N
); break;
60 case ISD::SELECT_CC
: R
= ScalarizeVecRes_SELECT_CC(N
); break;
61 case ISD::SETCC
: R
= ScalarizeVecRes_SETCC(N
); break;
62 case ISD::UNDEF
: R
= ScalarizeVecRes_UNDEF(N
); break;
63 case ISD::VECTOR_SHUFFLE
: R
= ScalarizeVecRes_VECTOR_SHUFFLE(N
); break;
64 case ISD::VSETCC
: R
= ScalarizeVecRes_VSETCC(N
); break;
84 case ISD::SIGN_EXTEND
:
85 case ISD::ZERO_EXTEND
:
87 R
= ScalarizeVecRes_UnaryOp(N
);
109 R
= ScalarizeVecRes_BinOp(N
);
113 // If R is null, the sub-method took care of registering the result.
115 SetScalarizedVector(SDValue(N
, ResNo
), R
);
118 SDValue
DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode
*N
) {
119 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
120 SDValue RHS
= GetScalarizedVector(N
->getOperand(1));
121 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(),
122 LHS
.getValueType(), LHS
, RHS
);
125 SDValue
DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode
*N
) {
126 EVT NewVT
= N
->getValueType(0).getVectorElementType();
127 return DAG
.getNode(ISD::BITCAST
, N
->getDebugLoc(),
128 NewVT
, N
->getOperand(0));
131 SDValue
DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode
*N
) {
132 EVT NewVT
= N
->getValueType(0).getVectorElementType();
133 SDValue Op0
= GetScalarizedVector(N
->getOperand(0));
134 return DAG
.getConvertRndSat(NewVT
, N
->getDebugLoc(),
135 Op0
, DAG
.getValueType(NewVT
),
136 DAG
.getValueType(Op0
.getValueType()),
139 cast
<CvtRndSatSDNode
>(N
)->getCvtCode());
142 SDValue
DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode
*N
) {
143 return DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, N
->getDebugLoc(),
144 N
->getValueType(0).getVectorElementType(),
145 N
->getOperand(0), N
->getOperand(1));
148 SDValue
DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode
*N
) {
149 SDValue Op
= GetScalarizedVector(N
->getOperand(0));
150 return DAG
.getNode(ISD::FPOWI
, N
->getDebugLoc(),
151 Op
.getValueType(), Op
, N
->getOperand(1));
154 SDValue
DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode
*N
) {
155 // The value to insert may have a wider type than the vector element type,
156 // so be sure to truncate it to the element type if necessary.
157 SDValue Op
= N
->getOperand(1);
158 EVT EltVT
= N
->getValueType(0).getVectorElementType();
159 if (Op
.getValueType() != EltVT
)
160 // FIXME: Can this happen for floating point types?
161 Op
= DAG
.getNode(ISD::TRUNCATE
, N
->getDebugLoc(), EltVT
, Op
);
165 SDValue
DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode
*N
) {
166 assert(N
->isUnindexed() && "Indexed vector load?");
168 SDValue Result
= DAG
.getLoad(ISD::UNINDEXED
,
169 N
->getExtensionType(),
170 N
->getValueType(0).getVectorElementType(),
172 N
->getChain(), N
->getBasePtr(),
173 DAG
.getUNDEF(N
->getBasePtr().getValueType()),
175 N
->getMemoryVT().getVectorElementType(),
176 N
->isVolatile(), N
->isNonTemporal(),
177 N
->getOriginalAlignment());
179 // Legalized the chain result - switch anything that used the old chain to
181 ReplaceValueWith(SDValue(N
, 1), Result
.getValue(1));
185 SDValue
DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode
*N
) {
186 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
187 EVT DestVT
= N
->getValueType(0).getVectorElementType();
188 SDValue Op
= GetScalarizedVector(N
->getOperand(0));
189 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), DestVT
, Op
);
192 SDValue
DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode
*N
) {
193 EVT EltVT
= N
->getValueType(0).getVectorElementType();
194 EVT ExtVT
= cast
<VTSDNode
>(N
->getOperand(1))->getVT().getVectorElementType();
195 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
196 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), EltVT
,
197 LHS
, DAG
.getValueType(ExtVT
));
200 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode
*N
) {
201 // If the operand is wider than the vector element type then it is implicitly
202 // truncated. Make that explicit here.
203 EVT EltVT
= N
->getValueType(0).getVectorElementType();
204 SDValue InOp
= N
->getOperand(0);
205 if (InOp
.getValueType() != EltVT
)
206 return DAG
.getNode(ISD::TRUNCATE
, N
->getDebugLoc(), EltVT
, InOp
);
210 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode
*N
) {
211 SDValue LHS
= GetScalarizedVector(N
->getOperand(1));
212 return DAG
.getNode(ISD::SELECT
, N
->getDebugLoc(),
213 LHS
.getValueType(), N
->getOperand(0), LHS
,
214 GetScalarizedVector(N
->getOperand(2)));
217 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode
*N
) {
218 SDValue LHS
= GetScalarizedVector(N
->getOperand(2));
219 return DAG
.getNode(ISD::SELECT_CC
, N
->getDebugLoc(), LHS
.getValueType(),
220 N
->getOperand(0), N
->getOperand(1),
221 LHS
, GetScalarizedVector(N
->getOperand(3)),
225 SDValue
DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode
*N
) {
226 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
227 SDValue RHS
= GetScalarizedVector(N
->getOperand(1));
228 DebugLoc DL
= N
->getDebugLoc();
230 // Turn it into a scalar SETCC.
231 return DAG
.getNode(ISD::SETCC
, DL
, MVT::i1
, LHS
, RHS
, N
->getOperand(2));
234 SDValue
DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode
*N
) {
235 return DAG
.getUNDEF(N
->getValueType(0).getVectorElementType());
238 SDValue
DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode
*N
) {
239 // Figure out if the scalar is the LHS or RHS and return it.
240 SDValue Arg
= N
->getOperand(2).getOperand(0);
241 if (Arg
.getOpcode() == ISD::UNDEF
)
242 return DAG
.getUNDEF(N
->getValueType(0).getVectorElementType());
243 unsigned Op
= !cast
<ConstantSDNode
>(Arg
)->isNullValue();
244 return GetScalarizedVector(N
->getOperand(Op
));
247 SDValue
DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode
*N
) {
248 SDValue LHS
= GetScalarizedVector(N
->getOperand(0));
249 SDValue RHS
= GetScalarizedVector(N
->getOperand(1));
250 EVT NVT
= N
->getValueType(0).getVectorElementType();
251 EVT SVT
= TLI
.getSetCCResultType(LHS
.getValueType());
252 DebugLoc DL
= N
->getDebugLoc();
254 // Turn it into a scalar SETCC.
255 SDValue Res
= DAG
.getNode(ISD::SETCC
, DL
, SVT
, LHS
, RHS
, N
->getOperand(2));
257 // VSETCC always returns a sign-extended value, while SETCC may not. The
258 // SETCC result type may not match the vector element type. Correct these.
259 if (NVT
.bitsLE(SVT
)) {
260 // The SETCC result type is bigger than the vector element type.
261 // Ensure the SETCC result is sign-extended.
262 if (TLI
.getBooleanContents() !=
263 TargetLowering::ZeroOrNegativeOneBooleanContent
)
264 Res
= DAG
.getNode(ISD::SIGN_EXTEND_INREG
, DL
, SVT
, Res
,
265 DAG
.getValueType(MVT::i1
));
266 // Truncate to the final type.
267 return DAG
.getNode(ISD::TRUNCATE
, DL
, NVT
, Res
);
270 // The SETCC result type is smaller than the vector element type.
271 // If the SetCC result is not sign-extended, chop it down to MVT::i1.
272 if (TLI
.getBooleanContents() !=
273 TargetLowering::ZeroOrNegativeOneBooleanContent
)
274 Res
= DAG
.getNode(ISD::TRUNCATE
, DL
, MVT::i1
, Res
);
275 // Sign extend to the final type.
276 return DAG
.getNode(ISD::SIGN_EXTEND
, DL
, NVT
, Res
);
280 //===----------------------------------------------------------------------===//
281 // Operand Vector Scalarization <1 x ty> -> ty.
282 //===----------------------------------------------------------------------===//
284 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode
*N
, unsigned OpNo
) {
285 DEBUG(dbgs() << "Scalarize node operand " << OpNo
<< ": ";
288 SDValue Res
= SDValue();
290 if (Res
.getNode() == 0) {
291 switch (N
->getOpcode()) {
294 dbgs() << "ScalarizeVectorOperand Op #" << OpNo
<< ": ";
298 llvm_unreachable("Do not know how to scalarize this operator's operand!");
300 Res
= ScalarizeVecOp_BITCAST(N
);
302 case ISD::CONCAT_VECTORS
:
303 Res
= ScalarizeVecOp_CONCAT_VECTORS(N
);
305 case ISD::EXTRACT_VECTOR_ELT
:
306 Res
= ScalarizeVecOp_EXTRACT_VECTOR_ELT(N
);
309 Res
= ScalarizeVecOp_STORE(cast
<StoreSDNode
>(N
), OpNo
);
314 // If the result is null, the sub-method took care of registering results etc.
315 if (!Res
.getNode()) return false;
317 // If the result is N, the sub-method updated N in place. Tell the legalizer
319 if (Res
.getNode() == N
)
322 assert(Res
.getValueType() == N
->getValueType(0) && N
->getNumValues() == 1 &&
323 "Invalid operand expansion");
325 ReplaceValueWith(SDValue(N
, 0), Res
);
329 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs
330 /// to be scalarized, it must be <1 x ty>. Convert the element instead.
331 SDValue
DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode
*N
) {
332 SDValue Elt
= GetScalarizedVector(N
->getOperand(0));
333 return DAG
.getNode(ISD::BITCAST
, N
->getDebugLoc(),
334 N
->getValueType(0), Elt
);
337 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
338 /// use a BUILD_VECTOR instead.
339 SDValue
DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode
*N
) {
340 SmallVector
<SDValue
, 8> Ops(N
->getNumOperands());
341 for (unsigned i
= 0, e
= N
->getNumOperands(); i
< e
; ++i
)
342 Ops
[i
] = GetScalarizedVector(N
->getOperand(i
));
343 return DAG
.getNode(ISD::BUILD_VECTOR
, N
->getDebugLoc(), N
->getValueType(0),
344 &Ops
[0], Ops
.size());
347 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
348 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
350 SDValue
DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode
*N
) {
351 SDValue Res
= GetScalarizedVector(N
->getOperand(0));
352 if (Res
.getValueType() != N
->getValueType(0))
353 Res
= DAG
.getNode(ISD::ANY_EXTEND
, N
->getDebugLoc(), N
->getValueType(0),
358 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
359 /// scalarized, it must be <1 x ty>. Just store the element.
360 SDValue
DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode
*N
, unsigned OpNo
){
361 assert(N
->isUnindexed() && "Indexed store of one-element vector?");
362 assert(OpNo
== 1 && "Do not know how to scalarize this operand!");
363 DebugLoc dl
= N
->getDebugLoc();
365 if (N
->isTruncatingStore())
366 return DAG
.getTruncStore(N
->getChain(), dl
,
367 GetScalarizedVector(N
->getOperand(1)),
368 N
->getBasePtr(), N
->getPointerInfo(),
369 N
->getMemoryVT().getVectorElementType(),
370 N
->isVolatile(), N
->isNonTemporal(),
373 return DAG
.getStore(N
->getChain(), dl
, GetScalarizedVector(N
->getOperand(1)),
374 N
->getBasePtr(), N
->getPointerInfo(),
375 N
->isVolatile(), N
->isNonTemporal(),
376 N
->getOriginalAlignment());
380 //===----------------------------------------------------------------------===//
381 // Result Vector Splitting
382 //===----------------------------------------------------------------------===//
384 /// SplitVectorResult - This method is called when the specified result of the
385 /// specified node is found to need vector splitting. At this point, the node
386 /// may also have invalid operands or may have other results that need
387 /// legalization, we just know that (at least) one result needs vector
389 void DAGTypeLegalizer::SplitVectorResult(SDNode
*N
, unsigned ResNo
) {
390 DEBUG(dbgs() << "Split node result: ";
395 switch (N
->getOpcode()) {
398 dbgs() << "SplitVectorResult #" << ResNo
<< ": ";
402 llvm_unreachable("Do not know how to split the result of this operator!");
404 case ISD::MERGE_VALUES
: SplitRes_MERGE_VALUES(N
, Lo
, Hi
); break;
405 case ISD::SELECT
: SplitRes_SELECT(N
, Lo
, Hi
); break;
406 case ISD::SELECT_CC
: SplitRes_SELECT_CC(N
, Lo
, Hi
); break;
407 case ISD::UNDEF
: SplitRes_UNDEF(N
, Lo
, Hi
); break;
409 case ISD::BITCAST
: SplitVecRes_BITCAST(N
, Lo
, Hi
); break;
410 case ISD::BUILD_VECTOR
: SplitVecRes_BUILD_VECTOR(N
, Lo
, Hi
); break;
411 case ISD::CONCAT_VECTORS
: SplitVecRes_CONCAT_VECTORS(N
, Lo
, Hi
); break;
412 case ISD::CONVERT_RNDSAT
: SplitVecRes_CONVERT_RNDSAT(N
, Lo
, Hi
); break;
413 case ISD::EXTRACT_SUBVECTOR
: SplitVecRes_EXTRACT_SUBVECTOR(N
, Lo
, Hi
); break;
414 case ISD::FP_ROUND_INREG
: SplitVecRes_InregOp(N
, Lo
, Hi
); break;
415 case ISD::FPOWI
: SplitVecRes_FPOWI(N
, Lo
, Hi
); break;
416 case ISD::INSERT_VECTOR_ELT
: SplitVecRes_INSERT_VECTOR_ELT(N
, Lo
, Hi
); break;
417 case ISD::SCALAR_TO_VECTOR
: SplitVecRes_SCALAR_TO_VECTOR(N
, Lo
, Hi
); break;
418 case ISD::SIGN_EXTEND_INREG
: SplitVecRes_InregOp(N
, Lo
, Hi
); break;
420 SplitVecRes_LOAD(cast
<LoadSDNode
>(N
), Lo
, Hi
);
424 SplitVecRes_SETCC(N
, Lo
, Hi
);
426 case ISD::VECTOR_SHUFFLE
:
427 SplitVecRes_VECTOR_SHUFFLE(cast
<ShuffleVectorSDNode
>(N
), Lo
, Hi
);
442 case ISD::FNEARBYINT
:
443 case ISD::FP_TO_SINT
:
444 case ISD::FP_TO_UINT
:
445 case ISD::SINT_TO_FP
:
446 case ISD::UINT_TO_FP
:
448 case ISD::SIGN_EXTEND
:
449 case ISD::ZERO_EXTEND
:
450 case ISD::ANY_EXTEND
:
456 SplitVecRes_UnaryOp(N
, Lo
, Hi
);
478 SplitVecRes_BinOp(N
, Lo
, Hi
);
482 // If Lo/Hi is null, the sub-method took care of registering results etc.
484 SetSplitVector(SDValue(N
, ResNo
), Lo
, Hi
);
487 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode
*N
, SDValue
&Lo
,
489 SDValue LHSLo
, LHSHi
;
490 GetSplitVector(N
->getOperand(0), LHSLo
, LHSHi
);
491 SDValue RHSLo
, RHSHi
;
492 GetSplitVector(N
->getOperand(1), RHSLo
, RHSHi
);
493 DebugLoc dl
= N
->getDebugLoc();
495 Lo
= DAG
.getNode(N
->getOpcode(), dl
, LHSLo
.getValueType(), LHSLo
, RHSLo
);
496 Hi
= DAG
.getNode(N
->getOpcode(), dl
, LHSHi
.getValueType(), LHSHi
, RHSHi
);
499 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode
*N
, SDValue
&Lo
,
501 // We know the result is a vector. The input may be either a vector or a
504 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
505 DebugLoc dl
= N
->getDebugLoc();
507 SDValue InOp
= N
->getOperand(0);
508 EVT InVT
= InOp
.getValueType();
510 // Handle some special cases efficiently.
511 switch (getTypeAction(InVT
)) {
513 assert(false && "Unknown type action!");
517 case ScalarizeVector
:
521 // A scalar to vector conversion, where the scalar needs expansion.
522 // If the vector is being split in two then we can just convert the
525 GetExpandedOp(InOp
, Lo
, Hi
);
526 if (TLI
.isBigEndian())
528 Lo
= DAG
.getNode(ISD::BITCAST
, dl
, LoVT
, Lo
);
529 Hi
= DAG
.getNode(ISD::BITCAST
, dl
, HiVT
, Hi
);
534 // If the input is a vector that needs to be split, convert each split
535 // piece of the input now.
536 GetSplitVector(InOp
, Lo
, Hi
);
537 Lo
= DAG
.getNode(ISD::BITCAST
, dl
, LoVT
, Lo
);
538 Hi
= DAG
.getNode(ISD::BITCAST
, dl
, HiVT
, Hi
);
542 // In the general case, convert the input to an integer and split it by hand.
543 EVT LoIntVT
= EVT::getIntegerVT(*DAG
.getContext(), LoVT
.getSizeInBits());
544 EVT HiIntVT
= EVT::getIntegerVT(*DAG
.getContext(), HiVT
.getSizeInBits());
545 if (TLI
.isBigEndian())
546 std::swap(LoIntVT
, HiIntVT
);
548 SplitInteger(BitConvertToInteger(InOp
), LoIntVT
, HiIntVT
, Lo
, Hi
);
550 if (TLI
.isBigEndian())
552 Lo
= DAG
.getNode(ISD::BITCAST
, dl
, LoVT
, Lo
);
553 Hi
= DAG
.getNode(ISD::BITCAST
, dl
, HiVT
, Hi
);
556 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode
*N
, SDValue
&Lo
,
559 DebugLoc dl
= N
->getDebugLoc();
560 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
561 unsigned LoNumElts
= LoVT
.getVectorNumElements();
562 SmallVector
<SDValue
, 8> LoOps(N
->op_begin(), N
->op_begin()+LoNumElts
);
563 Lo
= DAG
.getNode(ISD::BUILD_VECTOR
, dl
, LoVT
, &LoOps
[0], LoOps
.size());
565 SmallVector
<SDValue
, 8> HiOps(N
->op_begin()+LoNumElts
, N
->op_end());
566 Hi
= DAG
.getNode(ISD::BUILD_VECTOR
, dl
, HiVT
, &HiOps
[0], HiOps
.size());
569 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode
*N
, SDValue
&Lo
,
571 assert(!(N
->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
572 DebugLoc dl
= N
->getDebugLoc();
573 unsigned NumSubvectors
= N
->getNumOperands() / 2;
574 if (NumSubvectors
== 1) {
575 Lo
= N
->getOperand(0);
576 Hi
= N
->getOperand(1);
581 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
583 SmallVector
<SDValue
, 8> LoOps(N
->op_begin(), N
->op_begin()+NumSubvectors
);
584 Lo
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, LoVT
, &LoOps
[0], LoOps
.size());
586 SmallVector
<SDValue
, 8> HiOps(N
->op_begin()+NumSubvectors
, N
->op_end());
587 Hi
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, HiVT
, &HiOps
[0], HiOps
.size());
590 void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode
*N
, SDValue
&Lo
,
593 DebugLoc dl
= N
->getDebugLoc();
594 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
596 SDValue DTyOpLo
= DAG
.getValueType(LoVT
);
597 SDValue DTyOpHi
= DAG
.getValueType(HiVT
);
599 SDValue RndOp
= N
->getOperand(3);
600 SDValue SatOp
= N
->getOperand(4);
601 ISD::CvtCode CvtCode
= cast
<CvtRndSatSDNode
>(N
)->getCvtCode();
605 EVT InVT
= N
->getOperand(0).getValueType();
606 switch (getTypeAction(InVT
)) {
607 default: llvm_unreachable("Unexpected type action!");
609 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
610 LoVT
.getVectorNumElements());
611 VLo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
612 DAG
.getIntPtrConstant(0));
613 VHi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
614 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
618 GetSplitVector(N
->getOperand(0), VLo
, VHi
);
621 // If the result needs to be split and the input needs to be widened,
622 // the two types must have different lengths. Use the widened result
623 // and extract from it to do the split.
624 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
625 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
626 LoVT
.getVectorNumElements());
627 VLo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
628 DAG
.getIntPtrConstant(0));
629 VHi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
630 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
635 SDValue STyOpLo
= DAG
.getValueType(VLo
.getValueType());
636 SDValue STyOpHi
= DAG
.getValueType(VHi
.getValueType());
638 Lo
= DAG
.getConvertRndSat(LoVT
, dl
, VLo
, DTyOpLo
, STyOpLo
, RndOp
, SatOp
,
640 Hi
= DAG
.getConvertRndSat(HiVT
, dl
, VHi
, DTyOpHi
, STyOpHi
, RndOp
, SatOp
,
644 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode
*N
, SDValue
&Lo
,
646 SDValue Vec
= N
->getOperand(0);
647 SDValue Idx
= N
->getOperand(1);
648 DebugLoc dl
= N
->getDebugLoc();
651 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
653 Lo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, LoVT
, Vec
, Idx
);
654 uint64_t IdxVal
= cast
<ConstantSDNode
>(Idx
)->getZExtValue();
655 Hi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, HiVT
, Vec
,
656 DAG
.getIntPtrConstant(IdxVal
+ LoVT
.getVectorNumElements()));
659 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode
*N
, SDValue
&Lo
,
661 DebugLoc dl
= N
->getDebugLoc();
662 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
663 Lo
= DAG
.getNode(ISD::FPOWI
, dl
, Lo
.getValueType(), Lo
, N
->getOperand(1));
664 Hi
= DAG
.getNode(ISD::FPOWI
, dl
, Hi
.getValueType(), Hi
, N
->getOperand(1));
667 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode
*N
, SDValue
&Lo
,
669 SDValue LHSLo
, LHSHi
;
670 GetSplitVector(N
->getOperand(0), LHSLo
, LHSHi
);
671 DebugLoc dl
= N
->getDebugLoc();
674 GetSplitDestVTs(cast
<VTSDNode
>(N
->getOperand(1))->getVT(), LoVT
, HiVT
);
676 Lo
= DAG
.getNode(N
->getOpcode(), dl
, LHSLo
.getValueType(), LHSLo
,
677 DAG
.getValueType(LoVT
));
678 Hi
= DAG
.getNode(N
->getOpcode(), dl
, LHSHi
.getValueType(), LHSHi
,
679 DAG
.getValueType(HiVT
));
682 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode
*N
, SDValue
&Lo
,
684 SDValue Vec
= N
->getOperand(0);
685 SDValue Elt
= N
->getOperand(1);
686 SDValue Idx
= N
->getOperand(2);
687 DebugLoc dl
= N
->getDebugLoc();
688 GetSplitVector(Vec
, Lo
, Hi
);
690 if (ConstantSDNode
*CIdx
= dyn_cast
<ConstantSDNode
>(Idx
)) {
691 unsigned IdxVal
= CIdx
->getZExtValue();
692 unsigned LoNumElts
= Lo
.getValueType().getVectorNumElements();
693 if (IdxVal
< LoNumElts
)
694 Lo
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
,
695 Lo
.getValueType(), Lo
, Elt
, Idx
);
697 Hi
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
, Hi
.getValueType(), Hi
, Elt
,
698 DAG
.getIntPtrConstant(IdxVal
- LoNumElts
));
702 // Spill the vector to the stack.
703 EVT VecVT
= Vec
.getValueType();
704 EVT EltVT
= VecVT
.getVectorElementType();
705 SDValue StackPtr
= DAG
.CreateStackTemporary(VecVT
);
706 SDValue Store
= DAG
.getStore(DAG
.getEntryNode(), dl
, Vec
, StackPtr
,
707 MachinePointerInfo(), false, false, 0);
709 // Store the new element. This may be larger than the vector element type,
710 // so use a truncating store.
711 SDValue EltPtr
= GetVectorElementPointer(StackPtr
, EltVT
, Idx
);
712 const Type
*VecType
= VecVT
.getTypeForEVT(*DAG
.getContext());
714 TLI
.getTargetData()->getPrefTypeAlignment(VecType
);
715 Store
= DAG
.getTruncStore(Store
, dl
, Elt
, EltPtr
, MachinePointerInfo(), EltVT
,
718 // Load the Lo part from the stack slot.
719 Lo
= DAG
.getLoad(Lo
.getValueType(), dl
, Store
, StackPtr
, MachinePointerInfo(),
722 // Increment the pointer to the other part.
723 unsigned IncrementSize
= Lo
.getValueType().getSizeInBits() / 8;
724 StackPtr
= DAG
.getNode(ISD::ADD
, dl
, StackPtr
.getValueType(), StackPtr
,
725 DAG
.getIntPtrConstant(IncrementSize
));
727 // Load the Hi part from the stack slot.
728 Hi
= DAG
.getLoad(Hi
.getValueType(), dl
, Store
, StackPtr
, MachinePointerInfo(),
729 false, false, MinAlign(Alignment
, IncrementSize
));
732 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode
*N
, SDValue
&Lo
,
735 DebugLoc dl
= N
->getDebugLoc();
736 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
737 Lo
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, LoVT
, N
->getOperand(0));
738 Hi
= DAG
.getUNDEF(HiVT
);
741 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode
*LD
, SDValue
&Lo
,
743 assert(ISD::isUNINDEXEDLoad(LD
) && "Indexed load during type legalization!");
745 DebugLoc dl
= LD
->getDebugLoc();
746 GetSplitDestVTs(LD
->getValueType(0), LoVT
, HiVT
);
748 ISD::LoadExtType ExtType
= LD
->getExtensionType();
749 SDValue Ch
= LD
->getChain();
750 SDValue Ptr
= LD
->getBasePtr();
751 SDValue Offset
= DAG
.getUNDEF(Ptr
.getValueType());
752 EVT MemoryVT
= LD
->getMemoryVT();
753 unsigned Alignment
= LD
->getOriginalAlignment();
754 bool isVolatile
= LD
->isVolatile();
755 bool isNonTemporal
= LD
->isNonTemporal();
757 EVT LoMemVT
, HiMemVT
;
758 GetSplitDestVTs(MemoryVT
, LoMemVT
, HiMemVT
);
760 Lo
= DAG
.getLoad(ISD::UNINDEXED
, ExtType
, LoVT
, dl
, Ch
, Ptr
, Offset
,
761 LD
->getPointerInfo(), LoMemVT
, isVolatile
, isNonTemporal
,
764 unsigned IncrementSize
= LoMemVT
.getSizeInBits()/8;
765 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
766 DAG
.getIntPtrConstant(IncrementSize
));
767 Hi
= DAG
.getLoad(ISD::UNINDEXED
, ExtType
, HiVT
, dl
, Ch
, Ptr
, Offset
,
768 LD
->getPointerInfo().getWithOffset(IncrementSize
),
769 HiMemVT
, isVolatile
, isNonTemporal
, Alignment
);
771 // Build a factor node to remember that this load is independent of the
773 Ch
= DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
.getValue(1),
776 // Legalized the chain result - switch anything that used the old chain to
778 ReplaceValueWith(SDValue(LD
, 1), Ch
);
781 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode
*N
, SDValue
&Lo
, SDValue
&Hi
) {
783 DebugLoc DL
= N
->getDebugLoc();
784 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
787 EVT InVT
= N
->getOperand(0).getValueType();
788 SDValue LL
, LH
, RL
, RH
;
789 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
790 LoVT
.getVectorNumElements());
791 LL
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(0),
792 DAG
.getIntPtrConstant(0));
793 LH
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(0),
794 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
796 RL
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(1),
797 DAG
.getIntPtrConstant(0));
798 RH
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, DL
, InNVT
, N
->getOperand(1),
799 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
801 Lo
= DAG
.getNode(N
->getOpcode(), DL
, LoVT
, LL
, RL
, N
->getOperand(2));
802 Hi
= DAG
.getNode(N
->getOpcode(), DL
, HiVT
, LH
, RH
, N
->getOperand(2));
805 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode
*N
, SDValue
&Lo
,
807 // Get the dest types - they may not match the input types, e.g. int_to_fp.
809 DebugLoc dl
= N
->getDebugLoc();
810 GetSplitDestVTs(N
->getValueType(0), LoVT
, HiVT
);
813 EVT InVT
= N
->getOperand(0).getValueType();
814 switch (getTypeAction(InVT
)) {
815 default: llvm_unreachable("Unexpected type action!");
817 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
818 LoVT
.getVectorNumElements());
819 Lo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
820 DAG
.getIntPtrConstant(0));
821 Hi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, N
->getOperand(0),
822 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
826 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
829 // If the result needs to be split and the input needs to be widened,
830 // the two types must have different lengths. Use the widened result
831 // and extract from it to do the split.
832 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
833 EVT InNVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
.getVectorElementType(),
834 LoVT
.getVectorNumElements());
835 Lo
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
836 DAG
.getIntPtrConstant(0));
837 Hi
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InNVT
, InOp
,
838 DAG
.getIntPtrConstant(InNVT
.getVectorNumElements()));
843 Lo
= DAG
.getNode(N
->getOpcode(), dl
, LoVT
, Lo
);
844 Hi
= DAG
.getNode(N
->getOpcode(), dl
, HiVT
, Hi
);
847 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode
*N
,
848 SDValue
&Lo
, SDValue
&Hi
) {
849 // The low and high parts of the original input give four input vectors.
851 DebugLoc dl
= N
->getDebugLoc();
852 GetSplitVector(N
->getOperand(0), Inputs
[0], Inputs
[1]);
853 GetSplitVector(N
->getOperand(1), Inputs
[2], Inputs
[3]);
854 EVT NewVT
= Inputs
[0].getValueType();
855 unsigned NewElts
= NewVT
.getVectorNumElements();
857 // If Lo or Hi uses elements from at most two of the four input vectors, then
858 // express it as a vector shuffle of those two inputs. Otherwise extract the
859 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
860 SmallVector
<int, 16> Ops
;
861 for (unsigned High
= 0; High
< 2; ++High
) {
862 SDValue
&Output
= High
? Hi
: Lo
;
864 // Build a shuffle mask for the output, discovering on the fly which
865 // input vectors to use as shuffle operands (recorded in InputUsed).
866 // If building a suitable shuffle vector proves too hard, then bail
867 // out with useBuildVector set.
868 unsigned InputUsed
[2] = { -1U, -1U }; // Not yet discovered.
869 unsigned FirstMaskIdx
= High
* NewElts
;
870 bool useBuildVector
= false;
871 for (unsigned MaskOffset
= 0; MaskOffset
< NewElts
; ++MaskOffset
) {
872 // The mask element. This indexes into the input.
873 int Idx
= N
->getMaskElt(FirstMaskIdx
+ MaskOffset
);
875 // The input vector this mask element indexes into.
876 unsigned Input
= (unsigned)Idx
/ NewElts
;
878 if (Input
>= array_lengthof(Inputs
)) {
879 // The mask element does not index into any input vector.
884 // Turn the index into an offset from the start of the input vector.
885 Idx
-= Input
* NewElts
;
887 // Find or create a shuffle vector operand to hold this input.
889 for (OpNo
= 0; OpNo
< array_lengthof(InputUsed
); ++OpNo
) {
890 if (InputUsed
[OpNo
] == Input
) {
891 // This input vector is already an operand.
893 } else if (InputUsed
[OpNo
] == -1U) {
894 // Create a new operand for this input vector.
895 InputUsed
[OpNo
] = Input
;
900 if (OpNo
>= array_lengthof(InputUsed
)) {
901 // More than two input vectors used! Give up on trying to create a
902 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
903 useBuildVector
= true;
907 // Add the mask index for the new shuffle vector.
908 Ops
.push_back(Idx
+ OpNo
* NewElts
);
911 if (useBuildVector
) {
912 EVT EltVT
= NewVT
.getVectorElementType();
913 SmallVector
<SDValue
, 16> SVOps
;
915 // Extract the input elements by hand.
916 for (unsigned MaskOffset
= 0; MaskOffset
< NewElts
; ++MaskOffset
) {
917 // The mask element. This indexes into the input.
918 int Idx
= N
->getMaskElt(FirstMaskIdx
+ MaskOffset
);
920 // The input vector this mask element indexes into.
921 unsigned Input
= (unsigned)Idx
/ NewElts
;
923 if (Input
>= array_lengthof(Inputs
)) {
924 // The mask element is "undef" or indexes off the end of the input.
925 SVOps
.push_back(DAG
.getUNDEF(EltVT
));
929 // Turn the index into an offset from the start of the input vector.
930 Idx
-= Input
* NewElts
;
932 // Extract the vector element by hand.
933 SVOps
.push_back(DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
,
934 Inputs
[Input
], DAG
.getIntPtrConstant(Idx
)));
937 // Construct the Lo/Hi output using a BUILD_VECTOR.
938 Output
= DAG
.getNode(ISD::BUILD_VECTOR
,dl
,NewVT
, &SVOps
[0], SVOps
.size());
939 } else if (InputUsed
[0] == -1U) {
940 // No input vectors were used! The result is undefined.
941 Output
= DAG
.getUNDEF(NewVT
);
943 SDValue Op0
= Inputs
[InputUsed
[0]];
944 // If only one input was used, use an undefined vector for the other.
945 SDValue Op1
= InputUsed
[1] == -1U ?
946 DAG
.getUNDEF(NewVT
) : Inputs
[InputUsed
[1]];
947 // At least one input vector was used. Create a new shuffle vector.
948 Output
= DAG
.getVectorShuffle(NewVT
, dl
, Op0
, Op1
, &Ops
[0]);
956 //===----------------------------------------------------------------------===//
957 // Operand Vector Splitting
958 //===----------------------------------------------------------------------===//
960 /// SplitVectorOperand - This method is called when the specified operand of the
961 /// specified node is found to need vector splitting. At this point, all of the
962 /// result types of the node are known to be legal, but other operands of the
963 /// node may need legalization as well as the specified one.
964 bool DAGTypeLegalizer::SplitVectorOperand(SDNode
*N
, unsigned OpNo
) {
965 DEBUG(dbgs() << "Split node operand: ";
968 SDValue Res
= SDValue();
970 if (Res
.getNode() == 0) {
971 switch (N
->getOpcode()) {
974 dbgs() << "SplitVectorOperand Op #" << OpNo
<< ": ";
978 llvm_unreachable("Do not know how to split this operator's operand!");
980 case ISD::BITCAST
: Res
= SplitVecOp_BITCAST(N
); break;
981 case ISD::EXTRACT_SUBVECTOR
: Res
= SplitVecOp_EXTRACT_SUBVECTOR(N
); break;
982 case ISD::EXTRACT_VECTOR_ELT
:Res
= SplitVecOp_EXTRACT_VECTOR_ELT(N
); break;
983 case ISD::CONCAT_VECTORS
: Res
= SplitVecOp_CONCAT_VECTORS(N
); break;
985 Res
= SplitVecOp_STORE(cast
<StoreSDNode
>(N
), OpNo
);
991 case ISD::FP_TO_SINT
:
992 case ISD::FP_TO_UINT
:
993 case ISD::SINT_TO_FP
:
994 case ISD::UINT_TO_FP
:
996 case ISD::SIGN_EXTEND
:
997 case ISD::ZERO_EXTEND
:
998 case ISD::ANY_EXTEND
:
999 Res
= SplitVecOp_UnaryOp(N
);
1004 // If the result is null, the sub-method took care of registering results etc.
1005 if (!Res
.getNode()) return false;
1007 // If the result is N, the sub-method updated N in place. Tell the legalizer
1009 if (Res
.getNode() == N
)
1012 assert(Res
.getValueType() == N
->getValueType(0) && N
->getNumValues() == 1 &&
1013 "Invalid operand expansion");
1015 ReplaceValueWith(SDValue(N
, 0), Res
);
1019 SDValue
DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode
*N
) {
1020 // The result has a legal vector type, but the input needs splitting.
1021 EVT ResVT
= N
->getValueType(0);
1023 DebugLoc dl
= N
->getDebugLoc();
1024 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
1025 EVT InVT
= Lo
.getValueType();
1027 EVT OutVT
= EVT::getVectorVT(*DAG
.getContext(), ResVT
.getVectorElementType(),
1028 InVT
.getVectorNumElements());
1030 Lo
= DAG
.getNode(N
->getOpcode(), dl
, OutVT
, Lo
);
1031 Hi
= DAG
.getNode(N
->getOpcode(), dl
, OutVT
, Hi
);
1033 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, ResVT
, Lo
, Hi
);
1036 SDValue
DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode
*N
) {
1037 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1038 // end up being split all the way down to individual components. Convert the
1039 // split pieces into integers and reassemble.
1041 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
1042 Lo
= BitConvertToInteger(Lo
);
1043 Hi
= BitConvertToInteger(Hi
);
1045 if (TLI
.isBigEndian())
1048 return DAG
.getNode(ISD::BITCAST
, N
->getDebugLoc(), N
->getValueType(0),
1049 JoinIntegers(Lo
, Hi
));
1052 SDValue
DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode
*N
) {
1053 // We know that the extracted result type is legal.
1054 EVT SubVT
= N
->getValueType(0);
1055 SDValue Idx
= N
->getOperand(1);
1056 DebugLoc dl
= N
->getDebugLoc();
1058 GetSplitVector(N
->getOperand(0), Lo
, Hi
);
1060 uint64_t LoElts
= Lo
.getValueType().getVectorNumElements();
1061 uint64_t IdxVal
= cast
<ConstantSDNode
>(Idx
)->getZExtValue();
1063 if (IdxVal
< LoElts
) {
1064 assert(IdxVal
+ SubVT
.getVectorNumElements() <= LoElts
&&
1065 "Extracted subvector crosses vector split!");
1066 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, SubVT
, Lo
, Idx
);
1068 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, SubVT
, Hi
,
1069 DAG
.getConstant(IdxVal
- LoElts
, Idx
.getValueType()));
1073 SDValue
DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode
*N
) {
1074 SDValue Vec
= N
->getOperand(0);
1075 SDValue Idx
= N
->getOperand(1);
1076 EVT VecVT
= Vec
.getValueType();
1078 if (isa
<ConstantSDNode
>(Idx
)) {
1079 uint64_t IdxVal
= cast
<ConstantSDNode
>(Idx
)->getZExtValue();
1080 assert(IdxVal
< VecVT
.getVectorNumElements() && "Invalid vector index!");
1083 GetSplitVector(Vec
, Lo
, Hi
);
1085 uint64_t LoElts
= Lo
.getValueType().getVectorNumElements();
1087 if (IdxVal
< LoElts
)
1088 return SDValue(DAG
.UpdateNodeOperands(N
, Lo
, Idx
), 0);
1089 return SDValue(DAG
.UpdateNodeOperands(N
, Hi
,
1090 DAG
.getConstant(IdxVal
- LoElts
,
1091 Idx
.getValueType())), 0);
1094 // Store the vector to the stack.
1095 EVT EltVT
= VecVT
.getVectorElementType();
1096 DebugLoc dl
= N
->getDebugLoc();
1097 SDValue StackPtr
= DAG
.CreateStackTemporary(VecVT
);
1098 SDValue Store
= DAG
.getStore(DAG
.getEntryNode(), dl
, Vec
, StackPtr
,
1099 MachinePointerInfo(), false, false, 0);
1101 // Load back the required element.
1102 StackPtr
= GetVectorElementPointer(StackPtr
, EltVT
, Idx
);
1103 return DAG
.getExtLoad(ISD::EXTLOAD
, N
->getValueType(0), dl
, Store
, StackPtr
,
1104 MachinePointerInfo(), EltVT
, false, false, 0);
1107 SDValue
DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode
*N
, unsigned OpNo
) {
1108 assert(N
->isUnindexed() && "Indexed store of vector?");
1109 assert(OpNo
== 1 && "Can only split the stored value");
1110 DebugLoc DL
= N
->getDebugLoc();
1112 bool isTruncating
= N
->isTruncatingStore();
1113 SDValue Ch
= N
->getChain();
1114 SDValue Ptr
= N
->getBasePtr();
1115 EVT MemoryVT
= N
->getMemoryVT();
1116 unsigned Alignment
= N
->getOriginalAlignment();
1117 bool isVol
= N
->isVolatile();
1118 bool isNT
= N
->isNonTemporal();
1120 GetSplitVector(N
->getOperand(1), Lo
, Hi
);
1122 EVT LoMemVT
, HiMemVT
;
1123 GetSplitDestVTs(MemoryVT
, LoMemVT
, HiMemVT
);
1125 unsigned IncrementSize
= LoMemVT
.getSizeInBits()/8;
1128 Lo
= DAG
.getTruncStore(Ch
, DL
, Lo
, Ptr
, N
->getPointerInfo(),
1129 LoMemVT
, isVol
, isNT
, Alignment
);
1131 Lo
= DAG
.getStore(Ch
, DL
, Lo
, Ptr
, N
->getPointerInfo(),
1132 isVol
, isNT
, Alignment
);
1134 // Increment the pointer to the other half.
1135 Ptr
= DAG
.getNode(ISD::ADD
, DL
, Ptr
.getValueType(), Ptr
,
1136 DAG
.getIntPtrConstant(IncrementSize
));
1139 Hi
= DAG
.getTruncStore(Ch
, DL
, Hi
, Ptr
,
1140 N
->getPointerInfo().getWithOffset(IncrementSize
),
1141 HiMemVT
, isVol
, isNT
, Alignment
);
1143 Hi
= DAG
.getStore(Ch
, DL
, Hi
, Ptr
,
1144 N
->getPointerInfo().getWithOffset(IncrementSize
),
1145 isVol
, isNT
, Alignment
);
1147 return DAG
.getNode(ISD::TokenFactor
, DL
, MVT::Other
, Lo
, Hi
);
1150 SDValue
DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode
*N
) {
1151 DebugLoc DL
= N
->getDebugLoc();
1153 // The input operands all must have the same type, and we know the result the
1154 // result type is valid. Convert this to a buildvector which extracts all the
1156 // TODO: If the input elements are power-two vectors, we could convert this to
1157 // a new CONCAT_VECTORS node with elements that are half-wide.
1158 SmallVector
<SDValue
, 32> Elts
;
1159 EVT EltVT
= N
->getValueType(0).getVectorElementType();
1160 for (unsigned op
= 0, e
= N
->getNumOperands(); op
!= e
; ++op
) {
1161 SDValue Op
= N
->getOperand(op
);
1162 for (unsigned i
= 0, e
= Op
.getValueType().getVectorNumElements();
1164 Elts
.push_back(DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, DL
, EltVT
,
1165 Op
, DAG
.getIntPtrConstant(i
)));
1170 return DAG
.getNode(ISD::BUILD_VECTOR
, DL
, N
->getValueType(0),
1171 &Elts
[0], Elts
.size());
1175 //===----------------------------------------------------------------------===//
1176 // Result Vector Widening
1177 //===----------------------------------------------------------------------===//
1179 void DAGTypeLegalizer::WidenVectorResult(SDNode
*N
, unsigned ResNo
) {
1180 DEBUG(dbgs() << "Widen node result " << ResNo
<< ": ";
1184 // See if the target wants to custom widen this node.
1185 if (CustomWidenLowerNode(N
, N
->getValueType(ResNo
)))
1188 SDValue Res
= SDValue();
1189 switch (N
->getOpcode()) {
1192 dbgs() << "WidenVectorResult #" << ResNo
<< ": ";
1196 llvm_unreachable("Do not know how to widen the result of this operator!");
1198 case ISD::BITCAST
: Res
= WidenVecRes_BITCAST(N
); break;
1199 case ISD::BUILD_VECTOR
: Res
= WidenVecRes_BUILD_VECTOR(N
); break;
1200 case ISD::CONCAT_VECTORS
: Res
= WidenVecRes_CONCAT_VECTORS(N
); break;
1201 case ISD::CONVERT_RNDSAT
: Res
= WidenVecRes_CONVERT_RNDSAT(N
); break;
1202 case ISD::EXTRACT_SUBVECTOR
: Res
= WidenVecRes_EXTRACT_SUBVECTOR(N
); break;
1203 case ISD::FP_ROUND_INREG
: Res
= WidenVecRes_InregOp(N
); break;
1204 case ISD::INSERT_VECTOR_ELT
: Res
= WidenVecRes_INSERT_VECTOR_ELT(N
); break;
1205 case ISD::LOAD
: Res
= WidenVecRes_LOAD(N
); break;
1206 case ISD::SCALAR_TO_VECTOR
: Res
= WidenVecRes_SCALAR_TO_VECTOR(N
); break;
1207 case ISD::SIGN_EXTEND_INREG
: Res
= WidenVecRes_InregOp(N
); break;
1208 case ISD::SELECT
: Res
= WidenVecRes_SELECT(N
); break;
1209 case ISD::SELECT_CC
: Res
= WidenVecRes_SELECT_CC(N
); break;
1210 case ISD::SETCC
: Res
= WidenVecRes_SETCC(N
); break;
1211 case ISD::UNDEF
: Res
= WidenVecRes_UNDEF(N
); break;
1212 case ISD::VECTOR_SHUFFLE
:
1213 Res
= WidenVecRes_VECTOR_SHUFFLE(cast
<ShuffleVectorSDNode
>(N
));
1216 Res
= WidenVecRes_VSETCC(N
);
1223 case ISD::FCOPYSIGN
:
1239 Res
= WidenVecRes_Binary(N
);
1243 Res
= WidenVecRes_POWI(N
);
1249 Res
= WidenVecRes_Shift(N
);
1253 case ISD::FP_TO_SINT
:
1254 case ISD::FP_TO_UINT
:
1255 case ISD::SINT_TO_FP
:
1256 case ISD::UINT_TO_FP
:
1258 case ISD::SIGN_EXTEND
:
1259 case ISD::ZERO_EXTEND
:
1260 case ISD::ANY_EXTEND
:
1261 Res
= WidenVecRes_Convert(N
);
1277 Res
= WidenVecRes_Unary(N
);
1281 // If Res is null, the sub-method took care of registering the result.
1283 SetWidenedVector(SDValue(N
, ResNo
), Res
);
1286 SDValue
DAGTypeLegalizer::WidenVecRes_Binary(SDNode
*N
) {
1287 // Binary op widening.
1288 unsigned Opcode
= N
->getOpcode();
1289 DebugLoc dl
= N
->getDebugLoc();
1290 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1291 EVT WidenEltVT
= WidenVT
.getVectorElementType();
1293 unsigned NumElts
= VT
.getVectorNumElements();
1294 while (!TLI
.isTypeLegal(VT
) && NumElts
!= 1) {
1295 NumElts
= NumElts
/ 2;
1296 VT
= EVT::getVectorVT(*DAG
.getContext(), WidenEltVT
, NumElts
);
1299 if (NumElts
!= 1 && !TLI
.canOpTrap(N
->getOpcode(), VT
)) {
1300 // Operation doesn't trap so just widen as normal.
1301 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1302 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1303 return DAG
.getNode(N
->getOpcode(), dl
, WidenVT
, InOp1
, InOp2
);
1306 // No legal vector version so unroll the vector operation and then widen.
1308 return DAG
.UnrollVectorOp(N
, WidenVT
.getVectorNumElements());
1310 // Since the operation can trap, apply operation on the original vector.
1312 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1313 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1314 unsigned CurNumElts
= N
->getValueType(0).getVectorNumElements();
1316 SmallVector
<SDValue
, 16> ConcatOps(CurNumElts
);
1317 unsigned ConcatEnd
= 0; // Current ConcatOps index.
1318 int Idx
= 0; // Current Idx into input vectors.
1320 // NumElts := greatest legal vector size (at most WidenVT)
1321 // while (orig. vector has unhandled elements) {
1322 // take munches of size NumElts from the beginning and add to ConcatOps
1323 // NumElts := next smaller supported vector size or 1
1325 while (CurNumElts
!= 0) {
1326 while (CurNumElts
>= NumElts
) {
1327 SDValue EOp1
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, VT
, InOp1
,
1328 DAG
.getIntPtrConstant(Idx
));
1329 SDValue EOp2
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, VT
, InOp2
,
1330 DAG
.getIntPtrConstant(Idx
));
1331 ConcatOps
[ConcatEnd
++] = DAG
.getNode(Opcode
, dl
, VT
, EOp1
, EOp2
);
1333 CurNumElts
-= NumElts
;
1336 NumElts
= NumElts
/ 2;
1337 VT
= EVT::getVectorVT(*DAG
.getContext(), WidenEltVT
, NumElts
);
1338 } while (!TLI
.isTypeLegal(VT
) && NumElts
!= 1);
1341 for (unsigned i
= 0; i
!= CurNumElts
; ++i
, ++Idx
) {
1342 SDValue EOp1
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, WidenEltVT
,
1343 InOp1
, DAG
.getIntPtrConstant(Idx
));
1344 SDValue EOp2
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, WidenEltVT
,
1345 InOp2
, DAG
.getIntPtrConstant(Idx
));
1346 ConcatOps
[ConcatEnd
++] = DAG
.getNode(Opcode
, dl
, WidenEltVT
,
1353 // Check to see if we have a single operation with the widen type.
1354 if (ConcatEnd
== 1) {
1355 VT
= ConcatOps
[0].getValueType();
1357 return ConcatOps
[0];
1360 // while (Some element of ConcatOps is not of type MaxVT) {
1361 // From the end of ConcatOps, collect elements of the same type and put
1362 // them into an op of the next larger supported type
1364 while (ConcatOps
[ConcatEnd
-1].getValueType() != MaxVT
) {
1365 Idx
= ConcatEnd
- 1;
1366 VT
= ConcatOps
[Idx
--].getValueType();
1367 while (Idx
>= 0 && ConcatOps
[Idx
].getValueType() == VT
)
1370 int NextSize
= VT
.isVector() ? VT
.getVectorNumElements() : 1;
1374 NextVT
= EVT::getVectorVT(*DAG
.getContext(), WidenEltVT
, NextSize
);
1375 } while (!TLI
.isTypeLegal(NextVT
));
1377 if (!VT
.isVector()) {
1378 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
1379 SDValue VecOp
= DAG
.getUNDEF(NextVT
);
1380 unsigned NumToInsert
= ConcatEnd
- Idx
- 1;
1381 for (unsigned i
= 0, OpIdx
= Idx
+1; i
< NumToInsert
; i
++, OpIdx
++) {
1382 VecOp
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
, NextVT
, VecOp
,
1383 ConcatOps
[OpIdx
], DAG
.getIntPtrConstant(i
));
1385 ConcatOps
[Idx
+1] = VecOp
;
1386 ConcatEnd
= Idx
+ 2;
1388 // Vector type, create a CONCAT_VECTORS of type NextVT
1389 SDValue undefVec
= DAG
.getUNDEF(VT
);
1390 unsigned OpsToConcat
= NextSize
/VT
.getVectorNumElements();
1391 SmallVector
<SDValue
, 16> SubConcatOps(OpsToConcat
);
1392 unsigned RealVals
= ConcatEnd
- Idx
- 1;
1393 unsigned SubConcatEnd
= 0;
1394 unsigned SubConcatIdx
= Idx
+ 1;
1395 while (SubConcatEnd
< RealVals
)
1396 SubConcatOps
[SubConcatEnd
++] = ConcatOps
[++Idx
];
1397 while (SubConcatEnd
< OpsToConcat
)
1398 SubConcatOps
[SubConcatEnd
++] = undefVec
;
1399 ConcatOps
[SubConcatIdx
] = DAG
.getNode(ISD::CONCAT_VECTORS
, dl
,
1400 NextVT
, &SubConcatOps
[0],
1402 ConcatEnd
= SubConcatIdx
+ 1;
1406 // Check to see if we have a single operation with the widen type.
1407 if (ConcatEnd
== 1) {
1408 VT
= ConcatOps
[0].getValueType();
1410 return ConcatOps
[0];
1413 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
1414 unsigned NumOps
= WidenVT
.getVectorNumElements()/MaxVT
.getVectorNumElements();
1415 if (NumOps
!= ConcatEnd
) {
1416 SDValue UndefVal
= DAG
.getUNDEF(MaxVT
);
1417 for (unsigned j
= ConcatEnd
; j
< NumOps
; ++j
)
1418 ConcatOps
[j
] = UndefVal
;
1420 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &ConcatOps
[0], NumOps
);
1423 SDValue
DAGTypeLegalizer::WidenVecRes_Convert(SDNode
*N
) {
1424 SDValue InOp
= N
->getOperand(0);
1425 DebugLoc dl
= N
->getDebugLoc();
1427 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1428 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1430 EVT InVT
= InOp
.getValueType();
1431 EVT InEltVT
= InVT
.getVectorElementType();
1432 EVT InWidenVT
= EVT::getVectorVT(*DAG
.getContext(), InEltVT
, WidenNumElts
);
1434 unsigned Opcode
= N
->getOpcode();
1435 unsigned InVTNumElts
= InVT
.getVectorNumElements();
1437 if (getTypeAction(InVT
) == WidenVector
) {
1438 InOp
= GetWidenedVector(N
->getOperand(0));
1439 InVT
= InOp
.getValueType();
1440 InVTNumElts
= InVT
.getVectorNumElements();
1441 if (InVTNumElts
== WidenNumElts
)
1442 return DAG
.getNode(Opcode
, dl
, WidenVT
, InOp
);
1445 if (TLI
.isTypeLegal(InWidenVT
)) {
1446 // Because the result and the input are different vector types, widening
1447 // the result could create a legal type but widening the input might make
1448 // it an illegal type that might lead to repeatedly splitting the input
1449 // and then widening it. To avoid this, we widen the input only if
1450 // it results in a legal type.
1451 if (WidenNumElts
% InVTNumElts
== 0) {
1452 // Widen the input and call convert on the widened input vector.
1453 unsigned NumConcat
= WidenNumElts
/InVTNumElts
;
1454 SmallVector
<SDValue
, 16> Ops(NumConcat
);
1456 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1457 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
1459 return DAG
.getNode(Opcode
, dl
, WidenVT
,
1460 DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, InWidenVT
,
1461 &Ops
[0], NumConcat
));
1464 if (InVTNumElts
% WidenNumElts
== 0) {
1465 // Extract the input and convert the shorten input vector.
1466 return DAG
.getNode(Opcode
, dl
, WidenVT
,
1467 DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InWidenVT
,
1468 InOp
, DAG
.getIntPtrConstant(0)));
1472 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1473 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1474 EVT EltVT
= WidenVT
.getVectorElementType();
1475 unsigned MinElts
= std::min(InVTNumElts
, WidenNumElts
);
1477 for (i
=0; i
< MinElts
; ++i
)
1478 Ops
[i
] = DAG
.getNode(Opcode
, dl
, EltVT
,
1479 DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, InEltVT
, InOp
,
1480 DAG
.getIntPtrConstant(i
)));
1482 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1483 for (; i
< WidenNumElts
; ++i
)
1486 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1489 SDValue
DAGTypeLegalizer::WidenVecRes_POWI(SDNode
*N
) {
1490 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1491 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1492 SDValue ShOp
= N
->getOperand(1);
1493 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), WidenVT
, InOp
, ShOp
);
1496 SDValue
DAGTypeLegalizer::WidenVecRes_Shift(SDNode
*N
) {
1497 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1498 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1499 SDValue ShOp
= N
->getOperand(1);
1501 EVT ShVT
= ShOp
.getValueType();
1502 if (getTypeAction(ShVT
) == WidenVector
) {
1503 ShOp
= GetWidenedVector(ShOp
);
1504 ShVT
= ShOp
.getValueType();
1506 EVT ShWidenVT
= EVT::getVectorVT(*DAG
.getContext(),
1507 ShVT
.getVectorElementType(),
1508 WidenVT
.getVectorNumElements());
1509 if (ShVT
!= ShWidenVT
)
1510 ShOp
= ModifyToType(ShOp
, ShWidenVT
);
1512 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), WidenVT
, InOp
, ShOp
);
1515 SDValue
DAGTypeLegalizer::WidenVecRes_Unary(SDNode
*N
) {
1516 // Unary op widening.
1517 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1518 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1519 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(), WidenVT
, InOp
);
1522 SDValue
DAGTypeLegalizer::WidenVecRes_InregOp(SDNode
*N
) {
1523 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1524 EVT ExtVT
= EVT::getVectorVT(*DAG
.getContext(),
1525 cast
<VTSDNode
>(N
->getOperand(1))->getVT()
1526 .getVectorElementType(),
1527 WidenVT
.getVectorNumElements());
1528 SDValue WidenLHS
= GetWidenedVector(N
->getOperand(0));
1529 return DAG
.getNode(N
->getOpcode(), N
->getDebugLoc(),
1530 WidenVT
, WidenLHS
, DAG
.getValueType(ExtVT
));
1533 SDValue
DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode
*N
) {
1534 SDValue InOp
= N
->getOperand(0);
1535 EVT InVT
= InOp
.getValueType();
1536 EVT VT
= N
->getValueType(0);
1537 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1538 DebugLoc dl
= N
->getDebugLoc();
1540 switch (getTypeAction(InVT
)) {
1542 assert(false && "Unknown type action!");
1546 case PromoteInteger
:
1547 // If the InOp is promoted to the same size, convert it. Otherwise,
1548 // fall out of the switch and widen the promoted input.
1549 InOp
= GetPromotedInteger(InOp
);
1550 InVT
= InOp
.getValueType();
1551 if (WidenVT
.bitsEq(InVT
))
1552 return DAG
.getNode(ISD::BITCAST
, dl
, WidenVT
, InOp
);
1557 case ScalarizeVector
:
1561 // If the InOp is widened to the same size, convert it. Otherwise, fall
1562 // out of the switch and widen the widened input.
1563 InOp
= GetWidenedVector(InOp
);
1564 InVT
= InOp
.getValueType();
1565 if (WidenVT
.bitsEq(InVT
))
1566 // The input widens to the same size. Convert to the widen value.
1567 return DAG
.getNode(ISD::BITCAST
, dl
, WidenVT
, InOp
);
1571 unsigned WidenSize
= WidenVT
.getSizeInBits();
1572 unsigned InSize
= InVT
.getSizeInBits();
1573 // x86mmx is not an acceptable vector element type, so don't try.
1574 if (WidenSize
% InSize
== 0 && InVT
!= MVT::x86mmx
) {
1575 // Determine new input vector type. The new input vector type will use
1576 // the same element type (if its a vector) or use the input type as a
1577 // vector. It is the same size as the type to widen to.
1579 unsigned NewNumElts
= WidenSize
/ InSize
;
1580 if (InVT
.isVector()) {
1581 EVT InEltVT
= InVT
.getVectorElementType();
1582 NewInVT
= EVT::getVectorVT(*DAG
.getContext(), InEltVT
,
1583 WidenSize
/ InEltVT
.getSizeInBits());
1585 NewInVT
= EVT::getVectorVT(*DAG
.getContext(), InVT
, NewNumElts
);
1588 if (TLI
.isTypeLegal(NewInVT
)) {
1589 // Because the result and the input are different vector types, widening
1590 // the result could create a legal type but widening the input might make
1591 // it an illegal type that might lead to repeatedly splitting the input
1592 // and then widening it. To avoid this, we widen the input only if
1593 // it results in a legal type.
1594 SmallVector
<SDValue
, 16> Ops(NewNumElts
);
1595 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1597 for (unsigned i
= 1; i
< NewNumElts
; ++i
)
1601 if (InVT
.isVector())
1602 NewVec
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
,
1603 NewInVT
, &Ops
[0], NewNumElts
);
1605 NewVec
= DAG
.getNode(ISD::BUILD_VECTOR
, dl
,
1606 NewInVT
, &Ops
[0], NewNumElts
);
1607 return DAG
.getNode(ISD::BITCAST
, dl
, WidenVT
, NewVec
);
1611 return CreateStackStoreLoad(InOp
, WidenVT
);
1614 SDValue
DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode
*N
) {
1615 DebugLoc dl
= N
->getDebugLoc();
1616 // Build a vector with undefined for the new nodes.
1617 EVT VT
= N
->getValueType(0);
1618 EVT EltVT
= VT
.getVectorElementType();
1619 unsigned NumElts
= VT
.getVectorNumElements();
1621 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1622 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1624 SmallVector
<SDValue
, 16> NewOps(N
->op_begin(), N
->op_end());
1625 NewOps
.reserve(WidenNumElts
);
1626 for (unsigned i
= NumElts
; i
< WidenNumElts
; ++i
)
1627 NewOps
.push_back(DAG
.getUNDEF(EltVT
));
1629 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &NewOps
[0], NewOps
.size());
1632 SDValue
DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode
*N
) {
1633 EVT InVT
= N
->getOperand(0).getValueType();
1634 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1635 DebugLoc dl
= N
->getDebugLoc();
1636 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1637 unsigned NumOperands
= N
->getNumOperands();
1639 bool InputWidened
= false; // Indicates we need to widen the input.
1640 if (getTypeAction(InVT
) != WidenVector
) {
1641 if (WidenVT
.getVectorNumElements() % InVT
.getVectorNumElements() == 0) {
1642 // Add undef vectors to widen to correct length.
1643 unsigned NumConcat
= WidenVT
.getVectorNumElements() /
1644 InVT
.getVectorNumElements();
1645 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1646 SmallVector
<SDValue
, 16> Ops(NumConcat
);
1647 for (unsigned i
=0; i
< NumOperands
; ++i
)
1648 Ops
[i
] = N
->getOperand(i
);
1649 for (unsigned i
= NumOperands
; i
!= NumConcat
; ++i
)
1651 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &Ops
[0], NumConcat
);
1654 InputWidened
= true;
1655 if (WidenVT
== TLI
.getTypeToTransformTo(*DAG
.getContext(), InVT
)) {
1656 // The inputs and the result are widen to the same value.
1658 for (i
=1; i
< NumOperands
; ++i
)
1659 if (N
->getOperand(i
).getOpcode() != ISD::UNDEF
)
1662 if (i
> NumOperands
)
1663 // Everything but the first operand is an UNDEF so just return the
1664 // widened first operand.
1665 return GetWidenedVector(N
->getOperand(0));
1667 if (NumOperands
== 2) {
1668 // Replace concat of two operands with a shuffle.
1669 SmallVector
<int, 16> MaskOps(WidenNumElts
);
1670 for (unsigned i
=0; i
< WidenNumElts
/2; ++i
) {
1672 MaskOps
[i
+WidenNumElts
/2] = i
+WidenNumElts
;
1674 return DAG
.getVectorShuffle(WidenVT
, dl
,
1675 GetWidenedVector(N
->getOperand(0)),
1676 GetWidenedVector(N
->getOperand(1)),
1682 // Fall back to use extracts and build vector.
1683 EVT EltVT
= WidenVT
.getVectorElementType();
1684 unsigned NumInElts
= InVT
.getVectorNumElements();
1685 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1687 for (unsigned i
=0; i
< NumOperands
; ++i
) {
1688 SDValue InOp
= N
->getOperand(i
);
1690 InOp
= GetWidenedVector(InOp
);
1691 for (unsigned j
=0; j
< NumInElts
; ++j
)
1692 Ops
[Idx
++] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
1693 DAG
.getIntPtrConstant(j
));
1695 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1696 for (; Idx
< WidenNumElts
; ++Idx
)
1697 Ops
[Idx
] = UndefVal
;
1698 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1701 SDValue
DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode
*N
) {
1702 DebugLoc dl
= N
->getDebugLoc();
1703 SDValue InOp
= N
->getOperand(0);
1704 SDValue RndOp
= N
->getOperand(3);
1705 SDValue SatOp
= N
->getOperand(4);
1707 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1708 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1710 EVT InVT
= InOp
.getValueType();
1711 EVT InEltVT
= InVT
.getVectorElementType();
1712 EVT InWidenVT
= EVT::getVectorVT(*DAG
.getContext(), InEltVT
, WidenNumElts
);
1714 SDValue DTyOp
= DAG
.getValueType(WidenVT
);
1715 SDValue STyOp
= DAG
.getValueType(InWidenVT
);
1716 ISD::CvtCode CvtCode
= cast
<CvtRndSatSDNode
>(N
)->getCvtCode();
1718 unsigned InVTNumElts
= InVT
.getVectorNumElements();
1719 if (getTypeAction(InVT
) == WidenVector
) {
1720 InOp
= GetWidenedVector(InOp
);
1721 InVT
= InOp
.getValueType();
1722 InVTNumElts
= InVT
.getVectorNumElements();
1723 if (InVTNumElts
== WidenNumElts
)
1724 return DAG
.getConvertRndSat(WidenVT
, dl
, InOp
, DTyOp
, STyOp
, RndOp
,
1728 if (TLI
.isTypeLegal(InWidenVT
)) {
1729 // Because the result and the input are different vector types, widening
1730 // the result could create a legal type but widening the input might make
1731 // it an illegal type that might lead to repeatedly splitting the input
1732 // and then widening it. To avoid this, we widen the input only if
1733 // it results in a legal type.
1734 if (WidenNumElts
% InVTNumElts
== 0) {
1735 // Widen the input and call convert on the widened input vector.
1736 unsigned NumConcat
= WidenNumElts
/InVTNumElts
;
1737 SmallVector
<SDValue
, 16> Ops(NumConcat
);
1739 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
1740 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
1743 InOp
= DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, InWidenVT
, &Ops
[0],NumConcat
);
1744 return DAG
.getConvertRndSat(WidenVT
, dl
, InOp
, DTyOp
, STyOp
, RndOp
,
1748 if (InVTNumElts
% WidenNumElts
== 0) {
1749 // Extract the input and convert the shorten input vector.
1750 InOp
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, InWidenVT
, InOp
,
1751 DAG
.getIntPtrConstant(0));
1752 return DAG
.getConvertRndSat(WidenVT
, dl
, InOp
, DTyOp
, STyOp
, RndOp
,
1757 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1758 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1759 EVT EltVT
= WidenVT
.getVectorElementType();
1760 DTyOp
= DAG
.getValueType(EltVT
);
1761 STyOp
= DAG
.getValueType(InEltVT
);
1763 unsigned MinElts
= std::min(InVTNumElts
, WidenNumElts
);
1765 for (i
=0; i
< MinElts
; ++i
) {
1766 SDValue ExtVal
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, InEltVT
, InOp
,
1767 DAG
.getIntPtrConstant(i
));
1768 Ops
[i
] = DAG
.getConvertRndSat(WidenVT
, dl
, ExtVal
, DTyOp
, STyOp
, RndOp
,
1772 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1773 for (; i
< WidenNumElts
; ++i
)
1776 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1779 SDValue
DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode
*N
) {
1780 EVT VT
= N
->getValueType(0);
1781 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1782 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1783 SDValue InOp
= N
->getOperand(0);
1784 SDValue Idx
= N
->getOperand(1);
1785 DebugLoc dl
= N
->getDebugLoc();
1787 if (getTypeAction(InOp
.getValueType()) == WidenVector
)
1788 InOp
= GetWidenedVector(InOp
);
1790 EVT InVT
= InOp
.getValueType();
1792 // Check if we can just return the input vector after widening.
1793 uint64_t IdxVal
= cast
<ConstantSDNode
>(Idx
)->getZExtValue();
1794 if (IdxVal
== 0 && InVT
== WidenVT
)
1797 // Check if we can extract from the vector.
1798 unsigned InNumElts
= InVT
.getVectorNumElements();
1799 if (IdxVal
% WidenNumElts
== 0 && IdxVal
+ WidenNumElts
< InNumElts
)
1800 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, WidenVT
, InOp
, Idx
);
1802 // We could try widening the input to the right length but for now, extract
1803 // the original elements, fill the rest with undefs and build a vector.
1804 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
1805 EVT EltVT
= VT
.getVectorElementType();
1806 unsigned NumElts
= VT
.getVectorNumElements();
1808 for (i
=0; i
< NumElts
; ++i
)
1809 Ops
[i
] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
1810 DAG
.getIntPtrConstant(IdxVal
+i
));
1812 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
1813 for (; i
< WidenNumElts
; ++i
)
1815 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], WidenNumElts
);
1818 SDValue
DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode
*N
) {
1819 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
1820 return DAG
.getNode(ISD::INSERT_VECTOR_ELT
, N
->getDebugLoc(),
1821 InOp
.getValueType(), InOp
,
1822 N
->getOperand(1), N
->getOperand(2));
1825 SDValue
DAGTypeLegalizer::WidenVecRes_LOAD(SDNode
*N
) {
1826 LoadSDNode
*LD
= cast
<LoadSDNode
>(N
);
1827 ISD::LoadExtType ExtType
= LD
->getExtensionType();
1830 SmallVector
<SDValue
, 16> LdChain
; // Chain for the series of load
1831 if (ExtType
!= ISD::NON_EXTLOAD
)
1832 Result
= GenWidenVectorExtLoads(LdChain
, LD
, ExtType
);
1834 Result
= GenWidenVectorLoads(LdChain
, LD
);
1836 // If we generate a single load, we can use that for the chain. Otherwise,
1837 // build a factor node to remember the multiple loads are independent and
1840 if (LdChain
.size() == 1)
1841 NewChain
= LdChain
[0];
1843 NewChain
= DAG
.getNode(ISD::TokenFactor
, LD
->getDebugLoc(), MVT::Other
,
1844 &LdChain
[0], LdChain
.size());
1846 // Modified the chain - switch anything that used the old chain to use
1848 ReplaceValueWith(SDValue(N
, 1), NewChain
);
1853 SDValue
DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode
*N
) {
1854 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1855 return DAG
.getNode(ISD::SCALAR_TO_VECTOR
, N
->getDebugLoc(),
1856 WidenVT
, N
->getOperand(0));
1859 SDValue
DAGTypeLegalizer::WidenVecRes_SELECT(SDNode
*N
) {
1860 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1861 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1863 SDValue Cond1
= N
->getOperand(0);
1864 EVT CondVT
= Cond1
.getValueType();
1865 if (CondVT
.isVector()) {
1866 EVT CondEltVT
= CondVT
.getVectorElementType();
1867 EVT CondWidenVT
= EVT::getVectorVT(*DAG
.getContext(),
1868 CondEltVT
, WidenNumElts
);
1869 if (getTypeAction(CondVT
) == WidenVector
)
1870 Cond1
= GetWidenedVector(Cond1
);
1872 if (Cond1
.getValueType() != CondWidenVT
)
1873 Cond1
= ModifyToType(Cond1
, CondWidenVT
);
1876 SDValue InOp1
= GetWidenedVector(N
->getOperand(1));
1877 SDValue InOp2
= GetWidenedVector(N
->getOperand(2));
1878 assert(InOp1
.getValueType() == WidenVT
&& InOp2
.getValueType() == WidenVT
);
1879 return DAG
.getNode(ISD::SELECT
, N
->getDebugLoc(),
1880 WidenVT
, Cond1
, InOp1
, InOp2
);
1883 SDValue
DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode
*N
) {
1884 SDValue InOp1
= GetWidenedVector(N
->getOperand(2));
1885 SDValue InOp2
= GetWidenedVector(N
->getOperand(3));
1886 return DAG
.getNode(ISD::SELECT_CC
, N
->getDebugLoc(),
1887 InOp1
.getValueType(), N
->getOperand(0),
1888 N
->getOperand(1), InOp1
, InOp2
, N
->getOperand(4));
1891 SDValue
DAGTypeLegalizer::WidenVecRes_SETCC(SDNode
*N
) {
1892 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1893 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1894 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1895 return DAG
.getNode(ISD::SETCC
, N
->getDebugLoc(), WidenVT
,
1896 InOp1
, InOp2
, N
->getOperand(2));
1899 SDValue
DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode
*N
) {
1900 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1901 return DAG
.getUNDEF(WidenVT
);
1904 SDValue
DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode
*N
) {
1905 EVT VT
= N
->getValueType(0);
1906 DebugLoc dl
= N
->getDebugLoc();
1908 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), VT
);
1909 unsigned NumElts
= VT
.getVectorNumElements();
1910 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1912 SDValue InOp1
= GetWidenedVector(N
->getOperand(0));
1913 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1915 // Adjust mask based on new input vector length.
1916 SmallVector
<int, 16> NewMask
;
1917 for (unsigned i
= 0; i
!= NumElts
; ++i
) {
1918 int Idx
= N
->getMaskElt(i
);
1919 if (Idx
< (int)NumElts
)
1920 NewMask
.push_back(Idx
);
1922 NewMask
.push_back(Idx
- NumElts
+ WidenNumElts
);
1924 for (unsigned i
= NumElts
; i
!= WidenNumElts
; ++i
)
1925 NewMask
.push_back(-1);
1926 return DAG
.getVectorShuffle(WidenVT
, dl
, InOp1
, InOp2
, &NewMask
[0]);
1929 SDValue
DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode
*N
) {
1930 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), N
->getValueType(0));
1931 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
1933 SDValue InOp1
= N
->getOperand(0);
1934 EVT InVT
= InOp1
.getValueType();
1935 assert(InVT
.isVector() && "can not widen non vector type");
1936 EVT WidenInVT
= EVT::getVectorVT(*DAG
.getContext(),
1937 InVT
.getVectorElementType(), WidenNumElts
);
1938 InOp1
= GetWidenedVector(InOp1
);
1939 SDValue InOp2
= GetWidenedVector(N
->getOperand(1));
1941 // Assume that the input and output will be widen appropriately. If not,
1942 // we will have to unroll it at some point.
1943 assert(InOp1
.getValueType() == WidenInVT
&&
1944 InOp2
.getValueType() == WidenInVT
&&
1945 "Input not widened to expected type!");
1946 return DAG
.getNode(ISD::VSETCC
, N
->getDebugLoc(),
1947 WidenVT
, InOp1
, InOp2
, N
->getOperand(2));
1951 //===----------------------------------------------------------------------===//
1952 // Widen Vector Operand
1953 //===----------------------------------------------------------------------===//
1954 bool DAGTypeLegalizer::WidenVectorOperand(SDNode
*N
, unsigned ResNo
) {
1955 DEBUG(dbgs() << "Widen node operand " << ResNo
<< ": ";
1958 SDValue Res
= SDValue();
1960 switch (N
->getOpcode()) {
1963 dbgs() << "WidenVectorOperand op #" << ResNo
<< ": ";
1967 llvm_unreachable("Do not know how to widen this operator's operand!");
1969 case ISD::BITCAST
: Res
= WidenVecOp_BITCAST(N
); break;
1970 case ISD::CONCAT_VECTORS
: Res
= WidenVecOp_CONCAT_VECTORS(N
); break;
1971 case ISD::EXTRACT_SUBVECTOR
: Res
= WidenVecOp_EXTRACT_SUBVECTOR(N
); break;
1972 case ISD::EXTRACT_VECTOR_ELT
: Res
= WidenVecOp_EXTRACT_VECTOR_ELT(N
); break;
1973 case ISD::STORE
: Res
= WidenVecOp_STORE(N
); break;
1976 case ISD::FP_TO_SINT
:
1977 case ISD::FP_TO_UINT
:
1978 case ISD::SINT_TO_FP
:
1979 case ISD::UINT_TO_FP
:
1981 case ISD::SIGN_EXTEND
:
1982 case ISD::ZERO_EXTEND
:
1983 case ISD::ANY_EXTEND
:
1984 Res
= WidenVecOp_Convert(N
);
1988 // If Res is null, the sub-method took care of registering the result.
1989 if (!Res
.getNode()) return false;
1991 // If the result is N, the sub-method updated N in place. Tell the legalizer
1993 if (Res
.getNode() == N
)
1997 assert(Res
.getValueType() == N
->getValueType(0) && N
->getNumValues() == 1 &&
1998 "Invalid operand expansion");
2000 ReplaceValueWith(SDValue(N
, 0), Res
);
2004 SDValue
DAGTypeLegalizer::WidenVecOp_Convert(SDNode
*N
) {
2005 // Since the result is legal and the input is illegal, it is unlikely
2006 // that we can fix the input to a legal type so unroll the convert
2007 // into some scalar code and create a nasty build vector.
2008 EVT VT
= N
->getValueType(0);
2009 EVT EltVT
= VT
.getVectorElementType();
2010 DebugLoc dl
= N
->getDebugLoc();
2011 unsigned NumElts
= VT
.getVectorNumElements();
2012 SDValue InOp
= N
->getOperand(0);
2013 if (getTypeAction(InOp
.getValueType()) == WidenVector
)
2014 InOp
= GetWidenedVector(InOp
);
2015 EVT InVT
= InOp
.getValueType();
2016 EVT InEltVT
= InVT
.getVectorElementType();
2018 unsigned Opcode
= N
->getOpcode();
2019 SmallVector
<SDValue
, 16> Ops(NumElts
);
2020 for (unsigned i
=0; i
< NumElts
; ++i
)
2021 Ops
[i
] = DAG
.getNode(Opcode
, dl
, EltVT
,
2022 DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, InEltVT
, InOp
,
2023 DAG
.getIntPtrConstant(i
)));
2025 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, VT
, &Ops
[0], NumElts
);
2028 SDValue
DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode
*N
) {
2029 EVT VT
= N
->getValueType(0);
2030 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
2031 EVT InWidenVT
= InOp
.getValueType();
2032 DebugLoc dl
= N
->getDebugLoc();
2034 // Check if we can convert between two legal vector types and extract.
2035 unsigned InWidenSize
= InWidenVT
.getSizeInBits();
2036 unsigned Size
= VT
.getSizeInBits();
2037 // x86mmx is not an acceptable vector element type, so don't try.
2038 if (InWidenSize
% Size
== 0 && !VT
.isVector() && VT
!= MVT::x86mmx
) {
2039 unsigned NewNumElts
= InWidenSize
/ Size
;
2040 EVT NewVT
= EVT::getVectorVT(*DAG
.getContext(), VT
, NewNumElts
);
2041 if (TLI
.isTypeLegal(NewVT
)) {
2042 SDValue BitOp
= DAG
.getNode(ISD::BITCAST
, dl
, NewVT
, InOp
);
2043 return DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, VT
, BitOp
,
2044 DAG
.getIntPtrConstant(0));
2048 return CreateStackStoreLoad(InOp
, VT
);
2051 SDValue
DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode
*N
) {
2052 // If the input vector is not legal, it is likely that we will not find a
2053 // legal vector of the same size. Replace the concatenate vector with a
2054 // nasty build vector.
2055 EVT VT
= N
->getValueType(0);
2056 EVT EltVT
= VT
.getVectorElementType();
2057 DebugLoc dl
= N
->getDebugLoc();
2058 unsigned NumElts
= VT
.getVectorNumElements();
2059 SmallVector
<SDValue
, 16> Ops(NumElts
);
2061 EVT InVT
= N
->getOperand(0).getValueType();
2062 unsigned NumInElts
= InVT
.getVectorNumElements();
2065 unsigned NumOperands
= N
->getNumOperands();
2066 for (unsigned i
=0; i
< NumOperands
; ++i
) {
2067 SDValue InOp
= N
->getOperand(i
);
2068 if (getTypeAction(InOp
.getValueType()) == WidenVector
)
2069 InOp
= GetWidenedVector(InOp
);
2070 for (unsigned j
=0; j
< NumInElts
; ++j
)
2071 Ops
[Idx
++] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
2072 DAG
.getIntPtrConstant(j
));
2074 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, VT
, &Ops
[0], NumElts
);
2077 SDValue
DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode
*N
) {
2078 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
2079 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, N
->getDebugLoc(),
2080 N
->getValueType(0), InOp
, N
->getOperand(1));
2083 SDValue
DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode
*N
) {
2084 SDValue InOp
= GetWidenedVector(N
->getOperand(0));
2085 return DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, N
->getDebugLoc(),
2086 N
->getValueType(0), InOp
, N
->getOperand(1));
2089 SDValue
DAGTypeLegalizer::WidenVecOp_STORE(SDNode
*N
) {
2090 // We have to widen the value but we want only to store the original
2092 StoreSDNode
*ST
= cast
<StoreSDNode
>(N
);
2094 SmallVector
<SDValue
, 16> StChain
;
2095 if (ST
->isTruncatingStore())
2096 GenWidenVectorTruncStores(StChain
, ST
);
2098 GenWidenVectorStores(StChain
, ST
);
2100 if (StChain
.size() == 1)
2103 return DAG
.getNode(ISD::TokenFactor
, ST
->getDebugLoc(),
2104 MVT::Other
,&StChain
[0],StChain
.size());
2107 //===----------------------------------------------------------------------===//
2108 // Vector Widening Utilities
2109 //===----------------------------------------------------------------------===//
2111 // Utility function to find the type to chop up a widen vector for load/store
2112 // TLI: Target lowering used to determine legal types.
2113 // Width: Width left need to load/store.
2114 // WidenVT: The widen vector type to load to/store from
2115 // Align: If 0, don't allow use of a wider type
2116 // WidenEx: If Align is not 0, the amount additional we can load/store from.
2118 static EVT
FindMemType(SelectionDAG
& DAG
, const TargetLowering
&TLI
,
2119 unsigned Width
, EVT WidenVT
,
2120 unsigned Align
= 0, unsigned WidenEx
= 0) {
2121 EVT WidenEltVT
= WidenVT
.getVectorElementType();
2122 unsigned WidenWidth
= WidenVT
.getSizeInBits();
2123 unsigned WidenEltWidth
= WidenEltVT
.getSizeInBits();
2124 unsigned AlignInBits
= Align
*8;
2126 // If we have one element to load/store, return it.
2127 EVT RetVT
= WidenEltVT
;
2128 if (Width
== WidenEltWidth
)
2131 // See if there is larger legal integer than the element type to load/store
2133 for (VT
= (unsigned)MVT::LAST_INTEGER_VALUETYPE
;
2134 VT
>= (unsigned)MVT::FIRST_INTEGER_VALUETYPE
; --VT
) {
2135 EVT
MemVT((MVT::SimpleValueType
) VT
);
2136 unsigned MemVTWidth
= MemVT
.getSizeInBits();
2137 if (MemVT
.getSizeInBits() <= WidenEltWidth
)
2139 if (TLI
.isTypeLegal(MemVT
) && (WidenWidth
% MemVTWidth
) == 0 &&
2140 (MemVTWidth
<= Width
||
2141 (Align
!=0 && MemVTWidth
<=AlignInBits
&& MemVTWidth
<=Width
+WidenEx
))) {
2147 // See if there is a larger vector type to load/store that has the same vector
2148 // element type and is evenly divisible with the WidenVT.
2149 for (VT
= (unsigned)MVT::LAST_VECTOR_VALUETYPE
;
2150 VT
>= (unsigned)MVT::FIRST_VECTOR_VALUETYPE
; --VT
) {
2151 EVT MemVT
= (MVT::SimpleValueType
) VT
;
2152 unsigned MemVTWidth
= MemVT
.getSizeInBits();
2153 if (TLI
.isTypeLegal(MemVT
) && WidenEltVT
== MemVT
.getVectorElementType() &&
2154 (WidenWidth
% MemVTWidth
) == 0 &&
2155 (MemVTWidth
<= Width
||
2156 (Align
!=0 && MemVTWidth
<=AlignInBits
&& MemVTWidth
<=Width
+WidenEx
))) {
2157 if (RetVT
.getSizeInBits() < MemVTWidth
|| MemVT
== WidenVT
)
2165 // Builds a vector type from scalar loads
2166 // VecTy: Resulting Vector type
2167 // LDOps: Load operators to build a vector type
2168 // [Start,End) the list of loads to use.
2169 static SDValue
BuildVectorFromScalar(SelectionDAG
& DAG
, EVT VecTy
,
2170 SmallVector
<SDValue
, 16>& LdOps
,
2171 unsigned Start
, unsigned End
) {
2172 DebugLoc dl
= LdOps
[Start
].getDebugLoc();
2173 EVT LdTy
= LdOps
[Start
].getValueType();
2174 unsigned Width
= VecTy
.getSizeInBits();
2175 unsigned NumElts
= Width
/ LdTy
.getSizeInBits();
2176 EVT NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), LdTy
, NumElts
);
2179 SDValue VecOp
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, NewVecVT
,LdOps
[Start
]);
2181 for (unsigned i
= Start
+ 1; i
!= End
; ++i
) {
2182 EVT NewLdTy
= LdOps
[i
].getValueType();
2183 if (NewLdTy
!= LdTy
) {
2184 NumElts
= Width
/ NewLdTy
.getSizeInBits();
2185 NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), NewLdTy
, NumElts
);
2186 VecOp
= DAG
.getNode(ISD::BITCAST
, dl
, NewVecVT
, VecOp
);
2187 // Readjust position and vector position based on new load type
2188 Idx
= Idx
* LdTy
.getSizeInBits() / NewLdTy
.getSizeInBits();
2191 VecOp
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, dl
, NewVecVT
, VecOp
, LdOps
[i
],
2192 DAG
.getIntPtrConstant(Idx
++));
2194 return DAG
.getNode(ISD::BITCAST
, dl
, VecTy
, VecOp
);
2197 SDValue
DAGTypeLegalizer::GenWidenVectorLoads(SmallVector
<SDValue
, 16> &LdChain
,
2199 // The strategy assumes that we can efficiently load powers of two widths.
2200 // The routines chops the vector into the largest vector loads with the same
2201 // element type or scalar loads and then recombines it to the widen vector
2203 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(),LD
->getValueType(0));
2204 unsigned WidenWidth
= WidenVT
.getSizeInBits();
2205 EVT LdVT
= LD
->getMemoryVT();
2206 DebugLoc dl
= LD
->getDebugLoc();
2207 assert(LdVT
.isVector() && WidenVT
.isVector());
2208 assert(LdVT
.getVectorElementType() == WidenVT
.getVectorElementType());
2211 SDValue Chain
= LD
->getChain();
2212 SDValue BasePtr
= LD
->getBasePtr();
2213 unsigned Align
= LD
->getAlignment();
2214 bool isVolatile
= LD
->isVolatile();
2215 bool isNonTemporal
= LD
->isNonTemporal();
2217 int LdWidth
= LdVT
.getSizeInBits();
2218 int WidthDiff
= WidenWidth
- LdWidth
; // Difference
2219 unsigned LdAlign
= (isVolatile
) ? 0 : Align
; // Allow wider loads
2221 // Find the vector type that can load from.
2222 EVT NewVT
= FindMemType(DAG
, TLI
, LdWidth
, WidenVT
, LdAlign
, WidthDiff
);
2223 int NewVTWidth
= NewVT
.getSizeInBits();
2224 SDValue LdOp
= DAG
.getLoad(NewVT
, dl
, Chain
, BasePtr
, LD
->getPointerInfo(),
2225 isVolatile
, isNonTemporal
, Align
);
2226 LdChain
.push_back(LdOp
.getValue(1));
2228 // Check if we can load the element with one instruction
2229 if (LdWidth
<= NewVTWidth
) {
2230 if (!NewVT
.isVector()) {
2231 unsigned NumElts
= WidenWidth
/ NewVTWidth
;
2232 EVT NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), NewVT
, NumElts
);
2233 SDValue VecOp
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, NewVecVT
, LdOp
);
2234 return DAG
.getNode(ISD::BITCAST
, dl
, WidenVT
, VecOp
);
2236 if (NewVT
== WidenVT
)
2239 assert(WidenWidth
% NewVTWidth
== 0);
2240 unsigned NumConcat
= WidenWidth
/ NewVTWidth
;
2241 SmallVector
<SDValue
, 16> ConcatOps(NumConcat
);
2242 SDValue UndefVal
= DAG
.getUNDEF(NewVT
);
2243 ConcatOps
[0] = LdOp
;
2244 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
2245 ConcatOps
[i
] = UndefVal
;
2246 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &ConcatOps
[0],
2250 // Load vector by using multiple loads from largest vector to scalar
2251 SmallVector
<SDValue
, 16> LdOps
;
2252 LdOps
.push_back(LdOp
);
2254 LdWidth
-= NewVTWidth
;
2255 unsigned Offset
= 0;
2257 while (LdWidth
> 0) {
2258 unsigned Increment
= NewVTWidth
/ 8;
2259 Offset
+= Increment
;
2260 BasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(), BasePtr
,
2261 DAG
.getIntPtrConstant(Increment
));
2263 if (LdWidth
< NewVTWidth
) {
2264 // Our current type we are using is too large, find a better size
2265 NewVT
= FindMemType(DAG
, TLI
, LdWidth
, WidenVT
, LdAlign
, WidthDiff
);
2266 NewVTWidth
= NewVT
.getSizeInBits();
2269 SDValue LdOp
= DAG
.getLoad(NewVT
, dl
, Chain
, BasePtr
,
2270 LD
->getPointerInfo().getWithOffset(Offset
),
2272 isNonTemporal
, MinAlign(Align
, Increment
));
2273 LdChain
.push_back(LdOp
.getValue(1));
2274 LdOps
.push_back(LdOp
);
2276 LdWidth
-= NewVTWidth
;
2279 // Build the vector from the loads operations
2280 unsigned End
= LdOps
.size();
2281 if (!LdOps
[0].getValueType().isVector())
2282 // All the loads are scalar loads.
2283 return BuildVectorFromScalar(DAG
, WidenVT
, LdOps
, 0, End
);
2285 // If the load contains vectors, build the vector using concat vector.
2286 // All of the vectors used to loads are power of 2 and the scalars load
2287 // can be combined to make a power of 2 vector.
2288 SmallVector
<SDValue
, 16> ConcatOps(End
);
2291 EVT LdTy
= LdOps
[i
].getValueType();
2292 // First combine the scalar loads to a vector
2293 if (!LdTy
.isVector()) {
2294 for (--i
; i
>= 0; --i
) {
2295 LdTy
= LdOps
[i
].getValueType();
2296 if (LdTy
.isVector())
2299 ConcatOps
[--Idx
] = BuildVectorFromScalar(DAG
, LdTy
, LdOps
, i
+1, End
);
2301 ConcatOps
[--Idx
] = LdOps
[i
];
2302 for (--i
; i
>= 0; --i
) {
2303 EVT NewLdTy
= LdOps
[i
].getValueType();
2304 if (NewLdTy
!= LdTy
) {
2305 // Create a larger vector
2306 ConcatOps
[End
-1] = DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, NewLdTy
,
2307 &ConcatOps
[Idx
], End
- Idx
);
2311 ConcatOps
[--Idx
] = LdOps
[i
];
2314 if (WidenWidth
== LdTy
.getSizeInBits()*(End
- Idx
))
2315 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
,
2316 &ConcatOps
[Idx
], End
- Idx
);
2318 // We need to fill the rest with undefs to build the vector
2319 unsigned NumOps
= WidenWidth
/ LdTy
.getSizeInBits();
2320 SmallVector
<SDValue
, 16> WidenOps(NumOps
);
2321 SDValue UndefVal
= DAG
.getUNDEF(LdTy
);
2324 for (; i
!= End
-Idx
; ++i
)
2325 WidenOps
[i
] = ConcatOps
[Idx
+i
];
2326 for (; i
!= NumOps
; ++i
)
2327 WidenOps
[i
] = UndefVal
;
2329 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, WidenVT
, &WidenOps
[0],NumOps
);
2333 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVector
<SDValue
, 16>& LdChain
,
2335 ISD::LoadExtType ExtType
) {
2336 // For extension loads, it may not be more efficient to chop up the vector
2337 // and then extended it. Instead, we unroll the load and build a new vector.
2338 EVT WidenVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(),LD
->getValueType(0));
2339 EVT LdVT
= LD
->getMemoryVT();
2340 DebugLoc dl
= LD
->getDebugLoc();
2341 assert(LdVT
.isVector() && WidenVT
.isVector());
2344 SDValue Chain
= LD
->getChain();
2345 SDValue BasePtr
= LD
->getBasePtr();
2346 unsigned Align
= LD
->getAlignment();
2347 bool isVolatile
= LD
->isVolatile();
2348 bool isNonTemporal
= LD
->isNonTemporal();
2350 EVT EltVT
= WidenVT
.getVectorElementType();
2351 EVT LdEltVT
= LdVT
.getVectorElementType();
2352 unsigned NumElts
= LdVT
.getVectorNumElements();
2354 // Load each element and widen
2355 unsigned WidenNumElts
= WidenVT
.getVectorNumElements();
2356 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
2357 unsigned Increment
= LdEltVT
.getSizeInBits() / 8;
2358 Ops
[0] = DAG
.getExtLoad(ExtType
, EltVT
, dl
, Chain
, BasePtr
,
2359 LD
->getPointerInfo(),
2360 LdEltVT
, isVolatile
, isNonTemporal
, Align
);
2361 LdChain
.push_back(Ops
[0].getValue(1));
2362 unsigned i
= 0, Offset
= Increment
;
2363 for (i
=1; i
< NumElts
; ++i
, Offset
+= Increment
) {
2364 SDValue NewBasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(),
2365 BasePtr
, DAG
.getIntPtrConstant(Offset
));
2366 Ops
[i
] = DAG
.getExtLoad(ExtType
, EltVT
, dl
, Chain
, NewBasePtr
,
2367 LD
->getPointerInfo().getWithOffset(Offset
), LdEltVT
,
2368 isVolatile
, isNonTemporal
, Align
);
2369 LdChain
.push_back(Ops
[i
].getValue(1));
2372 // Fill the rest with undefs
2373 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
2374 for (; i
!= WidenNumElts
; ++i
)
2377 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, WidenVT
, &Ops
[0], Ops
.size());
2381 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector
<SDValue
, 16>& StChain
,
2383 // The strategy assumes that we can efficiently store powers of two widths.
2384 // The routines chops the vector into the largest vector stores with the same
2385 // element type or scalar stores.
2386 SDValue Chain
= ST
->getChain();
2387 SDValue BasePtr
= ST
->getBasePtr();
2388 unsigned Align
= ST
->getAlignment();
2389 bool isVolatile
= ST
->isVolatile();
2390 bool isNonTemporal
= ST
->isNonTemporal();
2391 SDValue ValOp
= GetWidenedVector(ST
->getValue());
2392 DebugLoc dl
= ST
->getDebugLoc();
2394 EVT StVT
= ST
->getMemoryVT();
2395 unsigned StWidth
= StVT
.getSizeInBits();
2396 EVT ValVT
= ValOp
.getValueType();
2397 unsigned ValWidth
= ValVT
.getSizeInBits();
2398 EVT ValEltVT
= ValVT
.getVectorElementType();
2399 unsigned ValEltWidth
= ValEltVT
.getSizeInBits();
2400 assert(StVT
.getVectorElementType() == ValEltVT
);
2402 int Idx
= 0; // current index to store
2403 unsigned Offset
= 0; // offset from base to store
2404 while (StWidth
!= 0) {
2405 // Find the largest vector type we can store with
2406 EVT NewVT
= FindMemType(DAG
, TLI
, StWidth
, ValVT
);
2407 unsigned NewVTWidth
= NewVT
.getSizeInBits();
2408 unsigned Increment
= NewVTWidth
/ 8;
2409 if (NewVT
.isVector()) {
2410 unsigned NumVTElts
= NewVT
.getVectorNumElements();
2412 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, NewVT
, ValOp
,
2413 DAG
.getIntPtrConstant(Idx
));
2414 StChain
.push_back(DAG
.getStore(Chain
, dl
, EOp
, BasePtr
,
2415 ST
->getPointerInfo().getWithOffset(Offset
),
2416 isVolatile
, isNonTemporal
,
2417 MinAlign(Align
, Offset
)));
2418 StWidth
-= NewVTWidth
;
2419 Offset
+= Increment
;
2421 BasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(), BasePtr
,
2422 DAG
.getIntPtrConstant(Increment
));
2423 } while (StWidth
!= 0 && StWidth
>= NewVTWidth
);
2425 // Cast the vector to the scalar type we can store
2426 unsigned NumElts
= ValWidth
/ NewVTWidth
;
2427 EVT NewVecVT
= EVT::getVectorVT(*DAG
.getContext(), NewVT
, NumElts
);
2428 SDValue VecOp
= DAG
.getNode(ISD::BITCAST
, dl
, NewVecVT
, ValOp
);
2429 // Readjust index position based on new vector type
2430 Idx
= Idx
* ValEltWidth
/ NewVTWidth
;
2432 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, NewVT
, VecOp
,
2433 DAG
.getIntPtrConstant(Idx
++));
2434 StChain
.push_back(DAG
.getStore(Chain
, dl
, EOp
, BasePtr
,
2435 ST
->getPointerInfo().getWithOffset(Offset
),
2436 isVolatile
, isNonTemporal
,
2437 MinAlign(Align
, Offset
)));
2438 StWidth
-= NewVTWidth
;
2439 Offset
+= Increment
;
2440 BasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(), BasePtr
,
2441 DAG
.getIntPtrConstant(Increment
));
2442 } while (StWidth
!= 0 && StWidth
>= NewVTWidth
);
2443 // Restore index back to be relative to the original widen element type
2444 Idx
= Idx
* NewVTWidth
/ ValEltWidth
;
2450 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVector
<SDValue
, 16>& StChain
,
2452 // For extension loads, it may not be more efficient to truncate the vector
2453 // and then store it. Instead, we extract each element and then store it.
2454 SDValue Chain
= ST
->getChain();
2455 SDValue BasePtr
= ST
->getBasePtr();
2456 unsigned Align
= ST
->getAlignment();
2457 bool isVolatile
= ST
->isVolatile();
2458 bool isNonTemporal
= ST
->isNonTemporal();
2459 SDValue ValOp
= GetWidenedVector(ST
->getValue());
2460 DebugLoc dl
= ST
->getDebugLoc();
2462 EVT StVT
= ST
->getMemoryVT();
2463 EVT ValVT
= ValOp
.getValueType();
2465 // It must be true that we the widen vector type is bigger than where
2466 // we need to store.
2467 assert(StVT
.isVector() && ValOp
.getValueType().isVector());
2468 assert(StVT
.bitsLT(ValOp
.getValueType()));
2470 // For truncating stores, we can not play the tricks of chopping legal
2471 // vector types and bit cast it to the right type. Instead, we unroll
2473 EVT StEltVT
= StVT
.getVectorElementType();
2474 EVT ValEltVT
= ValVT
.getVectorElementType();
2475 unsigned Increment
= ValEltVT
.getSizeInBits() / 8;
2476 unsigned NumElts
= StVT
.getVectorNumElements();
2477 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, ValEltVT
, ValOp
,
2478 DAG
.getIntPtrConstant(0));
2479 StChain
.push_back(DAG
.getTruncStore(Chain
, dl
, EOp
, BasePtr
,
2480 ST
->getPointerInfo(), StEltVT
,
2481 isVolatile
, isNonTemporal
, Align
));
2482 unsigned Offset
= Increment
;
2483 for (unsigned i
=1; i
< NumElts
; ++i
, Offset
+= Increment
) {
2484 SDValue NewBasePtr
= DAG
.getNode(ISD::ADD
, dl
, BasePtr
.getValueType(),
2485 BasePtr
, DAG
.getIntPtrConstant(Offset
));
2486 SDValue EOp
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, ValEltVT
, ValOp
,
2487 DAG
.getIntPtrConstant(0));
2488 StChain
.push_back(DAG
.getTruncStore(Chain
, dl
, EOp
, NewBasePtr
,
2489 ST
->getPointerInfo().getWithOffset(Offset
),
2490 StEltVT
, isVolatile
, isNonTemporal
,
2491 MinAlign(Align
, Offset
)));
2495 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
2496 /// input vector must have the same element type as NVT.
2497 SDValue
DAGTypeLegalizer::ModifyToType(SDValue InOp
, EVT NVT
) {
2498 // Note that InOp might have been widened so it might already have
2499 // the right width or it might need be narrowed.
2500 EVT InVT
= InOp
.getValueType();
2501 assert(InVT
.getVectorElementType() == NVT
.getVectorElementType() &&
2502 "input and widen element type must match");
2503 DebugLoc dl
= InOp
.getDebugLoc();
2505 // Check if InOp already has the right width.
2509 unsigned InNumElts
= InVT
.getVectorNumElements();
2510 unsigned WidenNumElts
= NVT
.getVectorNumElements();
2511 if (WidenNumElts
> InNumElts
&& WidenNumElts
% InNumElts
== 0) {
2512 unsigned NumConcat
= WidenNumElts
/ InNumElts
;
2513 SmallVector
<SDValue
, 16> Ops(NumConcat
);
2514 SDValue UndefVal
= DAG
.getUNDEF(InVT
);
2516 for (unsigned i
= 1; i
!= NumConcat
; ++i
)
2519 return DAG
.getNode(ISD::CONCAT_VECTORS
, dl
, NVT
, &Ops
[0], NumConcat
);
2522 if (WidenNumElts
< InNumElts
&& InNumElts
% WidenNumElts
)
2523 return DAG
.getNode(ISD::EXTRACT_SUBVECTOR
, dl
, NVT
, InOp
,
2524 DAG
.getIntPtrConstant(0));
2526 // Fall back to extract and build.
2527 SmallVector
<SDValue
, 16> Ops(WidenNumElts
);
2528 EVT EltVT
= NVT
.getVectorElementType();
2529 unsigned MinNumElts
= std::min(WidenNumElts
, InNumElts
);
2531 for (Idx
= 0; Idx
< MinNumElts
; ++Idx
)
2532 Ops
[Idx
] = DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, InOp
,
2533 DAG
.getIntPtrConstant(Idx
));
2535 SDValue UndefVal
= DAG
.getUNDEF(EltVT
);
2536 for ( ; Idx
< WidenNumElts
; ++Idx
)
2537 Ops
[Idx
] = UndefVal
;
2538 return DAG
.getNode(ISD::BUILD_VECTOR
, dl
, NVT
, &Ops
[0], WidenNumElts
);