1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 /// GetFPLibCall - Return the right libcall for the given floating point type.
30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31 /// refactored away (see RTLIB::getPOWI for an example).
GetFPLibCall(EVT VT,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128)32 static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45 }
46
47 //===----------------------------------------------------------------------===//
48 // Convert Float Results to Integer
49 //===----------------------------------------------------------------------===//
50
SoftenFloatResult(SDNode * N,unsigned ResNo)51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58 #ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61 #endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::STRICT_FACOS:
80 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
81 case ISD::STRICT_FASIN:
82 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
83 case ISD::STRICT_FATAN:
84 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
85 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
86 case ISD::STRICT_FCEIL:
87 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
88 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
89 case ISD::STRICT_FCOS:
90 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
91 case ISD::STRICT_FCOSH:
92 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
93 case ISD::STRICT_FDIV:
94 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
95 case ISD::STRICT_FEXP:
96 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
97 case ISD::STRICT_FEXP2:
98 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
99 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
100 case ISD::STRICT_FFLOOR:
101 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
102 case ISD::STRICT_FLOG:
103 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
104 case ISD::STRICT_FLOG2:
105 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
106 case ISD::STRICT_FLOG10:
107 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
108 case ISD::STRICT_FMA:
109 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
110 case ISD::STRICT_FMUL:
111 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
112 case ISD::STRICT_FNEARBYINT:
113 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
114 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
115 case ISD::STRICT_FP_EXTEND:
116 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
117 case ISD::STRICT_FP_ROUND:
118 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
119 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
120 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
121 case ISD::STRICT_FPOW:
122 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
123 case ISD::STRICT_FPOWI:
124 case ISD::FPOWI:
125 case ISD::FLDEXP:
126 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
127 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
128 case ISD::STRICT_FREM:
129 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
130 case ISD::STRICT_FRINT:
131 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
132 case ISD::STRICT_FROUND:
133 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
134 case ISD::STRICT_FROUNDEVEN:
135 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
136 case ISD::STRICT_FSIN:
137 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
138 case ISD::STRICT_FSINH:
139 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
140 case ISD::STRICT_FSQRT:
141 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
142 case ISD::STRICT_FSUB:
143 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
144 case ISD::STRICT_FTAN:
145 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
146 case ISD::STRICT_FTANH:
147 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
148 case ISD::STRICT_FTRUNC:
149 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
150 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
151 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
152 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
153 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
154 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
155 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
156 case ISD::STRICT_SINT_TO_FP:
157 case ISD::STRICT_UINT_TO_FP:
158 case ISD::SINT_TO_FP:
159 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
160 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
161 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
162 case ISD::VECREDUCE_FADD:
163 case ISD::VECREDUCE_FMUL:
164 case ISD::VECREDUCE_FMIN:
165 case ISD::VECREDUCE_FMAX:
166 case ISD::VECREDUCE_FMAXIMUM:
167 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
168 case ISD::VECREDUCE_SEQ_FADD:
169 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
170 // clang-format on
171 }
172
173 // If R is null, the sub-method took care of registering the result.
174 if (R.getNode()) {
175 assert(R.getNode() != N);
176 SetSoftenedFloat(SDValue(N, ResNo), R);
177 }
178 }
179
SoftenFloatRes_Unary(SDNode * N,RTLIB::Libcall LC)180 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
181 bool IsStrict = N->isStrictFPOpcode();
182 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
183 unsigned Offset = IsStrict ? 1 : 0;
184 assert(N->getNumOperands() == (1 + Offset) &&
185 "Unexpected number of operands!");
186 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
187 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
188 TargetLowering::MakeLibCallOptions CallOptions;
189 EVT OpVT = N->getOperand(0 + Offset).getValueType();
190 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
192 CallOptions, SDLoc(N),
193 Chain);
194 if (IsStrict)
195 ReplaceValueWith(SDValue(N, 1), Tmp.second);
196 return Tmp.first;
197 }
198
SoftenFloatRes_Binary(SDNode * N,RTLIB::Libcall LC)199 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
200 bool IsStrict = N->isStrictFPOpcode();
201 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
202 unsigned Offset = IsStrict ? 1 : 0;
203 assert(N->getNumOperands() == (2 + Offset) &&
204 "Unexpected number of operands!");
205 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
206 GetSoftenedFloat(N->getOperand(1 + Offset)) };
207 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
208 TargetLowering::MakeLibCallOptions CallOptions;
209 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
210 N->getOperand(1 + Offset).getValueType() };
211 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
212 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
213 CallOptions, SDLoc(N),
214 Chain);
215 if (IsStrict)
216 ReplaceValueWith(SDValue(N, 1), Tmp.second);
217 return Tmp.first;
218 }
219
SoftenFloatRes_BITCAST(SDNode * N)220 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
221 return BitConvertToInteger(N->getOperand(0));
222 }
223
SoftenFloatRes_FREEZE(SDNode * N)224 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
225 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
226 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
227 GetSoftenedFloat(N->getOperand(0)));
228 }
229
SoftenFloatRes_ARITH_FENCE(SDNode * N)230 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
231 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
232 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
233 GetSoftenedFloat(N->getOperand(0)));
234 return NewFence;
235 }
236
SoftenFloatRes_MERGE_VALUES(SDNode * N,unsigned ResNo)237 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
238 unsigned ResNo) {
239 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
240 return BitConvertToInteger(Op);
241 }
242
SoftenFloatRes_BUILD_PAIR(SDNode * N)243 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
244 // Convert the inputs to integers, and build a new pair out of them.
245 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
246 TLI.getTypeToTransformTo(*DAG.getContext(),
247 N->getValueType(0)),
248 BitConvertToInteger(N->getOperand(0)),
249 BitConvertToInteger(N->getOperand(1)));
250 }
251
SoftenFloatRes_ConstantFP(SDNode * N)252 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
253 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
254 // In ppcf128, the high 64 bits are always first in memory regardless
255 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
256 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
257 // way. However, APInt's are serialized in an Endian-sensitive fashion,
258 // so on big-Endian targets, the two doubles are output in the wrong
259 // order. Fix this by manually flipping the order of the high 64 bits
260 // and the low 64 bits here.
261 if (DAG.getDataLayout().isBigEndian() &&
262 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
263 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
264 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
265 APInt Val(128, words);
266 return DAG.getConstant(Val, SDLoc(CN),
267 TLI.getTypeToTransformTo(*DAG.getContext(),
268 CN->getValueType(0)));
269 } else {
270 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
271 TLI.getTypeToTransformTo(*DAG.getContext(),
272 CN->getValueType(0)));
273 }
274 }
275
SoftenFloatRes_EXTRACT_ELEMENT(SDNode * N)276 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
277 SDValue Src = N->getOperand(0);
278 assert(Src.getValueType() == MVT::ppcf128 &&
279 "In floats only ppcf128 can be extracted by element!");
280 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
281 N->getValueType(0).changeTypeToInteger(),
282 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
283 }
284
SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode * N,unsigned ResNo)285 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
286 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
287 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
288 NewOp.getValueType().getVectorElementType(),
289 NewOp, N->getOperand(1));
290 }
291
SoftenFloatRes_FABS(SDNode * N)292 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
293 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
294 unsigned Size = NVT.getSizeInBits();
295
296 // Mask = ~(1 << (Size-1))
297 APInt API = APInt::getAllOnes(Size);
298 API.clearBit(Size - 1);
299 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
300 SDValue Op = GetSoftenedFloat(N->getOperand(0));
301 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
302 }
303
SoftenFloatRes_FMINNUM(SDNode * N)304 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
305 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
306 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::FMIN_F32,
309 RTLIB::FMIN_F64,
310 RTLIB::FMIN_F80,
311 RTLIB::FMIN_F128,
312 RTLIB::FMIN_PPCF128));
313 }
314
SoftenFloatRes_FMAXNUM(SDNode * N)315 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
316 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
317 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
318 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
319 RTLIB::FMAX_F32,
320 RTLIB::FMAX_F64,
321 RTLIB::FMAX_F80,
322 RTLIB::FMAX_F128,
323 RTLIB::FMAX_PPCF128));
324 }
325
SoftenFloatRes_FADD(SDNode * N)326 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
327 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
328 RTLIB::ADD_F32,
329 RTLIB::ADD_F64,
330 RTLIB::ADD_F80,
331 RTLIB::ADD_F128,
332 RTLIB::ADD_PPCF128));
333 }
334
SoftenFloatRes_FACOS(SDNode * N)335 SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
336 return SoftenFloatRes_Unary(
337 N, GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64,
338 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128));
339 }
340
SoftenFloatRes_FASIN(SDNode * N)341 SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
342 return SoftenFloatRes_Unary(
343 N, GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64,
344 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128));
345 }
346
SoftenFloatRes_FATAN(SDNode * N)347 SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
348 return SoftenFloatRes_Unary(
349 N, GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64,
350 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128));
351 }
352
SoftenFloatRes_FCBRT(SDNode * N)353 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
354 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
355 RTLIB::CBRT_F32,
356 RTLIB::CBRT_F64,
357 RTLIB::CBRT_F80,
358 RTLIB::CBRT_F128,
359 RTLIB::CBRT_PPCF128));
360 }
361
SoftenFloatRes_FCEIL(SDNode * N)362 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
363 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
364 RTLIB::CEIL_F32,
365 RTLIB::CEIL_F64,
366 RTLIB::CEIL_F80,
367 RTLIB::CEIL_F128,
368 RTLIB::CEIL_PPCF128));
369 }
370
SoftenFloatRes_FCOPYSIGN(SDNode * N)371 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
372 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
373 SDValue RHS = BitConvertToInteger(N->getOperand(1));
374 SDLoc dl(N);
375
376 EVT LVT = LHS.getValueType();
377 EVT RVT = RHS.getValueType();
378
379 unsigned LSize = LVT.getSizeInBits();
380 unsigned RSize = RVT.getSizeInBits();
381
382 // First get the sign bit of second operand.
383 SDValue SignBit = DAG.getNode(
384 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
385 DAG.getConstant(RSize - 1, dl,
386 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
387 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
388
389 // Shift right or sign-extend it if the two operands have different types.
390 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
391 if (SizeDiff > 0) {
392 SignBit =
393 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
394 DAG.getConstant(SizeDiff, dl,
395 TLI.getShiftAmountTy(SignBit.getValueType(),
396 DAG.getDataLayout())));
397 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
398 } else if (SizeDiff < 0) {
399 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
400 SignBit =
401 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
402 DAG.getConstant(-SizeDiff, dl,
403 TLI.getShiftAmountTy(SignBit.getValueType(),
404 DAG.getDataLayout())));
405 }
406
407 // Clear the sign bit of the first operand.
408 SDValue Mask = DAG.getNode(
409 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
410 DAG.getConstant(LSize - 1, dl,
411 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
412 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
413 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
414
415 // Or the value with the sign bit.
416 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
417 }
418
SoftenFloatRes_FCOS(SDNode * N)419 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
420 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
421 RTLIB::COS_F32,
422 RTLIB::COS_F64,
423 RTLIB::COS_F80,
424 RTLIB::COS_F128,
425 RTLIB::COS_PPCF128));
426 }
427
SoftenFloatRes_FCOSH(SDNode * N)428 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
429 return SoftenFloatRes_Unary(
430 N, GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64,
431 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128));
432 }
433
SoftenFloatRes_FDIV(SDNode * N)434 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
435 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
436 RTLIB::DIV_F32,
437 RTLIB::DIV_F64,
438 RTLIB::DIV_F80,
439 RTLIB::DIV_F128,
440 RTLIB::DIV_PPCF128));
441 }
442
SoftenFloatRes_FEXP(SDNode * N)443 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
444 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
445 RTLIB::EXP_F32,
446 RTLIB::EXP_F64,
447 RTLIB::EXP_F80,
448 RTLIB::EXP_F128,
449 RTLIB::EXP_PPCF128));
450 }
451
SoftenFloatRes_FEXP2(SDNode * N)452 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
453 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
454 RTLIB::EXP2_F32,
455 RTLIB::EXP2_F64,
456 RTLIB::EXP2_F80,
457 RTLIB::EXP2_F128,
458 RTLIB::EXP2_PPCF128));
459 }
460
SoftenFloatRes_FEXP10(SDNode * N)461 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
462 return SoftenFloatRes_Unary(
463 N,
464 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
465 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
466 }
467
SoftenFloatRes_FFLOOR(SDNode * N)468 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
469 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
470 RTLIB::FLOOR_F32,
471 RTLIB::FLOOR_F64,
472 RTLIB::FLOOR_F80,
473 RTLIB::FLOOR_F128,
474 RTLIB::FLOOR_PPCF128));
475 }
476
SoftenFloatRes_FLOG(SDNode * N)477 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
478 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
479 RTLIB::LOG_F32,
480 RTLIB::LOG_F64,
481 RTLIB::LOG_F80,
482 RTLIB::LOG_F128,
483 RTLIB::LOG_PPCF128));
484 }
485
SoftenFloatRes_FLOG2(SDNode * N)486 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
487 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
488 RTLIB::LOG2_F32,
489 RTLIB::LOG2_F64,
490 RTLIB::LOG2_F80,
491 RTLIB::LOG2_F128,
492 RTLIB::LOG2_PPCF128));
493 }
494
SoftenFloatRes_FLOG10(SDNode * N)495 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
497 RTLIB::LOG10_F32,
498 RTLIB::LOG10_F64,
499 RTLIB::LOG10_F80,
500 RTLIB::LOG10_F128,
501 RTLIB::LOG10_PPCF128));
502 }
503
SoftenFloatRes_FMA(SDNode * N)504 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
505 bool IsStrict = N->isStrictFPOpcode();
506 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
507 unsigned Offset = IsStrict ? 1 : 0;
508 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
509 GetSoftenedFloat(N->getOperand(1 + Offset)),
510 GetSoftenedFloat(N->getOperand(2 + Offset)) };
511 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
512 TargetLowering::MakeLibCallOptions CallOptions;
513 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
514 N->getOperand(1 + Offset).getValueType(),
515 N->getOperand(2 + Offset).getValueType() };
516 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
517 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
518 GetFPLibCall(N->getValueType(0),
519 RTLIB::FMA_F32,
520 RTLIB::FMA_F64,
521 RTLIB::FMA_F80,
522 RTLIB::FMA_F128,
523 RTLIB::FMA_PPCF128),
524 NVT, Ops, CallOptions, SDLoc(N), Chain);
525 if (IsStrict)
526 ReplaceValueWith(SDValue(N, 1), Tmp.second);
527 return Tmp.first;
528 }
529
SoftenFloatRes_FMUL(SDNode * N)530 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
531 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
532 RTLIB::MUL_F32,
533 RTLIB::MUL_F64,
534 RTLIB::MUL_F80,
535 RTLIB::MUL_F128,
536 RTLIB::MUL_PPCF128));
537 }
538
SoftenFloatRes_FNEARBYINT(SDNode * N)539 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
540 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
541 RTLIB::NEARBYINT_F32,
542 RTLIB::NEARBYINT_F64,
543 RTLIB::NEARBYINT_F80,
544 RTLIB::NEARBYINT_F128,
545 RTLIB::NEARBYINT_PPCF128));
546 }
547
SoftenFloatRes_FNEG(SDNode * N)548 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
549 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
550 SDLoc dl(N);
551
552 // Expand Y = FNEG(X) -> Y = X ^ sign mask
553 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
554 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
555 DAG.getConstant(SignMask, dl, NVT));
556 }
557
SoftenFloatRes_FP_EXTEND(SDNode * N)558 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
559 bool IsStrict = N->isStrictFPOpcode();
560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
561 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
562
563 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
564
565 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
566 Op = GetPromotedFloat(Op);
567 // If the promotion did the FP_EXTEND to the destination type for us,
568 // there's nothing left to do here.
569 if (Op.getValueType() == N->getValueType(0)) {
570 if (IsStrict)
571 ReplaceValueWith(SDValue(N, 1), Chain);
572 return BitConvertToInteger(Op);
573 }
574 }
575
576 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
577 // -> f32, so proceed in two stages. Also, it's entirely possible for both
578 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
579 // than FP16_TO_FP.
580 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
581 N->getValueType(0) != MVT::f32) {
582 if (IsStrict) {
583 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
584 { MVT::f32, MVT::Other }, { Chain, Op });
585 Chain = Op.getValue(1);
586 } else {
587 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
588 }
589 }
590
591 if (Op.getValueType() == MVT::bf16) {
592 // FIXME: Need ReplaceValueWith on chain in strict case
593 return SoftenFloatRes_BF16_TO_FP(N);
594 }
595
596 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
597 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
598 TargetLowering::MakeLibCallOptions CallOptions;
599 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
600 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
601 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
602 CallOptions, SDLoc(N),
603 Chain);
604 if (IsStrict)
605 ReplaceValueWith(SDValue(N, 1), Tmp.second);
606 return Tmp.first;
607 }
608
609 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
610 // nodes?
SoftenFloatRes_FP16_TO_FP(SDNode * N)611 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
612 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
613 SDValue Op = N->getOperand(0);
614 TargetLowering::MakeLibCallOptions CallOptions;
615 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
616 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
617 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
618 CallOptions, SDLoc(N)).first;
619 if (N->getValueType(0) == MVT::f32)
620 return Res32;
621
622 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
623 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
624 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
625 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
626 }
627
628 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
629 // nodes?
SoftenFloatRes_BF16_TO_FP(SDNode * N)630 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
631 assert(N->getValueType(0) == MVT::f32 &&
632 "Can only soften BF16_TO_FP with f32 result");
633 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
634 SDValue Op = N->getOperand(0);
635 SDLoc DL(N);
636 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
637 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
638 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
639 DAG.getShiftAmountConstant(16, NVT, DL));
640 return Res;
641 }
642
SoftenFloatRes_FP_ROUND(SDNode * N)643 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
644 bool IsStrict = N->isStrictFPOpcode();
645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
646 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
647 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
648 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
650 TargetLowering::MakeLibCallOptions CallOptions;
651 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
652 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
653 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
654 CallOptions, SDLoc(N),
655 Chain);
656 if (IsStrict)
657 ReplaceValueWith(SDValue(N, 1), Tmp.second);
658 return Tmp.first;
659 }
660
SoftenFloatRes_FPOW(SDNode * N)661 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
662 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
663 RTLIB::POW_F32,
664 RTLIB::POW_F64,
665 RTLIB::POW_F80,
666 RTLIB::POW_F128,
667 RTLIB::POW_PPCF128));
668 }
669
SoftenFloatRes_ExpOp(SDNode * N)670 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
671 bool IsStrict = N->isStrictFPOpcode();
672 unsigned Offset = IsStrict ? 1 : 0;
673 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
674 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
675 "Unsupported power type!");
676 bool IsPowI =
677 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
678
679 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
680 : RTLIB::getLDEXP(N->getValueType(0));
681 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
682 if (!TLI.getLibcallName(LC)) {
683 // Some targets don't have a powi libcall; use pow instead.
684 // FIXME: Implement this if some target needs it.
685 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
686 return DAG.getUNDEF(N->getValueType(0));
687 }
688
689 if (DAG.getLibInfo().getIntSize() !=
690 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
691 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
692 // would use the wrong type for the argument.
693 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
694 return DAG.getUNDEF(N->getValueType(0));
695 }
696
697 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
698 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
699 N->getOperand(1 + Offset) };
700 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
701 TargetLowering::MakeLibCallOptions CallOptions;
702 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
703 N->getOperand(1 + Offset).getValueType() };
704 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
705 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
706 CallOptions, SDLoc(N),
707 Chain);
708 if (IsStrict)
709 ReplaceValueWith(SDValue(N, 1), Tmp.second);
710 return Tmp.first;
711 }
712
SoftenFloatRes_FFREXP(SDNode * N)713 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
714 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
715 EVT VT0 = N->getValueType(0);
716 EVT VT1 = N->getValueType(1);
717 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
718
719 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
720 // If the exponent does not match with sizeof(int) a libcall would use the
721 // wrong type for the argument.
722 // TODO: Should be able to handle mismatches.
723 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
724 return DAG.getUNDEF(N->getValueType(0));
725 }
726
727 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
728 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
729
730 SDLoc DL(N);
731
732 TargetLowering::MakeLibCallOptions CallOptions;
733 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
734 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
735
736 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
737 // but we only really need to handle the 0th one for softening anyway.
738 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
739
740 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
741 /*Chain=*/SDValue());
742 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
743 auto PtrInfo =
744 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
745
746 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
747
748 ReplaceValueWith(SDValue(N, 1), LoadExp);
749 return ReturnVal;
750 }
751
SoftenFloatRes_FREM(SDNode * N)752 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
753 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
754 RTLIB::REM_F32,
755 RTLIB::REM_F64,
756 RTLIB::REM_F80,
757 RTLIB::REM_F128,
758 RTLIB::REM_PPCF128));
759 }
760
SoftenFloatRes_FRINT(SDNode * N)761 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
762 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
763 RTLIB::RINT_F32,
764 RTLIB::RINT_F64,
765 RTLIB::RINT_F80,
766 RTLIB::RINT_F128,
767 RTLIB::RINT_PPCF128));
768 }
769
SoftenFloatRes_FROUND(SDNode * N)770 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
771 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
772 RTLIB::ROUND_F32,
773 RTLIB::ROUND_F64,
774 RTLIB::ROUND_F80,
775 RTLIB::ROUND_F128,
776 RTLIB::ROUND_PPCF128));
777 }
778
SoftenFloatRes_FROUNDEVEN(SDNode * N)779 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
780 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
781 RTLIB::ROUNDEVEN_F32,
782 RTLIB::ROUNDEVEN_F64,
783 RTLIB::ROUNDEVEN_F80,
784 RTLIB::ROUNDEVEN_F128,
785 RTLIB::ROUNDEVEN_PPCF128));
786 }
787
SoftenFloatRes_FSIN(SDNode * N)788 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
789 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
790 RTLIB::SIN_F32,
791 RTLIB::SIN_F64,
792 RTLIB::SIN_F80,
793 RTLIB::SIN_F128,
794 RTLIB::SIN_PPCF128));
795 }
796
SoftenFloatRes_FSINH(SDNode * N)797 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
798 return SoftenFloatRes_Unary(
799 N, GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64,
800 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128));
801 }
802
SoftenFloatRes_FSQRT(SDNode * N)803 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
804 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
805 RTLIB::SQRT_F32,
806 RTLIB::SQRT_F64,
807 RTLIB::SQRT_F80,
808 RTLIB::SQRT_F128,
809 RTLIB::SQRT_PPCF128));
810 }
811
SoftenFloatRes_FSUB(SDNode * N)812 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
813 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
814 RTLIB::SUB_F32,
815 RTLIB::SUB_F64,
816 RTLIB::SUB_F80,
817 RTLIB::SUB_F128,
818 RTLIB::SUB_PPCF128));
819 }
820
SoftenFloatRes_FTAN(SDNode * N)821 SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
822 return SoftenFloatRes_Unary(
823 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
824 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
825 }
826
SoftenFloatRes_FTANH(SDNode * N)827 SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
828 return SoftenFloatRes_Unary(
829 N, GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64,
830 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128));
831 }
832
SoftenFloatRes_FTRUNC(SDNode * N)833 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
834 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
835 RTLIB::TRUNC_F32,
836 RTLIB::TRUNC_F64,
837 RTLIB::TRUNC_F80,
838 RTLIB::TRUNC_F128,
839 RTLIB::TRUNC_PPCF128));
840 }
841
SoftenFloatRes_LOAD(SDNode * N)842 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
843 LoadSDNode *L = cast<LoadSDNode>(N);
844 EVT VT = N->getValueType(0);
845 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
846 SDLoc dl(N);
847
848 auto MMOFlags =
849 L->getMemOperand()->getFlags() &
850 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
851 SDValue NewL;
852 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
853 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
854 L->getChain(), L->getBasePtr(), L->getOffset(),
855 L->getPointerInfo(), NVT, L->getOriginalAlign(),
856 MMOFlags, L->getAAInfo());
857 // Legalized the chain result - switch anything that used the old chain to
858 // use the new one.
859 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
860 return NewL;
861 }
862
863 // Do a non-extending load followed by FP_EXTEND.
864 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
865 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
866 L->getPointerInfo(), L->getMemoryVT(),
867 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
868 // Legalized the chain result - switch anything that used the old chain to
869 // use the new one.
870 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
871 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
872 return BitConvertToInteger(ExtendNode);
873 }
874
SoftenFloatRes_ATOMIC_LOAD(SDNode * N)875 SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
876 AtomicSDNode *L = cast<AtomicSDNode>(N);
877 EVT VT = N->getValueType(0);
878 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
879 SDLoc dl(N);
880
881 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
882 SDValue NewL =
883 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
884 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
885
886 // Legalized the chain result - switch anything that used the old chain to
887 // use the new one.
888 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
889 return NewL;
890 }
891
892 report_fatal_error("softening fp extending atomic load not handled");
893 }
894
SoftenFloatRes_SELECT(SDNode * N)895 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
896 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
897 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
898 return DAG.getSelect(SDLoc(N),
899 LHS.getValueType(), N->getOperand(0), LHS, RHS);
900 }
901
SoftenFloatRes_SELECT_CC(SDNode * N)902 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
903 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
904 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
905 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
906 LHS.getValueType(), N->getOperand(0),
907 N->getOperand(1), LHS, RHS, N->getOperand(4));
908 }
909
SoftenFloatRes_UNDEF(SDNode * N)910 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
911 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
912 N->getValueType(0)));
913 }
914
SoftenFloatRes_VAARG(SDNode * N)915 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
916 SDValue Chain = N->getOperand(0); // Get the chain.
917 SDValue Ptr = N->getOperand(1); // Get the pointer.
918 EVT VT = N->getValueType(0);
919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
920 SDLoc dl(N);
921
922 SDValue NewVAARG;
923 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
924 N->getConstantOperandVal(3));
925
926 // Legalized the chain result - switch anything that used the old chain to
927 // use the new one.
928 if (N != NewVAARG.getValue(1).getNode())
929 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
930 return NewVAARG;
931 }
932
SoftenFloatRes_XINT_TO_FP(SDNode * N)933 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
934 bool IsStrict = N->isStrictFPOpcode();
935 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
936 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
937 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
938 EVT RVT = N->getValueType(0);
939 EVT NVT = EVT();
940 SDLoc dl(N);
941
942 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
943 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
944 // match. Look for an appropriate libcall.
945 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
946 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
947 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
948 NVT = (MVT::SimpleValueType)t;
949 // The source needs to big enough to hold the operand.
950 if (NVT.bitsGE(SVT))
951 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
952 }
953 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
954
955 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
956 // Sign/zero extend the argument if the libcall takes a larger type.
957 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
958 NVT, N->getOperand(IsStrict ? 1 : 0));
959 TargetLowering::MakeLibCallOptions CallOptions;
960 CallOptions.setSExt(Signed);
961 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
962 std::pair<SDValue, SDValue> Tmp =
963 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
964 Op, CallOptions, dl, Chain);
965
966 if (IsStrict)
967 ReplaceValueWith(SDValue(N, 1), Tmp.second);
968 return Tmp.first;
969 }
970
SoftenFloatRes_VECREDUCE(SDNode * N)971 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
972 // Expand and soften recursively.
973 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
974 return SDValue();
975 }
976
SoftenFloatRes_VECREDUCE_SEQ(SDNode * N)977 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
978 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
979 return SDValue();
980 }
981
982 //===----------------------------------------------------------------------===//
983 // Convert Float Operand to Integer
984 //===----------------------------------------------------------------------===//
985
SoftenFloatOperand(SDNode * N,unsigned OpNo)986 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
987 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
988 SDValue Res = SDValue();
989
990 switch (N->getOpcode()) {
991 default:
992 #ifndef NDEBUG
993 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
994 N->dump(&DAG); dbgs() << "\n";
995 #endif
996 report_fatal_error("Do not know how to soften this operator's operand!");
997
998 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
999 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1000 case ISD::STRICT_FP_TO_FP16:
1001 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1002 case ISD::FP_TO_BF16:
1003 case ISD::STRICT_FP_TO_BF16:
1004 case ISD::STRICT_FP_ROUND:
1005 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1006 case ISD::STRICT_FP_TO_SINT:
1007 case ISD::STRICT_FP_TO_UINT:
1008 case ISD::FP_TO_SINT:
1009 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1010 case ISD::FP_TO_SINT_SAT:
1011 case ISD::FP_TO_UINT_SAT:
1012 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1013 case ISD::STRICT_LROUND:
1014 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1015 case ISD::STRICT_LLROUND:
1016 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1017 case ISD::STRICT_LRINT:
1018 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1019 case ISD::STRICT_LLRINT:
1020 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1021 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1022 case ISD::STRICT_FSETCC:
1023 case ISD::STRICT_FSETCCS:
1024 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1025 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1026 case ISD::ATOMIC_STORE:
1027 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1028 break;
1029 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1030 }
1031
1032 // If the result is null, the sub-method took care of registering results etc.
1033 if (!Res.getNode()) return false;
1034
1035 // If the result is N, the sub-method updated N in place. Tell the legalizer
1036 // core about this to re-analyze.
1037 if (Res.getNode() == N)
1038 return true;
1039
1040 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1041 "Invalid operand softening");
1042
1043 ReplaceValueWith(SDValue(N, 0), Res);
1044 return false;
1045 }
1046
SoftenFloatOp_BITCAST(SDNode * N)1047 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1048 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
1049
1050 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
1051 }
1052
SoftenFloatOp_FP_ROUND(SDNode * N)1053 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1054 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1055 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1056 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1057 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1058 N->getOpcode() == ISD::FP_TO_BF16 ||
1059 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1060 N->getOpcode() == ISD::STRICT_FP_ROUND);
1061
1062 bool IsStrict = N->isStrictFPOpcode();
1063 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1064 EVT SVT = Op.getValueType();
1065 EVT RVT = N->getValueType(0);
1066 EVT FloatRVT = RVT;
1067 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1068 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1069 FloatRVT = MVT::f16;
1070 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1071 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1072 FloatRVT = MVT::bf16;
1073
1074 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1075 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1076
1077 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1078 Op = GetSoftenedFloat(Op);
1079 TargetLowering::MakeLibCallOptions CallOptions;
1080 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1081 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1082 CallOptions, SDLoc(N),
1083 Chain);
1084 if (IsStrict) {
1085 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1086 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1087 return SDValue();
1088 }
1089 return Tmp.first;
1090 }
1091
SoftenFloatOp_BR_CC(SDNode * N)1092 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1093 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1094 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1095
1096 EVT VT = NewLHS.getValueType();
1097 NewLHS = GetSoftenedFloat(NewLHS);
1098 NewRHS = GetSoftenedFloat(NewRHS);
1099 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1100 N->getOperand(2), N->getOperand(3));
1101
1102 // If softenSetCCOperands returned a scalar, we need to compare the result
1103 // against zero to select between true and false values.
1104 if (!NewRHS.getNode()) {
1105 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1106 CCCode = ISD::SETNE;
1107 }
1108
1109 // Update N to have the operands specified.
1110 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1111 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1112 N->getOperand(4)),
1113 0);
1114 }
1115
1116 // Even if the result type is legal, no libcall may exactly match. (e.g. We
1117 // don't have FP-i8 conversions) This helper method looks for an appropriate
1118 // promoted libcall.
findFPToIntLibcall(EVT SrcVT,EVT RetVT,EVT & Promoted,bool Signed)1119 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1120 bool Signed) {
1121 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1122 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1123 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1124 ++IntVT) {
1125 Promoted = (MVT::SimpleValueType)IntVT;
1126 // The type needs to big enough to hold the result.
1127 if (Promoted.bitsGE(RetVT))
1128 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1129 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1130 }
1131 return LC;
1132 }
1133
SoftenFloatOp_FP_TO_XINT(SDNode * N)1134 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1135 bool IsStrict = N->isStrictFPOpcode();
1136 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1137 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1138
1139 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1140 EVT SVT = Op.getValueType();
1141 EVT RVT = N->getValueType(0);
1142 EVT NVT = EVT();
1143 SDLoc dl(N);
1144
1145 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1146 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1147 // match, eg. we don't have fp -> i8 conversions.
1148 // Look for an appropriate libcall.
1149 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1150 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1151 "Unsupported FP_TO_XINT!");
1152
1153 Op = GetSoftenedFloat(Op);
1154 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1155 TargetLowering::MakeLibCallOptions CallOptions;
1156 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1157 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1158 CallOptions, dl, Chain);
1159
1160 // Truncate the result if the libcall returns a larger type.
1161 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1162
1163 if (!IsStrict)
1164 return Res;
1165
1166 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1167 ReplaceValueWith(SDValue(N, 0), Res);
1168 return SDValue();
1169 }
1170
SoftenFloatOp_FP_TO_XINT_SAT(SDNode * N)1171 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1172 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1173 return Res;
1174 }
1175
SoftenFloatOp_SELECT_CC(SDNode * N)1176 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1177 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1178 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1179
1180 EVT VT = NewLHS.getValueType();
1181 NewLHS = GetSoftenedFloat(NewLHS);
1182 NewRHS = GetSoftenedFloat(NewRHS);
1183 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1184 N->getOperand(0), N->getOperand(1));
1185
1186 // If softenSetCCOperands returned a scalar, we need to compare the result
1187 // against zero to select between true and false values.
1188 if (!NewRHS.getNode()) {
1189 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1190 CCCode = ISD::SETNE;
1191 }
1192
1193 // Update N to have the operands specified.
1194 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1195 N->getOperand(2), N->getOperand(3),
1196 DAG.getCondCode(CCCode)),
1197 0);
1198 }
1199
SoftenFloatOp_SETCC(SDNode * N)1200 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1201 bool IsStrict = N->isStrictFPOpcode();
1202 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1203 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1204 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1205 ISD::CondCode CCCode =
1206 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1207
1208 EVT VT = Op0.getValueType();
1209 SDValue NewLHS = GetSoftenedFloat(Op0);
1210 SDValue NewRHS = GetSoftenedFloat(Op1);
1211 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1212 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1213
1214 // Update N to have the operands specified.
1215 if (NewRHS.getNode()) {
1216 if (IsStrict)
1217 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1218 NewRHS, DAG.getCondCode(CCCode));
1219 else
1220 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1221 DAG.getCondCode(CCCode)), 0);
1222 }
1223
1224 // Otherwise, softenSetCCOperands returned a scalar, use it.
1225 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1226 "Unexpected setcc expansion!");
1227
1228 if (IsStrict) {
1229 ReplaceValueWith(SDValue(N, 0), NewLHS);
1230 ReplaceValueWith(SDValue(N, 1), Chain);
1231 return SDValue();
1232 }
1233 return NewLHS;
1234 }
1235
SoftenFloatOp_STORE(SDNode * N,unsigned OpNo)1236 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1237 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1238 assert(OpNo == 1 && "Can only soften the stored value!");
1239 StoreSDNode *ST = cast<StoreSDNode>(N);
1240 SDValue Val = ST->getValue();
1241 SDLoc dl(N);
1242
1243 if (ST->isTruncatingStore())
1244 // Do an FP_ROUND followed by a non-truncating store.
1245 Val = BitConvertToInteger(
1246 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1247 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1248 else
1249 Val = GetSoftenedFloat(Val);
1250
1251 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1252 ST->getMemOperand());
1253 }
1254
SoftenFloatOp_ATOMIC_STORE(SDNode * N,unsigned OpNo)1255 SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1256 assert(OpNo == 1 && "Can only soften the stored value!");
1257 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1258 SDValue Val = ST->getVal();
1259 EVT VT = Val.getValueType();
1260 SDLoc dl(N);
1261
1262 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1263
1264 SDValue NewVal = GetSoftenedFloat(Val);
1265 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1266 ST->getBasePtr(), ST->getMemOperand());
1267 }
1268
SoftenFloatOp_FCOPYSIGN(SDNode * N)1269 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1270 SDValue LHS = N->getOperand(0);
1271 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1272 SDLoc dl(N);
1273
1274 EVT LVT = LHS.getValueType();
1275 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1276 EVT RVT = RHS.getValueType();
1277
1278 unsigned LSize = LVT.getSizeInBits();
1279 unsigned RSize = RVT.getSizeInBits();
1280
1281 // Shift right or sign-extend it if the two operands have different types.
1282 int SizeDiff = RSize - LSize;
1283 if (SizeDiff > 0) {
1284 RHS =
1285 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1286 DAG.getConstant(SizeDiff, dl,
1287 TLI.getShiftAmountTy(RHS.getValueType(),
1288 DAG.getDataLayout())));
1289 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1290 } else if (SizeDiff < 0) {
1291 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1292 RHS =
1293 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1294 DAG.getConstant(-SizeDiff, dl,
1295 TLI.getShiftAmountTy(RHS.getValueType(),
1296 DAG.getDataLayout())));
1297 }
1298
1299 RHS = DAG.getBitcast(LVT, RHS);
1300 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1301 }
1302
SoftenFloatOp_Unary(SDNode * N,RTLIB::Libcall LC)1303 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1304 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1305 bool IsStrict = N->isStrictFPOpcode();
1306 unsigned Offset = IsStrict ? 1 : 0;
1307 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1308 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1309 TargetLowering::MakeLibCallOptions CallOptions;
1310 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1311 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1312 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1313 CallOptions, SDLoc(N),
1314 Chain);
1315 if (IsStrict) {
1316 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1317 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1318 return SDValue();
1319 }
1320
1321 return Tmp.first;
1322 }
1323
SoftenFloatOp_LROUND(SDNode * N)1324 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1325 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1326 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1327 RTLIB::LROUND_F32,
1328 RTLIB::LROUND_F64,
1329 RTLIB::LROUND_F80,
1330 RTLIB::LROUND_F128,
1331 RTLIB::LROUND_PPCF128));
1332 }
1333
SoftenFloatOp_LLROUND(SDNode * N)1334 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1335 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1336 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1337 RTLIB::LLROUND_F32,
1338 RTLIB::LLROUND_F64,
1339 RTLIB::LLROUND_F80,
1340 RTLIB::LLROUND_F128,
1341 RTLIB::LLROUND_PPCF128));
1342 }
1343
SoftenFloatOp_LRINT(SDNode * N)1344 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1345 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1346 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1347 RTLIB::LRINT_F32,
1348 RTLIB::LRINT_F64,
1349 RTLIB::LRINT_F80,
1350 RTLIB::LRINT_F128,
1351 RTLIB::LRINT_PPCF128));
1352 }
1353
SoftenFloatOp_LLRINT(SDNode * N)1354 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1355 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1356 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1357 RTLIB::LLRINT_F32,
1358 RTLIB::LLRINT_F64,
1359 RTLIB::LLRINT_F80,
1360 RTLIB::LLRINT_F128,
1361 RTLIB::LLRINT_PPCF128));
1362 }
1363
1364 //===----------------------------------------------------------------------===//
1365 // Float Result Expansion
1366 //===----------------------------------------------------------------------===//
1367
1368 /// ExpandFloatResult - This method is called when the specified result of the
1369 /// specified node is found to need expansion. At this point, the node may also
1370 /// have invalid operands or may have other results that need promotion, we just
1371 /// know that (at least) one result needs expansion.
ExpandFloatResult(SDNode * N,unsigned ResNo)1372 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1373 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1374 SDValue Lo, Hi;
1375 Lo = Hi = SDValue();
1376
1377 // See if the target wants to custom expand this node.
1378 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1379 return;
1380
1381 switch (N->getOpcode()) {
1382 default:
1383 #ifndef NDEBUG
1384 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1385 N->dump(&DAG); dbgs() << "\n";
1386 #endif
1387 report_fatal_error("Do not know how to expand the result of this "
1388 "operator!");
1389 // clang-format off
1390 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1391 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1392 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1393
1394 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1395 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1396 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1397 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1398 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1399 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1400
1401 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1402 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1403 case ISD::STRICT_FMINNUM:
1404 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1405 case ISD::STRICT_FMAXNUM:
1406 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1407 case ISD::STRICT_FADD:
1408 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1409 case ISD::STRICT_FACOS:
1410 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1411 case ISD::STRICT_FASIN:
1412 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1413 case ISD::STRICT_FATAN:
1414 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1415 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1416 case ISD::STRICT_FCEIL:
1417 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1418 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1419 case ISD::STRICT_FCOS:
1420 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1421 case ISD::STRICT_FCOSH:
1422 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1423 case ISD::STRICT_FDIV:
1424 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1425 case ISD::STRICT_FEXP:
1426 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1427 case ISD::STRICT_FEXP2:
1428 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1429 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1430 case ISD::STRICT_FFLOOR:
1431 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1432 case ISD::STRICT_FLOG:
1433 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1434 case ISD::STRICT_FLOG2:
1435 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1436 case ISD::STRICT_FLOG10:
1437 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1438 case ISD::STRICT_FMA:
1439 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1440 case ISD::STRICT_FMUL:
1441 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1442 case ISD::STRICT_FNEARBYINT:
1443 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1444 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1445 case ISD::STRICT_FP_EXTEND:
1446 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1447 case ISD::STRICT_FPOW:
1448 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1449 case ISD::STRICT_FPOWI:
1450 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1451 case ISD::FLDEXP:
1452 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1453 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1454 case ISD::STRICT_FRINT:
1455 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1456 case ISD::STRICT_FROUND:
1457 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1458 case ISD::STRICT_FROUNDEVEN:
1459 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1460 case ISD::STRICT_FSIN:
1461 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1462 case ISD::STRICT_FSINH:
1463 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1464 case ISD::STRICT_FSQRT:
1465 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1466 case ISD::STRICT_FSUB:
1467 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1468 case ISD::STRICT_FTAN:
1469 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1470 case ISD::STRICT_FTANH:
1471 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1472 case ISD::STRICT_FTRUNC:
1473 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1474 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1475 case ISD::STRICT_SINT_TO_FP:
1476 case ISD::STRICT_UINT_TO_FP:
1477 case ISD::SINT_TO_FP:
1478 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1479 case ISD::STRICT_FREM:
1480 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1481 // clang-format on
1482 }
1483
1484 // If Lo/Hi is null, the sub-method took care of registering results etc.
1485 if (Lo.getNode())
1486 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1487 }
1488
ExpandFloatRes_ConstantFP(SDNode * N,SDValue & Lo,SDValue & Hi)1489 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1490 SDValue &Hi) {
1491 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1492 assert(NVT.getSizeInBits() == 64 &&
1493 "Do not know how to expand this float constant!");
1494 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1495 SDLoc dl(N);
1496 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1497 APInt(64, C.getRawData()[1])),
1498 dl, NVT);
1499 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1500 APInt(64, C.getRawData()[0])),
1501 dl, NVT);
1502 }
1503
ExpandFloatRes_Unary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1504 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1505 SDValue &Lo, SDValue &Hi) {
1506 bool IsStrict = N->isStrictFPOpcode();
1507 unsigned Offset = IsStrict ? 1 : 0;
1508 SDValue Op = N->getOperand(0 + Offset);
1509 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1510 TargetLowering::MakeLibCallOptions CallOptions;
1511 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1512 Op, CallOptions, SDLoc(N),
1513 Chain);
1514 if (IsStrict)
1515 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1516 GetPairElements(Tmp.first, Lo, Hi);
1517 }
1518
ExpandFloatRes_Binary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1519 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1520 SDValue &Lo, SDValue &Hi) {
1521 bool IsStrict = N->isStrictFPOpcode();
1522 unsigned Offset = IsStrict ? 1 : 0;
1523 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1524 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1525 TargetLowering::MakeLibCallOptions CallOptions;
1526 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1527 Ops, CallOptions, SDLoc(N),
1528 Chain);
1529 if (IsStrict)
1530 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1531 GetPairElements(Tmp.first, Lo, Hi);
1532 }
1533
ExpandFloatRes_FABS(SDNode * N,SDValue & Lo,SDValue & Hi)1534 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1535 SDValue &Hi) {
1536 assert(N->getValueType(0) == MVT::ppcf128 &&
1537 "Logic only correct for ppcf128!");
1538 SDLoc dl(N);
1539 SDValue Tmp;
1540 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1541 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1542 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1543 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1544 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1545 ISD::SETEQ);
1546 }
1547
ExpandFloatRes_FMINNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1548 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1549 SDValue &Hi) {
1550 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1551 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1552 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1553 RTLIB::FMIN_PPCF128), Lo, Hi);
1554 }
1555
ExpandFloatRes_FMAXNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1556 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1557 SDValue &Hi) {
1558 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1559 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1560 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1561 RTLIB::FMAX_PPCF128), Lo, Hi);
1562 }
1563
ExpandFloatRes_FADD(SDNode * N,SDValue & Lo,SDValue & Hi)1564 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1567 RTLIB::ADD_F32, RTLIB::ADD_F64,
1568 RTLIB::ADD_F80, RTLIB::ADD_F128,
1569 RTLIB::ADD_PPCF128), Lo, Hi);
1570 }
1571
ExpandFloatRes_FACOS(SDNode * N,SDValue & Lo,SDValue & Hi)1572 void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1573 SDValue &Hi) {
1574 ExpandFloatRes_Unary(N,
1575 GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32,
1576 RTLIB::ACOS_F64, RTLIB::ACOS_F80,
1577 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128),
1578 Lo, Hi);
1579 }
1580
ExpandFloatRes_FASIN(SDNode * N,SDValue & Lo,SDValue & Hi)1581 void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1582 SDValue &Hi) {
1583 ExpandFloatRes_Unary(N,
1584 GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32,
1585 RTLIB::ASIN_F64, RTLIB::ASIN_F80,
1586 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128),
1587 Lo, Hi);
1588 }
1589
ExpandFloatRes_FATAN(SDNode * N,SDValue & Lo,SDValue & Hi)1590 void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1591 SDValue &Hi) {
1592 ExpandFloatRes_Unary(N,
1593 GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32,
1594 RTLIB::ATAN_F64, RTLIB::ATAN_F80,
1595 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128),
1596 Lo, Hi);
1597 }
1598
ExpandFloatRes_FCBRT(SDNode * N,SDValue & Lo,SDValue & Hi)1599 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1600 SDValue &Hi) {
1601 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1602 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1603 RTLIB::CBRT_F128,
1604 RTLIB::CBRT_PPCF128), Lo, Hi);
1605 }
1606
ExpandFloatRes_FCEIL(SDNode * N,SDValue & Lo,SDValue & Hi)1607 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1608 SDValue &Lo, SDValue &Hi) {
1609 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1610 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1611 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1612 RTLIB::CEIL_PPCF128), Lo, Hi);
1613 }
1614
ExpandFloatRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1615 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1616 SDValue &Lo, SDValue &Hi) {
1617 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1618 RTLIB::COPYSIGN_F32,
1619 RTLIB::COPYSIGN_F64,
1620 RTLIB::COPYSIGN_F80,
1621 RTLIB::COPYSIGN_F128,
1622 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1623 }
1624
ExpandFloatRes_FCOS(SDNode * N,SDValue & Lo,SDValue & Hi)1625 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1626 SDValue &Lo, SDValue &Hi) {
1627 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1628 RTLIB::COS_F32, RTLIB::COS_F64,
1629 RTLIB::COS_F80, RTLIB::COS_F128,
1630 RTLIB::COS_PPCF128), Lo, Hi);
1631 }
1632
ExpandFloatRes_FCOSH(SDNode * N,SDValue & Lo,SDValue & Hi)1633 void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1634 SDValue &Hi) {
1635 ExpandFloatRes_Unary(N,
1636 GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32,
1637 RTLIB::COSH_F64, RTLIB::COSH_F80,
1638 RTLIB::COSH_F128, RTLIB::COSH_PPCF128),
1639 Lo, Hi);
1640 }
1641
ExpandFloatRes_FDIV(SDNode * N,SDValue & Lo,SDValue & Hi)1642 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1643 SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1645 RTLIB::DIV_F32,
1646 RTLIB::DIV_F64,
1647 RTLIB::DIV_F80,
1648 RTLIB::DIV_F128,
1649 RTLIB::DIV_PPCF128), Lo, Hi);
1650 }
1651
ExpandFloatRes_FEXP(SDNode * N,SDValue & Lo,SDValue & Hi)1652 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1653 SDValue &Lo, SDValue &Hi) {
1654 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1655 RTLIB::EXP_F32, RTLIB::EXP_F64,
1656 RTLIB::EXP_F80, RTLIB::EXP_F128,
1657 RTLIB::EXP_PPCF128), Lo, Hi);
1658 }
1659
ExpandFloatRes_FEXP2(SDNode * N,SDValue & Lo,SDValue & Hi)1660 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1661 SDValue &Lo, SDValue &Hi) {
1662 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1663 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1664 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1665 RTLIB::EXP2_PPCF128), Lo, Hi);
1666 }
1667
ExpandFloatRes_FEXP10(SDNode * N,SDValue & Lo,SDValue & Hi)1668 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1669 SDValue &Hi) {
1670 ExpandFloatRes_Unary(N,
1671 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1672 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1673 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1674 Lo, Hi);
1675 }
1676
ExpandFloatRes_FFLOOR(SDNode * N,SDValue & Lo,SDValue & Hi)1677 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1678 SDValue &Lo, SDValue &Hi) {
1679 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1680 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1681 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1682 RTLIB::FLOOR_PPCF128), Lo, Hi);
1683 }
1684
ExpandFloatRes_FLOG(SDNode * N,SDValue & Lo,SDValue & Hi)1685 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1686 SDValue &Lo, SDValue &Hi) {
1687 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1688 RTLIB::LOG_F32, RTLIB::LOG_F64,
1689 RTLIB::LOG_F80, RTLIB::LOG_F128,
1690 RTLIB::LOG_PPCF128), Lo, Hi);
1691 }
1692
ExpandFloatRes_FLOG2(SDNode * N,SDValue & Lo,SDValue & Hi)1693 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1694 SDValue &Lo, SDValue &Hi) {
1695 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1696 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1697 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1698 RTLIB::LOG2_PPCF128), Lo, Hi);
1699 }
1700
ExpandFloatRes_FLOG10(SDNode * N,SDValue & Lo,SDValue & Hi)1701 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1702 SDValue &Lo, SDValue &Hi) {
1703 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1704 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1705 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1706 RTLIB::LOG10_PPCF128), Lo, Hi);
1707 }
1708
ExpandFloatRes_FMA(SDNode * N,SDValue & Lo,SDValue & Hi)1709 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1710 SDValue &Hi) {
1711 bool IsStrict = N->isStrictFPOpcode();
1712 unsigned Offset = IsStrict ? 1 : 0;
1713 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1714 N->getOperand(2 + Offset) };
1715 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1716 TargetLowering::MakeLibCallOptions CallOptions;
1717 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1718 RTLIB::FMA_F32,
1719 RTLIB::FMA_F64,
1720 RTLIB::FMA_F80,
1721 RTLIB::FMA_F128,
1722 RTLIB::FMA_PPCF128),
1723 N->getValueType(0), Ops, CallOptions,
1724 SDLoc(N), Chain);
1725 if (IsStrict)
1726 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1727 GetPairElements(Tmp.first, Lo, Hi);
1728 }
1729
ExpandFloatRes_FMUL(SDNode * N,SDValue & Lo,SDValue & Hi)1730 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1731 SDValue &Hi) {
1732 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1733 RTLIB::MUL_F32,
1734 RTLIB::MUL_F64,
1735 RTLIB::MUL_F80,
1736 RTLIB::MUL_F128,
1737 RTLIB::MUL_PPCF128), Lo, Hi);
1738 }
1739
ExpandFloatRes_FNEARBYINT(SDNode * N,SDValue & Lo,SDValue & Hi)1740 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1741 SDValue &Lo, SDValue &Hi) {
1742 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1743 RTLIB::NEARBYINT_F32,
1744 RTLIB::NEARBYINT_F64,
1745 RTLIB::NEARBYINT_F80,
1746 RTLIB::NEARBYINT_F128,
1747 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1748 }
1749
ExpandFloatRes_FNEG(SDNode * N,SDValue & Lo,SDValue & Hi)1750 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1751 SDValue &Hi) {
1752 SDLoc dl(N);
1753 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1754 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1755 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1756 }
1757
ExpandFloatRes_FP_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)1758 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1759 SDValue &Hi) {
1760 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1761 SDLoc dl(N);
1762 bool IsStrict = N->isStrictFPOpcode();
1763
1764 SDValue Chain;
1765 if (IsStrict) {
1766 // If the expanded type is the same as the input type, just bypass the node.
1767 if (NVT == N->getOperand(1).getValueType()) {
1768 Hi = N->getOperand(1);
1769 Chain = N->getOperand(0);
1770 } else {
1771 // Other we need to extend.
1772 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1773 { N->getOperand(0), N->getOperand(1) });
1774 Chain = Hi.getValue(1);
1775 }
1776 } else {
1777 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1778 }
1779
1780 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1781 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1782
1783 if (IsStrict)
1784 ReplaceValueWith(SDValue(N, 1), Chain);
1785 }
1786
ExpandFloatRes_FPOW(SDNode * N,SDValue & Lo,SDValue & Hi)1787 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1788 SDValue &Lo, SDValue &Hi) {
1789 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1790 RTLIB::POW_F32, RTLIB::POW_F64,
1791 RTLIB::POW_F80, RTLIB::POW_F128,
1792 RTLIB::POW_PPCF128), Lo, Hi);
1793 }
1794
ExpandFloatRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1795 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1796 SDValue &Lo, SDValue &Hi) {
1797 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1798 }
1799
ExpandFloatRes_FLDEXP(SDNode * N,SDValue & Lo,SDValue & Hi)1800 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1801 SDValue &Hi) {
1802 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1803 }
1804
ExpandFloatRes_FREEZE(SDNode * N,SDValue & Lo,SDValue & Hi)1805 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1806 SDValue &Lo, SDValue &Hi) {
1807 assert(N->getValueType(0) == MVT::ppcf128 &&
1808 "Logic only correct for ppcf128!");
1809
1810 SDLoc dl(N);
1811 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1812 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1813 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1814 }
1815
ExpandFloatRes_FREM(SDNode * N,SDValue & Lo,SDValue & Hi)1816 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1817 SDValue &Lo, SDValue &Hi) {
1818 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1819 RTLIB::REM_F32, RTLIB::REM_F64,
1820 RTLIB::REM_F80, RTLIB::REM_F128,
1821 RTLIB::REM_PPCF128), Lo, Hi);
1822 }
1823
ExpandFloatRes_FRINT(SDNode * N,SDValue & Lo,SDValue & Hi)1824 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1825 SDValue &Lo, SDValue &Hi) {
1826 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1827 RTLIB::RINT_F32, RTLIB::RINT_F64,
1828 RTLIB::RINT_F80, RTLIB::RINT_F128,
1829 RTLIB::RINT_PPCF128), Lo, Hi);
1830 }
1831
ExpandFloatRes_FROUND(SDNode * N,SDValue & Lo,SDValue & Hi)1832 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1833 SDValue &Lo, SDValue &Hi) {
1834 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1835 RTLIB::ROUND_F32,
1836 RTLIB::ROUND_F64,
1837 RTLIB::ROUND_F80,
1838 RTLIB::ROUND_F128,
1839 RTLIB::ROUND_PPCF128), Lo, Hi);
1840 }
1841
ExpandFloatRes_FROUNDEVEN(SDNode * N,SDValue & Lo,SDValue & Hi)1842 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1843 SDValue &Lo, SDValue &Hi) {
1844 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1845 RTLIB::ROUNDEVEN_F32,
1846 RTLIB::ROUNDEVEN_F64,
1847 RTLIB::ROUNDEVEN_F80,
1848 RTLIB::ROUNDEVEN_F128,
1849 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1850 }
1851
ExpandFloatRes_FSIN(SDNode * N,SDValue & Lo,SDValue & Hi)1852 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1853 SDValue &Lo, SDValue &Hi) {
1854 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1855 RTLIB::SIN_F32, RTLIB::SIN_F64,
1856 RTLIB::SIN_F80, RTLIB::SIN_F128,
1857 RTLIB::SIN_PPCF128), Lo, Hi);
1858 }
1859
ExpandFloatRes_FSINH(SDNode * N,SDValue & Lo,SDValue & Hi)1860 void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1861 SDValue &Hi) {
1862 ExpandFloatRes_Unary(N,
1863 GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32,
1864 RTLIB::SINH_F64, RTLIB::SINH_F80,
1865 RTLIB::SINH_F128, RTLIB::SINH_PPCF128),
1866 Lo, Hi);
1867 }
1868
ExpandFloatRes_FSQRT(SDNode * N,SDValue & Lo,SDValue & Hi)1869 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1870 SDValue &Lo, SDValue &Hi) {
1871 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1872 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1873 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1874 RTLIB::SQRT_PPCF128), Lo, Hi);
1875 }
1876
ExpandFloatRes_FSUB(SDNode * N,SDValue & Lo,SDValue & Hi)1877 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1878 SDValue &Hi) {
1879 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1880 RTLIB::SUB_F32,
1881 RTLIB::SUB_F64,
1882 RTLIB::SUB_F80,
1883 RTLIB::SUB_F128,
1884 RTLIB::SUB_PPCF128), Lo, Hi);
1885 }
1886
ExpandFloatRes_FTAN(SDNode * N,SDValue & Lo,SDValue & Hi)1887 void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1888 SDValue &Hi) {
1889 ExpandFloatRes_Unary(N,
1890 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
1891 RTLIB::TAN_F64, RTLIB::TAN_F80,
1892 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
1893 Lo, Hi);
1894 }
1895
ExpandFloatRes_FTANH(SDNode * N,SDValue & Lo,SDValue & Hi)1896 void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
1897 SDValue &Hi) {
1898 ExpandFloatRes_Unary(N,
1899 GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32,
1900 RTLIB::TANH_F64, RTLIB::TANH_F80,
1901 RTLIB::TANH_F128, RTLIB::TANH_PPCF128),
1902 Lo, Hi);
1903 }
1904
ExpandFloatRes_FTRUNC(SDNode * N,SDValue & Lo,SDValue & Hi)1905 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1906 SDValue &Lo, SDValue &Hi) {
1907 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1908 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1909 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1910 RTLIB::TRUNC_PPCF128), Lo, Hi);
1911 }
1912
ExpandFloatRes_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)1913 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1914 SDValue &Hi) {
1915 if (ISD::isNormalLoad(N)) {
1916 ExpandRes_NormalLoad(N, Lo, Hi);
1917 return;
1918 }
1919
1920 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1921 LoadSDNode *LD = cast<LoadSDNode>(N);
1922 SDValue Chain = LD->getChain();
1923 SDValue Ptr = LD->getBasePtr();
1924 SDLoc dl(N);
1925
1926 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1927 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1928 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1929
1930 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1931 LD->getMemoryVT(), LD->getMemOperand());
1932
1933 // Remember the chain.
1934 Chain = Hi.getValue(1);
1935
1936 // The low part is zero.
1937 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1938 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1939
1940 // Modified the chain - switch anything that used the old chain to use the
1941 // new one.
1942 ReplaceValueWith(SDValue(LD, 1), Chain);
1943 }
1944
ExpandFloatRes_XINT_TO_FP(SDNode * N,SDValue & Lo,SDValue & Hi)1945 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1946 SDValue &Hi) {
1947 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1948 EVT VT = N->getValueType(0);
1949 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1950 bool Strict = N->isStrictFPOpcode();
1951 SDValue Src = N->getOperand(Strict ? 1 : 0);
1952 EVT SrcVT = Src.getValueType();
1953 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1954 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1955 SDLoc dl(N);
1956 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1957
1958 // TODO: Any other flags to propagate?
1959 SDNodeFlags Flags;
1960 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1961
1962 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1963 // When promoting partial word types to i32 we must honor the signedness,
1964 // though.
1965 if (SrcVT.bitsLE(MVT::i32)) {
1966 // The integer can be represented exactly in an f64.
1967 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1968 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1969 if (Strict) {
1970 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1971 {Chain, Src}, Flags);
1972 Chain = Hi.getValue(1);
1973 } else
1974 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1975 } else {
1976 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1977 if (SrcVT.bitsLE(MVT::i64)) {
1978 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1979 MVT::i64, Src);
1980 LC = RTLIB::SINTTOFP_I64_PPCF128;
1981 } else if (SrcVT.bitsLE(MVT::i128)) {
1982 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1983 LC = RTLIB::SINTTOFP_I128_PPCF128;
1984 }
1985 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1986
1987 TargetLowering::MakeLibCallOptions CallOptions;
1988 CallOptions.setSExt(true);
1989 std::pair<SDValue, SDValue> Tmp =
1990 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1991 if (Strict)
1992 Chain = Tmp.second;
1993 GetPairElements(Tmp.first, Lo, Hi);
1994 }
1995
1996 // No need to complement for unsigned 32-bit integers
1997 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1998 if (Strict)
1999 ReplaceValueWith(SDValue(N, 1), Chain);
2000
2001 return;
2002 }
2003
2004 // Unsigned - fix up the SINT_TO_FP value just calculated.
2005 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2006 // keep semantics correctness if the integer is not exactly representable
2007 // here. See ExpandLegalINT_TO_FP.
2008 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
2009 SrcVT = Src.getValueType();
2010
2011 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2012 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2013 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2014 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2015 ArrayRef<uint64_t> Parts;
2016
2017 switch (SrcVT.getSimpleVT().SimpleTy) {
2018 default:
2019 llvm_unreachable("Unsupported UINT_TO_FP!");
2020 case MVT::i32:
2021 Parts = TwoE32;
2022 break;
2023 case MVT::i64:
2024 Parts = TwoE64;
2025 break;
2026 case MVT::i128:
2027 Parts = TwoE128;
2028 break;
2029 }
2030
2031 // TODO: Are there other fast-math-flags to propagate to this FADD?
2032 SDValue NewLo = DAG.getConstantFP(
2033 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
2034 if (Strict) {
2035 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
2036 {Chain, Hi, NewLo}, Flags);
2037 Chain = Lo.getValue(1);
2038 ReplaceValueWith(SDValue(N, 1), Chain);
2039 } else
2040 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
2041 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
2042 Lo, Hi, ISD::SETLT);
2043 GetPairElements(Lo, Lo, Hi);
2044 }
2045
2046
2047 //===----------------------------------------------------------------------===//
2048 // Float Operand Expansion
2049 //===----------------------------------------------------------------------===//
2050
2051 /// ExpandFloatOperand - This method is called when the specified operand of the
2052 /// specified node is found to need expansion. At this point, all of the result
2053 /// types of the node are known to be legal, but other operands of the node may
2054 /// need promotion or expansion as well as the specified one.
ExpandFloatOperand(SDNode * N,unsigned OpNo)2055 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2056 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2057 SDValue Res = SDValue();
2058
2059 // See if the target wants to custom expand this node.
2060 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2061 return false;
2062
2063 switch (N->getOpcode()) {
2064 default:
2065 #ifndef NDEBUG
2066 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2067 N->dump(&DAG); dbgs() << "\n";
2068 #endif
2069 report_fatal_error("Do not know how to expand this operator's operand!");
2070
2071 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2072 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2073 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2074
2075 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2076 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2077 case ISD::STRICT_FP_ROUND:
2078 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2079 case ISD::STRICT_FP_TO_SINT:
2080 case ISD::STRICT_FP_TO_UINT:
2081 case ISD::FP_TO_SINT:
2082 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2083 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2084 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2085 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2086 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2087 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2088 case ISD::STRICT_FSETCC:
2089 case ISD::STRICT_FSETCCS:
2090 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2091 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
2092 OpNo); break;
2093 }
2094
2095 // If the result is null, the sub-method took care of registering results etc.
2096 if (!Res.getNode()) return false;
2097
2098 // If the result is N, the sub-method updated N in place. Tell the legalizer
2099 // core about this.
2100 if (Res.getNode() == N)
2101 return true;
2102
2103 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2104 "Invalid operand expansion");
2105
2106 ReplaceValueWith(SDValue(N, 0), Res);
2107 return false;
2108 }
2109
2110 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2111 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
FloatExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl,SDValue & Chain,bool IsSignaling)2112 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2113 SDValue &NewRHS,
2114 ISD::CondCode &CCCode,
2115 const SDLoc &dl, SDValue &Chain,
2116 bool IsSignaling) {
2117 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2118 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2119 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2120
2121 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2122
2123 // FIXME: This generated code sucks. We want to generate
2124 // FCMPU crN, hi1, hi2
2125 // BNE crN, L:
2126 // FCMPU crN, lo1, lo2
2127 // The following can be improved, but not that much.
2128 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2129 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2130 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
2131 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2132 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2133 RHSLo, CCCode, OutputChain, IsSignaling);
2134 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2135 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2136 Tmp1 =
2137 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2138 ISD::SETUNE, OutputChain, IsSignaling);
2139 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2140 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2141 RHSHi, CCCode, OutputChain, IsSignaling);
2142 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2143 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2144 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2145 NewRHS = SDValue(); // LHS is the result, not a compare.
2146 Chain = OutputChain;
2147 }
2148
ExpandFloatOp_BR_CC(SDNode * N)2149 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2150 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2151 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2152 SDValue Chain;
2153 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2154
2155 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2156 // against zero to select between true and false values.
2157 if (!NewRHS.getNode()) {
2158 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2159 CCCode = ISD::SETNE;
2160 }
2161
2162 // Update N to have the operands specified.
2163 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2164 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2165 N->getOperand(4)), 0);
2166 }
2167
ExpandFloatOp_FCOPYSIGN(SDNode * N)2168 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2169 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2170 "Logic only correct for ppcf128!");
2171 SDValue Lo, Hi;
2172 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2173 // The ppcf128 value is providing only the sign; take it from the
2174 // higher-order double (which must have the larger magnitude).
2175 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2176 N->getValueType(0), N->getOperand(0), Hi);
2177 }
2178
ExpandFloatOp_FP_ROUND(SDNode * N)2179 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2180 bool IsStrict = N->isStrictFPOpcode();
2181 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2182 "Logic only correct for ppcf128!");
2183 SDValue Lo, Hi;
2184 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2185
2186 if (!IsStrict)
2187 // Round it the rest of the way (e.g. to f32) if needed.
2188 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2189 N->getValueType(0), Hi, N->getOperand(1));
2190
2191 // Eliminate the node if the input float type is the same as the output float
2192 // type.
2193 if (Hi.getValueType() == N->getValueType(0)) {
2194 // Connect the output chain to the input chain, unlinking the node.
2195 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2196 ReplaceValueWith(SDValue(N, 0), Hi);
2197 return SDValue();
2198 }
2199
2200 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2201 {N->getValueType(0), MVT::Other},
2202 {N->getOperand(0), Hi, N->getOperand(2)});
2203 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2204 ReplaceValueWith(SDValue(N, 0), Expansion);
2205 return SDValue();
2206 }
2207
ExpandFloatOp_FP_TO_XINT(SDNode * N)2208 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2209 EVT RVT = N->getValueType(0);
2210 SDLoc dl(N);
2211
2212 bool IsStrict = N->isStrictFPOpcode();
2213 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2214 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2215 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2216 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2217
2218 EVT NVT;
2219 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2220 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2221 "Unsupported FP_TO_XINT!");
2222 TargetLowering::MakeLibCallOptions CallOptions;
2223 std::pair<SDValue, SDValue> Tmp =
2224 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2225 if (!IsStrict)
2226 return Tmp.first;
2227
2228 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2229 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2230 return SDValue();
2231 }
2232
ExpandFloatOp_SELECT_CC(SDNode * N)2233 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2234 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2235 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2236 SDValue Chain;
2237 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2238
2239 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2240 // against zero to select between true and false values.
2241 if (!NewRHS.getNode()) {
2242 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2243 CCCode = ISD::SETNE;
2244 }
2245
2246 // Update N to have the operands specified.
2247 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2248 N->getOperand(2), N->getOperand(3),
2249 DAG.getCondCode(CCCode)), 0);
2250 }
2251
ExpandFloatOp_SETCC(SDNode * N)2252 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2253 bool IsStrict = N->isStrictFPOpcode();
2254 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2255 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2256 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2257 ISD::CondCode CCCode =
2258 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2259 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2260 N->getOpcode() == ISD::STRICT_FSETCCS);
2261
2262 // FloatExpandSetCCOperands always returned a scalar.
2263 assert(!NewRHS.getNode() && "Expect to return scalar");
2264 assert(NewLHS.getValueType() == N->getValueType(0) &&
2265 "Unexpected setcc expansion!");
2266 if (Chain) {
2267 ReplaceValueWith(SDValue(N, 0), NewLHS);
2268 ReplaceValueWith(SDValue(N, 1), Chain);
2269 return SDValue();
2270 }
2271 return NewLHS;
2272 }
2273
ExpandFloatOp_STORE(SDNode * N,unsigned OpNo)2274 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2275 if (ISD::isNormalStore(N))
2276 return ExpandOp_NormalStore(N, OpNo);
2277
2278 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2279 assert(OpNo == 1 && "Can only expand the stored value so far");
2280 StoreSDNode *ST = cast<StoreSDNode>(N);
2281
2282 SDValue Chain = ST->getChain();
2283 SDValue Ptr = ST->getBasePtr();
2284
2285 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2286 ST->getValue().getValueType());
2287 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2288 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2289 (void)NVT;
2290
2291 SDValue Lo, Hi;
2292 GetExpandedOp(ST->getValue(), Lo, Hi);
2293
2294 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2295 ST->getMemoryVT(), ST->getMemOperand());
2296 }
2297
ExpandFloatOp_LROUND(SDNode * N)2298 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2299 EVT RVT = N->getValueType(0);
2300 EVT RetVT = N->getOperand(0).getValueType();
2301 TargetLowering::MakeLibCallOptions CallOptions;
2302 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2303 RTLIB::LROUND_F32,
2304 RTLIB::LROUND_F64,
2305 RTLIB::LROUND_F80,
2306 RTLIB::LROUND_F128,
2307 RTLIB::LROUND_PPCF128),
2308 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2309 }
2310
ExpandFloatOp_LLROUND(SDNode * N)2311 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2312 EVT RVT = N->getValueType(0);
2313 EVT RetVT = N->getOperand(0).getValueType();
2314 TargetLowering::MakeLibCallOptions CallOptions;
2315 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2316 RTLIB::LLROUND_F32,
2317 RTLIB::LLROUND_F64,
2318 RTLIB::LLROUND_F80,
2319 RTLIB::LLROUND_F128,
2320 RTLIB::LLROUND_PPCF128),
2321 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2322 }
2323
ExpandFloatOp_LRINT(SDNode * N)2324 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2325 EVT RVT = N->getValueType(0);
2326 EVT RetVT = N->getOperand(0).getValueType();
2327 TargetLowering::MakeLibCallOptions CallOptions;
2328 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2329 RTLIB::LRINT_F32,
2330 RTLIB::LRINT_F64,
2331 RTLIB::LRINT_F80,
2332 RTLIB::LRINT_F128,
2333 RTLIB::LRINT_PPCF128),
2334 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2335 }
2336
ExpandFloatOp_LLRINT(SDNode * N)2337 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2338 EVT RVT = N->getValueType(0);
2339 EVT RetVT = N->getOperand(0).getValueType();
2340 TargetLowering::MakeLibCallOptions CallOptions;
2341 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2342 RTLIB::LLRINT_F32,
2343 RTLIB::LLRINT_F64,
2344 RTLIB::LLRINT_F80,
2345 RTLIB::LLRINT_F128,
2346 RTLIB::LLRINT_PPCF128),
2347 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2348 }
2349
2350 //===----------------------------------------------------------------------===//
2351 // Float Operand Promotion
2352 //===----------------------------------------------------------------------===//
2353 //
2354
GetPromotionOpcode(EVT OpVT,EVT RetVT)2355 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2356 if (OpVT == MVT::f16) {
2357 return ISD::FP16_TO_FP;
2358 } else if (RetVT == MVT::f16) {
2359 return ISD::FP_TO_FP16;
2360 } else if (OpVT == MVT::bf16) {
2361 return ISD::BF16_TO_FP;
2362 } else if (RetVT == MVT::bf16) {
2363 return ISD::FP_TO_BF16;
2364 }
2365
2366 report_fatal_error("Attempt at an invalid promotion-related conversion");
2367 }
2368
GetPromotionOpcodeStrict(EVT OpVT,EVT RetVT)2369 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2370 if (OpVT == MVT::f16)
2371 return ISD::STRICT_FP16_TO_FP;
2372
2373 if (RetVT == MVT::f16)
2374 return ISD::STRICT_FP_TO_FP16;
2375
2376 if (OpVT == MVT::bf16)
2377 return ISD::STRICT_BF16_TO_FP;
2378
2379 if (RetVT == MVT::bf16)
2380 return ISD::STRICT_FP_TO_BF16;
2381
2382 report_fatal_error("Attempt at an invalid promotion-related conversion");
2383 }
2384
PromoteFloatOperand(SDNode * N,unsigned OpNo)2385 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2386 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2387 SDValue R = SDValue();
2388
2389 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2390 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2391 return false;
2392 }
2393
2394 // Nodes that use a promotion-requiring floating point operand, but doesn't
2395 // produce a promotion-requiring floating point result, need to be legalized
2396 // to use the promoted float operand. Nodes that produce at least one
2397 // promotion-requiring floating point result have their operands legalized as
2398 // a part of PromoteFloatResult.
2399 // clang-format off
2400 switch (N->getOpcode()) {
2401 default:
2402 #ifndef NDEBUG
2403 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2404 N->dump(&DAG); dbgs() << "\n";
2405 #endif
2406 report_fatal_error("Do not know how to promote this operator's operand!");
2407
2408 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2409 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2410 case ISD::FP_TO_SINT:
2411 case ISD::FP_TO_UINT:
2412 case ISD::LRINT:
2413 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2414 case ISD::FP_TO_SINT_SAT:
2415 case ISD::FP_TO_UINT_SAT:
2416 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2417 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2418 case ISD::STRICT_FP_EXTEND:
2419 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2420 break;
2421 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2422 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2423 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2424 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2425 }
2426 // clang-format on
2427
2428 if (R.getNode())
2429 ReplaceValueWith(SDValue(N, 0), R);
2430 return false;
2431 }
2432
PromoteFloatOp_BITCAST(SDNode * N,unsigned OpNo)2433 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2434 SDValue Op = N->getOperand(0);
2435 EVT OpVT = Op->getValueType(0);
2436
2437 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2438 EVT PromotedVT = Promoted->getValueType(0);
2439
2440 // Convert the promoted float value to the desired IVT.
2441 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2442 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2443 IVT, Promoted);
2444 // The final result type might not be an scalar so we need a bitcast. The
2445 // bitcast will be further legalized if needed.
2446 return DAG.getBitcast(N->getValueType(0), Convert);
2447 }
2448
2449 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2450 // PromoteFloatRes_FCOPYSIGN.
PromoteFloatOp_FCOPYSIGN(SDNode * N,unsigned OpNo)2451 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2452 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2453 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2454
2455 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2456 N->getOperand(0), Op1);
2457 }
2458
2459 // Convert the promoted float value to the desired integer type
PromoteFloatOp_UnaryOp(SDNode * N,unsigned OpNo)2460 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2461 SDValue Op = GetPromotedFloat(N->getOperand(0));
2462 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2463 }
2464
PromoteFloatOp_FP_TO_XINT_SAT(SDNode * N,unsigned OpNo)2465 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2466 unsigned OpNo) {
2467 SDValue Op = GetPromotedFloat(N->getOperand(0));
2468 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2469 N->getOperand(1));
2470 }
2471
PromoteFloatOp_FP_EXTEND(SDNode * N,unsigned OpNo)2472 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2473 SDValue Op = GetPromotedFloat(N->getOperand(0));
2474 EVT VT = N->getValueType(0);
2475
2476 // Desired VT is same as promoted type. Use promoted float directly.
2477 if (VT == Op->getValueType(0))
2478 return Op;
2479
2480 // Else, extend the promoted float value to the desired VT.
2481 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2482 }
2483
PromoteFloatOp_STRICT_FP_EXTEND(SDNode * N,unsigned OpNo)2484 SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2485 unsigned OpNo) {
2486 assert(OpNo == 1 && "Promoting unpromotable operand");
2487
2488 SDValue Op = GetPromotedFloat(N->getOperand(1));
2489 EVT VT = N->getValueType(0);
2490
2491 // Desired VT is same as promoted type. Use promoted float directly.
2492 if (VT == Op->getValueType(0)) {
2493 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2494 return Op;
2495 }
2496
2497 // Else, extend the promoted float value to the desired VT.
2498 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2499 N->getOperand(0), Op);
2500 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2501 return Res;
2502 }
2503
2504 // Promote the float operands used for comparison. The true- and false-
2505 // operands have the same type as the result and are promoted, if needed, by
2506 // PromoteFloatRes_SELECT_CC
PromoteFloatOp_SELECT_CC(SDNode * N,unsigned OpNo)2507 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2508 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2509 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2510
2511 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2512 LHS, RHS, N->getOperand(2), N->getOperand(3),
2513 N->getOperand(4));
2514 }
2515
2516 // Construct a SETCC that compares the promoted values and sets the conditional
2517 // code.
PromoteFloatOp_SETCC(SDNode * N,unsigned OpNo)2518 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2519 EVT VT = N->getValueType(0);
2520 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2521 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2522 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2523
2524 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2525
2526 }
2527
2528 // Lower the promoted Float down to the integer value of same size and construct
2529 // a STORE of the integer value.
PromoteFloatOp_STORE(SDNode * N,unsigned OpNo)2530 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2531 StoreSDNode *ST = cast<StoreSDNode>(N);
2532 SDValue Val = ST->getValue();
2533 SDLoc DL(N);
2534
2535 SDValue Promoted = GetPromotedFloat(Val);
2536 EVT VT = ST->getOperand(1).getValueType();
2537 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2538
2539 SDValue NewVal;
2540 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2541 IVT, Promoted);
2542
2543 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2544 ST->getMemOperand());
2545 }
2546
PromoteFloatOp_ATOMIC_STORE(SDNode * N,unsigned OpNo)2547 SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2548 unsigned OpNo) {
2549 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2550 SDValue Val = ST->getVal();
2551 SDLoc DL(N);
2552
2553 SDValue Promoted = GetPromotedFloat(Val);
2554 EVT VT = ST->getOperand(1).getValueType();
2555 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2556
2557 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2558 DL, IVT, Promoted);
2559
2560 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2561 ST->getBasePtr(), ST->getMemOperand());
2562 }
2563
2564 //===----------------------------------------------------------------------===//
2565 // Float Result Promotion
2566 //===----------------------------------------------------------------------===//
2567
PromoteFloatResult(SDNode * N,unsigned ResNo)2568 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2569 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2570 SDValue R = SDValue();
2571
2572 // See if the target wants to custom expand this node.
2573 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2574 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2575 return;
2576 }
2577
2578 switch (N->getOpcode()) {
2579 // These opcodes cannot appear if promotion of FP16 is done in the backend
2580 // instead of Clang
2581 case ISD::FP16_TO_FP:
2582 case ISD::FP_TO_FP16:
2583 default:
2584 #ifndef NDEBUG
2585 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2586 N->dump(&DAG); dbgs() << "\n";
2587 #endif
2588 report_fatal_error("Do not know how to promote this operator's result!");
2589
2590 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2591 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2592 case ISD::EXTRACT_VECTOR_ELT:
2593 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2594 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2595
2596 // Unary FP Operations
2597 case ISD::FABS:
2598 case ISD::FACOS:
2599 case ISD::FASIN:
2600 case ISD::FATAN:
2601 case ISD::FCBRT:
2602 case ISD::FCEIL:
2603 case ISD::FCOS:
2604 case ISD::FCOSH:
2605 case ISD::FEXP:
2606 case ISD::FEXP2:
2607 case ISD::FEXP10:
2608 case ISD::FFLOOR:
2609 case ISD::FLOG:
2610 case ISD::FLOG2:
2611 case ISD::FLOG10:
2612 case ISD::FNEARBYINT:
2613 case ISD::FNEG:
2614 case ISD::FRINT:
2615 case ISD::FROUND:
2616 case ISD::FROUNDEVEN:
2617 case ISD::FSIN:
2618 case ISD::FSINH:
2619 case ISD::FSQRT:
2620 case ISD::FTRUNC:
2621 case ISD::FTAN:
2622 case ISD::FTANH:
2623 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2624
2625 // Binary FP Operations
2626 case ISD::FADD:
2627 case ISD::FDIV:
2628 case ISD::FMAXIMUM:
2629 case ISD::FMINIMUM:
2630 case ISD::FMAXNUM:
2631 case ISD::FMINNUM:
2632 case ISD::FMAXNUM_IEEE:
2633 case ISD::FMINNUM_IEEE:
2634 case ISD::FMUL:
2635 case ISD::FPOW:
2636 case ISD::FREM:
2637 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2638
2639 case ISD::FMA: // FMA is same as FMAD
2640 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2641
2642 case ISD::FPOWI:
2643 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2644 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2645
2646 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2647 case ISD::STRICT_FP_ROUND:
2648 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2649 break;
2650 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2651 case ISD::ATOMIC_LOAD:
2652 R = PromoteFloatRes_ATOMIC_LOAD(N);
2653 break;
2654 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2655 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2656
2657 case ISD::SINT_TO_FP:
2658 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2659 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2660 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2661 case ISD::VECREDUCE_FADD:
2662 case ISD::VECREDUCE_FMUL:
2663 case ISD::VECREDUCE_FMIN:
2664 case ISD::VECREDUCE_FMAX:
2665 case ISD::VECREDUCE_FMAXIMUM:
2666 case ISD::VECREDUCE_FMINIMUM:
2667 R = PromoteFloatRes_VECREDUCE(N);
2668 break;
2669 case ISD::VECREDUCE_SEQ_FADD:
2670 case ISD::VECREDUCE_SEQ_FMUL:
2671 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2672 break;
2673 }
2674
2675 if (R.getNode())
2676 SetPromotedFloat(SDValue(N, ResNo), R);
2677 }
2678
2679 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2680 // At this point, it is not possible to determine if the bitcast value is
2681 // eventually stored to memory or promoted to f32 or promoted to a floating
2682 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2683 // STORE promotion handlers.
PromoteFloatRes_BITCAST(SDNode * N)2684 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2685 EVT VT = N->getValueType(0);
2686 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2687 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2688 // bitcast will be legalized further if necessary.
2689 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2690 N->getOperand(0).getValueType().getSizeInBits());
2691 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2692 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2693 }
2694
PromoteFloatRes_ConstantFP(SDNode * N)2695 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2696 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2697 EVT VT = N->getValueType(0);
2698 SDLoc DL(N);
2699
2700 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2701 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2702 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2703 IVT);
2704
2705 // Convert the Constant to the desired FP type
2706 // FIXME We might be able to do the conversion during compilation and get rid
2707 // of it from the object code
2708 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2709 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2710 }
2711
2712 // If the Index operand is a constant, try to redirect the extract operation to
2713 // the correct legalized vector. If not, bit-convert the input vector to
2714 // equivalent integer vector. Extract the element as an (bit-cast) integer
2715 // value and convert it to the promoted type.
PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode * N)2716 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2717 SDLoc DL(N);
2718
2719 // If the index is constant, try to extract the value from the legalized
2720 // vector type.
2721 if (isa<ConstantSDNode>(N->getOperand(1))) {
2722 SDValue Vec = N->getOperand(0);
2723 SDValue Idx = N->getOperand(1);
2724 EVT VecVT = Vec->getValueType(0);
2725 EVT EltVT = VecVT.getVectorElementType();
2726
2727 uint64_t IdxVal = Idx->getAsZExtVal();
2728
2729 switch (getTypeAction(VecVT)) {
2730 default: break;
2731 case TargetLowering::TypeScalarizeVector: {
2732 SDValue Res = GetScalarizedVector(N->getOperand(0));
2733 ReplaceValueWith(SDValue(N, 0), Res);
2734 return SDValue();
2735 }
2736 case TargetLowering::TypeWidenVector: {
2737 Vec = GetWidenedVector(Vec);
2738 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2739 ReplaceValueWith(SDValue(N, 0), Res);
2740 return SDValue();
2741 }
2742 case TargetLowering::TypeSplitVector: {
2743 SDValue Lo, Hi;
2744 GetSplitVector(Vec, Lo, Hi);
2745
2746 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2747 SDValue Res;
2748 if (IdxVal < LoElts)
2749 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2750 else
2751 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2752 DAG.getConstant(IdxVal - LoElts, DL,
2753 Idx.getValueType()));
2754 ReplaceValueWith(SDValue(N, 0), Res);
2755 return SDValue();
2756 }
2757
2758 }
2759 }
2760
2761 // Bit-convert the input vector to the equivalent integer vector
2762 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2763 EVT IVT = NewOp.getValueType().getVectorElementType();
2764
2765 // Extract the element as an (bit-cast) integer value
2766 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2767 NewOp, N->getOperand(1));
2768
2769 // Convert the element to the desired FP type
2770 EVT VT = N->getValueType(0);
2771 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2772 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2773 }
2774
2775 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2776 // needs promotion, so does the argument X. Note that Y, if needed, will be
2777 // handled during operand promotion.
PromoteFloatRes_FCOPYSIGN(SDNode * N)2778 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2779 EVT VT = N->getValueType(0);
2780 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2781 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2782
2783 SDValue Op1 = N->getOperand(1);
2784
2785 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2786 }
2787
2788 // Unary operation where the result and the operand have PromoteFloat type
2789 // action. Construct a new SDNode with the promoted float value of the old
2790 // operand.
PromoteFloatRes_UnaryOp(SDNode * N)2791 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2792 EVT VT = N->getValueType(0);
2793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2794 SDValue Op = GetPromotedFloat(N->getOperand(0));
2795
2796 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2797 }
2798
2799 // Binary operations where the result and both operands have PromoteFloat type
2800 // action. Construct a new SDNode with the promoted float values of the old
2801 // operands.
PromoteFloatRes_BinOp(SDNode * N)2802 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2803 EVT VT = N->getValueType(0);
2804 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2805 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2806 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2807 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2808 }
2809
PromoteFloatRes_FMAD(SDNode * N)2810 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2811 EVT VT = N->getValueType(0);
2812 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2813 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2814 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2815 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2816
2817 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2818 }
2819
2820 // Promote the Float (first) operand and retain the Integer (second) operand
PromoteFloatRes_ExpOp(SDNode * N)2821 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2822 EVT VT = N->getValueType(0);
2823 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2824 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2825 SDValue Op1 = N->getOperand(1);
2826
2827 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2828 }
2829
PromoteFloatRes_FFREXP(SDNode * N)2830 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2831 EVT VT = N->getValueType(0);
2832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2833 SDValue Op = GetPromotedFloat(N->getOperand(0));
2834 SDValue Res =
2835 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2836
2837 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2838 return Res;
2839 }
2840
2841 // Explicit operation to reduce precision. Reduce the value to half precision
2842 // and promote it back to the legal type.
PromoteFloatRes_FP_ROUND(SDNode * N)2843 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2844 SDLoc DL(N);
2845
2846 SDValue Op = N->getOperand(0);
2847 EVT VT = N->getValueType(0);
2848 EVT OpVT = Op->getValueType(0);
2849 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2850 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2851
2852 // Round promoted float to desired precision
2853 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2854 // Promote it back to the legal output type
2855 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2856 }
2857
2858 // Explicit operation to reduce precision. Reduce the value to half precision
2859 // and promote it back to the legal type.
PromoteFloatRes_STRICT_FP_ROUND(SDNode * N)2860 SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2861 SDLoc DL(N);
2862
2863 SDValue Chain = N->getOperand(0);
2864 SDValue Op = N->getOperand(1);
2865 EVT VT = N->getValueType(0);
2866 EVT OpVT = Op->getValueType(0);
2867 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2868 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2869
2870 // Round promoted float to desired precision
2871 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2872 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2873 // Promote it back to the legal output type
2874 SDValue Res =
2875 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2876 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2877 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2878 return Res;
2879 }
2880
PromoteFloatRes_LOAD(SDNode * N)2881 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2882 LoadSDNode *L = cast<LoadSDNode>(N);
2883 EVT VT = N->getValueType(0);
2884
2885 // Load the value as an integer value with the same number of bits.
2886 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2887 SDValue newL = DAG.getLoad(
2888 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2889 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2890 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2891 // Legalize the chain result by replacing uses of the old value chain with the
2892 // new one
2893 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2894
2895 // Convert the integer value to the desired FP type
2896 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2897 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2898 }
2899
PromoteFloatRes_ATOMIC_LOAD(SDNode * N)2900 SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
2901 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2902 EVT VT = AM->getValueType(0);
2903
2904 // Load the value as an integer value with the same number of bits.
2905 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2906 SDValue newL = DAG.getAtomic(
2907 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
2908 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
2909
2910 // Legalize the chain result by replacing uses of the old value chain with the
2911 // new one
2912 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2913
2914 // Convert the integer value to the desired FP type
2915 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2916 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
2917 }
2918
2919 // Construct a new SELECT node with the promoted true- and false- values.
PromoteFloatRes_SELECT(SDNode * N)2920 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2921 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2922 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2923
2924 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2925 N->getOperand(0), TrueVal, FalseVal);
2926 }
2927
2928 // Construct a new SELECT_CC node with the promoted true- and false- values.
2929 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
PromoteFloatRes_SELECT_CC(SDNode * N)2930 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2931 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2932 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2933
2934 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2935 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2936 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2937 }
2938
2939 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2940 // float type.
PromoteFloatRes_XINT_TO_FP(SDNode * N)2941 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2942 SDLoc DL(N);
2943 EVT VT = N->getValueType(0);
2944 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2945 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2946 // Round the value to the desired precision (that of the source type).
2947 return DAG.getNode(
2948 ISD::FP_EXTEND, DL, NVT,
2949 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2950 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2951 }
2952
PromoteFloatRes_UNDEF(SDNode * N)2953 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2954 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2955 N->getValueType(0)));
2956 }
2957
PromoteFloatRes_VECREDUCE(SDNode * N)2958 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2959 // Expand and promote recursively.
2960 // TODO: This is non-optimal, but dealing with the concurrently happening
2961 // vector-legalization is non-trivial. We could do something similar to
2962 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2963 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2964 return SDValue();
2965 }
2966
PromoteFloatRes_VECREDUCE_SEQ(SDNode * N)2967 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2968 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2969 return SDValue();
2970 }
2971
BitcastToInt_ATOMIC_SWAP(SDNode * N)2972 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2973 EVT VT = N->getValueType(0);
2974
2975 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2976 SDLoc SL(N);
2977
2978 SDValue CastVal = BitConvertToInteger(AM->getVal());
2979 EVT CastVT = CastVal.getValueType();
2980
2981 SDValue NewAtomic
2982 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2983 DAG.getVTList(CastVT, MVT::Other),
2984 { AM->getChain(), AM->getBasePtr(), CastVal },
2985 AM->getMemOperand());
2986
2987 SDValue Result = NewAtomic;
2988
2989 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2990 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2991 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2992 NewAtomic);
2993 }
2994
2995 // Legalize the chain result by replacing uses of the old value chain with the
2996 // new one
2997 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2998
2999 return Result;
3000
3001 }
3002
3003 //===----------------------------------------------------------------------===//
3004 // Half Result Soft Promotion
3005 //===----------------------------------------------------------------------===//
3006
SoftPromoteHalfResult(SDNode * N,unsigned ResNo)3007 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
3008 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
3009 N->dump(&DAG));
3010 SDValue R = SDValue();
3011
3012 // See if the target wants to custom expand this node.
3013 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
3014 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
3015 return;
3016 }
3017
3018 switch (N->getOpcode()) {
3019 default:
3020 #ifndef NDEBUG
3021 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
3022 N->dump(&DAG); dbgs() << "\n";
3023 #endif
3024 report_fatal_error("Do not know how to soft promote this operator's "
3025 "result!");
3026
3027 case ISD::ARITH_FENCE:
3028 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
3029 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
3030 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
3031 case ISD::EXTRACT_VECTOR_ELT:
3032 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
3033 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
3034 case ISD::STRICT_FP_ROUND:
3035 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
3036
3037 // Unary FP Operations
3038 case ISD::FABS:
3039 case ISD::FACOS:
3040 case ISD::FASIN:
3041 case ISD::FATAN:
3042 case ISD::FCBRT:
3043 case ISD::FCEIL:
3044 case ISD::FCOS:
3045 case ISD::FCOSH:
3046 case ISD::FEXP:
3047 case ISD::FEXP2:
3048 case ISD::FEXP10:
3049 case ISD::FFLOOR:
3050 case ISD::FLOG:
3051 case ISD::FLOG2:
3052 case ISD::FLOG10:
3053 case ISD::FNEARBYINT:
3054 case ISD::FNEG:
3055 case ISD::FREEZE:
3056 case ISD::FRINT:
3057 case ISD::FROUND:
3058 case ISD::FROUNDEVEN:
3059 case ISD::FSIN:
3060 case ISD::FSINH:
3061 case ISD::FSQRT:
3062 case ISD::FTRUNC:
3063 case ISD::FTAN:
3064 case ISD::FTANH:
3065 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3066
3067 // Binary FP Operations
3068 case ISD::FADD:
3069 case ISD::FDIV:
3070 case ISD::FMAXIMUM:
3071 case ISD::FMINIMUM:
3072 case ISD::FMAXNUM:
3073 case ISD::FMINNUM:
3074 case ISD::FMUL:
3075 case ISD::FPOW:
3076 case ISD::FREM:
3077 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
3078
3079 case ISD::FMA: // FMA is same as FMAD
3080 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
3081
3082 case ISD::FPOWI:
3083 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
3084
3085 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
3086
3087 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
3088 case ISD::ATOMIC_LOAD:
3089 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
3090 break;
3091 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
3092 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
3093 case ISD::SINT_TO_FP:
3094 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
3095 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
3096 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
3097 case ISD::VECREDUCE_FADD:
3098 case ISD::VECREDUCE_FMUL:
3099 case ISD::VECREDUCE_FMIN:
3100 case ISD::VECREDUCE_FMAX:
3101 case ISD::VECREDUCE_FMAXIMUM:
3102 case ISD::VECREDUCE_FMINIMUM:
3103 R = SoftPromoteHalfRes_VECREDUCE(N);
3104 break;
3105 case ISD::VECREDUCE_SEQ_FADD:
3106 case ISD::VECREDUCE_SEQ_FMUL:
3107 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
3108 break;
3109 }
3110
3111 if (R.getNode())
3112 SetSoftPromotedHalf(SDValue(N, ResNo), R);
3113 }
3114
SoftPromoteHalfRes_ARITH_FENCE(SDNode * N)3115 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
3116 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
3117 BitConvertToInteger(N->getOperand(0)));
3118 }
3119
SoftPromoteHalfRes_BITCAST(SDNode * N)3120 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
3121 return BitConvertToInteger(N->getOperand(0));
3122 }
3123
SoftPromoteHalfRes_ConstantFP(SDNode * N)3124 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
3125 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
3126
3127 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3128 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
3129 MVT::i16);
3130 }
3131
SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode * N)3132 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3133 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3134 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3135 NewOp.getValueType().getVectorElementType(), NewOp,
3136 N->getOperand(1));
3137 }
3138
SoftPromoteHalfRes_FCOPYSIGN(SDNode * N)3139 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3140 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
3141 SDValue RHS = BitConvertToInteger(N->getOperand(1));
3142 SDLoc dl(N);
3143
3144 EVT LVT = LHS.getValueType();
3145 EVT RVT = RHS.getValueType();
3146
3147 unsigned LSize = LVT.getSizeInBits();
3148 unsigned RSize = RVT.getSizeInBits();
3149
3150 // First get the sign bit of second operand.
3151 SDValue SignBit = DAG.getNode(
3152 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3153 DAG.getConstant(RSize - 1, dl,
3154 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3155 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3156
3157 // Shift right or sign-extend it if the two operands have different types.
3158 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3159 if (SizeDiff > 0) {
3160 SignBit =
3161 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3162 DAG.getConstant(SizeDiff, dl,
3163 TLI.getShiftAmountTy(SignBit.getValueType(),
3164 DAG.getDataLayout())));
3165 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3166 } else if (SizeDiff < 0) {
3167 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3168 SignBit =
3169 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3170 DAG.getConstant(-SizeDiff, dl,
3171 TLI.getShiftAmountTy(SignBit.getValueType(),
3172 DAG.getDataLayout())));
3173 }
3174
3175 // Clear the sign bit of the first operand.
3176 SDValue Mask = DAG.getNode(
3177 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3178 DAG.getConstant(LSize - 1, dl,
3179 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3180 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3181 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3182
3183 // Or the value with the sign bit.
3184 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3185 }
3186
SoftPromoteHalfRes_FMAD(SDNode * N)3187 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3188 EVT OVT = N->getValueType(0);
3189 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3190 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3191 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3192 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3193 SDLoc dl(N);
3194
3195 // Promote to the larger FP type.
3196 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3197 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3198 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3199 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3200
3201 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3202
3203 // Convert back to FP16 as an integer.
3204 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3205 }
3206
SoftPromoteHalfRes_ExpOp(SDNode * N)3207 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3208 EVT OVT = N->getValueType(0);
3209 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3210 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3211 SDValue Op1 = N->getOperand(1);
3212 SDLoc dl(N);
3213
3214 // Promote to the larger FP type.
3215 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3216
3217 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3218
3219 // Convert back to FP16 as an integer.
3220 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3221 }
3222
SoftPromoteHalfRes_FFREXP(SDNode * N)3223 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3224 EVT OVT = N->getValueType(0);
3225 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3226 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3227 SDLoc dl(N);
3228
3229 // Promote to the larger FP type.
3230 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3231
3232 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3233 DAG.getVTList(NVT, N->getValueType(1)), Op);
3234
3235 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3236
3237 // Convert back to FP16 as an integer.
3238 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3239 }
3240
SoftPromoteHalfRes_FP_ROUND(SDNode * N)3241 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3242 EVT RVT = N->getValueType(0);
3243 EVT SVT = N->getOperand(0).getValueType();
3244
3245 if (N->isStrictFPOpcode()) {
3246 // FIXME: assume we only have two f16 variants for now.
3247 unsigned Opcode;
3248 if (RVT == MVT::f16)
3249 Opcode = ISD::STRICT_FP_TO_FP16;
3250 else if (RVT == MVT::bf16)
3251 Opcode = ISD::STRICT_FP_TO_BF16;
3252 else
3253 llvm_unreachable("unknown half type");
3254 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3255 {N->getOperand(0), N->getOperand(1)});
3256 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3257 return Res;
3258 }
3259
3260 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3261 N->getOperand(0));
3262 }
3263
SoftPromoteHalfRes_LOAD(SDNode * N)3264 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3265 LoadSDNode *L = cast<LoadSDNode>(N);
3266
3267 // Load the value as an integer value with the same number of bits.
3268 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3269 SDValue NewL =
3270 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3271 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3272 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3273 L->getMemOperand()->getFlags(), L->getAAInfo());
3274 // Legalize the chain result by replacing uses of the old value chain with the
3275 // new one
3276 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3277 return NewL;
3278 }
3279
SoftPromoteHalfRes_ATOMIC_LOAD(SDNode * N)3280 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3281 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3282
3283 // Load the value as an integer value with the same number of bits.
3284 SDValue NewL = DAG.getAtomic(
3285 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3286 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3287
3288 // Legalize the chain result by replacing uses of the old value chain with the
3289 // new one
3290 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3291 return NewL;
3292 }
3293
SoftPromoteHalfRes_SELECT(SDNode * N)3294 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3295 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3296 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3297 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3298 Op2);
3299 }
3300
SoftPromoteHalfRes_SELECT_CC(SDNode * N)3301 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3302 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3303 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3304 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3305 N->getOperand(0), N->getOperand(1), Op2, Op3,
3306 N->getOperand(4));
3307 }
3308
SoftPromoteHalfRes_XINT_TO_FP(SDNode * N)3309 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3310 EVT OVT = N->getValueType(0);
3311 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3312 SDLoc dl(N);
3313
3314 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3315
3316 // Round the value to the softened type.
3317 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3318 }
3319
SoftPromoteHalfRes_UNDEF(SDNode * N)3320 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3321 return DAG.getUNDEF(MVT::i16);
3322 }
3323
SoftPromoteHalfRes_UnaryOp(SDNode * N)3324 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3325 EVT OVT = N->getValueType(0);
3326 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3327 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3328 SDLoc dl(N);
3329
3330 // Promote to the larger FP type.
3331 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3332
3333 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3334
3335 // Convert back to FP16 as an integer.
3336 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3337 }
3338
SoftPromoteHalfRes_BinOp(SDNode * N)3339 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3340 EVT OVT = N->getValueType(0);
3341 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3342 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3343 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3344 SDLoc dl(N);
3345
3346 // Promote to the larger FP type.
3347 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3348 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3349 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3350
3351 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3352
3353 // Convert back to FP16 as an integer.
3354 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3355 }
3356
SoftPromoteHalfRes_VECREDUCE(SDNode * N)3357 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3358 // Expand and soften recursively.
3359 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3360 return SDValue();
3361 }
3362
SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode * N)3363 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3364 // Expand and soften.
3365 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3366 return SDValue();
3367 }
3368
3369 //===----------------------------------------------------------------------===//
3370 // Half Operand Soft Promotion
3371 //===----------------------------------------------------------------------===//
3372
SoftPromoteHalfOperand(SDNode * N,unsigned OpNo)3373 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3374 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3375 N->dump(&DAG));
3376 SDValue Res = SDValue();
3377
3378 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3379 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3380 return false;
3381 }
3382
3383 // Nodes that use a promotion-requiring floating point operand, but doesn't
3384 // produce a soft promotion-requiring floating point result, need to be
3385 // legalized to use the soft promoted float operand. Nodes that produce at
3386 // least one soft promotion-requiring floating point result have their
3387 // operands legalized as a part of PromoteFloatResult.
3388 switch (N->getOpcode()) {
3389 default:
3390 #ifndef NDEBUG
3391 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3392 N->dump(&DAG); dbgs() << "\n";
3393 #endif
3394 report_fatal_error("Do not know how to soft promote this operator's "
3395 "operand!");
3396
3397 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3398 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3399 case ISD::FP_TO_SINT:
3400 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3401 case ISD::FP_TO_SINT_SAT:
3402 case ISD::FP_TO_UINT_SAT:
3403 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3404 case ISD::STRICT_FP_EXTEND:
3405 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3406 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3407 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3408 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3409 case ISD::ATOMIC_STORE:
3410 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3411 break;
3412 case ISD::STACKMAP:
3413 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3414 break;
3415 case ISD::PATCHPOINT:
3416 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3417 break;
3418 }
3419
3420 if (!Res.getNode())
3421 return false;
3422
3423 assert(Res.getNode() != N && "Expected a new node!");
3424
3425 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3426 "Invalid operand expansion");
3427
3428 ReplaceValueWith(SDValue(N, 0), Res);
3429 return false;
3430 }
3431
SoftPromoteHalfOp_BITCAST(SDNode * N)3432 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3433 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3434
3435 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3436 }
3437
SoftPromoteHalfOp_FCOPYSIGN(SDNode * N,unsigned OpNo)3438 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3439 unsigned OpNo) {
3440 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3441 SDValue Op1 = N->getOperand(1);
3442 EVT RVT = Op1.getValueType();
3443 SDLoc dl(N);
3444
3445 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3446
3447 Op1 = GetSoftPromotedHalf(Op1);
3448 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3449
3450 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3451 Op1);
3452 }
3453
SoftPromoteHalfOp_FP_EXTEND(SDNode * N)3454 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3455 EVT RVT = N->getValueType(0);
3456 bool IsStrict = N->isStrictFPOpcode();
3457 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3458 EVT SVT = Op.getValueType();
3459 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3460
3461 if (IsStrict) {
3462 unsigned Opcode;
3463 if (SVT == MVT::f16)
3464 Opcode = ISD::STRICT_FP16_TO_FP;
3465 else if (SVT == MVT::bf16)
3466 Opcode = ISD::STRICT_BF16_TO_FP;
3467 else
3468 llvm_unreachable("unknown half type");
3469 SDValue Res =
3470 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3471 {N->getOperand(0), Op});
3472 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3473 ReplaceValueWith(SDValue(N, 0), Res);
3474 return SDValue();
3475 }
3476
3477 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3478 }
3479
SoftPromoteHalfOp_FP_TO_XINT(SDNode * N)3480 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3481 EVT RVT = N->getValueType(0);
3482 SDValue Op = N->getOperand(0);
3483 EVT SVT = Op.getValueType();
3484 SDLoc dl(N);
3485
3486 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3487
3488 Op = GetSoftPromotedHalf(Op);
3489
3490 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3491
3492 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3493 }
3494
SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode * N)3495 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3496 EVT RVT = N->getValueType(0);
3497 SDValue Op = N->getOperand(0);
3498 EVT SVT = Op.getValueType();
3499 SDLoc dl(N);
3500
3501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3502
3503 Op = GetSoftPromotedHalf(Op);
3504
3505 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3506
3507 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3508 N->getOperand(1));
3509 }
3510
SoftPromoteHalfOp_SELECT_CC(SDNode * N,unsigned OpNo)3511 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3512 unsigned OpNo) {
3513 assert(OpNo == 0 && "Can only soften the comparison values");
3514 SDValue Op0 = N->getOperand(0);
3515 SDValue Op1 = N->getOperand(1);
3516 SDLoc dl(N);
3517
3518 EVT SVT = Op0.getValueType();
3519 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3520
3521 Op0 = GetSoftPromotedHalf(Op0);
3522 Op1 = GetSoftPromotedHalf(Op1);
3523
3524 // Promote to the larger FP type.
3525 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3526 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3527 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3528
3529 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3530 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3531 }
3532
SoftPromoteHalfOp_SETCC(SDNode * N)3533 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3534 SDValue Op0 = N->getOperand(0);
3535 SDValue Op1 = N->getOperand(1);
3536 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3537 SDLoc dl(N);
3538
3539 EVT SVT = Op0.getValueType();
3540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3541
3542 Op0 = GetSoftPromotedHalf(Op0);
3543 Op1 = GetSoftPromotedHalf(Op1);
3544
3545 // Promote to the larger FP type.
3546 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3547 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3548 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3549
3550 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3551 }
3552
SoftPromoteHalfOp_STORE(SDNode * N,unsigned OpNo)3553 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3554 assert(OpNo == 1 && "Can only soften the stored value!");
3555 StoreSDNode *ST = cast<StoreSDNode>(N);
3556 SDValue Val = ST->getValue();
3557 SDLoc dl(N);
3558
3559 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3560 SDValue Promoted = GetSoftPromotedHalf(Val);
3561 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3562 ST->getMemOperand());
3563 }
3564
SoftPromoteHalfOp_ATOMIC_STORE(SDNode * N,unsigned OpNo)3565 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3566 unsigned OpNo) {
3567 assert(OpNo == 1 && "Can only soften the stored value!");
3568 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3569 SDValue Val = ST->getVal();
3570 SDLoc dl(N);
3571
3572 SDValue Promoted = GetSoftPromotedHalf(Val);
3573 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3574 ST->getChain(), Promoted, ST->getBasePtr(),
3575 ST->getMemOperand());
3576 }
3577
SoftPromoteHalfOp_STACKMAP(SDNode * N,unsigned OpNo)3578 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3579 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3580 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3581 SDValue Op = N->getOperand(OpNo);
3582 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3583 SDValue NewNode =
3584 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3585
3586 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3587 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3588
3589 return SDValue(); // Signal that we replaced the node ourselves.
3590 }
3591
SoftPromoteHalfOp_PATCHPOINT(SDNode * N,unsigned OpNo)3592 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3593 unsigned OpNo) {
3594 assert(OpNo >= 7);
3595 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3596 SDValue Op = N->getOperand(OpNo);
3597 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3598 SDValue NewNode =
3599 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3600
3601 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3602 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3603
3604 return SDValue(); // Signal that we replaced the node ourselves.
3605 }
3606