1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 /// GetFPLibCall - Return the right libcall for the given floating point type.
30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31 /// refactored away (see RTLIB::getPOWI for an example).
GetFPLibCall(EVT VT,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128)32 static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45 }
46
47 //===----------------------------------------------------------------------===//
48 // Convert Float Results to Integer
49 //===----------------------------------------------------------------------===//
50
SoftenFloatResult(SDNode * N,unsigned ResNo)51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58 #ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61 #endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
78 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
79 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
80 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
81 case ISD::STRICT_FADD:
82 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
83 case ISD::STRICT_FACOS:
84 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
85 case ISD::STRICT_FASIN:
86 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
87 case ISD::STRICT_FATAN:
88 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
89 case ISD::STRICT_FATAN2:
90 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
91 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
92 case ISD::STRICT_FCEIL:
93 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
94 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
95 case ISD::STRICT_FCOS:
96 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
97 case ISD::STRICT_FCOSH:
98 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
99 case ISD::STRICT_FDIV:
100 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
101 case ISD::STRICT_FEXP:
102 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
103 case ISD::STRICT_FEXP2:
104 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
105 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
106 case ISD::STRICT_FFLOOR:
107 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
108 case ISD::STRICT_FLOG:
109 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
110 case ISD::STRICT_FLOG2:
111 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
112 case ISD::STRICT_FLOG10:
113 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
114 case ISD::STRICT_FMA:
115 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
116 case ISD::STRICT_FMUL:
117 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
118 case ISD::STRICT_FNEARBYINT:
119 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
120 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
121 case ISD::STRICT_FP_EXTEND:
122 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
123 case ISD::STRICT_FP_ROUND:
124 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
125 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
126 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
127 case ISD::STRICT_FPOW:
128 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
129 case ISD::STRICT_FPOWI:
130 case ISD::FPOWI:
131 case ISD::FLDEXP:
132 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
133 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
134 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
135 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
136 case ISD::STRICT_FREM:
137 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
138 case ISD::STRICT_FRINT:
139 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
140 case ISD::STRICT_FROUND:
141 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
142 case ISD::STRICT_FROUNDEVEN:
143 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
144 case ISD::STRICT_FSIN:
145 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
146 case ISD::STRICT_FSINH:
147 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
148 case ISD::STRICT_FSQRT:
149 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
150 case ISD::STRICT_FSUB:
151 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
152 case ISD::STRICT_FTAN:
153 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
154 case ISD::STRICT_FTANH:
155 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
156 case ISD::STRICT_FTRUNC:
157 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
158 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
159 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
160 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
161 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
162 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
163 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
164 case ISD::STRICT_SINT_TO_FP:
165 case ISD::STRICT_UINT_TO_FP:
166 case ISD::SINT_TO_FP:
167 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
168 case ISD::POISON:
169 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
170 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
171 case ISD::AssertNoFPClass: R = GetSoftenedFloat(N->getOperand(0)); break;
172 case ISD::VECREDUCE_FADD:
173 case ISD::VECREDUCE_FMUL:
174 case ISD::VECREDUCE_FMIN:
175 case ISD::VECREDUCE_FMAX:
176 case ISD::VECREDUCE_FMAXIMUM:
177 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
178 case ISD::VECREDUCE_SEQ_FADD:
179 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
180 // clang-format on
181 }
182
183 // If R is null, the sub-method took care of registering the result.
184 if (R.getNode()) {
185 assert(R.getNode() != N);
186 SetSoftenedFloat(SDValue(N, ResNo), R);
187 }
188 }
189
SoftenFloatRes_Unary(SDNode * N,RTLIB::Libcall LC)190 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
191 bool IsStrict = N->isStrictFPOpcode();
192 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
193 unsigned Offset = IsStrict ? 1 : 0;
194 assert(N->getNumOperands() == (1 + Offset) &&
195 "Unexpected number of operands!");
196 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
197 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
198 TargetLowering::MakeLibCallOptions CallOptions;
199 EVT OpVT = N->getOperand(0 + Offset).getValueType();
200 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
202 CallOptions, SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(SDValue(N, 1), Tmp.second);
206 return Tmp.first;
207 }
208
SoftenFloatRes_Binary(SDNode * N,RTLIB::Libcall LC)209 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
210 bool IsStrict = N->isStrictFPOpcode();
211 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
212 unsigned Offset = IsStrict ? 1 : 0;
213 assert(N->getNumOperands() == (2 + Offset) &&
214 "Unexpected number of operands!");
215 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
216 GetSoftenedFloat(N->getOperand(1 + Offset)) };
217 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
218 TargetLowering::MakeLibCallOptions CallOptions;
219 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
220 N->getOperand(1 + Offset).getValueType() };
221 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
222 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
223 CallOptions, SDLoc(N),
224 Chain);
225 if (IsStrict)
226 ReplaceValueWith(SDValue(N, 1), Tmp.second);
227 return Tmp.first;
228 }
229
SoftenFloatRes_BITCAST(SDNode * N)230 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
231 return BitConvertToInteger(N->getOperand(0));
232 }
233
SoftenFloatRes_FREEZE(SDNode * N)234 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
235 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
236 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
237 GetSoftenedFloat(N->getOperand(0)));
238 }
239
SoftenFloatRes_ARITH_FENCE(SDNode * N)240 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
241 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
242 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
243 GetSoftenedFloat(N->getOperand(0)));
244 return NewFence;
245 }
246
SoftenFloatRes_MERGE_VALUES(SDNode * N,unsigned ResNo)247 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
248 unsigned ResNo) {
249 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
250 return BitConvertToInteger(Op);
251 }
252
SoftenFloatRes_BUILD_PAIR(SDNode * N)253 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
254 // Convert the inputs to integers, and build a new pair out of them.
255 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
256 TLI.getTypeToTransformTo(*DAG.getContext(),
257 N->getValueType(0)),
258 BitConvertToInteger(N->getOperand(0)),
259 BitConvertToInteger(N->getOperand(1)));
260 }
261
SoftenFloatRes_ConstantFP(SDNode * N)262 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
263 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
264 // In ppcf128, the high 64 bits are always first in memory regardless
265 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
266 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
267 // way. However, APInt's are serialized in an Endian-sensitive fashion,
268 // so on big-Endian targets, the two doubles are output in the wrong
269 // order. Fix this by manually flipping the order of the high 64 bits
270 // and the low 64 bits here.
271 if (DAG.getDataLayout().isBigEndian() &&
272 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
273 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
274 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
275 APInt Val(128, words);
276 return DAG.getConstant(Val, SDLoc(CN),
277 TLI.getTypeToTransformTo(*DAG.getContext(),
278 CN->getValueType(0)));
279 } else {
280 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
281 TLI.getTypeToTransformTo(*DAG.getContext(),
282 CN->getValueType(0)));
283 }
284 }
285
SoftenFloatRes_EXTRACT_ELEMENT(SDNode * N)286 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
287 SDValue Src = N->getOperand(0);
288 assert(Src.getValueType() == MVT::ppcf128 &&
289 "In floats only ppcf128 can be extracted by element!");
290 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
291 N->getValueType(0).changeTypeToInteger(),
292 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
293 }
294
SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode * N,unsigned ResNo)295 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
296 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
297 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
298 NewOp.getValueType().getVectorElementType(),
299 NewOp, N->getOperand(1));
300 }
301
SoftenFloatRes_FABS(SDNode * N)302 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
303 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
304 unsigned Size = NVT.getSizeInBits();
305
306 // Mask = ~(1 << (Size-1))
307 APInt API = APInt::getAllOnes(Size);
308 API.clearBit(Size - 1);
309 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
310 SDValue Op = GetSoftenedFloat(N->getOperand(0));
311 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
312 }
313
SoftenFloatRes_FMINNUM(SDNode * N)314 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
315 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
316 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
317 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
318 RTLIB::FMIN_F32,
319 RTLIB::FMIN_F64,
320 RTLIB::FMIN_F80,
321 RTLIB::FMIN_F128,
322 RTLIB::FMIN_PPCF128));
323 }
324
SoftenFloatRes_FMAXNUM(SDNode * N)325 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
326 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
327 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
328 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
329 RTLIB::FMAX_F32,
330 RTLIB::FMAX_F64,
331 RTLIB::FMAX_F80,
332 RTLIB::FMAX_F128,
333 RTLIB::FMAX_PPCF128));
334 }
335
SoftenFloatRes_FMINIMUMNUM(SDNode * N)336 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
337 return SoftenFloatRes_Binary(
338 N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_NUM_F32,
339 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
340 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128));
341 }
342
SoftenFloatRes_FMAXIMUMNUM(SDNode * N)343 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
344 return SoftenFloatRes_Binary(
345 N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_NUM_F32,
346 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
347 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128));
348 }
349
SoftenFloatRes_FMINIMUM(SDNode * N)350 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
351 return SoftenFloatRes_Binary(
352 N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_F32,
353 RTLIB::FMINIMUM_F64, RTLIB::FMINIMUM_F80,
354 RTLIB::FMINIMUM_F128, RTLIB::FMINIMUM_PPCF128));
355 }
356
SoftenFloatRes_FMAXIMUM(SDNode * N)357 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
358 return SoftenFloatRes_Binary(
359 N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_F32,
360 RTLIB::FMAXIMUM_F64, RTLIB::FMAXIMUM_F80,
361 RTLIB::FMAXIMUM_F128, RTLIB::FMAXIMUM_PPCF128));
362 }
363
SoftenFloatRes_FADD(SDNode * N)364 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
365 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
366 RTLIB::ADD_F32,
367 RTLIB::ADD_F64,
368 RTLIB::ADD_F80,
369 RTLIB::ADD_F128,
370 RTLIB::ADD_PPCF128));
371 }
372
SoftenFloatRes_FACOS(SDNode * N)373 SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
374 return SoftenFloatRes_Unary(
375 N, GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64,
376 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128));
377 }
378
SoftenFloatRes_FASIN(SDNode * N)379 SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
380 return SoftenFloatRes_Unary(
381 N, GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64,
382 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128));
383 }
384
SoftenFloatRes_FATAN(SDNode * N)385 SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
386 return SoftenFloatRes_Unary(
387 N, GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64,
388 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128));
389 }
390
SoftenFloatRes_FATAN2(SDNode * N)391 SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
392 return SoftenFloatRes_Binary(
393 N,
394 GetFPLibCall(N->getValueType(0), RTLIB::ATAN2_F32, RTLIB::ATAN2_F64,
395 RTLIB::ATAN2_F80, RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128));
396 }
397
SoftenFloatRes_FCBRT(SDNode * N)398 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
399 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
400 RTLIB::CBRT_F32,
401 RTLIB::CBRT_F64,
402 RTLIB::CBRT_F80,
403 RTLIB::CBRT_F128,
404 RTLIB::CBRT_PPCF128));
405 }
406
SoftenFloatRes_FCEIL(SDNode * N)407 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
408 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
409 RTLIB::CEIL_F32,
410 RTLIB::CEIL_F64,
411 RTLIB::CEIL_F80,
412 RTLIB::CEIL_F128,
413 RTLIB::CEIL_PPCF128));
414 }
415
SoftenFloatRes_FCOPYSIGN(SDNode * N)416 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
417 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
418 SDValue RHS = BitConvertToInteger(N->getOperand(1));
419 SDLoc dl(N);
420
421 EVT LVT = LHS.getValueType();
422 EVT RVT = RHS.getValueType();
423
424 unsigned LSize = LVT.getSizeInBits();
425 unsigned RSize = RVT.getSizeInBits();
426
427 // First get the sign bit of second operand.
428 SDValue SignBit = DAG.getNode(
429 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
430 DAG.getConstant(RSize - 1, dl,
431 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
432 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
433
434 // Shift right or sign-extend it if the two operands have different types.
435 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
436 if (SizeDiff > 0) {
437 SignBit =
438 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
439 DAG.getConstant(SizeDiff, dl,
440 TLI.getShiftAmountTy(SignBit.getValueType(),
441 DAG.getDataLayout())));
442 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
443 } else if (SizeDiff < 0) {
444 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
445 SignBit =
446 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
447 DAG.getConstant(-SizeDiff, dl,
448 TLI.getShiftAmountTy(SignBit.getValueType(),
449 DAG.getDataLayout())));
450 }
451
452 // Clear the sign bit of the first operand.
453 SDValue Mask = DAG.getNode(
454 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
455 DAG.getConstant(LSize - 1, dl,
456 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
457 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
458 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
459
460 // Or the value with the sign bit.
461 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
462 }
463
SoftenFloatRes_FCOS(SDNode * N)464 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
465 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
466 RTLIB::COS_F32,
467 RTLIB::COS_F64,
468 RTLIB::COS_F80,
469 RTLIB::COS_F128,
470 RTLIB::COS_PPCF128));
471 }
472
SoftenFloatRes_FCOSH(SDNode * N)473 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
474 return SoftenFloatRes_Unary(
475 N, GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64,
476 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128));
477 }
478
SoftenFloatRes_FDIV(SDNode * N)479 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
480 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
481 RTLIB::DIV_F32,
482 RTLIB::DIV_F64,
483 RTLIB::DIV_F80,
484 RTLIB::DIV_F128,
485 RTLIB::DIV_PPCF128));
486 }
487
SoftenFloatRes_FEXP(SDNode * N)488 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
489 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
490 RTLIB::EXP_F32,
491 RTLIB::EXP_F64,
492 RTLIB::EXP_F80,
493 RTLIB::EXP_F128,
494 RTLIB::EXP_PPCF128));
495 }
496
SoftenFloatRes_FEXP2(SDNode * N)497 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
498 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
499 RTLIB::EXP2_F32,
500 RTLIB::EXP2_F64,
501 RTLIB::EXP2_F80,
502 RTLIB::EXP2_F128,
503 RTLIB::EXP2_PPCF128));
504 }
505
SoftenFloatRes_FEXP10(SDNode * N)506 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
507 return SoftenFloatRes_Unary(
508 N,
509 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
510 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
511 }
512
SoftenFloatRes_FFLOOR(SDNode * N)513 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
514 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
515 RTLIB::FLOOR_F32,
516 RTLIB::FLOOR_F64,
517 RTLIB::FLOOR_F80,
518 RTLIB::FLOOR_F128,
519 RTLIB::FLOOR_PPCF128));
520 }
521
SoftenFloatRes_FLOG(SDNode * N)522 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
523 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
524 RTLIB::LOG_F32,
525 RTLIB::LOG_F64,
526 RTLIB::LOG_F80,
527 RTLIB::LOG_F128,
528 RTLIB::LOG_PPCF128));
529 }
530
SoftenFloatRes_FLOG2(SDNode * N)531 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
532 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
533 RTLIB::LOG2_F32,
534 RTLIB::LOG2_F64,
535 RTLIB::LOG2_F80,
536 RTLIB::LOG2_F128,
537 RTLIB::LOG2_PPCF128));
538 }
539
SoftenFloatRes_FLOG10(SDNode * N)540 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
541 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
542 RTLIB::LOG10_F32,
543 RTLIB::LOG10_F64,
544 RTLIB::LOG10_F80,
545 RTLIB::LOG10_F128,
546 RTLIB::LOG10_PPCF128));
547 }
548
SoftenFloatRes_FMA(SDNode * N)549 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
550 bool IsStrict = N->isStrictFPOpcode();
551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
552 unsigned Offset = IsStrict ? 1 : 0;
553 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
554 GetSoftenedFloat(N->getOperand(1 + Offset)),
555 GetSoftenedFloat(N->getOperand(2 + Offset)) };
556 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
557 TargetLowering::MakeLibCallOptions CallOptions;
558 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
559 N->getOperand(1 + Offset).getValueType(),
560 N->getOperand(2 + Offset).getValueType() };
561 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
563 GetFPLibCall(N->getValueType(0),
564 RTLIB::FMA_F32,
565 RTLIB::FMA_F64,
566 RTLIB::FMA_F80,
567 RTLIB::FMA_F128,
568 RTLIB::FMA_PPCF128),
569 NVT, Ops, CallOptions, SDLoc(N), Chain);
570 if (IsStrict)
571 ReplaceValueWith(SDValue(N, 1), Tmp.second);
572 return Tmp.first;
573 }
574
SoftenFloatRes_FMUL(SDNode * N)575 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
576 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
577 RTLIB::MUL_F32,
578 RTLIB::MUL_F64,
579 RTLIB::MUL_F80,
580 RTLIB::MUL_F128,
581 RTLIB::MUL_PPCF128));
582 }
583
SoftenFloatRes_FNEARBYINT(SDNode * N)584 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
585 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
586 RTLIB::NEARBYINT_F32,
587 RTLIB::NEARBYINT_F64,
588 RTLIB::NEARBYINT_F80,
589 RTLIB::NEARBYINT_F128,
590 RTLIB::NEARBYINT_PPCF128));
591 }
592
SoftenFloatRes_FNEG(SDNode * N)593 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
594 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
595 SDLoc dl(N);
596
597 // Expand Y = FNEG(X) -> Y = X ^ sign mask
598 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
599 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
600 DAG.getConstant(SignMask, dl, NVT));
601 }
602
SoftenFloatRes_FP_EXTEND(SDNode * N)603 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
604 bool IsStrict = N->isStrictFPOpcode();
605 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
606 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
607
608 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
609
610 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
611 Op = GetPromotedFloat(Op);
612 // If the promotion did the FP_EXTEND to the destination type for us,
613 // there's nothing left to do here.
614 if (Op.getValueType() == N->getValueType(0)) {
615 if (IsStrict)
616 ReplaceValueWith(SDValue(N, 1), Chain);
617 return BitConvertToInteger(Op);
618 }
619 }
620
621 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
622 // -> f32, so proceed in two stages. Also, it's entirely possible for both
623 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
624 // than FP16_TO_FP.
625 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
626 N->getValueType(0) != MVT::f32) {
627 if (IsStrict) {
628 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
629 { MVT::f32, MVT::Other }, { Chain, Op });
630 Chain = Op.getValue(1);
631 } else {
632 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
633 }
634 }
635
636 if (Op.getValueType() == MVT::bf16) {
637 // FIXME: Need ReplaceValueWith on chain in strict case
638 return SoftenFloatRes_BF16_TO_FP(N);
639 }
640
641 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
643 TargetLowering::MakeLibCallOptions CallOptions;
644 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
645 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
646 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
647 CallOptions, SDLoc(N),
648 Chain);
649 if (IsStrict)
650 ReplaceValueWith(SDValue(N, 1), Tmp.second);
651 return Tmp.first;
652 }
653
654 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
655 // nodes?
SoftenFloatRes_FP16_TO_FP(SDNode * N)656 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
657 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
658 SDValue Op = N->getOperand(0);
659 TargetLowering::MakeLibCallOptions CallOptions;
660 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
661 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
662 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
663 CallOptions, SDLoc(N)).first;
664 if (N->getValueType(0) == MVT::f32)
665 return Res32;
666
667 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
668 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
669 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
670 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
671 }
672
673 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
674 // nodes?
SoftenFloatRes_BF16_TO_FP(SDNode * N)675 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
676 assert(N->getValueType(0) == MVT::f32 &&
677 "Can only soften BF16_TO_FP with f32 result");
678 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
679 SDValue Op = N->getOperand(0);
680 SDLoc DL(N);
681 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
682 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
683 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
684 DAG.getShiftAmountConstant(16, NVT, DL));
685 return Res;
686 }
687
SoftenFloatRes_FP_ROUND(SDNode * N)688 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
689 bool IsStrict = N->isStrictFPOpcode();
690 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
691 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
692 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
693 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
694 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
695 TargetLowering::MakeLibCallOptions CallOptions;
696 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
697 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
698 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
699 CallOptions, SDLoc(N),
700 Chain);
701 if (IsStrict)
702 ReplaceValueWith(SDValue(N, 1), Tmp.second);
703 return Tmp.first;
704 }
705
SoftenFloatRes_FPOW(SDNode * N)706 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
707 return SoftenFloatRes_Binary(N, RTLIB::getPOW(N->getValueType(0)));
708 }
709
SoftenFloatRes_ExpOp(SDNode * N)710 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
711 bool IsStrict = N->isStrictFPOpcode();
712 unsigned Offset = IsStrict ? 1 : 0;
713 bool IsPowI =
714 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
715 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
716
717 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
718 : RTLIB::getLDEXP(N->getValueType(0));
719 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
720 if (!TLI.getLibcallName(LC)) {
721 // Some targets don't have a powi libcall; use pow instead.
722 // FIXME: Implement this if some target needs it.
723 DAG.getContext()->emitError("do not know how to soften fpowi to fpow");
724 if (IsStrict)
725 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
726 return DAG.getPOISON(NVT);
727 }
728
729 if (DAG.getLibInfo().getIntSize() !=
730 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
731 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
732 // would use the wrong type for the argument.
733 DAG.getContext()->emitError("powi exponent does not match sizeof(int)");
734 if (IsStrict)
735 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
736 return DAG.getPOISON(NVT);
737 }
738
739 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
740 N->getOperand(1 + Offset) };
741 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
742 TargetLowering::MakeLibCallOptions CallOptions;
743 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
744 N->getOperand(1 + Offset).getValueType() };
745 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
746 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
747 CallOptions, SDLoc(N),
748 Chain);
749 if (IsStrict)
750 ReplaceValueWith(SDValue(N, 1), Tmp.second);
751 return Tmp.first;
752 }
753
SoftenFloatRes_FFREXP(SDNode * N)754 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
755 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
756 EVT VT0 = N->getValueType(0);
757 EVT VT1 = N->getValueType(1);
758 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
759 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
760 SDLoc DL(N);
761
762 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
763 // If the exponent does not match with sizeof(int) a libcall would use the
764 // wrong type for the argument.
765 // TODO: Should be able to handle mismatches.
766 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
767 SDValue PoisonExp = DAG.getPOISON(VT1);
768 ReplaceValueWith(SDValue(N, 1), PoisonExp);
769 return DAG.getMergeValues({DAG.getPOISON(NVT0), PoisonExp}, DL);
770 }
771
772 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
773
774 auto PointerTy = PointerType::getUnqual(*DAG.getContext());
775 TargetLowering::MakeLibCallOptions CallOptions;
776 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
777 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
778 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
779
780 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
781 // but we only really need to handle the 0th one for softening anyway.
782 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true)
783 .setOpsTypeOverrides(CallOpsTypeOverrides);
784
785 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
786 /*Chain=*/SDValue());
787 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
788 auto PtrInfo =
789 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
790
791 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
792
793 ReplaceValueWith(SDValue(N, 1), LoadExp);
794 return ReturnVal;
795 }
796
SoftenFloatRes_UnaryWithTwoFPResults(SDNode * N,RTLIB::Libcall LC,std::optional<unsigned> CallRetResNo)797 bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
798 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
799 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
800 EVT VT = N->getValueType(0);
801
802 assert(VT == N->getValueType(1) &&
803 "expected both return values to have the same type");
804
805 if (!TLI.getLibcallName(LC))
806 return false;
807
808 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
809
810 SDLoc DL(N);
811
812 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(N->getOperand(0))};
813 SmallVector<EVT, 3> OpsVT = {VT};
814
815 std::array<SDValue, 2> StackSlots;
816 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
817 auto PointerTy = PointerType::getUnqual(*DAG.getContext());
818 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
819 if (ResNum == CallRetResNo)
820 continue;
821 SDValue StackSlot = DAG.CreateStackTemporary(NVT);
822 Ops.push_back(StackSlot);
823 OpsVT.push_back(StackSlot.getValueType());
824 StackSlots[ResNum] = StackSlot;
825 CallOpsTypeOverrides.push_back(PointerTy);
826 }
827
828 TargetLowering::MakeLibCallOptions CallOptions;
829 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
830 // but since both returns have the same type it should be okay.
831 CallOptions.setTypeListBeforeSoften({OpsVT}, VT, true)
832 .setOpsTypeOverrides(CallOpsTypeOverrides);
833
834 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, DL,
835 /*Chain=*/SDValue());
836
837 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
838 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
839 auto PtrInfo =
840 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
841 return DAG.getLoad(NVT, DL, Chain, StackSlot, PtrInfo);
842 };
843
844 for (auto [ResNum, SlackSlot] : enumerate(StackSlots)) {
845 if (CallRetResNo == ResNum) {
846 SetSoftenedFloat(SDValue(N, ResNum), ReturnVal);
847 continue;
848 }
849 SetSoftenedFloat(SDValue(N, ResNum), CreateStackLoad(SlackSlot));
850 }
851
852 return true;
853 }
854
SoftenFloatRes_FSINCOS(SDNode * N)855 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
856 EVT VT = N->getValueType(0);
857 if (SoftenFloatRes_UnaryWithTwoFPResults(N, RTLIB::getSINCOS(VT)))
858 return SDValue();
859
860 // Fall back on softening the separate sin and cos calls if available.
861 RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
862 RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
863
864 SDValue SoftSin, SoftCos;
865 if (!TLI.getLibcallName(SinLC) || !TLI.getLibcallName(CosLC)) {
866 DAG.getContext()->emitError("do not know how to soften fsincos");
867
868 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
869 SoftSin = SoftCos = DAG.getPOISON(NVT);
870 } else {
871 SoftSin = SoftenFloatRes_Unary(N, SinLC);
872 SoftCos = SoftenFloatRes_Unary(N, CosLC);
873 }
874
875 SetSoftenedFloat(SDValue(N, 0), SoftSin);
876 SetSoftenedFloat(SDValue(N, 1), SoftCos);
877 return SDValue();
878 }
879
SoftenFloatRes_FMODF(SDNode * N)880 SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
881 EVT VT = N->getValueType(0);
882 if (SoftenFloatRes_UnaryWithTwoFPResults(N, RTLIB::getMODF(VT),
883 /*CallRetResNo=*/0))
884 return SDValue();
885
886 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
887 DAG.getContext()->emitError("do not know how to soften fmodf");
888 SDValue Poison = DAG.getPOISON(NVT);
889 SetSoftenedFloat(SDValue(N, 0), Poison);
890 SetSoftenedFloat(SDValue(N, 1), Poison);
891 return SDValue();
892 }
893
SoftenFloatRes_FREM(SDNode * N)894 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
895 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
896 RTLIB::REM_F32,
897 RTLIB::REM_F64,
898 RTLIB::REM_F80,
899 RTLIB::REM_F128,
900 RTLIB::REM_PPCF128));
901 }
902
SoftenFloatRes_FRINT(SDNode * N)903 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
904 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
905 RTLIB::RINT_F32,
906 RTLIB::RINT_F64,
907 RTLIB::RINT_F80,
908 RTLIB::RINT_F128,
909 RTLIB::RINT_PPCF128));
910 }
911
SoftenFloatRes_FROUND(SDNode * N)912 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
913 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
914 RTLIB::ROUND_F32,
915 RTLIB::ROUND_F64,
916 RTLIB::ROUND_F80,
917 RTLIB::ROUND_F128,
918 RTLIB::ROUND_PPCF128));
919 }
920
SoftenFloatRes_FROUNDEVEN(SDNode * N)921 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
922 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
923 RTLIB::ROUNDEVEN_F32,
924 RTLIB::ROUNDEVEN_F64,
925 RTLIB::ROUNDEVEN_F80,
926 RTLIB::ROUNDEVEN_F128,
927 RTLIB::ROUNDEVEN_PPCF128));
928 }
929
SoftenFloatRes_FSIN(SDNode * N)930 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
931 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
932 RTLIB::SIN_F32,
933 RTLIB::SIN_F64,
934 RTLIB::SIN_F80,
935 RTLIB::SIN_F128,
936 RTLIB::SIN_PPCF128));
937 }
938
SoftenFloatRes_FSINH(SDNode * N)939 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
940 return SoftenFloatRes_Unary(
941 N, GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64,
942 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128));
943 }
944
SoftenFloatRes_FSQRT(SDNode * N)945 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
946 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
947 RTLIB::SQRT_F32,
948 RTLIB::SQRT_F64,
949 RTLIB::SQRT_F80,
950 RTLIB::SQRT_F128,
951 RTLIB::SQRT_PPCF128));
952 }
953
SoftenFloatRes_FSUB(SDNode * N)954 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
955 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
956 RTLIB::SUB_F32,
957 RTLIB::SUB_F64,
958 RTLIB::SUB_F80,
959 RTLIB::SUB_F128,
960 RTLIB::SUB_PPCF128));
961 }
962
SoftenFloatRes_FTAN(SDNode * N)963 SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
964 return SoftenFloatRes_Unary(
965 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
966 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
967 }
968
SoftenFloatRes_FTANH(SDNode * N)969 SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
970 return SoftenFloatRes_Unary(
971 N, GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64,
972 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128));
973 }
974
SoftenFloatRes_FTRUNC(SDNode * N)975 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
976 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
977 RTLIB::TRUNC_F32,
978 RTLIB::TRUNC_F64,
979 RTLIB::TRUNC_F80,
980 RTLIB::TRUNC_F128,
981 RTLIB::TRUNC_PPCF128));
982 }
983
SoftenFloatRes_LOAD(SDNode * N)984 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
985 LoadSDNode *L = cast<LoadSDNode>(N);
986 EVT VT = N->getValueType(0);
987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
988 SDLoc dl(N);
989
990 auto MMOFlags =
991 L->getMemOperand()->getFlags() &
992 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
993 SDValue NewL;
994 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
995 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
996 L->getChain(), L->getBasePtr(), L->getOffset(),
997 L->getPointerInfo(), NVT, L->getBaseAlign(), MMOFlags,
998 L->getAAInfo());
999 // Legalized the chain result - switch anything that used the old chain to
1000 // use the new one.
1001 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
1002 return NewL;
1003 }
1004
1005 // Do a non-extending load followed by FP_EXTEND.
1006 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
1007 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
1008 L->getPointerInfo(), L->getMemoryVT(), L->getBaseAlign(),
1009 MMOFlags, L->getAAInfo());
1010 // Legalized the chain result - switch anything that used the old chain to
1011 // use the new one.
1012 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
1013 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
1014 return BitConvertToInteger(ExtendNode);
1015 }
1016
SoftenFloatRes_ATOMIC_LOAD(SDNode * N)1017 SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
1018 AtomicSDNode *L = cast<AtomicSDNode>(N);
1019 EVT VT = N->getValueType(0);
1020 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1021 SDLoc dl(N);
1022
1023 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
1024 SDValue NewL =
1025 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
1026 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
1027
1028 // Legalized the chain result - switch anything that used the old chain to
1029 // use the new one.
1030 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
1031 return NewL;
1032 }
1033
1034 report_fatal_error("softening fp extending atomic load not handled");
1035 }
1036
SoftenFloatRes_SELECT(SDNode * N)1037 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
1038 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
1039 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
1040 return DAG.getSelect(SDLoc(N),
1041 LHS.getValueType(), N->getOperand(0), LHS, RHS);
1042 }
1043
SoftenFloatRes_SELECT_CC(SDNode * N)1044 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
1045 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
1046 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
1047 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1048 LHS.getValueType(), N->getOperand(0),
1049 N->getOperand(1), LHS, RHS, N->getOperand(4));
1050 }
1051
SoftenFloatRes_UNDEF(SDNode * N)1052 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
1053 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1054 N->getValueType(0)));
1055 }
1056
SoftenFloatRes_VAARG(SDNode * N)1057 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
1058 SDValue Chain = N->getOperand(0); // Get the chain.
1059 SDValue Ptr = N->getOperand(1); // Get the pointer.
1060 EVT VT = N->getValueType(0);
1061 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1062 SDLoc dl(N);
1063
1064 SDValue NewVAARG;
1065 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
1066 N->getConstantOperandVal(3));
1067
1068 // Legalized the chain result - switch anything that used the old chain to
1069 // use the new one.
1070 if (N != NewVAARG.getValue(1).getNode())
1071 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
1072 return NewVAARG;
1073 }
1074
SoftenFloatRes_XINT_TO_FP(SDNode * N)1075 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1076 bool IsStrict = N->isStrictFPOpcode();
1077 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1078 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1079 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
1080 EVT RVT = N->getValueType(0);
1081 EVT NVT = EVT();
1082 SDLoc dl(N);
1083
1084 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1085 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1086 // match. Look for an appropriate libcall.
1087 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1088 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1089 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1090 NVT = (MVT::SimpleValueType)t;
1091 // The source needs to big enough to hold the operand.
1092 if (NVT.bitsGE(SVT))
1093 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
1094 }
1095 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1096
1097 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1098 // Sign/zero extend the argument if the libcall takes a larger type.
1099 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1100 NVT, N->getOperand(IsStrict ? 1 : 0));
1101 TargetLowering::MakeLibCallOptions CallOptions;
1102 CallOptions.setIsSigned(Signed);
1103 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1104 std::pair<SDValue, SDValue> Tmp =
1105 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
1106 Op, CallOptions, dl, Chain);
1107
1108 if (IsStrict)
1109 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1110 return Tmp.first;
1111 }
1112
SoftenFloatRes_VECREDUCE(SDNode * N)1113 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1114 // Expand and soften recursively.
1115 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
1116 return SDValue();
1117 }
1118
SoftenFloatRes_VECREDUCE_SEQ(SDNode * N)1119 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1120 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
1121 return SDValue();
1122 }
1123
1124 //===----------------------------------------------------------------------===//
1125 // Convert Float Operand to Integer
1126 //===----------------------------------------------------------------------===//
1127
SoftenFloatOperand(SDNode * N,unsigned OpNo)1128 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1129 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1130 SDValue Res = SDValue();
1131
1132 switch (N->getOpcode()) {
1133 default:
1134 #ifndef NDEBUG
1135 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1136 N->dump(&DAG); dbgs() << "\n";
1137 #endif
1138 report_fatal_error("Do not know how to soften this operator's operand!");
1139
1140 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1141 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1142 case ISD::STRICT_FP_TO_FP16:
1143 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1144 case ISD::FP_TO_BF16:
1145 case ISD::STRICT_FP_TO_BF16:
1146 case ISD::STRICT_FP_ROUND:
1147 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1148 case ISD::STRICT_FP_TO_SINT:
1149 case ISD::STRICT_FP_TO_UINT:
1150 case ISD::FP_TO_SINT:
1151 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1152 case ISD::FP_TO_SINT_SAT:
1153 case ISD::FP_TO_UINT_SAT:
1154 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1155 case ISD::STRICT_LROUND:
1156 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1157 case ISD::STRICT_LLROUND:
1158 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1159 case ISD::STRICT_LRINT:
1160 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1161 case ISD::STRICT_LLRINT:
1162 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1163 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1164 case ISD::STRICT_FSETCC:
1165 case ISD::STRICT_FSETCCS:
1166 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1167 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1168 case ISD::ATOMIC_STORE:
1169 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1170 break;
1171 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1172 case ISD::FAKE_USE:
1173 Res = SoftenFloatOp_FAKE_USE(N);
1174 break;
1175 }
1176
1177 // If the result is null, the sub-method took care of registering results etc.
1178 if (!Res.getNode()) return false;
1179
1180 // If the result is N, the sub-method updated N in place. Tell the legalizer
1181 // core about this to re-analyze.
1182 if (Res.getNode() == N)
1183 return true;
1184
1185 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1186 "Invalid operand softening");
1187
1188 ReplaceValueWith(SDValue(N, 0), Res);
1189 return false;
1190 }
1191
SoftenFloatOp_BITCAST(SDNode * N)1192 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1193 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
1194
1195 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
1196 }
1197
SoftenFloatOp_FP_ROUND(SDNode * N)1198 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1199 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1200 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1201 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1202 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1203 N->getOpcode() == ISD::FP_TO_BF16 ||
1204 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1205 N->getOpcode() == ISD::STRICT_FP_ROUND);
1206
1207 bool IsStrict = N->isStrictFPOpcode();
1208 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1209 EVT SVT = Op.getValueType();
1210 EVT RVT = N->getValueType(0);
1211 EVT FloatRVT = RVT;
1212 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1213 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1214 FloatRVT = MVT::f16;
1215 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1216 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1217 FloatRVT = MVT::bf16;
1218
1219 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1220 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1221
1222 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1223 Op = GetSoftenedFloat(Op);
1224 TargetLowering::MakeLibCallOptions CallOptions;
1225 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1226 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1227 CallOptions, SDLoc(N),
1228 Chain);
1229 if (IsStrict) {
1230 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1231 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1232 return SDValue();
1233 }
1234 return Tmp.first;
1235 }
1236
SoftenFloatOp_BR_CC(SDNode * N)1237 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1238 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1239 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1240
1241 EVT VT = NewLHS.getValueType();
1242 NewLHS = GetSoftenedFloat(NewLHS);
1243 NewRHS = GetSoftenedFloat(NewRHS);
1244 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1245 N->getOperand(2), N->getOperand(3));
1246
1247 // If softenSetCCOperands returned a scalar, we need to compare the result
1248 // against zero to select between true and false values.
1249 if (!NewRHS.getNode()) {
1250 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1251 CCCode = ISD::SETNE;
1252 }
1253
1254 // Update N to have the operands specified.
1255 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1256 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1257 N->getOperand(4)),
1258 0);
1259 }
1260
1261 // Even if the result type is legal, no libcall may exactly match. (e.g. We
1262 // don't have FP-i8 conversions) This helper method looks for an appropriate
1263 // promoted libcall.
findFPToIntLibcall(EVT SrcVT,EVT RetVT,EVT & Promoted,bool Signed)1264 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1265 bool Signed) {
1266 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1267 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1268 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1269 ++IntVT) {
1270 Promoted = (MVT::SimpleValueType)IntVT;
1271 // The type needs to big enough to hold the result.
1272 if (Promoted.bitsGE(RetVT))
1273 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1274 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1275 }
1276 return LC;
1277 }
1278
SoftenFloatOp_FP_TO_XINT(SDNode * N)1279 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1280 bool IsStrict = N->isStrictFPOpcode();
1281 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1282 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1283
1284 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1285 EVT SVT = Op.getValueType();
1286 EVT RVT = N->getValueType(0);
1287 EVT NVT = EVT();
1288 SDLoc dl(N);
1289
1290 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1291 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1292 // match, eg. we don't have fp -> i8 conversions.
1293 // Look for an appropriate libcall.
1294 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1295 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1296 "Unsupported FP_TO_XINT!");
1297
1298 Op = GetSoftenedFloat(Op);
1299 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1300 TargetLowering::MakeLibCallOptions CallOptions;
1301 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1302 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1303 CallOptions, dl, Chain);
1304
1305 // Truncate the result if the libcall returns a larger type.
1306 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1307
1308 if (!IsStrict)
1309 return Res;
1310
1311 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1312 ReplaceValueWith(SDValue(N, 0), Res);
1313 return SDValue();
1314 }
1315
SoftenFloatOp_FP_TO_XINT_SAT(SDNode * N)1316 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1317 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1318 return Res;
1319 }
1320
SoftenFloatOp_SELECT_CC(SDNode * N)1321 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1322 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1323 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1324
1325 EVT VT = NewLHS.getValueType();
1326 NewLHS = GetSoftenedFloat(NewLHS);
1327 NewRHS = GetSoftenedFloat(NewRHS);
1328 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1329 N->getOperand(0), N->getOperand(1));
1330
1331 // If softenSetCCOperands returned a scalar, we need to compare the result
1332 // against zero to select between true and false values.
1333 if (!NewRHS.getNode()) {
1334 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1335 CCCode = ISD::SETNE;
1336 }
1337
1338 // Update N to have the operands specified.
1339 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1340 N->getOperand(2), N->getOperand(3),
1341 DAG.getCondCode(CCCode)),
1342 0);
1343 }
1344
SoftenFloatOp_SETCC(SDNode * N)1345 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1346 bool IsStrict = N->isStrictFPOpcode();
1347 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1348 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1349 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1350 ISD::CondCode CCCode =
1351 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1352
1353 EVT VT = Op0.getValueType();
1354 SDValue NewLHS = GetSoftenedFloat(Op0);
1355 SDValue NewRHS = GetSoftenedFloat(Op1);
1356 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1357 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1358
1359 // Update N to have the operands specified.
1360 if (NewRHS.getNode()) {
1361 if (IsStrict)
1362 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1363 NewRHS, DAG.getCondCode(CCCode));
1364 else
1365 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1366 DAG.getCondCode(CCCode)), 0);
1367 }
1368
1369 // Otherwise, softenSetCCOperands returned a scalar, use it.
1370 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1371 "Unexpected setcc expansion!");
1372
1373 if (IsStrict) {
1374 ReplaceValueWith(SDValue(N, 0), NewLHS);
1375 ReplaceValueWith(SDValue(N, 1), Chain);
1376 return SDValue();
1377 }
1378 return NewLHS;
1379 }
1380
SoftenFloatOp_STORE(SDNode * N,unsigned OpNo)1381 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1382 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1383 assert(OpNo == 1 && "Can only soften the stored value!");
1384 StoreSDNode *ST = cast<StoreSDNode>(N);
1385 SDValue Val = ST->getValue();
1386 SDLoc dl(N);
1387
1388 if (ST->isTruncatingStore())
1389 // Do an FP_ROUND followed by a non-truncating store.
1390 Val = BitConvertToInteger(
1391 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1392 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1393 else
1394 Val = GetSoftenedFloat(Val);
1395
1396 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1397 ST->getMemOperand());
1398 }
1399
SoftenFloatOp_ATOMIC_STORE(SDNode * N,unsigned OpNo)1400 SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1401 assert(OpNo == 1 && "Can only soften the stored value!");
1402 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1403 SDValue Val = ST->getVal();
1404 EVT VT = Val.getValueType();
1405 SDLoc dl(N);
1406
1407 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1408
1409 SDValue NewVal = GetSoftenedFloat(Val);
1410 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1411 ST->getBasePtr(), ST->getMemOperand());
1412 }
1413
SoftenFloatOp_FCOPYSIGN(SDNode * N)1414 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1415 SDValue LHS = N->getOperand(0);
1416 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1417 SDLoc dl(N);
1418
1419 EVT LVT = LHS.getValueType();
1420 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1421 EVT RVT = RHS.getValueType();
1422
1423 unsigned LSize = LVT.getSizeInBits();
1424 unsigned RSize = RVT.getSizeInBits();
1425
1426 // Shift right or sign-extend it if the two operands have different types.
1427 int SizeDiff = RSize - LSize;
1428 if (SizeDiff > 0) {
1429 RHS =
1430 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1431 DAG.getConstant(SizeDiff, dl,
1432 TLI.getShiftAmountTy(RHS.getValueType(),
1433 DAG.getDataLayout())));
1434 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1435 } else if (SizeDiff < 0) {
1436 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1437 RHS =
1438 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1439 DAG.getConstant(-SizeDiff, dl,
1440 TLI.getShiftAmountTy(RHS.getValueType(),
1441 DAG.getDataLayout())));
1442 }
1443
1444 RHS = DAG.getBitcast(LVT, RHS);
1445 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1446 }
1447
SoftenFloatOp_Unary(SDNode * N,RTLIB::Libcall LC)1448 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1449 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1450 bool IsStrict = N->isStrictFPOpcode();
1451 unsigned Offset = IsStrict ? 1 : 0;
1452 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1453 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1454 TargetLowering::MakeLibCallOptions CallOptions;
1455 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1456 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1457 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1458 CallOptions, SDLoc(N),
1459 Chain);
1460 if (IsStrict) {
1461 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1462 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1463 return SDValue();
1464 }
1465
1466 return Tmp.first;
1467 }
1468
SoftenFloatOp_LROUND(SDNode * N)1469 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1470 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1471 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1472 RTLIB::LROUND_F32,
1473 RTLIB::LROUND_F64,
1474 RTLIB::LROUND_F80,
1475 RTLIB::LROUND_F128,
1476 RTLIB::LROUND_PPCF128));
1477 }
1478
SoftenFloatOp_LLROUND(SDNode * N)1479 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1480 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1481 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1482 RTLIB::LLROUND_F32,
1483 RTLIB::LLROUND_F64,
1484 RTLIB::LLROUND_F80,
1485 RTLIB::LLROUND_F128,
1486 RTLIB::LLROUND_PPCF128));
1487 }
1488
SoftenFloatOp_LRINT(SDNode * N)1489 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1490 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1491 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1492 RTLIB::LRINT_F32,
1493 RTLIB::LRINT_F64,
1494 RTLIB::LRINT_F80,
1495 RTLIB::LRINT_F128,
1496 RTLIB::LRINT_PPCF128));
1497 }
1498
SoftenFloatOp_LLRINT(SDNode * N)1499 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1500 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1501 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1502 RTLIB::LLRINT_F32,
1503 RTLIB::LLRINT_F64,
1504 RTLIB::LLRINT_F80,
1505 RTLIB::LLRINT_F128,
1506 RTLIB::LLRINT_PPCF128));
1507 }
1508
SoftenFloatOp_FAKE_USE(SDNode * N)1509 SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1510 SDValue Op1 = BitConvertToInteger(N->getOperand(1));
1511 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1512 N->getOperand(0), Op1);
1513 }
1514
1515 //===----------------------------------------------------------------------===//
1516 // Float Result Expansion
1517 //===----------------------------------------------------------------------===//
1518
1519 /// ExpandFloatResult - This method is called when the specified result of the
1520 /// specified node is found to need expansion. At this point, the node may also
1521 /// have invalid operands or may have other results that need promotion, we just
1522 /// know that (at least) one result needs expansion.
ExpandFloatResult(SDNode * N,unsigned ResNo)1523 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1524 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1525 SDValue Lo, Hi;
1526 Lo = Hi = SDValue();
1527
1528 // See if the target wants to custom expand this node.
1529 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1530 return;
1531
1532 switch (N->getOpcode()) {
1533 default:
1534 #ifndef NDEBUG
1535 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1536 N->dump(&DAG); dbgs() << "\n";
1537 #endif
1538 report_fatal_error("Do not know how to expand the result of this "
1539 "operator!");
1540 // clang-format off
1541 case ISD::POISON:
1542 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1543 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1544 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1545
1546 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1547 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1548 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1549 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1550 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1551 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1552
1553 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1554 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1555 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1556 case ISD::STRICT_FMINNUM:
1557 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1558 case ISD::STRICT_FMAXNUM:
1559 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1560 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1561 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1562 case ISD::STRICT_FADD:
1563 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1564 case ISD::STRICT_FACOS:
1565 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1566 case ISD::STRICT_FASIN:
1567 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1568 case ISD::STRICT_FATAN:
1569 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1570 case ISD::STRICT_FATAN2:
1571 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1572 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1573 case ISD::STRICT_FCEIL:
1574 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1575 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1576 case ISD::STRICT_FCOS:
1577 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1578 case ISD::STRICT_FCOSH:
1579 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1580 case ISD::STRICT_FDIV:
1581 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1582 case ISD::STRICT_FEXP:
1583 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1584 case ISD::STRICT_FEXP2:
1585 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1586 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1587 case ISD::STRICT_FFLOOR:
1588 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1589 case ISD::STRICT_FLOG:
1590 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1591 case ISD::STRICT_FLOG2:
1592 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1593 case ISD::STRICT_FLOG10:
1594 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1595 case ISD::STRICT_FMA:
1596 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1597 case ISD::STRICT_FMUL:
1598 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1599 case ISD::STRICT_FNEARBYINT:
1600 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1601 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1602 case ISD::STRICT_FP_EXTEND:
1603 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1604 case ISD::STRICT_FPOW:
1605 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1606 case ISD::STRICT_FPOWI:
1607 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1608 case ISD::FLDEXP:
1609 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1610 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1611 case ISD::STRICT_FRINT:
1612 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1613 case ISD::STRICT_FROUND:
1614 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1615 case ISD::STRICT_FROUNDEVEN:
1616 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1617 case ISD::STRICT_FSIN:
1618 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1619 case ISD::STRICT_FSINH:
1620 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1621 case ISD::STRICT_FSQRT:
1622 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1623 case ISD::STRICT_FSUB:
1624 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1625 case ISD::STRICT_FTAN:
1626 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1627 case ISD::STRICT_FTANH:
1628 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1629 case ISD::STRICT_FTRUNC:
1630 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1631 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1632 case ISD::STRICT_SINT_TO_FP:
1633 case ISD::STRICT_UINT_TO_FP:
1634 case ISD::SINT_TO_FP:
1635 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1636 case ISD::STRICT_FREM:
1637 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1638 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1639 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1640 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1641 // clang-format on
1642 }
1643
1644 // If Lo/Hi is null, the sub-method took care of registering results etc.
1645 if (Lo.getNode())
1646 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1647 }
1648
ExpandFloatRes_ConstantFP(SDNode * N,SDValue & Lo,SDValue & Hi)1649 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1650 SDValue &Hi) {
1651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1652 assert(NVT.getSizeInBits() == 64 &&
1653 "Do not know how to expand this float constant!");
1654 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1655 SDLoc dl(N);
1656 const fltSemantics &Sem = NVT.getFltSemantics();
1657 Lo = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 64)), dl, NVT);
1658 Hi = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 0)), dl, NVT);
1659 }
1660
ExpandFloatRes_Unary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1661 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1662 SDValue &Lo, SDValue &Hi) {
1663 bool IsStrict = N->isStrictFPOpcode();
1664 unsigned Offset = IsStrict ? 1 : 0;
1665 SDValue Op = N->getOperand(0 + Offset);
1666 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1667 TargetLowering::MakeLibCallOptions CallOptions;
1668 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1669 Op, CallOptions, SDLoc(N),
1670 Chain);
1671 if (IsStrict)
1672 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1673 GetPairElements(Tmp.first, Lo, Hi);
1674 }
1675
ExpandFloatRes_Binary(SDNode * N,RTLIB::Libcall LC,SDValue & Lo,SDValue & Hi)1676 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1677 SDValue &Lo, SDValue &Hi) {
1678 bool IsStrict = N->isStrictFPOpcode();
1679 unsigned Offset = IsStrict ? 1 : 0;
1680 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1681 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1682 TargetLowering::MakeLibCallOptions CallOptions;
1683 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1684 Ops, CallOptions, SDLoc(N),
1685 Chain);
1686 if (IsStrict)
1687 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1688 GetPairElements(Tmp.first, Lo, Hi);
1689 }
1690
ExpandFloatRes_FMODF(SDNode * N)1691 void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1692 ExpandFloatRes_UnaryWithTwoFPResults(N, RTLIB::getMODF(N->getValueType(0)),
1693 /*CallRetResNo=*/0);
1694 }
1695
ExpandFloatRes_FSINCOS(SDNode * N)1696 void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1697 ExpandFloatRes_UnaryWithTwoFPResults(N, RTLIB::getSINCOS(N->getValueType(0)));
1698 }
1699
ExpandFloatRes_FSINCOSPI(SDNode * N)1700 void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1701 ExpandFloatRes_UnaryWithTwoFPResults(N,
1702 RTLIB::getSINCOSPI(N->getValueType(0)));
1703 }
1704
ExpandFloatRes_UnaryWithTwoFPResults(SDNode * N,RTLIB::Libcall LC,std::optional<unsigned> CallRetResNo)1705 void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1706 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1707 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1708 SmallVector<SDValue> Results;
1709 DAG.expandMultipleResultFPLibCall(LC, N, Results, CallRetResNo);
1710 for (auto [ResNo, Res] : enumerate(Results)) {
1711 SDValue Lo, Hi;
1712 GetPairElements(Res, Lo, Hi);
1713 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1714 }
1715 }
1716
ExpandFloatRes_FABS(SDNode * N,SDValue & Lo,SDValue & Hi)1717 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1718 SDValue &Hi) {
1719 assert(N->getValueType(0) == MVT::ppcf128 &&
1720 "Logic only correct for ppcf128!");
1721 SDLoc dl(N);
1722 SDValue Tmp;
1723 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1724 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1725 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1726 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1727 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1728 ISD::SETEQ);
1729 }
1730
ExpandFloatRes_FMINNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1731 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1732 SDValue &Hi) {
1733 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1734 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1735 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1736 RTLIB::FMIN_PPCF128), Lo, Hi);
1737 }
1738
ExpandFloatRes_FMAXNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1739 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1740 SDValue &Hi) {
1741 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1742 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1743 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1744 RTLIB::FMAX_PPCF128), Lo, Hi);
1745 }
1746
ExpandFloatRes_FMINIMUMNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1747 void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1748 SDValue &Hi) {
1749 ExpandFloatRes_Binary(
1750 N,
1751 GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_NUM_F32,
1752 RTLIB::FMINIMUM_NUM_F64, RTLIB::FMINIMUM_NUM_F80,
1753 RTLIB::FMINIMUM_NUM_F128, RTLIB::FMINIMUM_NUM_PPCF128),
1754 Lo, Hi);
1755 }
1756
ExpandFloatRes_FMAXIMUMNUM(SDNode * N,SDValue & Lo,SDValue & Hi)1757 void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1758 SDValue &Hi) {
1759 ExpandFloatRes_Binary(
1760 N,
1761 GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_NUM_F32,
1762 RTLIB::FMAXIMUM_NUM_F64, RTLIB::FMAXIMUM_NUM_F80,
1763 RTLIB::FMAXIMUM_NUM_F128, RTLIB::FMAXIMUM_NUM_PPCF128),
1764 Lo, Hi);
1765 }
1766
ExpandFloatRes_FADD(SDNode * N,SDValue & Lo,SDValue & Hi)1767 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1768 SDValue &Hi) {
1769 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1770 RTLIB::ADD_F32, RTLIB::ADD_F64,
1771 RTLIB::ADD_F80, RTLIB::ADD_F128,
1772 RTLIB::ADD_PPCF128), Lo, Hi);
1773 }
1774
ExpandFloatRes_FACOS(SDNode * N,SDValue & Lo,SDValue & Hi)1775 void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1776 SDValue &Hi) {
1777 ExpandFloatRes_Unary(N,
1778 GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32,
1779 RTLIB::ACOS_F64, RTLIB::ACOS_F80,
1780 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128),
1781 Lo, Hi);
1782 }
1783
ExpandFloatRes_FASIN(SDNode * N,SDValue & Lo,SDValue & Hi)1784 void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1785 SDValue &Hi) {
1786 ExpandFloatRes_Unary(N,
1787 GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32,
1788 RTLIB::ASIN_F64, RTLIB::ASIN_F80,
1789 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128),
1790 Lo, Hi);
1791 }
1792
ExpandFloatRes_FATAN(SDNode * N,SDValue & Lo,SDValue & Hi)1793 void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1794 SDValue &Hi) {
1795 ExpandFloatRes_Unary(N,
1796 GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32,
1797 RTLIB::ATAN_F64, RTLIB::ATAN_F80,
1798 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128),
1799 Lo, Hi);
1800 }
1801
ExpandFloatRes_FATAN2(SDNode * N,SDValue & Lo,SDValue & Hi)1802 void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1803 SDValue &Hi) {
1804 ExpandFloatRes_Binary(N,
1805 GetFPLibCall(N->getValueType(0), RTLIB::ATAN2_F32,
1806 RTLIB::ATAN2_F64, RTLIB::ATAN2_F80,
1807 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128),
1808 Lo, Hi);
1809 }
1810
ExpandFloatRes_FCBRT(SDNode * N,SDValue & Lo,SDValue & Hi)1811 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1814 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1815 RTLIB::CBRT_F128,
1816 RTLIB::CBRT_PPCF128), Lo, Hi);
1817 }
1818
ExpandFloatRes_FCEIL(SDNode * N,SDValue & Lo,SDValue & Hi)1819 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1820 SDValue &Lo, SDValue &Hi) {
1821 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1822 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1823 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1824 RTLIB::CEIL_PPCF128), Lo, Hi);
1825 }
1826
ExpandFloatRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1827 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1828 SDValue &Lo, SDValue &Hi) {
1829 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1830 RTLIB::COPYSIGN_F32,
1831 RTLIB::COPYSIGN_F64,
1832 RTLIB::COPYSIGN_F80,
1833 RTLIB::COPYSIGN_F128,
1834 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1835 }
1836
ExpandFloatRes_FCOS(SDNode * N,SDValue & Lo,SDValue & Hi)1837 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1838 SDValue &Lo, SDValue &Hi) {
1839 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1840 RTLIB::COS_F32, RTLIB::COS_F64,
1841 RTLIB::COS_F80, RTLIB::COS_F128,
1842 RTLIB::COS_PPCF128), Lo, Hi);
1843 }
1844
ExpandFloatRes_FCOSH(SDNode * N,SDValue & Lo,SDValue & Hi)1845 void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1846 SDValue &Hi) {
1847 ExpandFloatRes_Unary(N,
1848 GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32,
1849 RTLIB::COSH_F64, RTLIB::COSH_F80,
1850 RTLIB::COSH_F128, RTLIB::COSH_PPCF128),
1851 Lo, Hi);
1852 }
1853
ExpandFloatRes_FDIV(SDNode * N,SDValue & Lo,SDValue & Hi)1854 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1855 SDValue &Hi) {
1856 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1857 RTLIB::DIV_F32,
1858 RTLIB::DIV_F64,
1859 RTLIB::DIV_F80,
1860 RTLIB::DIV_F128,
1861 RTLIB::DIV_PPCF128), Lo, Hi);
1862 }
1863
ExpandFloatRes_FEXP(SDNode * N,SDValue & Lo,SDValue & Hi)1864 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1865 SDValue &Lo, SDValue &Hi) {
1866 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1867 RTLIB::EXP_F32, RTLIB::EXP_F64,
1868 RTLIB::EXP_F80, RTLIB::EXP_F128,
1869 RTLIB::EXP_PPCF128), Lo, Hi);
1870 }
1871
ExpandFloatRes_FEXP2(SDNode * N,SDValue & Lo,SDValue & Hi)1872 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1873 SDValue &Lo, SDValue &Hi) {
1874 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1875 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1876 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1877 RTLIB::EXP2_PPCF128), Lo, Hi);
1878 }
1879
ExpandFloatRes_FEXP10(SDNode * N,SDValue & Lo,SDValue & Hi)1880 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1881 SDValue &Hi) {
1882 ExpandFloatRes_Unary(N,
1883 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1884 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1885 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1886 Lo, Hi);
1887 }
1888
ExpandFloatRes_FFLOOR(SDNode * N,SDValue & Lo,SDValue & Hi)1889 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1890 SDValue &Lo, SDValue &Hi) {
1891 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1892 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1893 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1894 RTLIB::FLOOR_PPCF128), Lo, Hi);
1895 }
1896
ExpandFloatRes_FLOG(SDNode * N,SDValue & Lo,SDValue & Hi)1897 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1898 SDValue &Lo, SDValue &Hi) {
1899 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1900 RTLIB::LOG_F32, RTLIB::LOG_F64,
1901 RTLIB::LOG_F80, RTLIB::LOG_F128,
1902 RTLIB::LOG_PPCF128), Lo, Hi);
1903 }
1904
ExpandFloatRes_FLOG2(SDNode * N,SDValue & Lo,SDValue & Hi)1905 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1906 SDValue &Lo, SDValue &Hi) {
1907 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1908 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1909 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1910 RTLIB::LOG2_PPCF128), Lo, Hi);
1911 }
1912
ExpandFloatRes_FLOG10(SDNode * N,SDValue & Lo,SDValue & Hi)1913 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1914 SDValue &Lo, SDValue &Hi) {
1915 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1916 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1917 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1918 RTLIB::LOG10_PPCF128), Lo, Hi);
1919 }
1920
ExpandFloatRes_FMA(SDNode * N,SDValue & Lo,SDValue & Hi)1921 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1922 SDValue &Hi) {
1923 bool IsStrict = N->isStrictFPOpcode();
1924 unsigned Offset = IsStrict ? 1 : 0;
1925 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1926 N->getOperand(2 + Offset) };
1927 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1928 TargetLowering::MakeLibCallOptions CallOptions;
1929 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1930 RTLIB::FMA_F32,
1931 RTLIB::FMA_F64,
1932 RTLIB::FMA_F80,
1933 RTLIB::FMA_F128,
1934 RTLIB::FMA_PPCF128),
1935 N->getValueType(0), Ops, CallOptions,
1936 SDLoc(N), Chain);
1937 if (IsStrict)
1938 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1939 GetPairElements(Tmp.first, Lo, Hi);
1940 }
1941
ExpandFloatRes_FMUL(SDNode * N,SDValue & Lo,SDValue & Hi)1942 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1943 SDValue &Hi) {
1944 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1945 RTLIB::MUL_F32,
1946 RTLIB::MUL_F64,
1947 RTLIB::MUL_F80,
1948 RTLIB::MUL_F128,
1949 RTLIB::MUL_PPCF128), Lo, Hi);
1950 }
1951
ExpandFloatRes_FNEARBYINT(SDNode * N,SDValue & Lo,SDValue & Hi)1952 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1953 SDValue &Lo, SDValue &Hi) {
1954 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1955 RTLIB::NEARBYINT_F32,
1956 RTLIB::NEARBYINT_F64,
1957 RTLIB::NEARBYINT_F80,
1958 RTLIB::NEARBYINT_F128,
1959 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1960 }
1961
ExpandFloatRes_FNEG(SDNode * N,SDValue & Lo,SDValue & Hi)1962 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1963 SDValue &Hi) {
1964 SDLoc dl(N);
1965 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1966 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1967 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1968 }
1969
ExpandFloatRes_AssertNoFPClass(SDNode * N,SDValue & Lo,SDValue & Hi)1970 void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
1971 SDValue &Hi) {
1972 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
1973 SDLoc dl(N);
1974 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1975 }
1976
ExpandFloatRes_FP_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)1977 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1978 SDValue &Hi) {
1979 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1980 SDLoc dl(N);
1981 bool IsStrict = N->isStrictFPOpcode();
1982
1983 SDValue Chain;
1984 if (IsStrict) {
1985 // If the expanded type is the same as the input type, just bypass the node.
1986 if (NVT == N->getOperand(1).getValueType()) {
1987 Hi = N->getOperand(1);
1988 Chain = N->getOperand(0);
1989 } else {
1990 // Other we need to extend.
1991 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1992 { N->getOperand(0), N->getOperand(1) });
1993 Chain = Hi.getValue(1);
1994 }
1995 } else {
1996 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1997 }
1998
1999 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
2000
2001 if (IsStrict)
2002 ReplaceValueWith(SDValue(N, 1), Chain);
2003 }
2004
ExpandFloatRes_FPOW(SDNode * N,SDValue & Lo,SDValue & Hi)2005 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
2006 SDValue &Lo, SDValue &Hi) {
2007 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
2008 RTLIB::POW_F32, RTLIB::POW_F64,
2009 RTLIB::POW_F80, RTLIB::POW_F128,
2010 RTLIB::POW_PPCF128), Lo, Hi);
2011 }
2012
ExpandFloatRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)2013 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
2014 SDValue &Lo, SDValue &Hi) {
2015 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
2016 }
2017
ExpandFloatRes_FLDEXP(SDNode * N,SDValue & Lo,SDValue & Hi)2018 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
2019 SDValue &Hi) {
2020 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
2021 }
2022
ExpandFloatRes_FREEZE(SDNode * N,SDValue & Lo,SDValue & Hi)2023 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
2024 SDValue &Lo, SDValue &Hi) {
2025 assert(N->getValueType(0) == MVT::ppcf128 &&
2026 "Logic only correct for ppcf128!");
2027
2028 SDLoc dl(N);
2029 GetExpandedFloat(N->getOperand(0), Lo, Hi);
2030 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
2031 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
2032 }
2033
ExpandFloatRes_FREM(SDNode * N,SDValue & Lo,SDValue & Hi)2034 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
2035 SDValue &Lo, SDValue &Hi) {
2036 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
2037 RTLIB::REM_F32, RTLIB::REM_F64,
2038 RTLIB::REM_F80, RTLIB::REM_F128,
2039 RTLIB::REM_PPCF128), Lo, Hi);
2040 }
2041
ExpandFloatRes_FRINT(SDNode * N,SDValue & Lo,SDValue & Hi)2042 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
2043 SDValue &Lo, SDValue &Hi) {
2044 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2045 RTLIB::RINT_F32, RTLIB::RINT_F64,
2046 RTLIB::RINT_F80, RTLIB::RINT_F128,
2047 RTLIB::RINT_PPCF128), Lo, Hi);
2048 }
2049
ExpandFloatRes_FROUND(SDNode * N,SDValue & Lo,SDValue & Hi)2050 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
2051 SDValue &Lo, SDValue &Hi) {
2052 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2053 RTLIB::ROUND_F32,
2054 RTLIB::ROUND_F64,
2055 RTLIB::ROUND_F80,
2056 RTLIB::ROUND_F128,
2057 RTLIB::ROUND_PPCF128), Lo, Hi);
2058 }
2059
ExpandFloatRes_FROUNDEVEN(SDNode * N,SDValue & Lo,SDValue & Hi)2060 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
2061 SDValue &Lo, SDValue &Hi) {
2062 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2063 RTLIB::ROUNDEVEN_F32,
2064 RTLIB::ROUNDEVEN_F64,
2065 RTLIB::ROUNDEVEN_F80,
2066 RTLIB::ROUNDEVEN_F128,
2067 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
2068 }
2069
ExpandFloatRes_FSIN(SDNode * N,SDValue & Lo,SDValue & Hi)2070 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
2071 SDValue &Lo, SDValue &Hi) {
2072 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2073 RTLIB::SIN_F32, RTLIB::SIN_F64,
2074 RTLIB::SIN_F80, RTLIB::SIN_F128,
2075 RTLIB::SIN_PPCF128), Lo, Hi);
2076 }
2077
ExpandFloatRes_FSINH(SDNode * N,SDValue & Lo,SDValue & Hi)2078 void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
2079 SDValue &Hi) {
2080 ExpandFloatRes_Unary(N,
2081 GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32,
2082 RTLIB::SINH_F64, RTLIB::SINH_F80,
2083 RTLIB::SINH_F128, RTLIB::SINH_PPCF128),
2084 Lo, Hi);
2085 }
2086
ExpandFloatRes_FSQRT(SDNode * N,SDValue & Lo,SDValue & Hi)2087 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
2088 SDValue &Lo, SDValue &Hi) {
2089 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2090 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
2091 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
2092 RTLIB::SQRT_PPCF128), Lo, Hi);
2093 }
2094
ExpandFloatRes_FSUB(SDNode * N,SDValue & Lo,SDValue & Hi)2095 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2096 SDValue &Hi) {
2097 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
2098 RTLIB::SUB_F32,
2099 RTLIB::SUB_F64,
2100 RTLIB::SUB_F80,
2101 RTLIB::SUB_F128,
2102 RTLIB::SUB_PPCF128), Lo, Hi);
2103 }
2104
ExpandFloatRes_FTAN(SDNode * N,SDValue & Lo,SDValue & Hi)2105 void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2106 SDValue &Hi) {
2107 ExpandFloatRes_Unary(N,
2108 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
2109 RTLIB::TAN_F64, RTLIB::TAN_F80,
2110 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
2111 Lo, Hi);
2112 }
2113
ExpandFloatRes_FTANH(SDNode * N,SDValue & Lo,SDValue & Hi)2114 void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2115 SDValue &Hi) {
2116 ExpandFloatRes_Unary(N,
2117 GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32,
2118 RTLIB::TANH_F64, RTLIB::TANH_F80,
2119 RTLIB::TANH_F128, RTLIB::TANH_PPCF128),
2120 Lo, Hi);
2121 }
2122
ExpandFloatRes_FTRUNC(SDNode * N,SDValue & Lo,SDValue & Hi)2123 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2124 SDValue &Lo, SDValue &Hi) {
2125 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
2126 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
2127 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
2128 RTLIB::TRUNC_PPCF128), Lo, Hi);
2129 }
2130
ExpandFloatRes_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)2131 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2132 SDValue &Hi) {
2133 if (ISD::isNormalLoad(N)) {
2134 ExpandRes_NormalLoad(N, Lo, Hi);
2135 return;
2136 }
2137
2138 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2139 LoadSDNode *LD = cast<LoadSDNode>(N);
2140 SDValue Chain = LD->getChain();
2141 SDValue Ptr = LD->getBasePtr();
2142 SDLoc dl(N);
2143
2144 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
2145 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2146 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2147
2148 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
2149 LD->getMemoryVT(), LD->getMemOperand());
2150
2151 // Remember the chain.
2152 Chain = Hi.getValue(1);
2153
2154 // The low part is zero.
2155 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
2156
2157 // Modified the chain - switch anything that used the old chain to use the
2158 // new one.
2159 ReplaceValueWith(SDValue(LD, 1), Chain);
2160 }
2161
ExpandFloatRes_XINT_TO_FP(SDNode * N,SDValue & Lo,SDValue & Hi)2162 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2163 SDValue &Hi) {
2164 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2165 EVT VT = N->getValueType(0);
2166 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2167 bool Strict = N->isStrictFPOpcode();
2168 SDValue Src = N->getOperand(Strict ? 1 : 0);
2169 EVT SrcVT = Src.getValueType();
2170 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2171 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2172 SDLoc dl(N);
2173 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
2174
2175 // TODO: Any other flags to propagate?
2176 SDNodeFlags Flags;
2177 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2178
2179 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2180 // When promoting partial word types to i32 we must honor the signedness,
2181 // though.
2182 if (SrcVT.bitsLE(MVT::i32)) {
2183 // The integer can be represented exactly in an f64.
2184 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT);
2185 if (Strict) {
2186 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
2187 {Chain, Src}, Flags);
2188 Chain = Hi.getValue(1);
2189 } else
2190 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
2191 } else {
2192 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2193 if (SrcVT.bitsLE(MVT::i64)) {
2194 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
2195 MVT::i64, Src);
2196 LC = RTLIB::SINTTOFP_I64_PPCF128;
2197 } else if (SrcVT.bitsLE(MVT::i128)) {
2198 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
2199 LC = RTLIB::SINTTOFP_I128_PPCF128;
2200 }
2201 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2202
2203 TargetLowering::MakeLibCallOptions CallOptions;
2204 CallOptions.setIsSigned(true);
2205 std::pair<SDValue, SDValue> Tmp =
2206 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
2207 if (Strict)
2208 Chain = Tmp.second;
2209 GetPairElements(Tmp.first, Lo, Hi);
2210 }
2211
2212 // No need to complement for unsigned 32-bit integers
2213 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
2214 if (Strict)
2215 ReplaceValueWith(SDValue(N, 1), Chain);
2216
2217 return;
2218 }
2219
2220 // Unsigned - fix up the SINT_TO_FP value just calculated.
2221 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2222 // keep semantics correctness if the integer is not exactly representable
2223 // here. See ExpandLegalINT_TO_FP.
2224 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
2225 SrcVT = Src.getValueType();
2226
2227 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2228 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2229 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2230 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2231 ArrayRef<uint64_t> Parts;
2232
2233 switch (SrcVT.getSimpleVT().SimpleTy) {
2234 default:
2235 llvm_unreachable("Unsupported UINT_TO_FP!");
2236 case MVT::i32:
2237 Parts = TwoE32;
2238 break;
2239 case MVT::i64:
2240 Parts = TwoE64;
2241 break;
2242 case MVT::i128:
2243 Parts = TwoE128;
2244 break;
2245 }
2246
2247 // TODO: Are there other fast-math-flags to propagate to this FADD?
2248 SDValue NewLo = DAG.getConstantFP(
2249 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
2250 if (Strict) {
2251 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
2252 {Chain, Hi, NewLo}, Flags);
2253 Chain = Lo.getValue(1);
2254 ReplaceValueWith(SDValue(N, 1), Chain);
2255 } else
2256 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
2257 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
2258 Lo, Hi, ISD::SETLT);
2259 GetPairElements(Lo, Lo, Hi);
2260 }
2261
2262
2263 //===----------------------------------------------------------------------===//
2264 // Float Operand Expansion
2265 //===----------------------------------------------------------------------===//
2266
2267 /// ExpandFloatOperand - This method is called when the specified operand of the
2268 /// specified node is found to need expansion. At this point, all of the result
2269 /// types of the node are known to be legal, but other operands of the node may
2270 /// need promotion or expansion as well as the specified one.
ExpandFloatOperand(SDNode * N,unsigned OpNo)2271 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2272 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2273 SDValue Res = SDValue();
2274
2275 // See if the target wants to custom expand this node.
2276 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2277 return false;
2278
2279 switch (N->getOpcode()) {
2280 default:
2281 #ifndef NDEBUG
2282 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2283 N->dump(&DAG); dbgs() << "\n";
2284 #endif
2285 report_fatal_error("Do not know how to expand this operator's operand!");
2286
2287 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2288 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2289 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2290
2291 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2292 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2293 case ISD::STRICT_FP_ROUND:
2294 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2295 case ISD::STRICT_FP_TO_SINT:
2296 case ISD::STRICT_FP_TO_UINT:
2297 case ISD::FP_TO_SINT:
2298 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2299 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2300 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2301 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2302 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2303 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2304 case ISD::STRICT_FSETCC:
2305 case ISD::STRICT_FSETCCS:
2306 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2307 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
2308 OpNo); break;
2309 }
2310
2311 // If the result is null, the sub-method took care of registering results etc.
2312 if (!Res.getNode()) return false;
2313
2314 // If the result is N, the sub-method updated N in place. Tell the legalizer
2315 // core about this.
2316 if (Res.getNode() == N)
2317 return true;
2318
2319 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2320 "Invalid operand expansion");
2321
2322 ReplaceValueWith(SDValue(N, 0), Res);
2323 return false;
2324 }
2325
2326 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2327 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
FloatExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl,SDValue & Chain,bool IsSignaling)2328 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2329 SDValue &NewRHS,
2330 ISD::CondCode &CCCode,
2331 const SDLoc &dl, SDValue &Chain,
2332 bool IsSignaling) {
2333 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2334 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2335 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2336
2337 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2338
2339 // FIXME: This generated code sucks. We want to generate
2340 // FCMPU crN, hi1, hi2
2341 // BNE crN, L:
2342 // FCMPU crN, lo1, lo2
2343 // The following can be improved, but not that much.
2344 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2345 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2346 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
2347 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2348 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2349 RHSLo, CCCode, OutputChain, IsSignaling);
2350 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2351 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2352 Tmp1 =
2353 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2354 ISD::SETUNE, OutputChain, IsSignaling);
2355 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2356 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2357 RHSHi, CCCode, OutputChain, IsSignaling);
2358 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2359 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2360 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2361 NewRHS = SDValue(); // LHS is the result, not a compare.
2362 Chain = OutputChain;
2363 }
2364
ExpandFloatOp_BR_CC(SDNode * N)2365 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2366 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2367 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2368 SDValue Chain;
2369 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2370
2371 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2372 // against zero to select between true and false values.
2373 if (!NewRHS.getNode()) {
2374 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2375 CCCode = ISD::SETNE;
2376 }
2377
2378 // Update N to have the operands specified.
2379 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2380 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2381 N->getOperand(4)), 0);
2382 }
2383
ExpandFloatOp_FCOPYSIGN(SDNode * N)2384 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2385 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2386 "Logic only correct for ppcf128!");
2387 SDValue Lo, Hi;
2388 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2389 // The ppcf128 value is providing only the sign; take it from the
2390 // higher-order double (which must have the larger magnitude).
2391 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2392 N->getValueType(0), N->getOperand(0), Hi);
2393 }
2394
ExpandFloatOp_FP_ROUND(SDNode * N)2395 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2396 bool IsStrict = N->isStrictFPOpcode();
2397 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2398 "Logic only correct for ppcf128!");
2399 SDValue Lo, Hi;
2400 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2401
2402 if (!IsStrict)
2403 // Round it the rest of the way (e.g. to f32) if needed.
2404 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2405 N->getValueType(0), Hi, N->getOperand(1));
2406
2407 // Eliminate the node if the input float type is the same as the output float
2408 // type.
2409 if (Hi.getValueType() == N->getValueType(0)) {
2410 // Connect the output chain to the input chain, unlinking the node.
2411 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2412 ReplaceValueWith(SDValue(N, 0), Hi);
2413 return SDValue();
2414 }
2415
2416 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2417 {N->getValueType(0), MVT::Other},
2418 {N->getOperand(0), Hi, N->getOperand(2)});
2419 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2420 ReplaceValueWith(SDValue(N, 0), Expansion);
2421 return SDValue();
2422 }
2423
ExpandFloatOp_FP_TO_XINT(SDNode * N)2424 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2425 EVT RVT = N->getValueType(0);
2426 SDLoc dl(N);
2427
2428 bool IsStrict = N->isStrictFPOpcode();
2429 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2430 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2431 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2432 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2433
2434 EVT NVT;
2435 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2436 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2437 "Unsupported FP_TO_XINT!");
2438 TargetLowering::MakeLibCallOptions CallOptions;
2439 std::pair<SDValue, SDValue> Tmp =
2440 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2441 if (!IsStrict)
2442 return Tmp.first;
2443
2444 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2445 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2446 return SDValue();
2447 }
2448
ExpandFloatOp_SELECT_CC(SDNode * N)2449 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2450 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2451 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2452 SDValue Chain;
2453 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2454
2455 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2456 // against zero to select between true and false values.
2457 if (!NewRHS.getNode()) {
2458 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2459 CCCode = ISD::SETNE;
2460 }
2461
2462 // Update N to have the operands specified.
2463 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2464 N->getOperand(2), N->getOperand(3),
2465 DAG.getCondCode(CCCode)), 0);
2466 }
2467
ExpandFloatOp_SETCC(SDNode * N)2468 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2469 bool IsStrict = N->isStrictFPOpcode();
2470 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2471 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2472 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2473 ISD::CondCode CCCode =
2474 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2475 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2476 N->getOpcode() == ISD::STRICT_FSETCCS);
2477
2478 // FloatExpandSetCCOperands always returned a scalar.
2479 assert(!NewRHS.getNode() && "Expect to return scalar");
2480 assert(NewLHS.getValueType() == N->getValueType(0) &&
2481 "Unexpected setcc expansion!");
2482 if (Chain) {
2483 ReplaceValueWith(SDValue(N, 0), NewLHS);
2484 ReplaceValueWith(SDValue(N, 1), Chain);
2485 return SDValue();
2486 }
2487 return NewLHS;
2488 }
2489
ExpandFloatOp_STORE(SDNode * N,unsigned OpNo)2490 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2491 if (ISD::isNormalStore(N))
2492 return ExpandOp_NormalStore(N, OpNo);
2493
2494 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2495 assert(OpNo == 1 && "Can only expand the stored value so far");
2496 StoreSDNode *ST = cast<StoreSDNode>(N);
2497
2498 SDValue Chain = ST->getChain();
2499 SDValue Ptr = ST->getBasePtr();
2500
2501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2502 ST->getValue().getValueType());
2503 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2504 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2505 (void)NVT;
2506
2507 SDValue Lo, Hi;
2508 GetExpandedOp(ST->getValue(), Lo, Hi);
2509
2510 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2511 ST->getMemoryVT(), ST->getMemOperand());
2512 }
2513
ExpandFloatOp_LROUND(SDNode * N)2514 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2515 EVT RVT = N->getValueType(0);
2516 EVT RetVT = N->getOperand(0).getValueType();
2517 TargetLowering::MakeLibCallOptions CallOptions;
2518 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2519 RTLIB::LROUND_F32,
2520 RTLIB::LROUND_F64,
2521 RTLIB::LROUND_F80,
2522 RTLIB::LROUND_F128,
2523 RTLIB::LROUND_PPCF128),
2524 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2525 }
2526
ExpandFloatOp_LLROUND(SDNode * N)2527 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2528 EVT RVT = N->getValueType(0);
2529 EVT RetVT = N->getOperand(0).getValueType();
2530 TargetLowering::MakeLibCallOptions CallOptions;
2531 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2532 RTLIB::LLROUND_F32,
2533 RTLIB::LLROUND_F64,
2534 RTLIB::LLROUND_F80,
2535 RTLIB::LLROUND_F128,
2536 RTLIB::LLROUND_PPCF128),
2537 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2538 }
2539
ExpandFloatOp_LRINT(SDNode * N)2540 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2541 EVT RVT = N->getValueType(0);
2542 EVT RetVT = N->getOperand(0).getValueType();
2543 TargetLowering::MakeLibCallOptions CallOptions;
2544 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2545 RTLIB::LRINT_F32,
2546 RTLIB::LRINT_F64,
2547 RTLIB::LRINT_F80,
2548 RTLIB::LRINT_F128,
2549 RTLIB::LRINT_PPCF128),
2550 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2551 }
2552
ExpandFloatOp_LLRINT(SDNode * N)2553 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2554 EVT RVT = N->getValueType(0);
2555 EVT RetVT = N->getOperand(0).getValueType();
2556 TargetLowering::MakeLibCallOptions CallOptions;
2557 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2558 RTLIB::LLRINT_F32,
2559 RTLIB::LLRINT_F64,
2560 RTLIB::LLRINT_F80,
2561 RTLIB::LLRINT_F128,
2562 RTLIB::LLRINT_PPCF128),
2563 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2564 }
2565
2566 //===----------------------------------------------------------------------===//
2567 // Float Operand Promotion
2568 //===----------------------------------------------------------------------===//
2569 //
2570
GetPromotionOpcode(EVT OpVT,EVT RetVT)2571 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2572 if (OpVT == MVT::f16)
2573 return ISD::FP16_TO_FP;
2574 if (RetVT == MVT::f16)
2575 return ISD::FP_TO_FP16;
2576 if (OpVT == MVT::bf16)
2577 return ISD::BF16_TO_FP;
2578 if (RetVT == MVT::bf16)
2579 return ISD::FP_TO_BF16;
2580 report_fatal_error("Attempt at an invalid promotion-related conversion");
2581 }
2582
GetPromotionOpcodeStrict(EVT OpVT,EVT RetVT)2583 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2584 if (OpVT == MVT::f16)
2585 return ISD::STRICT_FP16_TO_FP;
2586 if (RetVT == MVT::f16)
2587 return ISD::STRICT_FP_TO_FP16;
2588 if (OpVT == MVT::bf16)
2589 return ISD::STRICT_BF16_TO_FP;
2590 if (RetVT == MVT::bf16)
2591 return ISD::STRICT_FP_TO_BF16;
2592 report_fatal_error("Attempt at an invalid promotion-related conversion");
2593 }
2594
PromoteFloatOperand(SDNode * N,unsigned OpNo)2595 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2596 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2597 SDValue R = SDValue();
2598
2599 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2600 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2601 return false;
2602 }
2603
2604 // Nodes that use a promotion-requiring floating point operand, but doesn't
2605 // produce a promotion-requiring floating point result, need to be legalized
2606 // to use the promoted float operand. Nodes that produce at least one
2607 // promotion-requiring floating point result have their operands legalized as
2608 // a part of PromoteFloatResult.
2609 // clang-format off
2610 switch (N->getOpcode()) {
2611 default:
2612 #ifndef NDEBUG
2613 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2614 N->dump(&DAG); dbgs() << "\n";
2615 #endif
2616 report_fatal_error("Do not know how to promote this operator's operand!");
2617
2618 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2619 case ISD::FAKE_USE:
2620 R = PromoteFloatOp_FAKE_USE(N, OpNo);
2621 break;
2622 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2623 case ISD::FP_TO_SINT:
2624 case ISD::FP_TO_UINT:
2625 case ISD::LROUND:
2626 case ISD::LLROUND:
2627 case ISD::LRINT:
2628 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2629 case ISD::AssertNoFPClass: R = PromoteFloatOp_AssertNoFPClass(N, OpNo); break;
2630 case ISD::FP_TO_SINT_SAT:
2631 case ISD::FP_TO_UINT_SAT:
2632 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2633 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2634 case ISD::STRICT_FP_EXTEND:
2635 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2636 break;
2637 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2638 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2639 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2640 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2641 }
2642 // clang-format on
2643
2644 if (R.getNode())
2645 ReplaceValueWith(SDValue(N, 0), R);
2646 return false;
2647 }
2648
PromoteFloatOp_BITCAST(SDNode * N,unsigned OpNo)2649 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2650 SDValue Op = N->getOperand(0);
2651 EVT OpVT = Op->getValueType(0);
2652
2653 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2654 EVT PromotedVT = Promoted->getValueType(0);
2655
2656 // Convert the promoted float value to the desired IVT.
2657 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2658 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2659 IVT, Promoted);
2660 // The final result type might not be an scalar so we need a bitcast. The
2661 // bitcast will be further legalized if needed.
2662 return DAG.getBitcast(N->getValueType(0), Convert);
2663 }
2664
PromoteFloatOp_FAKE_USE(SDNode * N,unsigned OpNo)2665 SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo) {
2666 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2667 SDValue Op = GetPromotedFloat(N->getOperand(OpNo));
2668 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0),
2669 Op);
2670 }
2671
2672 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2673 // PromoteFloatRes_FCOPYSIGN.
PromoteFloatOp_FCOPYSIGN(SDNode * N,unsigned OpNo)2674 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2675 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2676 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2677
2678 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2679 N->getOperand(0), Op1);
2680 }
2681
2682 // Convert the promoted float value to the desired integer type
PromoteFloatOp_UnaryOp(SDNode * N,unsigned OpNo)2683 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2684 SDValue Op = GetPromotedFloat(N->getOperand(0));
2685 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2686 }
2687
2688 // Convert the promoted float value to the desired integer type
PromoteFloatOp_AssertNoFPClass(SDNode * N,unsigned OpNo)2689 SDValue DAGTypeLegalizer::PromoteFloatOp_AssertNoFPClass(SDNode *N,
2690 unsigned OpNo) {
2691 return GetPromotedFloat(N->getOperand(0));
2692 }
2693
PromoteFloatOp_FP_TO_XINT_SAT(SDNode * N,unsigned OpNo)2694 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2695 unsigned OpNo) {
2696 SDValue Op = GetPromotedFloat(N->getOperand(0));
2697 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2698 N->getOperand(1));
2699 }
2700
PromoteFloatOp_FP_EXTEND(SDNode * N,unsigned OpNo)2701 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2702 SDValue Op = GetPromotedFloat(N->getOperand(0));
2703 EVT VT = N->getValueType(0);
2704
2705 // Desired VT is same as promoted type. Use promoted float directly.
2706 if (VT == Op->getValueType(0))
2707 return Op;
2708
2709 // Else, extend the promoted float value to the desired VT.
2710 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2711 }
2712
PromoteFloatOp_STRICT_FP_EXTEND(SDNode * N,unsigned OpNo)2713 SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2714 unsigned OpNo) {
2715 assert(OpNo == 1 && "Promoting unpromotable operand");
2716
2717 SDValue Op = GetPromotedFloat(N->getOperand(1));
2718 EVT VT = N->getValueType(0);
2719
2720 // Desired VT is same as promoted type. Use promoted float directly.
2721 if (VT == Op->getValueType(0)) {
2722 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2723 return Op;
2724 }
2725
2726 // Else, extend the promoted float value to the desired VT.
2727 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2728 N->getOperand(0), Op);
2729 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2730 return Res;
2731 }
2732
2733 // Promote the float operands used for comparison. The true- and false-
2734 // operands have the same type as the result and are promoted, if needed, by
2735 // PromoteFloatRes_SELECT_CC
PromoteFloatOp_SELECT_CC(SDNode * N,unsigned OpNo)2736 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2737 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2738 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2739
2740 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2741 LHS, RHS, N->getOperand(2), N->getOperand(3),
2742 N->getOperand(4));
2743 }
2744
2745 // Construct a SETCC that compares the promoted values and sets the conditional
2746 // code.
PromoteFloatOp_SETCC(SDNode * N,unsigned OpNo)2747 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2748 EVT VT = N->getValueType(0);
2749 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2750 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2751 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2752
2753 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2754
2755 }
2756
2757 // Lower the promoted Float down to the integer value of same size and construct
2758 // a STORE of the integer value.
PromoteFloatOp_STORE(SDNode * N,unsigned OpNo)2759 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2760 StoreSDNode *ST = cast<StoreSDNode>(N);
2761 SDValue Val = ST->getValue();
2762 SDLoc DL(N);
2763
2764 SDValue Promoted = GetPromotedFloat(Val);
2765 EVT VT = ST->getOperand(1).getValueType();
2766 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2767
2768 SDValue NewVal;
2769 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2770 IVT, Promoted);
2771
2772 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2773 ST->getMemOperand());
2774 }
2775
PromoteFloatOp_ATOMIC_STORE(SDNode * N,unsigned OpNo)2776 SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2777 unsigned OpNo) {
2778 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2779 SDValue Val = ST->getVal();
2780 SDLoc DL(N);
2781
2782 SDValue Promoted = GetPromotedFloat(Val);
2783 EVT VT = ST->getOperand(1).getValueType();
2784 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2785
2786 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2787 DL, IVT, Promoted);
2788
2789 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2790 ST->getBasePtr(), ST->getMemOperand());
2791 }
2792
2793 //===----------------------------------------------------------------------===//
2794 // Float Result Promotion
2795 //===----------------------------------------------------------------------===//
2796
PromoteFloatResult(SDNode * N,unsigned ResNo)2797 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2798 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2799 SDValue R = SDValue();
2800
2801 // See if the target wants to custom expand this node.
2802 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2803 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2804 return;
2805 }
2806
2807 switch (N->getOpcode()) {
2808 // These opcodes cannot appear if promotion of FP16 is done in the backend
2809 // instead of Clang
2810 case ISD::FP16_TO_FP:
2811 case ISD::FP_TO_FP16:
2812 default:
2813 #ifndef NDEBUG
2814 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2815 N->dump(&DAG); dbgs() << "\n";
2816 #endif
2817 report_fatal_error("Do not know how to promote this operator's result!");
2818
2819 case ISD::BITCAST:
2820 R = PromoteFloatRes_BITCAST(N);
2821 break;
2822 case ISD::FREEZE:
2823 R = PromoteFloatRes_FREEZE(N);
2824 break;
2825 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2826 case ISD::EXTRACT_VECTOR_ELT:
2827 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2828 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2829
2830 // Unary FP Operations
2831 case ISD::FABS:
2832 case ISD::FACOS:
2833 case ISD::FASIN:
2834 case ISD::FATAN:
2835 case ISD::FCBRT:
2836 case ISD::FCEIL:
2837 case ISD::FCOS:
2838 case ISD::FCOSH:
2839 case ISD::FEXP:
2840 case ISD::FEXP2:
2841 case ISD::FEXP10:
2842 case ISD::FFLOOR:
2843 case ISD::FLOG:
2844 case ISD::FLOG2:
2845 case ISD::FLOG10:
2846 case ISD::FNEARBYINT:
2847 case ISD::FNEG:
2848 case ISD::FRINT:
2849 case ISD::FROUND:
2850 case ISD::FROUNDEVEN:
2851 case ISD::FSIN:
2852 case ISD::FSINH:
2853 case ISD::FSQRT:
2854 case ISD::FTRUNC:
2855 case ISD::FTAN:
2856 case ISD::FTANH:
2857 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2858 case ISD::AssertNoFPClass:
2859 R = PromoteFloatRes_AssertNoFPClass(N);
2860 break;
2861
2862 // Binary FP Operations
2863 case ISD::FADD:
2864 case ISD::FDIV:
2865 case ISD::FMAXIMUM:
2866 case ISD::FMINIMUM:
2867 case ISD::FMAXIMUMNUM:
2868 case ISD::FMINIMUMNUM:
2869 case ISD::FMAXNUM:
2870 case ISD::FMINNUM:
2871 case ISD::FMAXNUM_IEEE:
2872 case ISD::FMINNUM_IEEE:
2873 case ISD::FMUL:
2874 case ISD::FPOW:
2875 case ISD::FATAN2:
2876 case ISD::FREM:
2877 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2878
2879 case ISD::FMA: // FMA is same as FMAD
2880 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2881
2882 case ISD::FPOWI:
2883 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2884 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2885
2886 case ISD::FMODF:
2887 case ISD::FSINCOS:
2888 case ISD::FSINCOSPI:
2889 R = PromoteFloatRes_UnaryWithTwoFPResults(N);
2890 break;
2891 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2892 case ISD::STRICT_FP_ROUND:
2893 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2894 break;
2895 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2896 case ISD::ATOMIC_LOAD:
2897 R = PromoteFloatRes_ATOMIC_LOAD(N);
2898 break;
2899 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2900 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2901
2902 case ISD::SINT_TO_FP:
2903 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2904 case ISD::POISON:
2905 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2906 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2907 case ISD::VECREDUCE_FADD:
2908 case ISD::VECREDUCE_FMUL:
2909 case ISD::VECREDUCE_FMIN:
2910 case ISD::VECREDUCE_FMAX:
2911 case ISD::VECREDUCE_FMAXIMUM:
2912 case ISD::VECREDUCE_FMINIMUM:
2913 R = PromoteFloatRes_VECREDUCE(N);
2914 break;
2915 case ISD::VECREDUCE_SEQ_FADD:
2916 case ISD::VECREDUCE_SEQ_FMUL:
2917 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2918 break;
2919 }
2920
2921 if (R.getNode())
2922 SetPromotedFloat(SDValue(N, ResNo), R);
2923 }
2924
2925 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2926 // At this point, it is not possible to determine if the bitcast value is
2927 // eventually stored to memory or promoted to f32 or promoted to a floating
2928 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2929 // STORE promotion handlers.
PromoteFloatRes_BITCAST(SDNode * N)2930 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2931 EVT VT = N->getValueType(0);
2932 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2933 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2934 // bitcast will be legalized further if necessary.
2935 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2936 N->getOperand(0).getValueType().getSizeInBits());
2937 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2938 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2939 }
2940
PromoteFloatRes_FREEZE(SDNode * N)2941 SDValue DAGTypeLegalizer::PromoteFloatRes_FREEZE(SDNode *N) {
2942 EVT VT = N->getValueType(0);
2943 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2944 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2945 // bitcast will be legalized further if necessary.
2946 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2947 N->getOperand(0).getValueType().getSizeInBits());
2948 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2949 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT,
2950 DAG.getFreeze(Cast));
2951 }
2952
PromoteFloatRes_ConstantFP(SDNode * N)2953 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2954 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2955 EVT VT = N->getValueType(0);
2956 SDLoc DL(N);
2957
2958 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2959 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2960 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2961 IVT);
2962
2963 // Convert the Constant to the desired FP type
2964 // FIXME We might be able to do the conversion during compilation and get rid
2965 // of it from the object code
2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2967 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2968 }
2969
2970 // If the Index operand is a constant, try to redirect the extract operation to
2971 // the correct legalized vector. If not, bit-convert the input vector to
2972 // equivalent integer vector. Extract the element as an (bit-cast) integer
2973 // value and convert it to the promoted type.
PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode * N)2974 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2975 SDLoc DL(N);
2976
2977 // If the index is constant, try to extract the value from the legalized
2978 // vector type.
2979 if (isa<ConstantSDNode>(N->getOperand(1))) {
2980 SDValue Vec = N->getOperand(0);
2981 SDValue Idx = N->getOperand(1);
2982 EVT VecVT = Vec->getValueType(0);
2983 EVT EltVT = VecVT.getVectorElementType();
2984
2985 uint64_t IdxVal = Idx->getAsZExtVal();
2986
2987 switch (getTypeAction(VecVT)) {
2988 default: break;
2989 case TargetLowering::TypeScalarizeVector: {
2990 SDValue Res = GetScalarizedVector(N->getOperand(0));
2991 ReplaceValueWith(SDValue(N, 0), Res);
2992 return SDValue();
2993 }
2994 case TargetLowering::TypeWidenVector: {
2995 Vec = GetWidenedVector(Vec);
2996 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2997 ReplaceValueWith(SDValue(N, 0), Res);
2998 return SDValue();
2999 }
3000 case TargetLowering::TypeSplitVector: {
3001 SDValue Lo, Hi;
3002 GetSplitVector(Vec, Lo, Hi);
3003
3004 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
3005 SDValue Res;
3006 if (IdxVal < LoElts)
3007 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
3008 else
3009 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
3010 DAG.getConstant(IdxVal - LoElts, DL,
3011 Idx.getValueType()));
3012 ReplaceValueWith(SDValue(N, 0), Res);
3013 return SDValue();
3014 }
3015
3016 }
3017 }
3018
3019 // Bit-convert the input vector to the equivalent integer vector
3020 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3021 EVT IVT = NewOp.getValueType().getVectorElementType();
3022
3023 // Extract the element as an (bit-cast) integer value
3024 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
3025 NewOp, N->getOperand(1));
3026
3027 // Convert the element to the desired FP type
3028 EVT VT = N->getValueType(0);
3029 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3030 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
3031 }
3032
3033 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
3034 // needs promotion, so does the argument X. Note that Y, if needed, will be
3035 // handled during operand promotion.
PromoteFloatRes_FCOPYSIGN(SDNode * N)3036 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
3037 EVT VT = N->getValueType(0);
3038 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3039 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3040
3041 SDValue Op1 = N->getOperand(1);
3042
3043 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
3044 }
3045
3046 // Unary operation where the result and the operand have PromoteFloat type
3047 // action. Construct a new SDNode with the promoted float value of the old
3048 // operand.
PromoteFloatRes_UnaryOp(SDNode * N)3049 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
3050 EVT VT = N->getValueType(0);
3051 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3052 SDValue Op = GetPromotedFloat(N->getOperand(0));
3053 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
3054 }
3055
3056 // Unary operation with a more non-float operand where the result and the
3057 // operand have PromoteFloat type action. Construct a new SDNode with the
3058 // promoted float value of the old operand.
PromoteFloatRes_AssertNoFPClass(SDNode * N)3059 SDValue DAGTypeLegalizer::PromoteFloatRes_AssertNoFPClass(SDNode *N) {
3060 return GetPromotedFloat(N->getOperand(0));
3061 }
3062
3063 // Binary operations where the result and both operands have PromoteFloat type
3064 // action. Construct a new SDNode with the promoted float values of the old
3065 // operands.
PromoteFloatRes_BinOp(SDNode * N)3066 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
3067 EVT VT = N->getValueType(0);
3068 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3069 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3070 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
3071 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
3072 }
3073
PromoteFloatRes_FMAD(SDNode * N)3074 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
3075 EVT VT = N->getValueType(0);
3076 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3077 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3078 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
3079 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
3080
3081 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
3082 }
3083
3084 // Promote the Float (first) operand and retain the Integer (second) operand
PromoteFloatRes_ExpOp(SDNode * N)3085 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
3086 EVT VT = N->getValueType(0);
3087 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3088 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
3089 SDValue Op1 = N->getOperand(1);
3090
3091 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
3092 }
3093
PromoteFloatRes_FFREXP(SDNode * N)3094 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
3095 EVT VT = N->getValueType(0);
3096 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3097 SDValue Op = GetPromotedFloat(N->getOperand(0));
3098 SDValue Res =
3099 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
3100
3101 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3102 return Res;
3103 }
3104
PromoteFloatRes_UnaryWithTwoFPResults(SDNode * N)3105 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N) {
3106 EVT VT = N->getValueType(0);
3107 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3108 SDValue Op = GetPromotedFloat(N->getOperand(0));
3109 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, NVT}, Op);
3110
3111 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3112 ++ResNum) {
3113 SetPromotedFloat(SDValue(N, ResNum), Res.getValue(ResNum));
3114 }
3115
3116 return SDValue();
3117 }
3118
3119 // Explicit operation to reduce precision. Reduce the value to half precision
3120 // and promote it back to the legal type.
PromoteFloatRes_FP_ROUND(SDNode * N)3121 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
3122 SDLoc DL(N);
3123
3124 SDValue Op = N->getOperand(0);
3125 EVT VT = N->getValueType(0);
3126 EVT OpVT = Op->getValueType(0);
3127 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3128 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3129
3130 // Round promoted float to desired precision
3131 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
3132 // Promote it back to the legal output type
3133 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
3134 }
3135
3136 // Explicit operation to reduce precision. Reduce the value to half precision
3137 // and promote it back to the legal type.
PromoteFloatRes_STRICT_FP_ROUND(SDNode * N)3138 SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
3139 SDLoc DL(N);
3140
3141 SDValue Chain = N->getOperand(0);
3142 SDValue Op = N->getOperand(1);
3143 EVT VT = N->getValueType(0);
3144 EVT OpVT = Op->getValueType(0);
3145 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3146 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3147
3148 // Round promoted float to desired precision
3149 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
3150 DAG.getVTList(IVT, MVT::Other), Chain, Op);
3151 // Promote it back to the legal output type
3152 SDValue Res =
3153 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
3154 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
3155 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3156 return Res;
3157 }
3158
PromoteFloatRes_LOAD(SDNode * N)3159 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
3160 LoadSDNode *L = cast<LoadSDNode>(N);
3161 EVT VT = N->getValueType(0);
3162
3163 // Load the value as an integer value with the same number of bits.
3164 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3165 SDValue newL = DAG.getLoad(
3166 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
3167 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
3168 L->getBaseAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
3169 // Legalize the chain result by replacing uses of the old value chain with the
3170 // new one
3171 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
3172
3173 // Convert the integer value to the desired FP type
3174 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3175 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
3176 }
3177
PromoteFloatRes_ATOMIC_LOAD(SDNode * N)3178 SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
3179 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3180 EVT VT = AM->getValueType(0);
3181
3182 // Load the value as an integer value with the same number of bits.
3183 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3184 SDValue newL = DAG.getAtomic(
3185 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
3186 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3187
3188 // Legalize the chain result by replacing uses of the old value chain with the
3189 // new one
3190 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
3191
3192 // Convert the integer value to the desired FP type
3193 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3194 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
3195 }
3196
3197 // Construct a new SELECT node with the promoted true- and false- values.
PromoteFloatRes_SELECT(SDNode * N)3198 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
3199 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
3200 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
3201
3202 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
3203 N->getOperand(0), TrueVal, FalseVal);
3204 }
3205
3206 // Construct a new SELECT_CC node with the promoted true- and false- values.
3207 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
PromoteFloatRes_SELECT_CC(SDNode * N)3208 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
3209 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
3210 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
3211
3212 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3213 TrueVal.getNode()->getValueType(0), N->getOperand(0),
3214 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
3215 }
3216
3217 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
3218 // float type.
PromoteFloatRes_XINT_TO_FP(SDNode * N)3219 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
3220 SDLoc DL(N);
3221 EVT VT = N->getValueType(0);
3222 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3223 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
3224 // Round the value to the desired precision (that of the source type).
3225 return DAG.getNode(
3226 ISD::FP_EXTEND, DL, NVT,
3227 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
3228 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
3229 }
3230
PromoteFloatRes_UNDEF(SDNode * N)3231 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
3232 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
3233 N->getValueType(0)));
3234 }
3235
PromoteFloatRes_VECREDUCE(SDNode * N)3236 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
3237 // Expand and promote recursively.
3238 // TODO: This is non-optimal, but dealing with the concurrently happening
3239 // vector-legalization is non-trivial. We could do something similar to
3240 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
3241 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3242 return SDValue();
3243 }
3244
PromoteFloatRes_VECREDUCE_SEQ(SDNode * N)3245 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
3246 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3247 return SDValue();
3248 }
3249
BitcastToInt_ATOMIC_SWAP(SDNode * N)3250 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
3251 EVT VT = N->getValueType(0);
3252
3253 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3254 SDLoc SL(N);
3255
3256 SDValue CastVal = BitConvertToInteger(AM->getVal());
3257 EVT CastVT = CastVal.getValueType();
3258
3259 SDValue NewAtomic
3260 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
3261 DAG.getVTList(CastVT, MVT::Other),
3262 { AM->getChain(), AM->getBasePtr(), CastVal },
3263 AM->getMemOperand());
3264
3265 SDValue Result = NewAtomic;
3266
3267 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
3268 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3269 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
3270 NewAtomic);
3271 }
3272
3273 // Legalize the chain result by replacing uses of the old value chain with the
3274 // new one
3275 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
3276
3277 return Result;
3278
3279 }
3280
3281 //===----------------------------------------------------------------------===//
3282 // Half Result Soft Promotion
3283 //===----------------------------------------------------------------------===//
3284
SoftPromoteHalfResult(SDNode * N,unsigned ResNo)3285 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
3286 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
3287 N->dump(&DAG));
3288 SDValue R = SDValue();
3289
3290 // See if the target wants to custom expand this node.
3291 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
3292 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
3293 return;
3294 }
3295
3296 switch (N->getOpcode()) {
3297 default:
3298 #ifndef NDEBUG
3299 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
3300 N->dump(&DAG); dbgs() << "\n";
3301 #endif
3302 report_fatal_error("Do not know how to soft promote this operator's "
3303 "result!");
3304
3305 case ISD::ARITH_FENCE:
3306 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
3307 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
3308 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
3309 case ISD::EXTRACT_VECTOR_ELT:
3310 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
3311 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
3312 case ISD::STRICT_FP_ROUND:
3313 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
3314
3315 // Unary FP Operations
3316 case ISD::FABS:
3317 case ISD::FACOS:
3318 case ISD::FASIN:
3319 case ISD::FATAN:
3320 case ISD::FCBRT:
3321 case ISD::FCEIL:
3322 case ISD::FCOS:
3323 case ISD::FCOSH:
3324 case ISD::FEXP:
3325 case ISD::FEXP2:
3326 case ISD::FEXP10:
3327 case ISD::FFLOOR:
3328 case ISD::FLOG:
3329 case ISD::FLOG2:
3330 case ISD::FLOG10:
3331 case ISD::FNEARBYINT:
3332 case ISD::FNEG:
3333 case ISD::FREEZE:
3334 case ISD::FRINT:
3335 case ISD::FROUND:
3336 case ISD::FROUNDEVEN:
3337 case ISD::FSIN:
3338 case ISD::FSINH:
3339 case ISD::FSQRT:
3340 case ISD::FTRUNC:
3341 case ISD::FTAN:
3342 case ISD::FTANH:
3343 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3344 case ISD::AssertNoFPClass:
3345 R = SoftPromoteHalfRes_AssertNoFPClass(N);
3346 break;
3347
3348 // Binary FP Operations
3349 case ISD::FADD:
3350 case ISD::FDIV:
3351 case ISD::FMAXIMUM:
3352 case ISD::FMINIMUM:
3353 case ISD::FMAXIMUMNUM:
3354 case ISD::FMINIMUMNUM:
3355 case ISD::FMAXNUM:
3356 case ISD::FMINNUM:
3357 case ISD::FMUL:
3358 case ISD::FPOW:
3359 case ISD::FATAN2:
3360 case ISD::FREM:
3361 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
3362
3363 case ISD::FMA: // FMA is same as FMAD
3364 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
3365
3366 case ISD::FPOWI:
3367 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
3368
3369 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
3370
3371 case ISD::FMODF:
3372 case ISD::FSINCOS:
3373 case ISD::FSINCOSPI:
3374 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
3375 break;
3376
3377 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
3378 case ISD::ATOMIC_LOAD:
3379 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
3380 break;
3381 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
3382 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
3383 case ISD::STRICT_SINT_TO_FP:
3384 case ISD::STRICT_UINT_TO_FP:
3385 case ISD::SINT_TO_FP:
3386 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
3387 case ISD::POISON:
3388 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
3389 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
3390 case ISD::VECREDUCE_FADD:
3391 case ISD::VECREDUCE_FMUL:
3392 case ISD::VECREDUCE_FMIN:
3393 case ISD::VECREDUCE_FMAX:
3394 case ISD::VECREDUCE_FMAXIMUM:
3395 case ISD::VECREDUCE_FMINIMUM:
3396 R = SoftPromoteHalfRes_VECREDUCE(N);
3397 break;
3398 case ISD::VECREDUCE_SEQ_FADD:
3399 case ISD::VECREDUCE_SEQ_FMUL:
3400 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
3401 break;
3402 }
3403
3404 if (R.getNode())
3405 SetSoftPromotedHalf(SDValue(N, ResNo), R);
3406 }
3407
SoftPromoteHalfRes_ARITH_FENCE(SDNode * N)3408 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
3409 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
3410 BitConvertToInteger(N->getOperand(0)));
3411 }
3412
SoftPromoteHalfRes_BITCAST(SDNode * N)3413 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
3414 return BitConvertToInteger(N->getOperand(0));
3415 }
3416
SoftPromoteHalfRes_ConstantFP(SDNode * N)3417 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
3418 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
3419
3420 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3421 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
3422 MVT::i16);
3423 }
3424
SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode * N)3425 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3426 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3427 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
3428 NewOp.getValueType().getVectorElementType(), NewOp,
3429 N->getOperand(1));
3430 }
3431
SoftPromoteHalfRes_FCOPYSIGN(SDNode * N)3432 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3433 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
3434 SDValue RHS = BitConvertToInteger(N->getOperand(1));
3435 SDLoc dl(N);
3436
3437 EVT LVT = LHS.getValueType();
3438 EVT RVT = RHS.getValueType();
3439
3440 unsigned LSize = LVT.getSizeInBits();
3441 unsigned RSize = RVT.getSizeInBits();
3442
3443 // First get the sign bit of second operand.
3444 SDValue SignBit = DAG.getNode(
3445 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3446 DAG.getConstant(RSize - 1, dl,
3447 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3448 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3449
3450 // Shift right or sign-extend it if the two operands have different types.
3451 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3452 if (SizeDiff > 0) {
3453 SignBit =
3454 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3455 DAG.getConstant(SizeDiff, dl,
3456 TLI.getShiftAmountTy(SignBit.getValueType(),
3457 DAG.getDataLayout())));
3458 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3459 } else if (SizeDiff < 0) {
3460 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3461 SignBit =
3462 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3463 DAG.getConstant(-SizeDiff, dl,
3464 TLI.getShiftAmountTy(SignBit.getValueType(),
3465 DAG.getDataLayout())));
3466 }
3467
3468 // Clear the sign bit of the first operand.
3469 SDValue Mask = DAG.getNode(
3470 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3471 DAG.getConstant(LSize - 1, dl,
3472 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3473 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3474 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3475
3476 // Or the value with the sign bit.
3477 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3478 }
3479
SoftPromoteHalfRes_FMAD(SDNode * N)3480 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3481 EVT OVT = N->getValueType(0);
3482 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3483 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3484 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3485 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3486 SDLoc dl(N);
3487
3488 // Promote to the larger FP type.
3489 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3490 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3491 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3492 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3493
3494 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3495
3496 // Convert back to FP16 as an integer.
3497 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3498 }
3499
SoftPromoteHalfRes_ExpOp(SDNode * N)3500 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3501 EVT OVT = N->getValueType(0);
3502 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3503 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3504 SDValue Op1 = N->getOperand(1);
3505 SDLoc dl(N);
3506
3507 // Promote to the larger FP type.
3508 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3509
3510 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3511
3512 // Convert back to FP16 as an integer.
3513 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3514 }
3515
SoftPromoteHalfRes_FFREXP(SDNode * N)3516 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3517 EVT OVT = N->getValueType(0);
3518 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3519 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3520 SDLoc dl(N);
3521
3522 // Promote to the larger FP type.
3523 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3524
3525 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3526 DAG.getVTList(NVT, N->getValueType(1)), Op);
3527
3528 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3529
3530 // Convert back to FP16 as an integer.
3531 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3532 }
3533
SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode * N)3534 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
3535 EVT OVT = N->getValueType(0);
3536 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3537 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3538 SDLoc dl(N);
3539
3540 // Promote to the larger FP type.
3541 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3542 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, NVT), Op);
3543
3544 // Convert back to FP16 as an integer.
3545 ISD::NodeType Truncate = GetPromotionOpcode(NVT, OVT);
3546 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3547 ++ResNum) {
3548 SDValue Trunc = DAG.getNode(Truncate, dl, MVT::i16, Res.getValue(ResNum));
3549 SetSoftPromotedHalf(SDValue(N, ResNum), Trunc);
3550 }
3551
3552 return SDValue();
3553 }
3554
SoftPromoteHalfRes_FP_ROUND(SDNode * N)3555 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3556 EVT RVT = N->getValueType(0);
3557 bool IsStrict = N->isStrictFPOpcode();
3558 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3559 EVT SVT = Op.getValueType();
3560
3561 // If the input type needs to be softened, do that now so that call lowering
3562 // will see the f16 type.
3563 if (getTypeAction(SVT) == TargetLowering::TypeSoftenFloat) {
3564 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, RVT);
3565 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
3566
3567 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3568 Op = GetSoftenedFloat(Op);
3569 TargetLowering::MakeLibCallOptions CallOptions;
3570 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
3571 std::pair<SDValue, SDValue> Tmp =
3572 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N), Chain);
3573 if (IsStrict)
3574 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3575 return DAG.getNode(ISD::BITCAST, SDLoc(N), MVT::i16, Tmp.first);
3576 }
3577
3578 if (IsStrict) {
3579 SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
3580 {MVT::i16, MVT::Other}, {N->getOperand(0), Op});
3581 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3582 return Res;
3583 }
3584
3585 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3586 N->getOperand(0));
3587 }
3588
SoftPromoteHalfRes_LOAD(SDNode * N)3589 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3590 LoadSDNode *L = cast<LoadSDNode>(N);
3591
3592 // Load the value as an integer value with the same number of bits.
3593 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3594 SDValue NewL =
3595 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3596 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3597 L->getPointerInfo(), MVT::i16, L->getBaseAlign(),
3598 L->getMemOperand()->getFlags(), L->getAAInfo());
3599 // Legalize the chain result by replacing uses of the old value chain with the
3600 // new one
3601 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3602 return NewL;
3603 }
3604
SoftPromoteHalfRes_ATOMIC_LOAD(SDNode * N)3605 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3606 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3607
3608 // Load the value as an integer value with the same number of bits.
3609 SDValue NewL = DAG.getAtomic(
3610 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3611 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3612
3613 // Legalize the chain result by replacing uses of the old value chain with the
3614 // new one
3615 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3616 return NewL;
3617 }
3618
SoftPromoteHalfRes_SELECT(SDNode * N)3619 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3620 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3621 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3622 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3623 Op2);
3624 }
3625
SoftPromoteHalfRes_SELECT_CC(SDNode * N)3626 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3627 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3628 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3629 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3630 N->getOperand(0), N->getOperand(1), Op2, Op3,
3631 N->getOperand(4));
3632 }
3633
SoftPromoteHalfRes_XINT_TO_FP(SDNode * N)3634 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3635 EVT OVT = N->getValueType(0);
3636 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3637 SDLoc dl(N);
3638
3639 if (N->isStrictFPOpcode()) {
3640 SDValue Op = DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other},
3641 {N->getOperand(0), N->getOperand(1)});
3642 Op = DAG.getNode(GetPromotionOpcodeStrict(NVT, OVT), dl,
3643 {MVT::i16, MVT::Other}, {Op.getValue(1), Op});
3644 ReplaceValueWith(SDValue(N, 1), Op.getValue(1));
3645 return Op;
3646 }
3647
3648 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3649
3650 // Round the value to the softened type.
3651 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3652 }
3653
SoftPromoteHalfRes_UNDEF(SDNode * N)3654 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3655 return DAG.getUNDEF(MVT::i16);
3656 }
3657
SoftPromoteHalfRes_UnaryOp(SDNode * N)3658 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3659 EVT OVT = N->getValueType(0);
3660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3661 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3662 SDLoc dl(N);
3663
3664 // Promote to the larger FP type.
3665 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3666
3667 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3668
3669 // Convert back to FP16 as an integer.
3670 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3671 }
3672
SoftPromoteHalfRes_AssertNoFPClass(SDNode * N)3673 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
3674 return GetSoftPromotedHalf(N->getOperand(0));
3675 }
3676
SoftPromoteHalfRes_BinOp(SDNode * N)3677 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3678 EVT OVT = N->getValueType(0);
3679 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3680 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3681 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3682 SDLoc dl(N);
3683
3684 // Promote to the larger FP type.
3685 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3686 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3687 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3688
3689 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3690
3691 // Convert back to FP16 as an integer.
3692 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3693 }
3694
SoftPromoteHalfRes_VECREDUCE(SDNode * N)3695 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3696 // Expand and soften recursively.
3697 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3698 return SDValue();
3699 }
3700
SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode * N)3701 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3702 // Expand and soften.
3703 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3704 return SDValue();
3705 }
3706
3707 //===----------------------------------------------------------------------===//
3708 // Half Operand Soft Promotion
3709 //===----------------------------------------------------------------------===//
3710
SoftPromoteHalfOperand(SDNode * N,unsigned OpNo)3711 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3712 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3713 N->dump(&DAG));
3714 SDValue Res = SDValue();
3715
3716 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3717 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3718 return false;
3719 }
3720
3721 // Nodes that use a promotion-requiring floating point operand, but doesn't
3722 // produce a soft promotion-requiring floating point result, need to be
3723 // legalized to use the soft promoted float operand. Nodes that produce at
3724 // least one soft promotion-requiring floating point result have their
3725 // operands legalized as a part of PromoteFloatResult.
3726 switch (N->getOpcode()) {
3727 default:
3728 #ifndef NDEBUG
3729 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3730 N->dump(&DAG); dbgs() << "\n";
3731 #endif
3732 report_fatal_error("Do not know how to soft promote this operator's "
3733 "operand!");
3734
3735 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3736 case ISD::FAKE_USE:
3737 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3738 break;
3739 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3740 case ISD::STRICT_FP_TO_SINT:
3741 case ISD::STRICT_FP_TO_UINT:
3742 case ISD::FP_TO_SINT:
3743 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3744 case ISD::FP_TO_SINT_SAT:
3745 case ISD::FP_TO_UINT_SAT:
3746 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3747 case ISD::STRICT_FP_EXTEND:
3748 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3749 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3750 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3751 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3752 case ISD::ATOMIC_STORE:
3753 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3754 break;
3755 case ISD::STACKMAP:
3756 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3757 break;
3758 case ISD::PATCHPOINT:
3759 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3760 break;
3761 }
3762
3763 if (!Res.getNode())
3764 return false;
3765
3766 assert(Res.getNode() != N && "Expected a new node!");
3767
3768 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3769 "Invalid operand expansion");
3770
3771 ReplaceValueWith(SDValue(N, 0), Res);
3772 return false;
3773 }
3774
SoftPromoteHalfOp_BITCAST(SDNode * N)3775 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3776 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3777
3778 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3779 }
3780
SoftPromoteHalfOp_FAKE_USE(SDNode * N,unsigned OpNo)3781 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3782 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3783 SDValue Op = GetSoftPromotedHalf(N->getOperand(OpNo));
3784 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0),
3785 Op);
3786 }
3787
SoftPromoteHalfOp_FCOPYSIGN(SDNode * N,unsigned OpNo)3788 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3789 unsigned OpNo) {
3790 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3791 SDValue Op1 = N->getOperand(1);
3792 EVT RVT = Op1.getValueType();
3793 SDLoc dl(N);
3794
3795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3796
3797 Op1 = GetSoftPromotedHalf(Op1);
3798 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3799
3800 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3801 Op1);
3802 }
3803
SoftPromoteHalfOp_FP_EXTEND(SDNode * N)3804 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3805 EVT RVT = N->getValueType(0);
3806 bool IsStrict = N->isStrictFPOpcode();
3807 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3808 EVT SVT = Op.getValueType();
3809 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3810
3811 if (IsStrict) {
3812 SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N),
3813 {RVT, MVT::Other}, {N->getOperand(0), Op});
3814 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3815 ReplaceValueWith(SDValue(N, 0), Res);
3816 return SDValue();
3817 }
3818
3819 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3820 }
3821
SoftPromoteHalfOp_FP_TO_XINT(SDNode * N)3822 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3823 EVT RVT = N->getValueType(0);
3824 bool IsStrict = N->isStrictFPOpcode();
3825 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3826 EVT SVT = Op.getValueType();
3827 SDLoc dl(N);
3828
3829 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3830 Op = GetSoftPromotedHalf(Op);
3831
3832 if (IsStrict) {
3833 Op = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), dl, {NVT, MVT::Other},
3834 {N->getOperand(0), Op});
3835 Op = DAG.getNode(N->getOpcode(), dl, {RVT, MVT::Other},
3836 {Op.getValue(1), Op});
3837 ReplaceValueWith(SDValue(N, 1), Op.getValue(1));
3838 ReplaceValueWith(SDValue(N, 0), Op);
3839 return SDValue();
3840 }
3841
3842 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3843 return DAG.getNode(N->getOpcode(), dl, RVT, Res);
3844 }
3845
SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode * N)3846 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3847 EVT RVT = N->getValueType(0);
3848 SDValue Op = N->getOperand(0);
3849 EVT SVT = Op.getValueType();
3850 SDLoc dl(N);
3851
3852 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3853
3854 Op = GetSoftPromotedHalf(Op);
3855
3856 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3857
3858 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3859 N->getOperand(1));
3860 }
3861
SoftPromoteHalfOp_SELECT_CC(SDNode * N,unsigned OpNo)3862 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3863 unsigned OpNo) {
3864 assert(OpNo == 0 && "Can only soften the comparison values");
3865 SDValue Op0 = N->getOperand(0);
3866 SDValue Op1 = N->getOperand(1);
3867 SDLoc dl(N);
3868
3869 EVT SVT = Op0.getValueType();
3870 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3871
3872 Op0 = GetSoftPromotedHalf(Op0);
3873 Op1 = GetSoftPromotedHalf(Op1);
3874
3875 // Promote to the larger FP type.
3876 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3877 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3878 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3879
3880 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3881 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3882 }
3883
SoftPromoteHalfOp_SETCC(SDNode * N)3884 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3885 SDValue Op0 = N->getOperand(0);
3886 SDValue Op1 = N->getOperand(1);
3887 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3888 SDLoc dl(N);
3889
3890 EVT SVT = Op0.getValueType();
3891 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3892
3893 Op0 = GetSoftPromotedHalf(Op0);
3894 Op1 = GetSoftPromotedHalf(Op1);
3895
3896 // Promote to the larger FP type.
3897 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3898 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3899 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3900
3901 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3902 }
3903
SoftPromoteHalfOp_STORE(SDNode * N,unsigned OpNo)3904 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3905 assert(OpNo == 1 && "Can only soften the stored value!");
3906 StoreSDNode *ST = cast<StoreSDNode>(N);
3907 SDValue Val = ST->getValue();
3908 SDLoc dl(N);
3909
3910 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3911 SDValue Promoted = GetSoftPromotedHalf(Val);
3912 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3913 ST->getMemOperand());
3914 }
3915
SoftPromoteHalfOp_ATOMIC_STORE(SDNode * N,unsigned OpNo)3916 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3917 unsigned OpNo) {
3918 assert(OpNo == 1 && "Can only soften the stored value!");
3919 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3920 SDValue Val = ST->getVal();
3921 SDLoc dl(N);
3922
3923 SDValue Promoted = GetSoftPromotedHalf(Val);
3924 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3925 ST->getChain(), Promoted, ST->getBasePtr(),
3926 ST->getMemOperand());
3927 }
3928
SoftPromoteHalfOp_STACKMAP(SDNode * N,unsigned OpNo)3929 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3930 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3931 SmallVector<SDValue> NewOps(N->ops());
3932 SDValue Op = N->getOperand(OpNo);
3933 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3934 SDValue NewNode =
3935 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3936
3937 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3938 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3939
3940 return SDValue(); // Signal that we replaced the node ourselves.
3941 }
3942
SoftPromoteHalfOp_PATCHPOINT(SDNode * N,unsigned OpNo)3943 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3944 unsigned OpNo) {
3945 assert(OpNo >= 7);
3946 SmallVector<SDValue> NewOps(N->ops());
3947 SDValue Op = N->getOperand(OpNo);
3948 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3949 SDValue NewNode =
3950 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3951
3952 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3953 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3954
3955 return SDValue(); // Signal that we replaced the node ourselves.
3956 }
3957