Lines Matching refs:ISD
75 setOperationAction(ISD::LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
76 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
78 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); in AMDGPUTargetLowering()
79 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
81 setOperationAction(ISD::LOAD, MVT::v3f32, Promote); in AMDGPUTargetLowering()
82 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
84 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); in AMDGPUTargetLowering()
85 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
87 setOperationAction(ISD::LOAD, MVT::v5f32, Promote); in AMDGPUTargetLowering()
88 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
90 setOperationAction(ISD::LOAD, MVT::v6f32, Promote); in AMDGPUTargetLowering()
91 AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
93 setOperationAction(ISD::LOAD, MVT::v7f32, Promote); in AMDGPUTargetLowering()
94 AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
96 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); in AMDGPUTargetLowering()
97 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
99 setOperationAction(ISD::LOAD, MVT::v9f32, Promote); in AMDGPUTargetLowering()
100 AddPromotedToType(ISD::LOAD, MVT::v9f32, MVT::v9i32); in AMDGPUTargetLowering()
102 setOperationAction(ISD::LOAD, MVT::v10f32, Promote); in AMDGPUTargetLowering()
103 AddPromotedToType(ISD::LOAD, MVT::v10f32, MVT::v10i32); in AMDGPUTargetLowering()
105 setOperationAction(ISD::LOAD, MVT::v11f32, Promote); in AMDGPUTargetLowering()
106 AddPromotedToType(ISD::LOAD, MVT::v11f32, MVT::v11i32); in AMDGPUTargetLowering()
108 setOperationAction(ISD::LOAD, MVT::v12f32, Promote); in AMDGPUTargetLowering()
109 AddPromotedToType(ISD::LOAD, MVT::v12f32, MVT::v12i32); in AMDGPUTargetLowering()
111 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); in AMDGPUTargetLowering()
112 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
114 setOperationAction(ISD::LOAD, MVT::v32f32, Promote); in AMDGPUTargetLowering()
115 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
117 setOperationAction(ISD::LOAD, MVT::i64, Promote); in AMDGPUTargetLowering()
118 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
120 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); in AMDGPUTargetLowering()
121 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
123 setOperationAction(ISD::LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
124 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
126 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); in AMDGPUTargetLowering()
127 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
129 setOperationAction(ISD::LOAD, MVT::v3i64, Promote); in AMDGPUTargetLowering()
130 AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
132 setOperationAction(ISD::LOAD, MVT::v4i64, Promote); in AMDGPUTargetLowering()
133 AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
135 setOperationAction(ISD::LOAD, MVT::v3f64, Promote); in AMDGPUTargetLowering()
136 AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
138 setOperationAction(ISD::LOAD, MVT::v4f64, Promote); in AMDGPUTargetLowering()
139 AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
141 setOperationAction(ISD::LOAD, MVT::v8i64, Promote); in AMDGPUTargetLowering()
142 AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
144 setOperationAction(ISD::LOAD, MVT::v8f64, Promote); in AMDGPUTargetLowering()
145 AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
147 setOperationAction(ISD::LOAD, MVT::v16i64, Promote); in AMDGPUTargetLowering()
148 AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
150 setOperationAction(ISD::LOAD, MVT::v16f64, Promote); in AMDGPUTargetLowering()
151 AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
153 setOperationAction(ISD::LOAD, MVT::i128, Promote); in AMDGPUTargetLowering()
154 AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32); in AMDGPUTargetLowering()
157 setOperationAction(ISD::ATOMIC_LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
158 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
160 setOperationAction(ISD::ATOMIC_LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
161 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f64, MVT::i64); in AMDGPUTargetLowering()
163 setOperationAction(ISD::ATOMIC_LOAD, MVT::f16, Promote); in AMDGPUTargetLowering()
164 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::f16, MVT::i16); in AMDGPUTargetLowering()
166 setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote); in AMDGPUTargetLowering()
167 AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16); in AMDGPUTargetLowering()
169 setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
170 AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
172 setOperationAction(ISD::ATOMIC_STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
173 AddPromotedToType(ISD::ATOMIC_STORE, MVT::f64, MVT::i64); in AMDGPUTargetLowering()
175 setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote); in AMDGPUTargetLowering()
176 AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16); in AMDGPUTargetLowering()
178 setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote); in AMDGPUTargetLowering()
179 AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16); in AMDGPUTargetLowering()
184 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64, VT, in AMDGPUTargetLowering()
191 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) { in AMDGPUTargetLowering()
202 setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, MemVT, in AMDGPUTargetLowering()
205 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
206 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand); in AMDGPUTargetLowering()
207 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
208 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2bf16, Expand); in AMDGPUTargetLowering()
209 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand); in AMDGPUTargetLowering()
210 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3bf16, Expand); in AMDGPUTargetLowering()
211 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
212 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4bf16, Expand); in AMDGPUTargetLowering()
213 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
214 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8bf16, Expand); in AMDGPUTargetLowering()
215 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand); in AMDGPUTargetLowering()
216 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16bf16, Expand); in AMDGPUTargetLowering()
217 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand); in AMDGPUTargetLowering()
218 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32bf16, Expand); in AMDGPUTargetLowering()
220 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
221 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
222 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand); in AMDGPUTargetLowering()
223 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
224 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
225 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand); in AMDGPUTargetLowering()
227 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
228 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand); in AMDGPUTargetLowering()
229 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
230 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2bf16, Expand); in AMDGPUTargetLowering()
231 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand); in AMDGPUTargetLowering()
232 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3bf16, Expand); in AMDGPUTargetLowering()
233 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
234 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4bf16, Expand); in AMDGPUTargetLowering()
235 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
236 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8bf16, Expand); in AMDGPUTargetLowering()
237 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand); in AMDGPUTargetLowering()
238 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16bf16, Expand); in AMDGPUTargetLowering()
240 setOperationAction(ISD::STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
241 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
243 setOperationAction(ISD::STORE, MVT::v2f32, Promote); in AMDGPUTargetLowering()
244 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
246 setOperationAction(ISD::STORE, MVT::v3f32, Promote); in AMDGPUTargetLowering()
247 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
249 setOperationAction(ISD::STORE, MVT::v4f32, Promote); in AMDGPUTargetLowering()
250 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
252 setOperationAction(ISD::STORE, MVT::v5f32, Promote); in AMDGPUTargetLowering()
253 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
255 setOperationAction(ISD::STORE, MVT::v6f32, Promote); in AMDGPUTargetLowering()
256 AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
258 setOperationAction(ISD::STORE, MVT::v7f32, Promote); in AMDGPUTargetLowering()
259 AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
261 setOperationAction(ISD::STORE, MVT::v8f32, Promote); in AMDGPUTargetLowering()
262 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
264 setOperationAction(ISD::STORE, MVT::v9f32, Promote); in AMDGPUTargetLowering()
265 AddPromotedToType(ISD::STORE, MVT::v9f32, MVT::v9i32); in AMDGPUTargetLowering()
267 setOperationAction(ISD::STORE, MVT::v10f32, Promote); in AMDGPUTargetLowering()
268 AddPromotedToType(ISD::STORE, MVT::v10f32, MVT::v10i32); in AMDGPUTargetLowering()
270 setOperationAction(ISD::STORE, MVT::v11f32, Promote); in AMDGPUTargetLowering()
271 AddPromotedToType(ISD::STORE, MVT::v11f32, MVT::v11i32); in AMDGPUTargetLowering()
273 setOperationAction(ISD::STORE, MVT::v12f32, Promote); in AMDGPUTargetLowering()
274 AddPromotedToType(ISD::STORE, MVT::v12f32, MVT::v12i32); in AMDGPUTargetLowering()
276 setOperationAction(ISD::STORE, MVT::v16f32, Promote); in AMDGPUTargetLowering()
277 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
279 setOperationAction(ISD::STORE, MVT::v32f32, Promote); in AMDGPUTargetLowering()
280 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
282 setOperationAction(ISD::STORE, MVT::i64, Promote); in AMDGPUTargetLowering()
283 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
285 setOperationAction(ISD::STORE, MVT::v2i64, Promote); in AMDGPUTargetLowering()
286 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
288 setOperationAction(ISD::STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
289 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
291 setOperationAction(ISD::STORE, MVT::v2f64, Promote); in AMDGPUTargetLowering()
292 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
294 setOperationAction(ISD::STORE, MVT::v3i64, Promote); in AMDGPUTargetLowering()
295 AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
297 setOperationAction(ISD::STORE, MVT::v3f64, Promote); in AMDGPUTargetLowering()
298 AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
300 setOperationAction(ISD::STORE, MVT::v4i64, Promote); in AMDGPUTargetLowering()
301 AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
303 setOperationAction(ISD::STORE, MVT::v4f64, Promote); in AMDGPUTargetLowering()
304 AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
306 setOperationAction(ISD::STORE, MVT::v8i64, Promote); in AMDGPUTargetLowering()
307 AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
309 setOperationAction(ISD::STORE, MVT::v8f64, Promote); in AMDGPUTargetLowering()
310 AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
312 setOperationAction(ISD::STORE, MVT::v16i64, Promote); in AMDGPUTargetLowering()
313 AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
315 setOperationAction(ISD::STORE, MVT::v16f64, Promote); in AMDGPUTargetLowering()
316 AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
318 setOperationAction(ISD::STORE, MVT::i128, Promote); in AMDGPUTargetLowering()
319 AddPromotedToType(ISD::STORE, MVT::i128, MVT::v4i32); in AMDGPUTargetLowering()
383 setOperationAction(ISD::Constant, {MVT::i32, MVT::i64}, Legal); in AMDGPUTargetLowering()
384 setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
386 setOperationAction({ISD::BR_JT, ISD::BRIND}, MVT::Other, Expand); in AMDGPUTargetLowering()
390 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in AMDGPUTargetLowering()
394 setOperationAction({ISD::FCEIL, ISD::FPOW, ISD::FABS, ISD::FFLOOR, in AMDGPUTargetLowering()
395 ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM}, in AMDGPUTargetLowering()
398 setOperationAction(ISD::FLOG2, MVT::f32, Custom); in AMDGPUTargetLowering()
399 setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
402 {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, MVT::f32, in AMDGPUTargetLowering()
405 setOperationAction(ISD::FNEARBYINT, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
407 setOperationAction(ISD::FRINT, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
409 setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
412 setOperationAction(ISD::IS_FPCLASS, {MVT::f16, MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
414 setOperationAction(ISD::IS_FPCLASS, {MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
415 setOperationAction({ISD::FLOG2, ISD::FEXP2}, MVT::f16, Custom); in AMDGPUTargetLowering()
418 setOperationAction({ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, MVT::f16, in AMDGPUTargetLowering()
425 ISD::IS_FPCLASS, in AMDGPUTargetLowering()
432 setOperationAction(ISD::FSUB, MVT::f64, Expand); in AMDGPUTargetLowering()
434 setOperationAction(ISD::CONCAT_VECTORS, in AMDGPUTargetLowering()
444 ISD::EXTRACT_SUBVECTOR, in AMDGPUTargetLowering()
457 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in AMDGPUTargetLowering()
458 setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom); in AMDGPUTargetLowering()
463 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT, in AMDGPUTargetLowering()
467 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, VT, Custom); in AMDGPUTargetLowering()
470 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); in AMDGPUTargetLowering()
472 setOperationAction({ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Expand); in AMDGPUTargetLowering()
475 setOperationAction({ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT, Legal); in AMDGPUTargetLowering()
479 setOperationAction(ISD::FSHR, MVT::i32, Legal); in AMDGPUTargetLowering()
482 setOperationAction(ISD::ROTL, {MVT::i32, MVT::i64}, Expand); in AMDGPUTargetLowering()
483 setOperationAction(ISD::ROTR, MVT::i64, Expand); in AMDGPUTargetLowering()
485 setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i16, Expand); in AMDGPUTargetLowering()
487 setOperationAction({ISD::MUL, ISD::MULHU, ISD::MULHS}, MVT::i64, Expand); in AMDGPUTargetLowering()
489 {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, in AMDGPUTargetLowering()
491 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in AMDGPUTargetLowering()
493 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i32, in AMDGPUTargetLowering()
497 {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, in AMDGPUTargetLowering()
501 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, VT, Custom); in AMDGPUTargetLowering()
509 setOperationAction({ISD::ADD, ISD::AND, ISD::FP_TO_SINT, in AMDGPUTargetLowering()
510 ISD::FP_TO_UINT, ISD::MUL, ISD::MULHU, in AMDGPUTargetLowering()
511 ISD::MULHS, ISD::OR, ISD::SHL, in AMDGPUTargetLowering()
512 ISD::SRA, ISD::SRL, ISD::ROTL, in AMDGPUTargetLowering()
513 ISD::ROTR, ISD::SUB, ISD::SINT_TO_FP, in AMDGPUTargetLowering()
514 ISD::UINT_TO_FP, ISD::SDIV, ISD::UDIV, in AMDGPUTargetLowering()
515 ISD::SREM, ISD::UREM, ISD::SMUL_LOHI, in AMDGPUTargetLowering()
516 ISD::UMUL_LOHI, ISD::SDIVREM, ISD::UDIVREM, in AMDGPUTargetLowering()
517 ISD::SELECT, ISD::VSELECT, ISD::SELECT_CC, in AMDGPUTargetLowering()
518 ISD::XOR, ISD::BSWAP, ISD::CTPOP, in AMDGPUTargetLowering()
519 ISD::CTTZ, ISD::CTLZ, ISD::VECTOR_SHUFFLE, in AMDGPUTargetLowering()
520 ISD::SETCC}, in AMDGPUTargetLowering()
530 {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM, in AMDGPUTargetLowering()
531 ISD::FADD, ISD::FCEIL, ISD::FCOS, in AMDGPUTargetLowering()
532 ISD::FDIV, ISD::FEXP2, ISD::FEXP, in AMDGPUTargetLowering()
533 ISD::FEXP10, ISD::FLOG2, ISD::FREM, in AMDGPUTargetLowering()
534 ISD::FLOG, ISD::FLOG10, ISD::FPOW, in AMDGPUTargetLowering()
535 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL, in AMDGPUTargetLowering()
536 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT, in AMDGPUTargetLowering()
537 ISD::FSQRT, ISD::FSIN, ISD::FSUB, in AMDGPUTargetLowering()
538 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC, in AMDGPUTargetLowering()
539 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC, in AMDGPUTargetLowering()
540 ISD::FCANONICALIZE, ISD::FROUNDEVEN}, in AMDGPUTargetLowering()
547 setOperationAction(ISD::SELECT, MVT::v2f32, Promote); in AMDGPUTargetLowering()
548 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
550 setOperationAction(ISD::SELECT, MVT::v3f32, Promote); in AMDGPUTargetLowering()
551 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
553 setOperationAction(ISD::SELECT, MVT::v4f32, Promote); in AMDGPUTargetLowering()
554 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
556 setOperationAction(ISD::SELECT, MVT::v5f32, Promote); in AMDGPUTargetLowering()
557 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
559 setOperationAction(ISD::SELECT, MVT::v6f32, Promote); in AMDGPUTargetLowering()
560 AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
562 setOperationAction(ISD::SELECT, MVT::v7f32, Promote); in AMDGPUTargetLowering()
563 AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
565 setOperationAction(ISD::SELECT, MVT::v9f32, Promote); in AMDGPUTargetLowering()
566 AddPromotedToType(ISD::SELECT, MVT::v9f32, MVT::v9i32); in AMDGPUTargetLowering()
568 setOperationAction(ISD::SELECT, MVT::v10f32, Promote); in AMDGPUTargetLowering()
569 AddPromotedToType(ISD::SELECT, MVT::v10f32, MVT::v10i32); in AMDGPUTargetLowering()
571 setOperationAction(ISD::SELECT, MVT::v11f32, Promote); in AMDGPUTargetLowering()
572 AddPromotedToType(ISD::SELECT, MVT::v11f32, MVT::v11i32); in AMDGPUTargetLowering()
574 setOperationAction(ISD::SELECT, MVT::v12f32, Promote); in AMDGPUTargetLowering()
575 AddPromotedToType(ISD::SELECT, MVT::v12f32, MVT::v12i32); in AMDGPUTargetLowering()
613 setTargetDAGCombine({ISD::BITCAST, ISD::SHL, in AMDGPUTargetLowering()
614 ISD::SRA, ISD::SRL, in AMDGPUTargetLowering()
615 ISD::TRUNCATE, ISD::MUL, in AMDGPUTargetLowering()
616 ISD::SMUL_LOHI, ISD::UMUL_LOHI, in AMDGPUTargetLowering()
617 ISD::MULHU, ISD::MULHS, in AMDGPUTargetLowering()
618 ISD::SELECT, ISD::SELECT_CC, in AMDGPUTargetLowering()
619 ISD::STORE, ISD::FADD, in AMDGPUTargetLowering()
620 ISD::FSUB, ISD::FNEG, in AMDGPUTargetLowering()
621 ISD::FABS, ISD::AssertZext, in AMDGPUTargetLowering()
622 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN}); in AMDGPUTargetLowering()
647 case ISD::FADD: in fnegFoldsIntoOpcode()
648 case ISD::FSUB: in fnegFoldsIntoOpcode()
649 case ISD::FMUL: in fnegFoldsIntoOpcode()
650 case ISD::FMA: in fnegFoldsIntoOpcode()
651 case ISD::FMAD: in fnegFoldsIntoOpcode()
652 case ISD::FMINNUM: in fnegFoldsIntoOpcode()
653 case ISD::FMAXNUM: in fnegFoldsIntoOpcode()
654 case ISD::FMINNUM_IEEE: in fnegFoldsIntoOpcode()
655 case ISD::FMAXNUM_IEEE: in fnegFoldsIntoOpcode()
656 case ISD::FMINIMUM: in fnegFoldsIntoOpcode()
657 case ISD::FMAXIMUM: in fnegFoldsIntoOpcode()
658 case ISD::SELECT: in fnegFoldsIntoOpcode()
659 case ISD::FSIN: in fnegFoldsIntoOpcode()
660 case ISD::FTRUNC: in fnegFoldsIntoOpcode()
661 case ISD::FRINT: in fnegFoldsIntoOpcode()
662 case ISD::FNEARBYINT: in fnegFoldsIntoOpcode()
663 case ISD::FROUNDEVEN: in fnegFoldsIntoOpcode()
664 case ISD::FCANONICALIZE: in fnegFoldsIntoOpcode()
675 case ISD::BITCAST: in fnegFoldsIntoOpcode()
684 if (Opc == ISD::BITCAST) { in fnegFoldsIntoOp()
688 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) { in fnegFoldsIntoOp()
693 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32; in fnegFoldsIntoOp()
704 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) || in opMustUseVOP3Encoding()
724 case ISD::CopyToReg: in hasSourceMods()
725 case ISD::FDIV: in hasSourceMods()
726 case ISD::FREM: in hasSourceMods()
727 case ISD::INLINEASM: in hasSourceMods()
728 case ISD::INLINEASM_BR: in hasSourceMods()
730 case ISD::INTRINSIC_W_CHAIN: in hasSourceMods()
735 case ISD::BITCAST: in hasSourceMods()
737 case ISD::INTRINSIC_WO_CHAIN: { in hasSourceMods()
749 case ISD::SELECT: in hasSourceMods()
783 ISD::NodeType ExtendKind) const { in getTypeForExtReturn()
817 ISD::LoadExtType ExtTy, in shouldReduceLoadWidth()
890 case ISD::EntryToken: in isSDNodeAlwaysUniform()
891 case ISD::TokenFactor: in isSDNodeAlwaysUniform()
893 case ISD::INTRINSIC_WO_CHAIN: { in isSDNodeAlwaysUniform()
897 case ISD::LOAD: in isSDNodeAlwaysUniform()
913 case ISD::FMA: in getNegatedExpression()
914 case ISD::FMAD: { in getNegatedExpression()
1031 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
1032 N->getOpcode() == ISD::SRL) && in isDesirableToCommuteWithShift()
1037 N->getOpcode() != ISD::SHL || N->getOperand(0).getOpcode() != ISD::OR) in isDesirableToCommuteWithShift()
1042 (N->use_begin()->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
1043 N->use_begin()->getOpcode() == ISD::SRL)) in isDesirableToCommuteWithShift()
1048 if (LHS.getOpcode() != ISD::SHL) in isDesirableToCommuteWithShift()
1053 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD && in isDesirableToCommuteWithShift()
1055 RHSLd->getExtensionType() == ISD::ZEXTLOAD; in isDesirableToCommuteWithShift()
1148 const SmallVectorImpl<ISD::InputArg> &Ins) const { in analyzeFormalArgumentsCompute()
1267 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
1322 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
1345 for (ISD::InputArg &Arg : CLI.Ins) in lowerUnhandledCall()
1376 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); in LowerOperation()
1377 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); in LowerOperation()
1378 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); in LowerOperation()
1379 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); in LowerOperation()
1380 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG); in LowerOperation()
1381 case ISD::FREM: return LowerFREM(Op, DAG); in LowerOperation()
1382 case ISD::FCEIL: return LowerFCEIL(Op, DAG); in LowerOperation()
1383 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG); in LowerOperation()
1384 case ISD::FRINT: return LowerFRINT(Op, DAG); in LowerOperation()
1385 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG); in LowerOperation()
1386 case ISD::FROUNDEVEN: in LowerOperation()
1388 case ISD::FROUND: return LowerFROUND(Op, DAG); in LowerOperation()
1389 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); in LowerOperation()
1390 case ISD::FLOG2: in LowerOperation()
1392 case ISD::FLOG: in LowerOperation()
1393 case ISD::FLOG10: in LowerOperation()
1395 case ISD::FEXP: in LowerOperation()
1396 case ISD::FEXP10: in LowerOperation()
1398 case ISD::FEXP2: in LowerOperation()
1400 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); in LowerOperation()
1401 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); in LowerOperation()
1402 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG); in LowerOperation()
1403 case ISD::FP_TO_SINT: in LowerOperation()
1404 case ISD::FP_TO_UINT: in LowerOperation()
1406 case ISD::CTTZ: in LowerOperation()
1407 case ISD::CTTZ_ZERO_UNDEF: in LowerOperation()
1408 case ISD::CTLZ: in LowerOperation()
1409 case ISD::CTLZ_ZERO_UNDEF: in LowerOperation()
1411 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); in LowerOperation()
1420 case ISD::SIGN_EXTEND_INREG: in ReplaceNodeResults()
1428 case ISD::FLOG2: in ReplaceNodeResults()
1432 case ISD::FLOG: in ReplaceNodeResults()
1433 case ISD::FLOG10: in ReplaceNodeResults()
1437 case ISD::FEXP2: in ReplaceNodeResults()
1441 case ISD::FEXP: in ReplaceNodeResults()
1442 case ISD::FEXP10: in ReplaceNodeResults()
1446 case ISD::CTLZ: in ReplaceNodeResults()
1447 case ISD::CTLZ_ZERO_UNDEF: in ReplaceNodeResults()
1487 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode()); in LowerGlobalAddress()
1488 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerGlobalAddress()
1522 SDValue NewIn = DAG.getNode(ISD::BITCAST, SL, NewEltVT, In); in LowerCONCAT_VECTORS()
1532 return DAG.getNode(ISD::BITCAST, SL, VT, BV); in LowerCONCAT_VECTORS()
1560 SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0)); in LowerEXTRACT_SUBVECTOR()
1568 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp); in LowerEXTRACT_SUBVECTOR()
1579 if (Val.getOpcode() == ISD::FNEG) in peekFNeg()
1586 if (Val.getOpcode() == ISD::FNEG) in peekFPSignOps()
1588 if (Val.getOpcode() == ISD::FABS) in peekFPSignOps()
1590 if (Val.getOpcode() == ISD::FCOPYSIGN) in peekFPSignOps()
1599 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); in combineFMinMaxLegacyImpl()
1601 case ISD::SETOEQ: in combineFMinMaxLegacyImpl()
1602 case ISD::SETONE: in combineFMinMaxLegacyImpl()
1603 case ISD::SETUNE: in combineFMinMaxLegacyImpl()
1604 case ISD::SETNE: in combineFMinMaxLegacyImpl()
1605 case ISD::SETUEQ: in combineFMinMaxLegacyImpl()
1606 case ISD::SETEQ: in combineFMinMaxLegacyImpl()
1607 case ISD::SETFALSE: in combineFMinMaxLegacyImpl()
1608 case ISD::SETFALSE2: in combineFMinMaxLegacyImpl()
1609 case ISD::SETTRUE: in combineFMinMaxLegacyImpl()
1610 case ISD::SETTRUE2: in combineFMinMaxLegacyImpl()
1611 case ISD::SETUO: in combineFMinMaxLegacyImpl()
1612 case ISD::SETO: in combineFMinMaxLegacyImpl()
1614 case ISD::SETULE: in combineFMinMaxLegacyImpl()
1615 case ISD::SETULT: { in combineFMinMaxLegacyImpl()
1620 case ISD::SETOLE: in combineFMinMaxLegacyImpl()
1621 case ISD::SETOLT: in combineFMinMaxLegacyImpl()
1622 case ISD::SETLE: in combineFMinMaxLegacyImpl()
1623 case ISD::SETLT: { in combineFMinMaxLegacyImpl()
1639 case ISD::SETUGE: in combineFMinMaxLegacyImpl()
1640 case ISD::SETUGT: { in combineFMinMaxLegacyImpl()
1645 case ISD::SETGT: in combineFMinMaxLegacyImpl()
1646 case ISD::SETGE: in combineFMinMaxLegacyImpl()
1647 case ISD::SETOGE: in combineFMinMaxLegacyImpl()
1648 case ISD::SETOGT: { in combineFMinMaxLegacyImpl()
1657 case ISD::SETCC_INVALID: in combineFMinMaxLegacyImpl()
1694 return DAG.getNode(ISD::FNEG, DL, VT, Combined); in combineFMinMaxLegacy()
1706 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1711 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1712 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1720 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1722 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1728 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1730 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1759 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, in splitVector()
1762 HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL, in splitVector()
1811 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad); in SplitVectorLoad()
1813 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad, in SplitVectorLoad()
1816 HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL, in SplitVectorLoad()
1821 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1855 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad, in WidenOrSplitVectorLoad()
1899 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1928 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; in LowerDIVREM24()
1929 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT; in LowerDIVREM24()
1935 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerDIVREM24()
1938 jq = DAG.getNode(ISD::SRA, DL, VT, jq, in LowerDIVREM24()
1942 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); in LowerDIVREM24()
1957 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT, in LowerDIVREM24()
1961 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq); in LowerDIVREM24()
1964 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq); in LowerDIVREM24()
1976 unsigned OpCode = !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA in LowerDIVREM24()
1978 : (unsigned)ISD::FMAD; in LowerDIVREM24()
1985 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr); in LowerDIVREM24()
1988 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb); in LowerDIVREM24()
1993 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE); in LowerDIVREM24()
1996 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); in LowerDIVREM24()
1999 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); in LowerDIVREM24()
2002 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); in LowerDIVREM24()
2003 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); in LowerDIVREM24()
2009 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize); in LowerDIVREM24()
2010 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize); in LowerDIVREM24()
2013 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask); in LowerDIVREM24()
2014 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask); in LowerDIVREM24()
2045 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), in LowerUDIVREM64()
2051 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); in LowerUDIVREM64()
2052 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); in LowerUDIVREM64()
2065 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA in LowerUDIVREM64()
2067 ? (unsigned)ISD::FMAD in LowerUDIVREM64()
2070 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); in LowerUDIVREM64()
2071 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); in LowerUDIVREM64()
2076 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, in LowerUDIVREM64()
2078 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, in LowerUDIVREM64()
2080 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); in LowerUDIVREM64()
2084 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2); in LowerUDIVREM64()
2085 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc); in LowerUDIVREM64()
2095 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS); in LowerUDIVREM64()
2096 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64); in LowerUDIVREM64()
2097 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1); in LowerUDIVREM64()
2101 SDValue Add1_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Lo, in LowerUDIVREM64()
2103 SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi, in LowerUDIVREM64()
2109 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1); in LowerUDIVREM64()
2110 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2); in LowerUDIVREM64()
2114 SDValue Add2_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Lo, in LowerUDIVREM64()
2116 SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi, in LowerUDIVREM64()
2121 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2); in LowerUDIVREM64()
2123 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3); in LowerUDIVREM64()
2127 SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo, in LowerUDIVREM64()
2129 SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi, in LowerUDIVREM64()
2131 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi); in LowerUDIVREM64()
2137 ISD::SETUGE); in LowerUDIVREM64()
2139 ISD::SETUGE); in LowerUDIVREM64()
2140 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ); in LowerUDIVREM64()
2147 SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo, in LowerUDIVREM64()
2149 SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi, in LowerUDIVREM64()
2151 SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi, in LowerUDIVREM64()
2156 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64); in LowerUDIVREM64()
2159 ISD::SETUGE); in LowerUDIVREM64()
2161 ISD::SETUGE); in LowerUDIVREM64()
2162 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ); in LowerUDIVREM64()
2165 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64); in LowerUDIVREM64()
2167 SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo, in LowerUDIVREM64()
2169 SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi, in LowerUDIVREM64()
2171 SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi, in LowerUDIVREM64()
2179 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE); in LowerUDIVREM64()
2180 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE); in LowerUDIVREM64()
2182 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE); in LowerUDIVREM64()
2183 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE); in LowerUDIVREM64()
2193 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo); in LowerUDIVREM64()
2194 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo); in LowerUDIVREM64()
2196 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ); in LowerUDIVREM64()
2198 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); in LowerUDIVREM64()
2200 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ); in LowerUDIVREM64()
2209 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS); in LowerUDIVREM64()
2210 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One); in LowerUDIVREM64()
2211 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); in LowerUDIVREM64()
2214 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); in LowerUDIVREM64()
2216 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); in LowerUDIVREM64()
2219 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE); in LowerUDIVREM64()
2221 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT); in LowerUDIVREM64()
2224 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); in LowerUDIVREM64()
2225 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE); in LowerUDIVREM64()
2229 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); in LowerUDIVREM64()
2260 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y); in LowerUDIVREM()
2261 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z); in LowerUDIVREM()
2262 Z = DAG.getNode(ISD::ADD, DL, VT, Z, in LowerUDIVREM()
2263 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ)); in LowerUDIVREM()
2266 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z); in LowerUDIVREM()
2268 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y)); in LowerUDIVREM()
2273 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); in LowerUDIVREM()
2274 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2275 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
2276 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2277 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
2280 Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); in LowerUDIVREM()
2281 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2282 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
2283 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2284 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
2311 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero); in LowerSDIVREM()
2312 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero); in LowerSDIVREM()
2313 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), in LowerSDIVREM()
2316 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), in LowerSDIVREM()
2317 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) in LowerSDIVREM()
2322 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT); in LowerSDIVREM()
2323 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT); in LowerSDIVREM()
2324 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); in LowerSDIVREM()
2327 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); in LowerSDIVREM()
2328 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); in LowerSDIVREM()
2330 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); in LowerSDIVREM()
2331 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); in LowerSDIVREM()
2333 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); in LowerSDIVREM()
2336 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); in LowerSDIVREM()
2337 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); in LowerSDIVREM()
2339 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); in LowerSDIVREM()
2340 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); in LowerSDIVREM()
2357 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); in LowerFREM()
2358 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); in LowerFREM()
2359 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); in LowerFREM()
2361 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); in LowerFREM()
2372 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
2380 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); in LowerFCEIL()
2381 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFCEIL()
2382 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFCEIL()
2384 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
2386 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2398 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2422 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2426 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2428 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2432 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2434 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2441 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); in LowerFTRUNC()
2442 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); in LowerFTRUNC()
2444 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2445 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2447 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2459 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFROUNDEVEN()
2463 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFROUNDEVEN()
2464 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFROUNDEVEN()
2466 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFROUNDEVEN()
2473 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); in LowerFROUNDEVEN()
2483 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), Op.getValueType(), in LowerFNEARBYINT()
2490 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), VT, Arg); in LowerFRINT()
2503 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); in LowerFROUND()
2507 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); in LowerFROUND()
2509 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); in LowerFROUND()
2518 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); in LowerFROUND()
2519 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero); in LowerFROUND()
2521 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X); in LowerFROUND()
2522 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset); in LowerFROUND()
2533 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2541 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); in LowerFFLOOR()
2542 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFFLOOR()
2543 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFFLOOR()
2545 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2547 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2553 case ISD::FP_EXTEND: in valueIsKnownNeverF32Denorm()
2555 case ISD::FP16_TO_FP: in valueIsKnownNeverF32Denorm()
2556 case ISD::FFREXP: in valueIsKnownNeverF32Denorm()
2558 case ISD::INTRINSIC_WO_CHAIN: { in valueIsKnownNeverF32Denorm()
2604 SmallestNormal, ISD::SETOLT); in getIsLtSmallestNormal()
2616 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags); in getIsFinite()
2619 Inf, ISD::SETOLT); in getIsFinite()
2638 SmallestNormal, ISD::SETOLT); in getScaledLogInput()
2643 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags); in getScaledLogInput()
2645 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags); in getScaledLogInput()
2664 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in LowerFLOG2()
2666 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, in LowerFLOG2()
2680 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero); in LowerFLOG2()
2681 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags); in LowerFLOG2()
2686 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags); in getMad()
2687 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags); in getMad()
2697 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10; in LowerFLOGCommon()
2698 assert(IsLog10 || Op.getOpcode() == ISD::FLOG); in LowerFLOGCommon()
2706 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags); in LowerFLOGCommon()
2711 return DAG.getNode(ISD::FP_ROUND, DL, VT, Lowered, in LowerFLOGCommon()
2737 R = DAG.getNode(ISD::FMUL, DL, VT, Y, C, Flags); in LowerFLOGCommon()
2738 SDValue NegR = DAG.getNode(ISD::FNEG, DL, VT, R, Flags); in LowerFLOGCommon()
2739 SDValue FMA0 = DAG.getNode(ISD::FMA, DL, VT, Y, C, NegR, Flags); in LowerFLOGCommon()
2740 SDValue FMA1 = DAG.getNode(ISD::FMA, DL, VT, Y, CC, FMA0, Flags); in LowerFLOGCommon()
2741 R = DAG.getNode(ISD::FADD, DL, VT, R, FMA1, Flags); in LowerFLOGCommon()
2754 SDValue YAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Y); in LowerFLOGCommon()
2756 SDValue YHInt = DAG.getNode(ISD::AND, DL, MVT::i32, YAsInt, MaskConst); in LowerFLOGCommon()
2757 SDValue YH = DAG.getNode(ISD::BITCAST, DL, MVT::f32, YHInt); in LowerFLOGCommon()
2758 SDValue YT = DAG.getNode(ISD::FSUB, DL, VT, Y, YH, Flags); in LowerFLOGCommon()
2760 SDValue YTCT = DAG.getNode(ISD::FMUL, DL, VT, YT, CT, Flags); in LowerFLOGCommon()
2772 R = DAG.getNode(ISD::SELECT, DL, VT, IsFinite, R, Y, Flags); in LowerFLOGCommon()
2780 DAG.getNode(ISD::SELECT, DL, VT, IsScaled, ShiftK, Zero, Flags); in LowerFLOGCommon()
2781 R = DAG.getNode(ISD::FSUB, DL, VT, R, Shift, Flags); in LowerFLOGCommon()
2798 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2; in LowerFLOGUnsafe()
2812 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled, in LowerFLOGUnsafe()
2818 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset, in LowerFLOGUnsafe()
2820 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags); in LowerFLOGUnsafe()
2821 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset); in LowerFLOGUnsafe()
2828 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand, in LowerFLOGUnsafe()
2844 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2846 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, in lowerFEXP2()
2864 DAG.getSetCC(SL, SetCCVT, Src, RangeCheckConst, ISD::SETOLT); in lowerFEXP2()
2870 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero); in lowerFEXP2()
2872 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags); in lowerFEXP2()
2878 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One); in lowerFEXP2()
2880 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags); in lowerFEXP2()
2891 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Log2E, Flags); in lowerFEXPUnsafe()
2893 : (unsigned)ISD::FEXP2, in lowerFEXPUnsafe()
2900 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); in lowerFEXPUnsafe()
2904 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); in lowerFEXPUnsafe()
2907 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); in lowerFEXPUnsafe()
2909 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags); in lowerFEXPUnsafe()
2915 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags); in lowerFEXPUnsafe()
2917 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2, in lowerFEXPUnsafe()
2927 const unsigned Exp2Op = VT == MVT::f32 ? AMDGPUISD::EXP : ISD::FEXP2; in lowerFEXP10Unsafe()
2934 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags); in lowerFEXP10Unsafe()
2936 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags); in lowerFEXP10Unsafe()
2938 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1); in lowerFEXP10Unsafe()
2950 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); in lowerFEXP10Unsafe()
2953 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); in lowerFEXP10Unsafe()
2955 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); in lowerFEXP10Unsafe()
2960 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags); in lowerFEXP10Unsafe()
2962 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags); in lowerFEXP10Unsafe()
2965 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags); in lowerFEXP10Unsafe()
2969 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags); in lowerFEXP10Unsafe()
2971 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps, in lowerFEXP10Unsafe()
2980 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10; in lowerFEXP()
2994 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags); in lowerFEXP()
2996 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered, in lowerFEXP()
3046 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags); in lowerFEXP()
3047 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags); in lowerFEXP()
3048 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags); in lowerFEXP()
3049 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags); in lowerFEXP()
3060 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X); in lowerFEXP()
3062 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst); in lowerFEXP()
3063 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt); in lowerFEXP()
3064 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags); in lowerFEXP()
3066 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags); in lowerFEXP()
3068 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags); in lowerFEXP()
3073 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags); in lowerFEXP()
3076 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract); in lowerFEXP()
3078 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags); in lowerFEXP()
3079 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E); in lowerFEXP()
3082 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags); in lowerFEXP()
3090 DAG.getSetCC(SL, SetCCVT, X, UnderflowCheckConst, ISD::SETOLT); in lowerFEXP()
3092 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R); in lowerFEXP()
3099 DAG.getSetCC(SL, SetCCVT, X, OverflowCheckConst, ISD::SETOGT); in lowerFEXP()
3102 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R); in lowerFEXP()
3109 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF; in isCtlzOpc()
3113 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF; in isCttzOpc()
3133 if (Opc == ISD::CTLZ_ZERO_UNDEF) { in lowerCTLZResults()
3134 NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg); in lowerCTLZResults()
3135 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits); in lowerCTLZResults()
3138 NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); in lowerCTLZResults()
3140 NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits); in lowerCTLZResults()
3143 return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp); in lowerCTLZResults()
3154 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF || in LowerCTLZ_CTTZ()
3155 Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF; in LowerCTLZ_CTTZ()
3173 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal); in LowerCTLZ_CTTZ()
3175 return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr); in LowerCTLZ_CTTZ()
3189 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT; in LowerCTLZ_CTTZ()
3197 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); in LowerCTLZ_CTTZ()
3200 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); in LowerCTLZ_CTTZ()
3203 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
3264 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), in LowerINT_TO_FP32()
3267 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3273 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3275 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); in LowerINT_TO_FP32()
3280 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, in LowerINT_TO_FP32()
3283 DAG.getNode(ISD::XOR, SL, MVT::i64, in LowerINT_TO_FP32()
3284 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); in LowerINT_TO_FP32()
3288 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3292 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); in LowerINT_TO_FP32()
3297 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, in LowerINT_TO_FP32()
3300 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); in LowerINT_TO_FP32()
3303 (Signed && Subtarget->isGCN()) ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; in LowerINT_TO_FP32()
3308 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3312 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt); in LowerINT_TO_FP32()
3317 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3320 DAG.getNode(ISD::ADD, SL, MVT::i32, in LowerINT_TO_FP32()
3321 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); in LowerINT_TO_FP32()
3324 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, in LowerINT_TO_FP32()
3325 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), in LowerINT_TO_FP32()
3327 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); in LowerINT_TO_FP32()
3329 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); in LowerINT_TO_FP32()
3340 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, in LowerINT_TO_FP64()
3343 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
3345 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
3348 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
3364 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); in LowerUINT_TO_FP()
3365 return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext); in LowerUINT_TO_FP()
3370 SDValue ToF32 = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f32, Src); in LowerUINT_TO_FP()
3372 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerUINT_TO_FP()
3385 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerUINT_TO_FP()
3410 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); in LowerSINT_TO_FP()
3411 return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext); in LowerSINT_TO_FP()
3416 SDValue ToF32 = DAG.getNode(ISD::SINT_TO_FP, SL, MVT::f32, Src); in LowerSINT_TO_FP()
3418 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerSINT_TO_FP()
3434 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerSINT_TO_FP()
3464 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src); in LowerFP_TO_INT64()
3472 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, in LowerFP_TO_INT64()
3473 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), in LowerFP_TO_INT64()
3475 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc); in LowerFP_TO_INT64()
3493 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0); in LowerFP_TO_INT64()
3495 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul); in LowerFP_TO_INT64()
3497 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc); in LowerFP_TO_INT64()
3499 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT in LowerFP_TO_INT64()
3500 : ISD::FP_TO_UINT, in LowerFP_TO_INT64()
3502 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP_TO_INT64()
3504 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3510 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3514 DAG.getNode(ISD::SUB, SL, MVT::i64, in LowerFP_TO_INT64()
3515 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); in LowerFP_TO_INT64()
3542 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); in LowerFP_TO_FP16()
3543 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, in LowerFP_TO_FP16()
3547 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3549 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, in LowerFP_TO_FP16()
3553 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, in LowerFP_TO_FP16()
3556 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3558 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, in LowerFP_TO_FP16()
3561 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3563 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); in LowerFP_TO_FP16()
3565 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ); in LowerFP_TO_FP16()
3566 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); in LowerFP_TO_FP16()
3569 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, in LowerFP_TO_FP16()
3571 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); in LowerFP_TO_FP16()
3574 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
3575 DAG.getNode(ISD::SHL, DL, MVT::i32, E, in LowerFP_TO_FP16()
3579 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerFP_TO_FP16()
3581 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); in LowerFP_TO_FP16()
3582 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, in LowerFP_TO_FP16()
3585 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
3588 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); in LowerFP_TO_FP16()
3589 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); in LowerFP_TO_FP16()
3590 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE); in LowerFP_TO_FP16()
3591 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); in LowerFP_TO_FP16()
3593 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT); in LowerFP_TO_FP16()
3594 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, in LowerFP_TO_FP16()
3596 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, in LowerFP_TO_FP16()
3599 One, Zero, ISD::SETEQ); in LowerFP_TO_FP16()
3601 One, Zero, ISD::SETGT); in LowerFP_TO_FP16()
3602 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); in LowerFP_TO_FP16()
3603 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); in LowerFP_TO_FP16()
3606 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); in LowerFP_TO_FP16()
3608 I, V, ISD::SETEQ); in LowerFP_TO_FP16()
3611 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3613 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, in LowerFP_TO_FP16()
3616 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); in LowerFP_TO_FP16()
3633 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); in LowerFP_TO_INT()
3642 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32); in LowerFP_TO_INT()
3649 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) { in LowerFP_TO_INT()
3654 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerFP_TO_INT()
3659 return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT); in LowerFP_TO_INT()
3682 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp); in LowerSIGN_EXTEND_INREG()
3706 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN; in simplifyMul24()
3803 if (!LN->isSimple() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN)) in performLoadCombine()
3843 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); in performLoadCombine()
3856 if (!SN->isSimple() || !ISD::isNormalStore(SN)) in performStoreCombine()
3894 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val); in performStoreCombine()
3896 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); in performStoreCombine()
3914 if (N0.getOpcode() == ISD::TRUNCATE) { in performAssertSZExtCombine()
3923 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg); in performAssertSZExtCombine()
3986 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
4008 case ISD::ZERO_EXTEND: in performShlCombine()
4009 case ISD::SIGN_EXTEND: in performShlCombine()
4010 case ISD::ANY_EXTEND: { in performShlCombine()
4014 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) { in performShlCombine()
4019 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
4030 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0)); in performShlCombine()
4048 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
4049 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
4054 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
4073 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4077 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4083 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4086 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4106 if (LHS.getOpcode() == ISD::AND) { in performSrlCombine()
4112 ISD::AND, SL, VT, in performSrlCombine()
4113 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), in performSrlCombine()
4114 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); in performSrlCombine()
4133 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
4137 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
4148 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) { in performTruncateCombine()
4150 if (Vec.getOpcode() == ISD::BUILD_VECTOR) { in performTruncateCombine()
4155 Elt0 = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
4159 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); in performTruncateCombine()
4167 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) { in performTruncateCombine()
4171 if (BV.getOpcode() == ISD::BUILD_VECTOR && in performTruncateCombine()
4176 SrcElt = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
4180 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); in performTruncateCombine()
4193 (Src.getOpcode() == ISD::SRL || in performTruncateCombine()
4194 Src.getOpcode() == ISD::SRA || in performTruncateCombine()
4195 Src.getOpcode() == ISD::SHL)) { in performTruncateCombine()
4204 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits()); in performTruncateCombine()
4211 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT, in performTruncateCombine()
4222 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); in performTruncateCombine()
4247 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); in getMul24()
4253 if (V->getOpcode() != ISD::ADD) in getAddOneOp()
4261 assert(N->getOpcode() == ISD::MUL); in performMulCombine()
4291 return U->getOpcode() == ISD::MUL; in performMulCombine()
4302 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1); in performMulCombine()
4307 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0); in performMulCombine()
4318 if (N0.getOpcode() == ISD::ANY_EXTEND) in performMulCombine()
4321 if (N1.getOpcode() == ISD::ANY_EXTEND) in performMulCombine()
4352 bool Signed = N->getOpcode() == ISD::SMUL_LOHI; in performMulLoHiCombine()
4360 if (N0.getOpcode() == ISD::ANY_EXTEND) in performMulLoHiCombine()
4362 if (N1.getOpcode() == ISD::ANY_EXTEND) in performMulLoHiCombine()
4470 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); in getFFBX_U32()
4474 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX); in getFFBX_U32()
4493 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in performCtlz_CttzCombine()
4498 if (CCOpcode == ISD::SETEQ && in performCtlz_CttzCombine()
4508 if (CCOpcode == ISD::SETNE && in performCtlz_CttzCombine()
4529 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, in distributeOpThroughSelect()
4551 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) || in foldFreeOpFromSelect()
4552 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) { in foldFreeOpFromSelect()
4561 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) { in foldFreeOpFromSelect()
4568 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS && in foldFreeOpFromSelect()
4581 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(NewLHS.getNode())) in foldFreeOpFromSelect()
4583 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL) in foldFreeOpFromSelect()
4588 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative()) in foldFreeOpFromSelect()
4597 if (NewLHS.getOpcode() == ISD::FABS && in foldFreeOpFromSelect()
4604 if (LHS.getOpcode() == ISD::FNEG) in foldFreeOpFromSelect()
4605 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in foldFreeOpFromSelect()
4610 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, in foldFreeOpFromSelect()
4626 if (Cond.getOpcode() != ISD::SETCC) in performSelectCombine()
4646 ISD::CondCode NewCC = in performSelectCombine()
4650 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); in performSelectCombine()
4703 case ISD::FMAXNUM: in inverseMinMax()
4704 return ISD::FMINNUM; in inverseMinMax()
4705 case ISD::FMINNUM: in inverseMinMax()
4706 return ISD::FMAXNUM; in inverseMinMax()
4707 case ISD::FMAXNUM_IEEE: in inverseMinMax()
4708 return ISD::FMINNUM_IEEE; in inverseMinMax()
4709 case ISD::FMINNUM_IEEE: in inverseMinMax()
4710 return ISD::FMAXNUM_IEEE; in inverseMinMax()
4711 case ISD::FMAXIMUM: in inverseMinMax()
4712 return ISD::FMINIMUM; in inverseMinMax()
4713 case ISD::FMINIMUM: in inverseMinMax()
4714 return ISD::FMAXIMUM; in inverseMinMax()
4758 case ISD::FADD: { in performFNegCombine()
4766 if (LHS.getOpcode() != ISD::FNEG) in performFNegCombine()
4767 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
4771 if (RHS.getOpcode() != ISD::FNEG) in performFNegCombine()
4772 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4776 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
4777 if (Res.getOpcode() != ISD::FADD) in performFNegCombine()
4780 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4783 case ISD::FMUL: in performFNegCombine()
4790 if (LHS.getOpcode() == ISD::FNEG) in performFNegCombine()
4792 else if (RHS.getOpcode() == ISD::FNEG) in performFNegCombine()
4795 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4801 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4804 case ISD::FMA: in performFNegCombine()
4805 case ISD::FMAD: { in performFNegCombine()
4815 if (LHS.getOpcode() == ISD::FNEG) in performFNegCombine()
4817 else if (MHS.getOpcode() == ISD::FNEG) in performFNegCombine()
4820 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); in performFNegCombine()
4822 if (RHS.getOpcode() != ISD::FNEG) in performFNegCombine()
4823 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4831 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4834 case ISD::FMAXNUM: in performFNegCombine()
4835 case ISD::FMINNUM: in performFNegCombine()
4836 case ISD::FMAXNUM_IEEE: in performFNegCombine()
4837 case ISD::FMINNUM_IEEE: in performFNegCombine()
4838 case ISD::FMINIMUM: in performFNegCombine()
4839 case ISD::FMAXIMUM: in performFNegCombine()
4855 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
4856 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4863 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4869 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); in performFNegCombine()
4876 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res); in performFNegCombine()
4885 case ISD::FP_EXTEND: in performFNegCombine()
4886 case ISD::FTRUNC: in performFNegCombine()
4887 case ISD::FRINT: in performFNegCombine()
4888 case ISD::FNEARBYINT: // XXX - Should fround be handled? in performFNegCombine()
4889 case ISD::FROUNDEVEN: in performFNegCombine()
4890 case ISD::FSIN: in performFNegCombine()
4891 case ISD::FCANONICALIZE: in performFNegCombine()
4897 if (CvtSrc.getOpcode() == ISD::FNEG) { in performFNegCombine()
4908 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
4911 case ISD::FP_ROUND: { in performFNegCombine()
4914 if (CvtSrc.getOpcode() == ISD::FNEG) { in performFNegCombine()
4916 return DAG.getNode(ISD::FP_ROUND, SL, VT, in performFNegCombine()
4924 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
4925 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); in performFNegCombine()
4927 case ISD::FP16_TO_FP: { in performFNegCombine()
4937 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src, in performFNegCombine()
4939 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg); in performFNegCombine()
4941 case ISD::SELECT: { in performFNegCombine()
4946 case ISD::BITCAST: { in performFNegCombine()
4949 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) { in performFNegCombine()
4964 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits); in performFNegCombine()
4965 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi); in performFNegCombine()
4967 DAG.getNode(ISD::BITCAST, SL, HighBits.getValueType(), NegHi); in performFNegCombine()
4973 DAG.getNode(ISD::BUILD_VECTOR, SL, BCSrc.getValueType(), Ops); in performFNegCombine()
4974 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build); in performFNegCombine()
4977 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result)); in performFNegCombine()
4981 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 && in performFNegCombine()
4989 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1)); in performFNegCombine()
4991 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2)); in performFNegCombine()
4993 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS); in performFNegCombine()
4994 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS); in performFNegCombine()
4996 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS, in performFNegCombine()
5016 case ISD::FP16_TO_FP: { in performFAbsCombine()
5023 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src, in performFAbsCombine()
5025 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs); in performFAbsCombine()
5052 case ISD::BITCAST: { in PerformDAGCombine()
5062 if (Src.getOpcode() == ISD::BUILD_VECTOR && in PerformDAGCombine()
5064 isOperationLegal(ISD::BUILD_VECTOR, DestVT))) { in PerformDAGCombine()
5075 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt)); in PerformDAGCombine()
5094 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5097 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV); in PerformDAGCombine()
5104 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5108 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec); in PerformDAGCombine()
5113 case ISD::SHL: { in PerformDAGCombine()
5119 case ISD::SRL: { in PerformDAGCombine()
5125 case ISD::SRA: { in PerformDAGCombine()
5131 case ISD::TRUNCATE: in PerformDAGCombine()
5133 case ISD::MUL: in PerformDAGCombine()
5144 case ISD::SMUL_LOHI: in PerformDAGCombine()
5145 case ISD::UMUL_LOHI: in PerformDAGCombine()
5147 case ISD::MULHS: in PerformDAGCombine()
5149 case ISD::MULHU: in PerformDAGCombine()
5151 case ISD::SELECT: in PerformDAGCombine()
5153 case ISD::FNEG: in PerformDAGCombine()
5155 case ISD::FABS: in PerformDAGCombine()
5194 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
5220 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
5241 case ISD::LOAD: in PerformDAGCombine()
5243 case ISD::STORE: in PerformDAGCombine()
5248 case ISD::AssertZext: in PerformDAGCombine()
5249 case ISD::AssertSext: in PerformDAGCombine()
5251 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
5356 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); in storeStackInputValue()
5377 V = DAG.getNode(ISD::SRL, SL, VT, V, in loadInputValue()
5379 return DAG.getNode(ISD::AND, SL, VT, V, in loadInputValue()
5776 case ISD::INTRINSIC_WO_CHAIN: { in computeKnownBitsForTargetNode()
5942 case ISD::FLDEXP: in isKnownNeverNaNForTargetNode()
5958 case ISD::INTRINSIC_WO_CHAIN: { in isKnownNeverNaNForTargetNode()