Lines Matching refs:ISD
159 static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, in handleCMSEValue()
163 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Arg.ArgVT, Value); in handleCMSEValue()
165 DAG.getNode(Arg.Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in handleCMSEValue()
172 setOperationAction(ISD::LOAD, VT, Promote); in addTypeForNEON()
173 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT); in addTypeForNEON()
175 setOperationAction(ISD::STORE, VT, Promote); in addTypeForNEON()
176 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT); in addTypeForNEON()
181 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
182 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
183 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
185 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in addTypeForNEON()
186 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in addTypeForNEON()
187 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForNEON()
188 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForNEON()
190 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addTypeForNEON()
191 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addTypeForNEON()
192 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addTypeForNEON()
193 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addTypeForNEON()
195 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
196 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
197 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
198 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in addTypeForNEON()
199 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
200 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
201 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
202 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in addTypeForNEON()
204 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
205 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
206 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
210 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
211 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
212 setOperationAction(ISD::FDIV, VT, Expand); in addTypeForNEON()
213 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
214 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
215 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
216 setOperationAction(ISD::SDIVREM, VT, Expand); in addTypeForNEON()
217 setOperationAction(ISD::UDIVREM, VT, Expand); in addTypeForNEON()
220 for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX, in addTypeForNEON()
221 ISD::UMIN, ISD::UMAX}) in addTypeForNEON()
224 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}) in addTypeForNEON()
239 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc) in setAllExpand()
245 setOperationAction(ISD::BITCAST, VT, Legal); in setAllExpand()
246 setOperationAction(ISD::LOAD, VT, Legal); in setAllExpand()
247 setOperationAction(ISD::STORE, VT, Legal); in setAllExpand()
248 setOperationAction(ISD::UNDEF, VT, Legal); in setAllExpand()
253 setLoadExtAction(ISD::EXTLOAD, From, To, Action); in addAllExtLoads()
254 setLoadExtAction(ISD::ZEXTLOAD, From, To, Action); in addAllExtLoads()
255 setLoadExtAction(ISD::SEXTLOAD, From, To, Action); in addAllExtLoads()
263 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
264 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
265 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
266 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
267 setOperationAction(ISD::SHL, VT, Custom); in addMVEVectorTypes()
268 setOperationAction(ISD::SRA, VT, Custom); in addMVEVectorTypes()
269 setOperationAction(ISD::SRL, VT, Custom); in addMVEVectorTypes()
270 setOperationAction(ISD::SMIN, VT, Legal); in addMVEVectorTypes()
271 setOperationAction(ISD::SMAX, VT, Legal); in addMVEVectorTypes()
272 setOperationAction(ISD::UMIN, VT, Legal); in addMVEVectorTypes()
273 setOperationAction(ISD::UMAX, VT, Legal); in addMVEVectorTypes()
274 setOperationAction(ISD::ABS, VT, Legal); in addMVEVectorTypes()
275 setOperationAction(ISD::SETCC, VT, Custom); in addMVEVectorTypes()
276 setOperationAction(ISD::MLOAD, VT, Custom); in addMVEVectorTypes()
277 setOperationAction(ISD::MSTORE, VT, Legal); in addMVEVectorTypes()
278 setOperationAction(ISD::CTLZ, VT, Legal); in addMVEVectorTypes()
279 setOperationAction(ISD::CTTZ, VT, Custom); in addMVEVectorTypes()
280 setOperationAction(ISD::BITREVERSE, VT, Legal); in addMVEVectorTypes()
281 setOperationAction(ISD::BSWAP, VT, Legal); in addMVEVectorTypes()
282 setOperationAction(ISD::SADDSAT, VT, Legal); in addMVEVectorTypes()
283 setOperationAction(ISD::UADDSAT, VT, Legal); in addMVEVectorTypes()
284 setOperationAction(ISD::SSUBSAT, VT, Legal); in addMVEVectorTypes()
285 setOperationAction(ISD::USUBSAT, VT, Legal); in addMVEVectorTypes()
286 setOperationAction(ISD::ABDS, VT, Legal); in addMVEVectorTypes()
287 setOperationAction(ISD::ABDU, VT, Legal); in addMVEVectorTypes()
288 setOperationAction(ISD::AVGFLOORS, VT, Legal); in addMVEVectorTypes()
289 setOperationAction(ISD::AVGFLOORU, VT, Legal); in addMVEVectorTypes()
290 setOperationAction(ISD::AVGCEILS, VT, Legal); in addMVEVectorTypes()
291 setOperationAction(ISD::AVGCEILU, VT, Legal); in addMVEVectorTypes()
294 setOperationAction(ISD::UDIV, VT, Expand); in addMVEVectorTypes()
295 setOperationAction(ISD::SDIV, VT, Expand); in addMVEVectorTypes()
296 setOperationAction(ISD::UREM, VT, Expand); in addMVEVectorTypes()
297 setOperationAction(ISD::SREM, VT, Expand); in addMVEVectorTypes()
298 setOperationAction(ISD::UDIVREM, VT, Expand); in addMVEVectorTypes()
299 setOperationAction(ISD::SDIVREM, VT, Expand); in addMVEVectorTypes()
300 setOperationAction(ISD::CTPOP, VT, Expand); in addMVEVectorTypes()
301 setOperationAction(ISD::SELECT, VT, Expand); in addMVEVectorTypes()
302 setOperationAction(ISD::SELECT_CC, VT, Expand); in addMVEVectorTypes()
305 setOperationAction(ISD::VECREDUCE_ADD, VT, Legal); in addMVEVectorTypes()
306 setOperationAction(ISD::VECREDUCE_SMAX, VT, Legal); in addMVEVectorTypes()
307 setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal); in addMVEVectorTypes()
308 setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal); in addMVEVectorTypes()
309 setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal); in addMVEVectorTypes()
310 setOperationAction(ISD::VECREDUCE_MUL, VT, Custom); in addMVEVectorTypes()
311 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in addMVEVectorTypes()
312 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in addMVEVectorTypes()
313 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in addMVEVectorTypes()
316 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addMVEVectorTypes()
317 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addMVEVectorTypes()
318 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addMVEVectorTypes()
319 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addMVEVectorTypes()
321 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom); in addMVEVectorTypes()
322 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom); in addMVEVectorTypes()
326 for (unsigned im = (unsigned)ISD::PRE_INC; in addMVEVectorTypes()
327 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in addMVEVectorTypes()
342 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
343 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
344 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getVectorElementType(), Custom); in addMVEVectorTypes()
345 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
346 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
347 setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom); in addMVEVectorTypes()
348 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal); in addMVEVectorTypes()
349 setOperationAction(ISD::SETCC, VT, Custom); in addMVEVectorTypes()
350 setOperationAction(ISD::MLOAD, VT, Custom); in addMVEVectorTypes()
351 setOperationAction(ISD::MSTORE, VT, Legal); in addMVEVectorTypes()
352 setOperationAction(ISD::SELECT, VT, Expand); in addMVEVectorTypes()
353 setOperationAction(ISD::SELECT_CC, VT, Expand); in addMVEVectorTypes()
356 for (unsigned im = (unsigned)ISD::PRE_INC; in addMVEVectorTypes()
357 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in addMVEVectorTypes()
365 setOperationAction(ISD::FMINNUM, VT, Legal); in addMVEVectorTypes()
366 setOperationAction(ISD::FMAXNUM, VT, Legal); in addMVEVectorTypes()
367 setOperationAction(ISD::FROUND, VT, Legal); in addMVEVectorTypes()
368 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in addMVEVectorTypes()
369 setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom); in addMVEVectorTypes()
370 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in addMVEVectorTypes()
371 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in addMVEVectorTypes()
374 setOperationAction(ISD::FDIV, VT, Expand); in addMVEVectorTypes()
375 setOperationAction(ISD::FREM, VT, Expand); in addMVEVectorTypes()
376 setOperationAction(ISD::FSQRT, VT, Expand); in addMVEVectorTypes()
377 setOperationAction(ISD::FSIN, VT, Expand); in addMVEVectorTypes()
378 setOperationAction(ISD::FCOS, VT, Expand); in addMVEVectorTypes()
379 setOperationAction(ISD::FTAN, VT, Expand); in addMVEVectorTypes()
380 setOperationAction(ISD::FPOW, VT, Expand); in addMVEVectorTypes()
381 setOperationAction(ISD::FLOG, VT, Expand); in addMVEVectorTypes()
382 setOperationAction(ISD::FLOG2, VT, Expand); in addMVEVectorTypes()
383 setOperationAction(ISD::FLOG10, VT, Expand); in addMVEVectorTypes()
384 setOperationAction(ISD::FEXP, VT, Expand); in addMVEVectorTypes()
385 setOperationAction(ISD::FEXP2, VT, Expand); in addMVEVectorTypes()
386 setOperationAction(ISD::FEXP10, VT, Expand); in addMVEVectorTypes()
387 setOperationAction(ISD::FNEARBYINT, VT, Expand); in addMVEVectorTypes()
393 setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f16, Custom); in addMVEVectorTypes()
394 setOperationAction(ISD::VECREDUCE_FMUL, MVT::v4f16, Custom); in addMVEVectorTypes()
395 setOperationAction(ISD::VECREDUCE_FMIN, MVT::v4f16, Custom); in addMVEVectorTypes()
396 setOperationAction(ISD::VECREDUCE_FMAX, MVT::v4f16, Custom); in addMVEVectorTypes()
397 setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f16, Custom); in addMVEVectorTypes()
398 setOperationAction(ISD::VECREDUCE_FMUL, MVT::v2f16, Custom); in addMVEVectorTypes()
399 setOperationAction(ISD::VECREDUCE_FMIN, MVT::v2f16, Custom); in addMVEVectorTypes()
400 setOperationAction(ISD::VECREDUCE_FMAX, MVT::v2f16, Custom); in addMVEVectorTypes()
409 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
410 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
411 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
412 setOperationAction(ISD::VSELECT, VT, Legal); in addMVEVectorTypes()
413 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
415 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); in addMVEVectorTypes()
418 setOperationAction(ISD::AND, MVT::v2i64, Legal); in addMVEVectorTypes()
419 setOperationAction(ISD::OR, MVT::v2i64, Legal); in addMVEVectorTypes()
420 setOperationAction(ISD::XOR, MVT::v2i64, Legal); in addMVEVectorTypes()
428 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal); in addMVEVectorTypes()
429 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal); in addMVEVectorTypes()
430 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal); in addMVEVectorTypes()
431 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i8, Legal); in addMVEVectorTypes()
432 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i16, Legal); in addMVEVectorTypes()
440 for (unsigned im = (unsigned)ISD::PRE_INC; in addMVEVectorTypes()
441 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in addMVEVectorTypes()
454 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
455 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
456 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addMVEVectorTypes()
457 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in addMVEVectorTypes()
458 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
459 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
460 setOperationAction(ISD::SETCC, VT, Custom); in addMVEVectorTypes()
461 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); in addMVEVectorTypes()
462 setOperationAction(ISD::LOAD, VT, Custom); in addMVEVectorTypes()
463 setOperationAction(ISD::STORE, VT, Custom); in addMVEVectorTypes()
464 setOperationAction(ISD::TRUNCATE, VT, Custom); in addMVEVectorTypes()
465 setOperationAction(ISD::VSELECT, VT, Expand); in addMVEVectorTypes()
466 setOperationAction(ISD::SELECT, VT, Expand); in addMVEVectorTypes()
467 setOperationAction(ISD::SELECT_CC, VT, Expand); in addMVEVectorTypes()
470 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addMVEVectorTypes()
471 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addMVEVectorTypes()
472 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addMVEVectorTypes()
473 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addMVEVectorTypes()
476 setOperationAction(ISD::SETCC, MVT::v2i1, Expand); in addMVEVectorTypes()
477 setOperationAction(ISD::TRUNCATE, MVT::v2i1, Expand); in addMVEVectorTypes()
478 setOperationAction(ISD::AND, MVT::v2i1, Expand); in addMVEVectorTypes()
479 setOperationAction(ISD::OR, MVT::v2i1, Expand); in addMVEVectorTypes()
480 setOperationAction(ISD::XOR, MVT::v2i1, Expand); in addMVEVectorTypes()
481 setOperationAction(ISD::SINT_TO_FP, MVT::v2i1, Expand); in addMVEVectorTypes()
482 setOperationAction(ISD::UINT_TO_FP, MVT::v2i1, Expand); in addMVEVectorTypes()
483 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Expand); in addMVEVectorTypes()
484 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Expand); in addMVEVectorTypes()
486 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); in addMVEVectorTypes()
487 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in addMVEVectorTypes()
488 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in addMVEVectorTypes()
489 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); in addMVEVectorTypes()
490 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); in addMVEVectorTypes()
491 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in addMVEVectorTypes()
492 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); in addMVEVectorTypes()
493 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); in addMVEVectorTypes()
521 const ISD::CondCode Cond; in ARMTargetLowering()
524 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
525 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
526 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
527 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
530 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
531 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
532 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
533 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
536 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE }, in ARMTargetLowering()
537 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE }, in ARMTargetLowering()
538 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE }, in ARMTargetLowering()
539 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE }, in ARMTargetLowering()
540 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE }, in ARMTargetLowering()
541 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE }, in ARMTargetLowering()
542 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE }, in ARMTargetLowering()
545 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE }, in ARMTargetLowering()
546 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE }, in ARMTargetLowering()
547 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE }, in ARMTargetLowering()
548 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE }, in ARMTargetLowering()
549 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE }, in ARMTargetLowering()
550 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE }, in ARMTargetLowering()
551 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE }, in ARMTargetLowering()
556 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
557 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
558 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
559 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
562 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
563 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
570 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
571 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
572 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
573 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID }, in ARMTargetLowering()
578 if (LC.Cond != ISD::SETCC_INVALID) in ARMTargetLowering()
592 const ISD::CondCode Cond; in ARMTargetLowering()
596 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
597 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
598 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
599 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
603 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
604 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ }, in ARMTargetLowering()
605 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
606 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
607 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
608 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
609 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
613 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
614 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
615 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
616 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
620 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
621 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ }, in ARMTargetLowering()
622 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
623 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
624 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
625 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
626 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE }, in ARMTargetLowering()
630 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
631 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
632 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
633 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
634 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
635 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
636 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
637 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
641 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
642 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
643 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
647 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
648 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
649 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
650 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
651 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
652 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
653 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
654 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
658 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
659 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
660 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
661 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
665 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
666 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
667 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
668 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
669 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
670 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
671 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
672 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
678 if (LC.Cond != ISD::SETCC_INVALID) in ARMTargetLowering()
689 const ISD::CondCode Cond; in ARMTargetLowering()
693 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
694 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
695 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID }, in ARMTargetLowering()
701 if (LC.Cond != ISD::SETCC_INVALID) in ARMTargetLowering()
781 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom); in ARMTargetLowering()
782 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom); in ARMTargetLowering()
783 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom); in ARMTargetLowering()
784 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom); in ARMTargetLowering()
794 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in ARMTargetLowering()
795 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in ARMTargetLowering()
797 setOperationAction(ISD::FMINNUM, MVT::f16, Legal); in ARMTargetLowering()
798 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal); in ARMTargetLowering()
805 setOperationAction(ISD::BITCAST, MVT::bf16, Custom); in ARMTargetLowering()
814 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in ARMTargetLowering()
815 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in ARMTargetLowering()
817 setOperationAction(ISD::BSWAP, VT, Expand); in ARMTargetLowering()
820 setOperationAction(ISD::ConstantFP, MVT::f32, Custom); in ARMTargetLowering()
821 setOperationAction(ISD::ConstantFP, MVT::f64, Custom); in ARMTargetLowering()
823 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom); in ARMTargetLowering()
824 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom); in ARMTargetLowering()
831 setTargetDAGCombine({ISD::BRCOND, ISD::BR_CC}); in ARMTargetLowering()
862 setOperationAction(ISD::FADD, MVT::v2f64, Expand); in ARMTargetLowering()
863 setOperationAction(ISD::FSUB, MVT::v2f64, Expand); in ARMTargetLowering()
864 setOperationAction(ISD::FMUL, MVT::v2f64, Expand); in ARMTargetLowering()
867 setOperationAction(ISD::FDIV, MVT::v2f64, Expand); in ARMTargetLowering()
868 setOperationAction(ISD::FREM, MVT::v2f64, Expand); in ARMTargetLowering()
872 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand); in ARMTargetLowering()
875 setOperationAction(ISD::SETCC, MVT::v2f64, Expand); in ARMTargetLowering()
877 setOperationAction(ISD::FNEG, MVT::v2f64, Expand); in ARMTargetLowering()
878 setOperationAction(ISD::FABS, MVT::v2f64, Expand); in ARMTargetLowering()
879 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand); in ARMTargetLowering()
880 setOperationAction(ISD::FSIN, MVT::v2f64, Expand); in ARMTargetLowering()
881 setOperationAction(ISD::FCOS, MVT::v2f64, Expand); in ARMTargetLowering()
882 setOperationAction(ISD::FTAN, MVT::v2f64, Expand); in ARMTargetLowering()
883 setOperationAction(ISD::FPOW, MVT::v2f64, Expand); in ARMTargetLowering()
884 setOperationAction(ISD::FLOG, MVT::v2f64, Expand); in ARMTargetLowering()
885 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand); in ARMTargetLowering()
886 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand); in ARMTargetLowering()
887 setOperationAction(ISD::FEXP, MVT::v2f64, Expand); in ARMTargetLowering()
888 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand); in ARMTargetLowering()
889 setOperationAction(ISD::FEXP10, MVT::v2f64, Expand); in ARMTargetLowering()
891 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand); in ARMTargetLowering()
892 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand); in ARMTargetLowering()
893 setOperationAction(ISD::FRINT, MVT::v2f64, Expand); in ARMTargetLowering()
894 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand); in ARMTargetLowering()
895 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand); in ARMTargetLowering()
896 setOperationAction(ISD::FMA, MVT::v2f64, Expand); in ARMTargetLowering()
902 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); in ARMTargetLowering()
903 setOperationAction(ISD::FSIN, MVT::v4f32, Expand); in ARMTargetLowering()
904 setOperationAction(ISD::FCOS, MVT::v4f32, Expand); in ARMTargetLowering()
905 setOperationAction(ISD::FTAN, MVT::v4f32, Expand); in ARMTargetLowering()
906 setOperationAction(ISD::FPOW, MVT::v4f32, Expand); in ARMTargetLowering()
907 setOperationAction(ISD::FLOG, MVT::v4f32, Expand); in ARMTargetLowering()
908 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand); in ARMTargetLowering()
909 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand); in ARMTargetLowering()
910 setOperationAction(ISD::FEXP, MVT::v4f32, Expand); in ARMTargetLowering()
911 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand); in ARMTargetLowering()
912 setOperationAction(ISD::FEXP10, MVT::v4f32, Expand); in ARMTargetLowering()
913 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand); in ARMTargetLowering()
914 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand); in ARMTargetLowering()
915 setOperationAction(ISD::FRINT, MVT::v4f32, Expand); in ARMTargetLowering()
916 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); in ARMTargetLowering()
917 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand); in ARMTargetLowering()
920 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand); in ARMTargetLowering()
921 setOperationAction(ISD::FSIN, MVT::v2f32, Expand); in ARMTargetLowering()
922 setOperationAction(ISD::FCOS, MVT::v2f32, Expand); in ARMTargetLowering()
923 setOperationAction(ISD::FTAN, MVT::v2f32, Expand); in ARMTargetLowering()
924 setOperationAction(ISD::FPOW, MVT::v2f32, Expand); in ARMTargetLowering()
925 setOperationAction(ISD::FLOG, MVT::v2f32, Expand); in ARMTargetLowering()
926 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand); in ARMTargetLowering()
927 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand); in ARMTargetLowering()
928 setOperationAction(ISD::FEXP, MVT::v2f32, Expand); in ARMTargetLowering()
929 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand); in ARMTargetLowering()
930 setOperationAction(ISD::FEXP10, MVT::v2f32, Expand); in ARMTargetLowering()
931 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand); in ARMTargetLowering()
932 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand); in ARMTargetLowering()
933 setOperationAction(ISD::FRINT, MVT::v2f32, Expand); in ARMTargetLowering()
934 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand); in ARMTargetLowering()
935 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand); in ARMTargetLowering()
938 setOperationAction(ISD::MUL, MVT::v1i64, Expand); in ARMTargetLowering()
940 setOperationAction(ISD::MUL, MVT::v8i16, Custom); in ARMTargetLowering()
941 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in ARMTargetLowering()
942 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in ARMTargetLowering()
944 setOperationAction(ISD::SDIV, MVT::v4i16, Custom); in ARMTargetLowering()
945 setOperationAction(ISD::SDIV, MVT::v8i8, Custom); in ARMTargetLowering()
946 setOperationAction(ISD::UDIV, MVT::v4i16, Custom); in ARMTargetLowering()
947 setOperationAction(ISD::UDIV, MVT::v8i8, Custom); in ARMTargetLowering()
952 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); in ARMTargetLowering()
953 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom); in ARMTargetLowering()
954 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); in ARMTargetLowering()
955 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); in ARMTargetLowering()
956 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom); in ARMTargetLowering()
957 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom); in ARMTargetLowering()
958 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom); in ARMTargetLowering()
959 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom); in ARMTargetLowering()
961 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); in ARMTargetLowering()
962 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand); in ARMTargetLowering()
967 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom); in ARMTargetLowering()
968 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); in ARMTargetLowering()
969 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom); in ARMTargetLowering()
970 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom); in ARMTargetLowering()
971 setOperationAction(ISD::CTPOP, MVT::v1i64, Custom); in ARMTargetLowering()
972 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom); in ARMTargetLowering()
974 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); in ARMTargetLowering()
975 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); in ARMTargetLowering()
978 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom); in ARMTargetLowering()
979 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom); in ARMTargetLowering()
980 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom); in ARMTargetLowering()
981 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom); in ARMTargetLowering()
983 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom); in ARMTargetLowering()
984 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom); in ARMTargetLowering()
985 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom); in ARMTargetLowering()
986 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom); in ARMTargetLowering()
988 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom); in ARMTargetLowering()
989 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom); in ARMTargetLowering()
990 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom); in ARMTargetLowering()
991 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom); in ARMTargetLowering()
993 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom); in ARMTargetLowering()
994 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom); in ARMTargetLowering()
995 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); in ARMTargetLowering()
996 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom); in ARMTargetLowering()
999 setOperationAction(ISD::MULHS, VT, Expand); in ARMTargetLowering()
1000 setOperationAction(ISD::MULHU, VT, Expand); in ARMTargetLowering()
1005 setOperationAction(ISD::FMA, MVT::v2f32, Expand); in ARMTargetLowering()
1006 setOperationAction(ISD::FMA, MVT::v4f32, Expand); in ARMTargetLowering()
1009 setTargetDAGCombine({ISD::SHL, ISD::SRL, ISD::SRA, ISD::FP_TO_SINT, in ARMTargetLowering()
1010 ISD::FP_TO_UINT, ISD::FMUL, ISD::LOAD}); in ARMTargetLowering()
1016 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
1017 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
1018 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
1024 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in ARMTargetLowering()
1025 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in ARMTargetLowering()
1026 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in ARMTargetLowering()
1027 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in ARMTargetLowering()
1033 {ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, ISD::INSERT_SUBVECTOR, in ARMTargetLowering()
1034 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT, in ARMTargetLowering()
1035 ISD::SIGN_EXTEND_INREG, ISD::STORE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND, in ARMTargetLowering()
1036 ISD::ANY_EXTEND, ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN, in ARMTargetLowering()
1037 ISD::INTRINSIC_VOID, ISD::VECREDUCE_ADD, ISD::ADD, ISD::BITCAST}); in ARMTargetLowering()
1040 setTargetDAGCombine({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX, in ARMTargetLowering()
1041 ISD::FP_EXTEND, ISD::SELECT, ISD::SELECT_CC, in ARMTargetLowering()
1042 ISD::SETCC}); in ARMTargetLowering()
1045 setTargetDAGCombine(ISD::FADD); in ARMTargetLowering()
1053 setOperationAction(ISD::FADD, MVT::f64, Expand); in ARMTargetLowering()
1054 setOperationAction(ISD::FSUB, MVT::f64, Expand); in ARMTargetLowering()
1055 setOperationAction(ISD::FMUL, MVT::f64, Expand); in ARMTargetLowering()
1056 setOperationAction(ISD::FMA, MVT::f64, Expand); in ARMTargetLowering()
1057 setOperationAction(ISD::FDIV, MVT::f64, Expand); in ARMTargetLowering()
1058 setOperationAction(ISD::FREM, MVT::f64, Expand); in ARMTargetLowering()
1059 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in ARMTargetLowering()
1060 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand); in ARMTargetLowering()
1061 setOperationAction(ISD::FNEG, MVT::f64, Expand); in ARMTargetLowering()
1062 setOperationAction(ISD::FABS, MVT::f64, Expand); in ARMTargetLowering()
1063 setOperationAction(ISD::FSQRT, MVT::f64, Expand); in ARMTargetLowering()
1064 setOperationAction(ISD::FSIN, MVT::f64, Expand); in ARMTargetLowering()
1065 setOperationAction(ISD::FCOS, MVT::f64, Expand); in ARMTargetLowering()
1066 setOperationAction(ISD::FPOW, MVT::f64, Expand); in ARMTargetLowering()
1067 setOperationAction(ISD::FLOG, MVT::f64, Expand); in ARMTargetLowering()
1068 setOperationAction(ISD::FLOG2, MVT::f64, Expand); in ARMTargetLowering()
1069 setOperationAction(ISD::FLOG10, MVT::f64, Expand); in ARMTargetLowering()
1070 setOperationAction(ISD::FEXP, MVT::f64, Expand); in ARMTargetLowering()
1071 setOperationAction(ISD::FEXP2, MVT::f64, Expand); in ARMTargetLowering()
1072 setOperationAction(ISD::FEXP10, MVT::f64, Expand); in ARMTargetLowering()
1073 setOperationAction(ISD::FCEIL, MVT::f64, Expand); in ARMTargetLowering()
1074 setOperationAction(ISD::FTRUNC, MVT::f64, Expand); in ARMTargetLowering()
1075 setOperationAction(ISD::FRINT, MVT::f64, Expand); in ARMTargetLowering()
1076 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); in ARMTargetLowering()
1077 setOperationAction(ISD::FFLOOR, MVT::f64, Expand); in ARMTargetLowering()
1078 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); in ARMTargetLowering()
1079 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); in ARMTargetLowering()
1080 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in ARMTargetLowering()
1081 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in ARMTargetLowering()
1082 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom); in ARMTargetLowering()
1083 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom); in ARMTargetLowering()
1084 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); in ARMTargetLowering()
1085 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); in ARMTargetLowering()
1086 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); in ARMTargetLowering()
1087 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::f64, Custom); in ARMTargetLowering()
1088 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::f64, Custom); in ARMTargetLowering()
1089 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom); in ARMTargetLowering()
1093 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom); in ARMTargetLowering()
1094 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom); in ARMTargetLowering()
1096 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); in ARMTargetLowering()
1097 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom); in ARMTargetLowering()
1102 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom); in ARMTargetLowering()
1103 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom); in ARMTargetLowering()
1110 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in ARMTargetLowering()
1111 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in ARMTargetLowering()
1121 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in ARMTargetLowering()
1125 for (unsigned im = (unsigned)ISD::PRE_INC; in ARMTargetLowering()
1126 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in ARMTargetLowering()
1138 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal); in ARMTargetLowering()
1139 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal); in ARMTargetLowering()
1142 setOperationAction(ISD::SADDO, MVT::i32, Custom); in ARMTargetLowering()
1143 setOperationAction(ISD::UADDO, MVT::i32, Custom); in ARMTargetLowering()
1144 setOperationAction(ISD::SSUBO, MVT::i32, Custom); in ARMTargetLowering()
1145 setOperationAction(ISD::USUBO, MVT::i32, Custom); in ARMTargetLowering()
1147 setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom); in ARMTargetLowering()
1148 setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom); in ARMTargetLowering()
1150 setOperationAction(ISD::SADDSAT, MVT::i8, Custom); in ARMTargetLowering()
1151 setOperationAction(ISD::SSUBSAT, MVT::i8, Custom); in ARMTargetLowering()
1152 setOperationAction(ISD::SADDSAT, MVT::i16, Custom); in ARMTargetLowering()
1153 setOperationAction(ISD::SSUBSAT, MVT::i16, Custom); in ARMTargetLowering()
1154 setOperationAction(ISD::UADDSAT, MVT::i8, Custom); in ARMTargetLowering()
1155 setOperationAction(ISD::USUBSAT, MVT::i8, Custom); in ARMTargetLowering()
1156 setOperationAction(ISD::UADDSAT, MVT::i16, Custom); in ARMTargetLowering()
1157 setOperationAction(ISD::USUBSAT, MVT::i16, Custom); in ARMTargetLowering()
1160 setOperationAction(ISD::SADDSAT, MVT::i32, Legal); in ARMTargetLowering()
1161 setOperationAction(ISD::SSUBSAT, MVT::i32, Legal); in ARMTargetLowering()
1165 setOperationAction(ISD::MUL, MVT::i64, Expand); in ARMTargetLowering()
1166 setOperationAction(ISD::MULHU, MVT::i32, Expand); in ARMTargetLowering()
1168 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); in ARMTargetLowering()
1169 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); in ARMTargetLowering()
1173 setOperationAction(ISD::MULHS, MVT::i32, Expand); in ARMTargetLowering()
1175 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); in ARMTargetLowering()
1176 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); in ARMTargetLowering()
1177 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); in ARMTargetLowering()
1178 setOperationAction(ISD::SRL, MVT::i64, Custom); in ARMTargetLowering()
1179 setOperationAction(ISD::SRA, MVT::i64, Custom); in ARMTargetLowering()
1180 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in ARMTargetLowering()
1181 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); in ARMTargetLowering()
1182 setOperationAction(ISD::LOAD, MVT::i64, Custom); in ARMTargetLowering()
1183 setOperationAction(ISD::STORE, MVT::i64, Custom); in ARMTargetLowering()
1188 setOperationAction(ISD::SHL, MVT::i64, Custom); in ARMTargetLowering()
1192 setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand); in ARMTargetLowering()
1193 setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand); in ARMTargetLowering()
1194 setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand); in ARMTargetLowering()
1198 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); in ARMTargetLowering()
1201 setOperationAction(ISD::ROTL, MVT::i32, Expand); in ARMTargetLowering()
1203 setOperationAction(ISD::ROTL, VT, Expand); in ARMTargetLowering()
1204 setOperationAction(ISD::ROTR, VT, Expand); in ARMTargetLowering()
1206 setOperationAction(ISD::CTTZ, MVT::i32, Custom); in ARMTargetLowering()
1207 setOperationAction(ISD::CTPOP, MVT::i32, Expand); in ARMTargetLowering()
1209 setOperationAction(ISD::CTLZ, MVT::i32, Expand); in ARMTargetLowering()
1210 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, LibCall); in ARMTargetLowering()
1218 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom); in ARMTargetLowering()
1222 setOperationAction(ISD::BSWAP, MVT::i32, Expand); in ARMTargetLowering()
1228 setOperationAction(ISD::SDIV, MVT::i32, LibCall); in ARMTargetLowering()
1229 setOperationAction(ISD::UDIV, MVT::i32, LibCall); in ARMTargetLowering()
1233 setOperationAction(ISD::SDIV, MVT::i32, Custom); in ARMTargetLowering()
1234 setOperationAction(ISD::UDIV, MVT::i32, Custom); in ARMTargetLowering()
1236 setOperationAction(ISD::SDIV, MVT::i64, Custom); in ARMTargetLowering()
1237 setOperationAction(ISD::UDIV, MVT::i64, Custom); in ARMTargetLowering()
1240 setOperationAction(ISD::SREM, MVT::i32, Expand); in ARMTargetLowering()
1241 setOperationAction(ISD::UREM, MVT::i32, Expand); in ARMTargetLowering()
1247 setOperationAction(ISD::SREM, MVT::i64, Custom); in ARMTargetLowering()
1248 setOperationAction(ISD::UREM, MVT::i64, Custom); in ARMTargetLowering()
1295 setOperationAction(ISD::SDIVREM, MVT::i32, Custom); in ARMTargetLowering()
1296 setOperationAction(ISD::UDIVREM, MVT::i32, Custom); in ARMTargetLowering()
1297 setOperationAction(ISD::SDIVREM, MVT::i64, Custom); in ARMTargetLowering()
1298 setOperationAction(ISD::UDIVREM, MVT::i64, Custom); in ARMTargetLowering()
1300 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); in ARMTargetLowering()
1301 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); in ARMTargetLowering()
1304 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); in ARMTargetLowering()
1305 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); in ARMTargetLowering()
1306 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); in ARMTargetLowering()
1307 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); in ARMTargetLowering()
1309 setOperationAction(ISD::TRAP, MVT::Other, Legal); in ARMTargetLowering()
1310 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in ARMTargetLowering()
1313 setOperationAction(ISD::VASTART, MVT::Other, Custom); in ARMTargetLowering()
1314 setOperationAction(ISD::VAARG, MVT::Other, Expand); in ARMTargetLowering()
1315 setOperationAction(ISD::VACOPY, MVT::Other, Expand); in ARMTargetLowering()
1316 setOperationAction(ISD::VAEND, MVT::Other, Expand); in ARMTargetLowering()
1317 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in ARMTargetLowering()
1318 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in ARMTargetLowering()
1321 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in ARMTargetLowering()
1323 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); in ARMTargetLowering()
1332 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); in ARMTargetLowering()
1334 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); in ARMTargetLowering()
1350 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, in ARMTargetLowering()
1354 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall); in ARMTargetLowering()
1355 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall); in ARMTargetLowering()
1356 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall); in ARMTargetLowering()
1357 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, LibCall); in ARMTargetLowering()
1358 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, LibCall); in ARMTargetLowering()
1359 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall); in ARMTargetLowering()
1360 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall); in ARMTargetLowering()
1361 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, LibCall); in ARMTargetLowering()
1362 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, LibCall); in ARMTargetLowering()
1363 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, LibCall); in ARMTargetLowering()
1364 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, LibCall); in ARMTargetLowering()
1365 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, LibCall); in ARMTargetLowering()
1369 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom); in ARMTargetLowering()
1370 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom); in ARMTargetLowering()
1400 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in ARMTargetLowering()
1404 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); in ARMTargetLowering()
1405 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); in ARMTargetLowering()
1407 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); in ARMTargetLowering()
1413 setOperationAction(ISD::BITCAST, MVT::i64, Custom); in ARMTargetLowering()
1414 setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom); in ARMTargetLowering()
1415 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); in ARMTargetLowering()
1416 setOperationAction(ISD::GET_FPENV, MVT::i32, Legal); in ARMTargetLowering()
1417 setOperationAction(ISD::SET_FPENV, MVT::i32, Legal); in ARMTargetLowering()
1418 setOperationAction(ISD::RESET_FPENV, MVT::Other, Legal); in ARMTargetLowering()
1419 setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal); in ARMTargetLowering()
1420 setOperationAction(ISD::SET_FPMODE, MVT::i32, Custom); in ARMTargetLowering()
1421 setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom); in ARMTargetLowering()
1425 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in ARMTargetLowering()
1426 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); in ARMTargetLowering()
1427 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); in ARMTargetLowering()
1428 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom); in ARMTargetLowering()
1432 setOperationAction(ISD::SETCC, MVT::i32, Expand); in ARMTargetLowering()
1433 setOperationAction(ISD::SETCC, MVT::f32, Expand); in ARMTargetLowering()
1434 setOperationAction(ISD::SETCC, MVT::f64, Expand); in ARMTargetLowering()
1435 setOperationAction(ISD::SELECT, MVT::i32, Custom); in ARMTargetLowering()
1436 setOperationAction(ISD::SELECT, MVT::f32, Custom); in ARMTargetLowering()
1437 setOperationAction(ISD::SELECT, MVT::f64, Custom); in ARMTargetLowering()
1438 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); in ARMTargetLowering()
1439 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); in ARMTargetLowering()
1440 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); in ARMTargetLowering()
1442 setOperationAction(ISD::SETCC, MVT::f16, Expand); in ARMTargetLowering()
1443 setOperationAction(ISD::SELECT, MVT::f16, Custom); in ARMTargetLowering()
1444 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom); in ARMTargetLowering()
1447 setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom); in ARMTargetLowering()
1449 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in ARMTargetLowering()
1450 setOperationAction(ISD::BR_CC, MVT::i32, Custom); in ARMTargetLowering()
1452 setOperationAction(ISD::BR_CC, MVT::f16, Custom); in ARMTargetLowering()
1453 setOperationAction(ISD::BR_CC, MVT::f32, Custom); in ARMTargetLowering()
1454 setOperationAction(ISD::BR_CC, MVT::f64, Custom); in ARMTargetLowering()
1455 setOperationAction(ISD::BR_JT, MVT::Other, Custom); in ARMTargetLowering()
1458 setOperationAction(ISD::FSIN, MVT::f64, Expand); in ARMTargetLowering()
1459 setOperationAction(ISD::FSIN, MVT::f32, Expand); in ARMTargetLowering()
1460 setOperationAction(ISD::FCOS, MVT::f32, Expand); in ARMTargetLowering()
1461 setOperationAction(ISD::FCOS, MVT::f64, Expand); in ARMTargetLowering()
1462 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in ARMTargetLowering()
1463 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in ARMTargetLowering()
1464 setOperationAction(ISD::FREM, MVT::f64, Expand); in ARMTargetLowering()
1465 setOperationAction(ISD::FREM, MVT::f32, Expand); in ARMTargetLowering()
1468 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in ARMTargetLowering()
1469 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in ARMTargetLowering()
1471 setOperationAction(ISD::FPOW, MVT::f64, Expand); in ARMTargetLowering()
1472 setOperationAction(ISD::FPOW, MVT::f32, Expand); in ARMTargetLowering()
1475 setOperationAction(ISD::FMA, MVT::f64, Expand); in ARMTargetLowering()
1476 setOperationAction(ISD::FMA, MVT::f32, Expand); in ARMTargetLowering()
1483 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in ARMTargetLowering()
1484 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); in ARMTargetLowering()
1489 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); in ARMTargetLowering()
1490 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); in ARMTargetLowering()
1494 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom); in ARMTargetLowering()
1495 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom); in ARMTargetLowering()
1496 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom); in ARMTargetLowering()
1497 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom); in ARMTargetLowering()
1498 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom); in ARMTargetLowering()
1499 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom); in ARMTargetLowering()
1505 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in ARMTargetLowering()
1506 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in ARMTargetLowering()
1511 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); in ARMTargetLowering()
1512 setOperationAction(ISD::FCEIL, MVT::f32, Legal); in ARMTargetLowering()
1513 setOperationAction(ISD::FROUND, MVT::f32, Legal); in ARMTargetLowering()
1514 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); in ARMTargetLowering()
1515 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); in ARMTargetLowering()
1516 setOperationAction(ISD::FRINT, MVT::f32, Legal); in ARMTargetLowering()
1517 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); in ARMTargetLowering()
1518 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); in ARMTargetLowering()
1520 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal); in ARMTargetLowering()
1521 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal); in ARMTargetLowering()
1522 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); in ARMTargetLowering()
1523 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); in ARMTargetLowering()
1527 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); in ARMTargetLowering()
1528 setOperationAction(ISD::FCEIL, MVT::f64, Legal); in ARMTargetLowering()
1529 setOperationAction(ISD::FROUND, MVT::f64, Legal); in ARMTargetLowering()
1530 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); in ARMTargetLowering()
1531 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); in ARMTargetLowering()
1532 setOperationAction(ISD::FRINT, MVT::f64, Legal); in ARMTargetLowering()
1533 setOperationAction(ISD::FMINNUM, MVT::f64, Legal); in ARMTargetLowering()
1534 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); in ARMTargetLowering()
1540 setOperationAction(ISD::FREM, MVT::f16, Promote); in ARMTargetLowering()
1541 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand); in ARMTargetLowering()
1542 setOperationAction(ISD::FSIN, MVT::f16, Promote); in ARMTargetLowering()
1543 setOperationAction(ISD::FCOS, MVT::f16, Promote); in ARMTargetLowering()
1544 setOperationAction(ISD::FTAN, MVT::f16, Promote); in ARMTargetLowering()
1545 setOperationAction(ISD::FSINCOS, MVT::f16, Promote); in ARMTargetLowering()
1546 setOperationAction(ISD::FPOWI, MVT::f16, Promote); in ARMTargetLowering()
1547 setOperationAction(ISD::FPOW, MVT::f16, Promote); in ARMTargetLowering()
1548 setOperationAction(ISD::FEXP, MVT::f16, Promote); in ARMTargetLowering()
1549 setOperationAction(ISD::FEXP2, MVT::f16, Promote); in ARMTargetLowering()
1550 setOperationAction(ISD::FEXP10, MVT::f16, Promote); in ARMTargetLowering()
1551 setOperationAction(ISD::FLOG, MVT::f16, Promote); in ARMTargetLowering()
1552 setOperationAction(ISD::FLOG10, MVT::f16, Promote); in ARMTargetLowering()
1553 setOperationAction(ISD::FLOG2, MVT::f16, Promote); in ARMTargetLowering()
1555 setOperationAction(ISD::FROUND, MVT::f16, Legal); in ARMTargetLowering()
1561 setOperationAction(ISD::FMINIMUM, MVT::f32, Legal); in ARMTargetLowering()
1562 setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal); in ARMTargetLowering()
1563 setOperationAction(ISD::FMINIMUM, MVT::f16, Legal); in ARMTargetLowering()
1564 setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal); in ARMTargetLowering()
1565 setOperationAction(ISD::FMINIMUM, MVT::v2f32, Legal); in ARMTargetLowering()
1566 setOperationAction(ISD::FMAXIMUM, MVT::v2f32, Legal); in ARMTargetLowering()
1567 setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal); in ARMTargetLowering()
1568 setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal); in ARMTargetLowering()
1571 setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal); in ARMTargetLowering()
1572 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal); in ARMTargetLowering()
1573 setOperationAction(ISD::FMINNUM, MVT::v8f16, Legal); in ARMTargetLowering()
1574 setOperationAction(ISD::FMAXNUM, MVT::v8f16, Legal); in ARMTargetLowering()
1576 setOperationAction(ISD::FMINIMUM, MVT::v4f16, Legal); in ARMTargetLowering()
1577 setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal); in ARMTargetLowering()
1578 setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal); in ARMTargetLowering()
1579 setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal); in ARMTargetLowering()
1586 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP}) in ARMTargetLowering()
1593 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP}) in ARMTargetLowering()
1600 {ISD::ADD, ISD::SUB, ISD::MUL, ISD::AND, ISD::OR, ISD::XOR}); in ARMTargetLowering()
1603 setTargetDAGCombine(ISD::VSELECT); in ARMTargetLowering()
1606 setTargetDAGCombine(ISD::SRL); in ARMTargetLowering()
1608 setTargetDAGCombine(ISD::SHL); in ARMTargetLowering()
1612 setTargetDAGCombine({ISD::SMIN, ISD::SMAX}); in ARMTargetLowering()
2006 if (Op.getOpcode() != ISD::SRL) in isSRL16()
2014 if (Op.getOpcode() != ISD::SRA) in isSRA16()
2022 if (Op.getOpcode() != ISD::SHL) in isSHL16()
2040 static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) { in IntCCToARMCC()
2043 case ISD::SETNE: return ARMCC::NE; in IntCCToARMCC()
2044 case ISD::SETEQ: return ARMCC::EQ; in IntCCToARMCC()
2045 case ISD::SETGT: return ARMCC::GT; in IntCCToARMCC()
2046 case ISD::SETGE: return ARMCC::GE; in IntCCToARMCC()
2047 case ISD::SETLT: return ARMCC::LT; in IntCCToARMCC()
2048 case ISD::SETLE: return ARMCC::LE; in IntCCToARMCC()
2049 case ISD::SETUGT: return ARMCC::HI; in IntCCToARMCC()
2050 case ISD::SETUGE: return ARMCC::HS; in IntCCToARMCC()
2051 case ISD::SETULT: return ARMCC::LO; in IntCCToARMCC()
2052 case ISD::SETULE: return ARMCC::LS; in IntCCToARMCC()
2057 static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, in FPCCToARMCC()
2062 case ISD::SETEQ: in FPCCToARMCC()
2063 case ISD::SETOEQ: CondCode = ARMCC::EQ; break; in FPCCToARMCC()
2064 case ISD::SETGT: in FPCCToARMCC()
2065 case ISD::SETOGT: CondCode = ARMCC::GT; break; in FPCCToARMCC()
2066 case ISD::SETGE: in FPCCToARMCC()
2067 case ISD::SETOGE: CondCode = ARMCC::GE; break; in FPCCToARMCC()
2068 case ISD::SETOLT: CondCode = ARMCC::MI; break; in FPCCToARMCC()
2069 case ISD::SETOLE: CondCode = ARMCC::LS; break; in FPCCToARMCC()
2070 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break; in FPCCToARMCC()
2071 case ISD::SETO: CondCode = ARMCC::VC; break; in FPCCToARMCC()
2072 case ISD::SETUO: CondCode = ARMCC::VS; break; in FPCCToARMCC()
2073 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break; in FPCCToARMCC()
2074 case ISD::SETUGT: CondCode = ARMCC::HI; break; in FPCCToARMCC()
2075 case ISD::SETUGE: CondCode = ARMCC::PL; break; in FPCCToARMCC()
2076 case ISD::SETLT: in FPCCToARMCC()
2077 case ISD::SETULT: CondCode = ARMCC::LT; break; in FPCCToARMCC()
2078 case ISD::SETLE: in FPCCToARMCC()
2079 case ISD::SETULE: CondCode = ARMCC::LE; break; in FPCCToARMCC()
2080 case ISD::SETNE: in FPCCToARMCC()
2081 case ISD::SETUNE: CondCode = ARMCC::NE; break; in FPCCToARMCC()
2174 Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()), in MoveToHPR()
2179 Val = DAG.getNode(ISD::TRUNCATE, dl, in MoveToHPR()
2181 Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val); in MoveToHPR()
2193 Val = DAG.getNode(ISD::BITCAST, dl, in MoveFromHPR()
2195 Val = DAG.getNode(ISD::ZERO_EXTEND, dl, in MoveFromHPR()
2198 return DAG.getNode(ISD::BITCAST, dl, LocVT, Val); in MoveFromHPR()
2205 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, in LowerCallResult()
2245 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64); in LowerCallResult()
2246 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val, in LowerCallResult()
2260 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val, in LowerCallResult()
2274 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val); in LowerCallResult()
2289 const ISD::InputArg &Arg = Ins[VA.getValNo()]; in LowerCallResult()
2318 DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), in computeAddrForCallArg()
2370 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; in LowerCall()
2372 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; in LowerCall()
2507 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; in LowerCall()
2515 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); in LowerCall()
2518 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); in LowerCall()
2521 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); in LowerCall()
2524 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); in LowerCall()
2548 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg); in LowerCall()
2549 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask); in LowerCall()
2550 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); in LowerCall()
2556 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, in LowerCall()
2558 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, in LowerCall()
2611 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); in LowerCall()
2633 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset); in LowerCall()
2657 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
2966 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, in MatchingStackOffset()
2971 if (Arg.getOpcode() == ISD::CopyFromReg) { in MatchingStackOffset()
3017 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; in IsEligibleForTailCallOptimization()
3019 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; in IsEligibleForTailCallOptimization()
3115 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; in IsEligibleForTailCallOptimization()
3152 const SmallVectorImpl<ISD::OutputArg> &Outs, in CanLowerReturn()
3195 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
3249 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) { in LowerReturn()
3251 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) { in LowerReturn()
3253 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) { in LowerReturn()
3266 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); in LowerReturn()
3280 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg); in LowerReturn()
3281 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask); in LowerReturn()
3282 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg); in LowerReturn()
3290 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, in LowerReturn()
3309 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg, in LowerReturn()
3377 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
3388 if (U->getOpcode() != ISD::CopyToReg) in isUsedByReturnOnly()
3410 } else if (Copy->getOpcode() == ISD::BITCAST) { in isUsedByReturnOnly()
3415 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0)) in isUsedByReturnOnly()
3464 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops); in LowerWRITE_REGISTER()
3639 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, in LowerGlobalTLSAddressWindows()
3648 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL)); in LowerGlobalTLSAddressWindows()
3660 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex, in LowerGlobalTLSAddressWindows()
3663 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot), in LowerGlobalTLSAddressWindows()
3675 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset); in LowerGlobalTLSAddressWindows()
3769 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); in LowerToTLSExecModels()
3978 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr); in LowerGlobalAddressELF()
4137 DAG.getNode(ISD::SRA, dl, VTy, Operand, DAG.getConstant(31, dl, VTy)); in LowerINTRINSIC_WO_CHAIN()
4138 SDValue XOR = DAG.getNode(ISD::XOR, dl, VTy, SRA, Operand); in LowerINTRINSIC_WO_CHAIN()
4140 DAG.getNode(ISD::SHL, dl, VTy, XOR, DAG.getConstant(1, dl, VTy)); in LowerINTRINSIC_WO_CHAIN()
4142 DAG.getNode(ISD::OR, dl, VTy, SHL, DAG.getConstant(1, dl, VTy)); in LowerINTRINSIC_WO_CHAIN()
4143 SDValue Result = DAG.getNode(ISD::CTLZ, dl, VTy, OR); in LowerINTRINSIC_WO_CHAIN()
4156 SDValue SRAHi = DAG.getNode(ISD::SRA, dl, VTy, Hi, Constant31); in LowerINTRINSIC_WO_CHAIN()
4157 SDValue XORHi = DAG.getNode(ISD::XOR, dl, VTy, SRAHi, Hi); in LowerINTRINSIC_WO_CHAIN()
4158 SDValue SHLHi = DAG.getNode(ISD::SHL, dl, VTy, XORHi, Constant1); in LowerINTRINSIC_WO_CHAIN()
4159 SDValue ORHi = DAG.getNode(ISD::OR, dl, VTy, SHLHi, Constant1); in LowerINTRINSIC_WO_CHAIN()
4160 SDValue CLSHi = DAG.getNode(ISD::CTLZ, dl, VTy, ORHi); in LowerINTRINSIC_WO_CHAIN()
4162 DAG.getSetCC(dl, MVT::i1, CLSHi, Constant31, ISD::CondCode::SETEQ); in LowerINTRINSIC_WO_CHAIN()
4164 DAG.getSetCC(dl, MVT::i1, Hi, Constant0, ISD::CondCode::SETEQ); in LowerINTRINSIC_WO_CHAIN()
4167 SDValue CLZAdjustedLo = DAG.getNode(ISD::CTLZ, dl, VTy, AdjustedLo); in LowerINTRINSIC_WO_CHAIN()
4170 DAG.getNode(ISD::ADD, dl, VTy, CLZAdjustedLo, Constant31), CLSHi); in LowerINTRINSIC_WO_CHAIN()
4197 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4201 return DAG.getNode(ISD::ABDS, SDLoc(Op), Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4205 return DAG.getNode(ISD::ABDU, SDLoc(Op), Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4217 ? ISD::FMINNUM : ISD::FMAXNUM; in LowerINTRINSIC_WO_CHAIN()
4226 ? ISD::UMIN : ISD::UMAX; in LowerINTRINSIC_WO_CHAIN()
4235 ? ISD::SMIN : ISD::SMAX; in LowerINTRINSIC_WO_CHAIN()
4240 ? ISD::FMINIMUM : ISD::FMAXIMUM; in LowerINTRINSIC_WO_CHAIN()
4297 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0), in LowerATOMIC_FENCE()
4433 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT)); in StoreByValRegs()
4437 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); in StoreByValRegs()
4468 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val); in splitValueIntoRegisterParts()
4469 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val); in splitValueIntoRegisterParts()
4470 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in splitValueIntoRegisterParts()
4485 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val); in joinRegisterPartsIntoValue()
4486 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val); in joinRegisterPartsIntoValue()
4487 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in joinRegisterPartsIntoValue()
4495 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, in LowerFormalArguments()
4527 ISD::ArgFlagsTy Flags = Ins[Index].Flags; in LowerFormalArguments()
4579 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64); in LowerFormalArguments()
4580 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue, in LowerFormalArguments()
4582 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue, in LowerFormalArguments()
4623 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
4638 const ISD::InputArg &Arg = Ins[VA.getValNo()]; in LowerFormalArguments()
4655 ISD::ArgFlagsTy Flags = Ins[index].Flags; in LowerFormalArguments()
4725 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { in isFloatingPointZero()
4733 } else if (Op->getOpcode() == ISD::BITCAST && in isFloatingPointZero()
4747 SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, in getARMCmp()
4756 case ISD::SETLT: in getARMCmp()
4757 case ISD::SETGE: in getARMCmp()
4759 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; in getARMCmp()
4763 case ISD::SETULT: in getARMCmp()
4764 case ISD::SETUGE: in getARMCmp()
4766 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; in getARMCmp()
4770 case ISD::SETLE: in getARMCmp()
4771 case ISD::SETGT: in getARMCmp()
4773 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; in getARMCmp()
4777 case ISD::SETULE: in getARMCmp()
4778 case ISD::SETUGT: in getARMCmp()
4780 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in getARMCmp()
4790 CC = ISD::getSetCCSwappedOperands(CC); in getARMCmp()
4815 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND && in getARMCmp()
4826 LHS = DAG.getNode(ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt); in getARMCmp()
4838 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL && in getARMCmp()
4840 CC == ISD::SETUGT && isa<ConstantSDNode>(LHS.getOperand(1)) && in getARMCmp()
4943 case ISD::SADDO: in getARMXALUOOp()
4945 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS); in getARMXALUOOp()
4948 case ISD::UADDO: in getARMXALUOOp()
4957 case ISD::SSUBO: in getARMXALUOOp()
4959 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS); in getARMXALUOOp()
4962 case ISD::USUBO: in getARMXALUOOp()
4964 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS); in getARMXALUOOp()
4967 case ISD::UMULO: in getARMXALUOOp()
4970 Value = DAG.getNode(ISD::UMUL_LOHI, dl, in getARMXALUOOp()
4977 case ISD::SMULO: in getARMXALUOOp()
4981 Value = DAG.getNode(ISD::SMUL_LOHI, dl, in getARMXALUOOp()
4985 DAG.getNode(ISD::SRA, dl, Op.getValueType(), in getARMXALUOOp()
5015 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); in LowerSignedALUO()
5059 case ISD::UADDO: in LowerUnsignedALUO()
5064 case ISD::USUBO: { in LowerUnsignedALUO()
5070 Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32, in LowerUnsignedALUO()
5076 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); in LowerUnsignedALUO()
5093 case ISD::UADDSAT: in LowerADDSUBSAT()
5096 case ISD::SADDSAT: in LowerADDSUBSAT()
5099 case ISD::USUBSAT: in LowerADDSUBSAT()
5102 case ISD::SSUBSAT: in LowerADDSUBSAT()
5109 case ISD::UADDSAT: in LowerADDSUBSAT()
5112 case ISD::SADDSAT: in LowerADDSUBSAT()
5115 case ISD::USUBSAT: in LowerADDSUBSAT()
5118 case ISD::SSUBSAT: in LowerADDSUBSAT()
5130 return DAG.getNode(ISD::TRUNCATE, dl, VT, Add); in LowerADDSUBSAT()
5141 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerSELECT()
5142 Opc == ISD::USUBO)) { in LowerSELECT()
5194 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond, in LowerSELECT()
5199 SelectTrue, SelectFalse, ISD::SETNE); in LowerSELECT()
5202 static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, in checkVSELConstraints()
5206 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE || in checkVSELConstraints()
5207 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE) in checkVSELConstraints()
5211 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT || in checkVSELConstraints()
5212 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT) in checkVSELConstraints()
5217 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT || in checkVSELConstraints()
5218 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT) in checkVSELConstraints()
5229 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE || in checkVSELConstraints()
5230 CC == ISD::SETUGT) { in checkVSELConstraints()
5238 if (CC == ISD::SETO) { in checkVSELConstraints()
5246 if (CC == ISD::SETUNE || CC == ISD::SETNE) { in checkVSELConstraints()
5278 static bool isGTorGE(ISD::CondCode CC) { in isGTorGE()
5279 return CC == ISD::SETGT || CC == ISD::SETGE; in isGTorGE()
5282 static bool isLTorLE(ISD::CondCode CC) { in isLTorLE()
5283 return CC == ISD::SETLT || CC == ISD::SETLE; in isLTorLE()
5294 const ISD::CondCode CC, const SDValue K) { in isLowerSaturate()
5324 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSaturatingConditional()
5327 if (Op2.getOpcode() != ISD::SELECT_CC) in LowerSaturatingConditional()
5334 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get(); in LowerSaturatingConditional()
5389 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in isLowerSaturatingConditional()
5447 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue, in LowerSELECT_CC()
5450 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV, in LowerSELECT_CC()
5452 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV); in LowerSELECT_CC()
5454 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV); in LowerSELECT_CC()
5459 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSELECT_CC()
5481 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
5491 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
5500 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
5521 CC = ISD::SETNE; in LowerSELECT_CC()
5542 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
5614 return ISD::isNormalLoad(N); in canChangeToInt()
5646 SDValue NewPtr = DAG.getNode(ISD::ADD, dl, in expandf64Toi32()
5663 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in OptimizeVFPBrcond()
5677 if (CC == ISD::SETOEQ) in OptimizeVFPBrcond()
5678 CC = ISD::SETEQ; in OptimizeVFPBrcond()
5679 else if (CC == ISD::SETUNE) in OptimizeVFPBrcond()
5680 CC = ISD::SETNE; in OptimizeVFPBrcond()
5685 LHS = DAG.getNode(ISD::AND, dl, MVT::i32, in OptimizeVFPBrcond()
5687 RHS = DAG.getNode(ISD::AND, dl, MVT::i32, in OptimizeVFPBrcond()
5699 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask); in OptimizeVFPBrcond()
5700 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask); in OptimizeVFPBrcond()
5720 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) && in LowerBRCOND()
5723 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerBRCOND()
5724 Opc == ISD::USUBO || OptimizeMul)) { in LowerBRCOND()
5750 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in LowerBR_CC()
5764 CC = ISD::SETNE; in LowerBR_CC()
5771 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) && in LowerBR_CC()
5774 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerBR_CC()
5775 Opc == ISD::USUBO || OptimizeMul) && in LowerBR_CC()
5776 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in LowerBR_CC()
5786 if ((CC == ISD::SETNE) != isOneConstant(RHS)) { in LowerBR_CC()
5808 (CC == ISD::SETEQ || CC == ISD::SETOEQ || in LowerBR_CC()
5809 CC == ISD::SETNE || CC == ISD::SETUNE)) { in LowerBR_CC()
5841 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy)); in LowerBR_JT()
5842 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index); in LowerBR_JT()
5856 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr); in LowerBR_JT()
5894 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op); in LowerVectorFP_TO_INT()
5907 if (Op.getOpcode() == ISD::FP_TO_SINT || in LowerFP_TO_INT()
5908 Op.getOpcode() == ISD::STRICT_FP_TO_SINT) in LowerFP_TO_INT()
5927 DAG.getNode(Op.getOpcode() == ISD::STRICT_FP_TO_SINT ? ISD::FP_TO_SINT in LowerFP_TO_INT()
5928 : ISD::FP_TO_UINT, in LowerFP_TO_INT()
5961 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT; in LowerFP_TO_INT_SAT()
5965 SDValue Max = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, CVT, in LowerFP_TO_INT_SAT()
5968 Max = DAG.getNode(ISD::SMAX, DL, VT, Max, in LowerFP_TO_INT_SAT()
6003 case ISD::SINT_TO_FP: in LowerVectorINT_TO_FP()
6004 CastOpc = ISD::SIGN_EXTEND; in LowerVectorINT_TO_FP()
6005 Opc = ISD::SINT_TO_FP; in LowerVectorINT_TO_FP()
6007 case ISD::UINT_TO_FP: in LowerVectorINT_TO_FP()
6008 CastOpc = ISD::ZERO_EXTEND; in LowerVectorINT_TO_FP()
6009 Opc = ISD::UINT_TO_FP; in LowerVectorINT_TO_FP()
6023 if (Op.getOpcode() == ISD::SINT_TO_FP) in LowerINT_TO_FP()
6044 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST || in LowerFCOPYSIGN()
6056 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask), in LowerFCOPYSIGN()
6059 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0); in LowerFCOPYSIGN()
6061 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1); in LowerFCOPYSIGN()
6064 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1), in LowerFCOPYSIGN()
6068 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1), in LowerFCOPYSIGN()
6070 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0); in LowerFCOPYSIGN()
6071 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1); in LowerFCOPYSIGN()
6076 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask, in LowerFCOPYSIGN()
6077 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes)); in LowerFCOPYSIGN()
6079 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT, in LowerFCOPYSIGN()
6080 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask), in LowerFCOPYSIGN()
6081 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot)); in LowerFCOPYSIGN()
6083 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res); in LowerFCOPYSIGN()
6084 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, in LowerFCOPYSIGN()
6087 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res); in LowerFCOPYSIGN()
6097 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1); in LowerFCOPYSIGN()
6102 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1); in LowerFCOPYSIGN()
6104 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32, in LowerFCOPYSIGN()
6105 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2); in LowerFCOPYSIGN()
6106 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, in LowerFCOPYSIGN()
6107 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1)); in LowerFCOPYSIGN()
6114 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2); in LowerFCOPYSIGN()
6115 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1); in LowerFCOPYSIGN()
6134 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), in LowerRETURNADDR()
6184 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL, in ExpandREAD_REGISTER()
6189 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0), in ExpandREAD_REGISTER()
6212 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in CombineVMOVDRRCandidateWithVecOp()
6239 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc); in CombineVMOVDRRCandidateWithVecOp()
6240 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast, in CombineVMOVDRRCandidateWithVecOp()
6263 DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op)); in ExpandBITCAST()
6268 ISD::TRUNCATE, SDLoc(N), DstVT, in ExpandBITCAST()
6282 return DAG.getNode(ISD::BITCAST, dl, DstVT, in ExpandBITCAST()
6298 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1)); in ExpandBITCAST()
6316 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in getZeroVector()
6332 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; in LowerShiftRightParts()
6334 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); in LowerShiftRightParts()
6336 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, in LowerShiftRightParts()
6338 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
6339 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, in LowerShiftRightParts()
6341 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
6342 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftRightParts()
6345 ISD::SETGE, ARMcc, DAG, dl); in LowerShiftRightParts()
6350 SDValue HiBigShift = Opc == ISD::SRA in LowerShiftRightParts()
6355 ISD::SETGE, ARMcc, DAG, dl); in LowerShiftRightParts()
6377 assert(Op.getOpcode() == ISD::SHL_PARTS); in LowerShiftLeftParts()
6378 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, in LowerShiftLeftParts()
6380 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
6381 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
6382 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftLeftParts()
6384 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, in LowerShiftLeftParts()
6386 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
6388 ISD::SETGE, ARMcc, DAG, dl); in LowerShiftLeftParts()
6393 ISD::SETGE, ARMcc, DAG, dl); in LowerShiftLeftParts()
6394 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
6414 DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, {MVT::i32, MVT::Other}, Ops); in LowerGET_ROUNDING()
6416 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR, in LowerGET_ROUNDING()
6418 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds, in LowerGET_ROUNDING()
6420 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE, in LowerGET_ROUNDING()
6442 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
6444 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
6446 RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
6453 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops); in LowerSET_ROUNDING()
6459 FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR, in LowerSET_ROUNDING()
6461 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue); in LowerSET_ROUNDING()
6464 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); in LowerSET_ROUNDING()
6478 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops); in LowerSET_FPMODE()
6483 DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR, in LowerSET_FPMODE()
6486 DAG.getNode(ISD::AND, DL, MVT::i32, Mode, in LowerSET_FPMODE()
6488 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCRMasked, InputMasked); in LowerSET_FPMODE()
6492 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); in LowerSET_FPMODE()
6505 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops); in LowerRESET_FPMODE()
6510 ISD::AND, DL, MVT::i32, FPSCR, in LowerRESET_FPMODE()
6515 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); in LowerRESET_FPMODE()
6526 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X); in LowerCTTZ()
6527 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX); in LowerCTTZ()
6535 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); in LowerCTTZ()
6536 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); in LowerCTTZ()
6540 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) { in LowerCTTZ()
6546 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB); in LowerCTTZ()
6547 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ); in LowerCTTZ()
6558 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF); in LowerCTTZ()
6562 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); in LowerCTTZ()
6564 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); in LowerCTTZ()
6570 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); in LowerCTTZ()
6571 return DAG.getNode(ISD::CTLZ, dl, VT, rbit); in LowerCTTZ()
6587 Res = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Res); in LowerCTPOP()
6601 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, Ops); in LowerCTPOP()
6612 while (Op.getOpcode() == ISD::BITCAST) in getVShiftImm()
6675 if (N->getOpcode() == ISD::SHL) { in LowerShift()
6683 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && in LowerShift()
6688 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM); in LowerShift()
6697 ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1)); in LowerShift()
6699 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu); in LowerShift()
6712 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA || in Expand64BitShift()
6713 N->getOpcode() == ISD::SHL) && in Expand64BitShift()
6732 if (ShOpc == ISD::SRL) { in Expand64BitShift()
6736 ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, in Expand64BitShift()
6741 } else if (ShOpc == ISD::SRA) in Expand64BitShift()
6753 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); in Expand64BitShift()
6757 if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL) in Expand64BitShift()
6770 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_GLUE:ARMISD::SRA_GLUE; in Expand64BitShift()
6777 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); in Expand64BitShift()
6790 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerVSETCC()
6812 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) { in LowerVSETCC()
6817 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0); in LowerVSETCC()
6818 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1); in LowerVSETCC()
6819 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1, in LowerVSETCC()
6820 DAG.getCondCode(ISD::SETEQ)); in LowerVSETCC()
6822 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed); in LowerVSETCC()
6823 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged); in LowerVSETCC()
6824 if (SetCCOpcode == ISD::SETNE) in LowerVSETCC()
6837 case ISD::SETUNE: in LowerVSETCC()
6838 case ISD::SETNE: in LowerVSETCC()
6844 case ISD::SETOEQ: in LowerVSETCC()
6845 case ISD::SETEQ: Opc = ARMCC::EQ; break; in LowerVSETCC()
6846 case ISD::SETOLT: in LowerVSETCC()
6847 case ISD::SETLT: Swap = true; [[fallthrough]]; in LowerVSETCC()
6848 case ISD::SETOGT: in LowerVSETCC()
6849 case ISD::SETGT: Opc = ARMCC::GT; break; in LowerVSETCC()
6850 case ISD::SETOLE: in LowerVSETCC()
6851 case ISD::SETLE: Swap = true; [[fallthrough]]; in LowerVSETCC()
6852 case ISD::SETOGE: in LowerVSETCC()
6853 case ISD::SETGE: Opc = ARMCC::GE; break; in LowerVSETCC()
6854 case ISD::SETUGE: Swap = true; [[fallthrough]]; in LowerVSETCC()
6855 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break; in LowerVSETCC()
6856 case ISD::SETUGT: Swap = true; [[fallthrough]]; in LowerVSETCC()
6857 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break; in LowerVSETCC()
6858 case ISD::SETUEQ: Invert = true; [[fallthrough]]; in LowerVSETCC()
6859 case ISD::SETONE: { in LowerVSETCC()
6865 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1); in LowerVSETCC()
6870 case ISD::SETUO: Invert = true; [[fallthrough]]; in LowerVSETCC()
6871 case ISD::SETO: { in LowerVSETCC()
6877 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1); in LowerVSETCC()
6887 case ISD::SETNE: in LowerVSETCC()
6893 case ISD::SETEQ: Opc = ARMCC::EQ; break; in LowerVSETCC()
6894 case ISD::SETLT: Swap = true; [[fallthrough]]; in LowerVSETCC()
6895 case ISD::SETGT: Opc = ARMCC::GT; break; in LowerVSETCC()
6896 case ISD::SETLE: Swap = true; [[fallthrough]]; in LowerVSETCC()
6897 case ISD::SETGE: Opc = ARMCC::GE; break; in LowerVSETCC()
6898 case ISD::SETULT: Swap = true; [[fallthrough]]; in LowerVSETCC()
6899 case ISD::SETUGT: Opc = ARMCC::HI; break; in LowerVSETCC()
6900 case ISD::SETULE: Swap = true; [[fallthrough]]; in LowerVSETCC()
6901 case ISD::SETUGE: Opc = ARMCC::HS; break; in LowerVSETCC()
6907 if (ISD::isBuildVectorAllZeros(Op1.getNode())) in LowerVSETCC()
6909 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) in LowerVSETCC()
6913 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST) in LowerVSETCC()
6916 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) { in LowerVSETCC()
6917 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0)); in LowerVSETCC()
6918 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1)); in LowerVSETCC()
6932 if (ISD::isBuildVectorAllZeros(Op0.getNode()) && in LowerVSETCC()
6943 if (ISD::isBuildVectorAllZeros(Op1.getNode()) && in LowerVSETCC()
6971 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerSETCCCARRY()
7200 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant, in LowerConstantFP()
7226 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant); in LowerConstantFP()
7229 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, in LowerConstantFP()
7231 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant, in LowerConstantFP()
7243 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant); in LowerConstantFP()
7246 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, in LowerConstantFP()
7248 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant, in LowerConstantFP()
7669 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in LowerBuildVectorOfFPTrunc()
7682 if (BV.getOperand(0).getOpcode() != ISD::FP_ROUND || in LowerBuildVectorOfFPTrunc()
7683 BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || in LowerBuildVectorOfFPTrunc()
7686 if (BV.getOperand(1).getOpcode() != ISD::FP_ROUND || in LowerBuildVectorOfFPTrunc()
7687 BV.getOperand(1).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || in LowerBuildVectorOfFPTrunc()
7698 return Trunc.getOpcode() == ISD::FP_ROUND && in LowerBuildVectorOfFPTrunc()
7699 Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && in LowerBuildVectorOfFPTrunc()
7722 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in LowerBuildVectorOfFPExt()
7735 if (BV.getOperand(0).getOpcode() != ISD::FP_EXTEND || in LowerBuildVectorOfFPExt()
7736 BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT) in LowerBuildVectorOfFPExt()
7746 return Trunc.getOpcode() == ISD::FP_EXTEND && in LowerBuildVectorOfFPExt()
7747 Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && in LowerBuildVectorOfFPExt()
7811 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp, in LowerBUILD_VECTOR_i1()
7834 Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V, in LowerBUILD_VECTOR_i1()
7853 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 || in LowerBUILD_VECTORToVIDUP()
7863 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 || in LowerBUILD_VECTORToVIDUP()
7878 case ISD::ADD: in IsQRMVEInstruction()
7879 case ISD::MUL: in IsQRMVEInstruction()
7880 case ISD::SADDSAT: in IsQRMVEInstruction()
7881 case ISD::UADDSAT: in IsQRMVEInstruction()
7883 case ISD::SUB: in IsQRMVEInstruction()
7884 case ISD::SSUBSAT: in IsQRMVEInstruction()
7885 case ISD::USUBSAT: in IsQRMVEInstruction()
7887 case ISD::INTRINSIC_WO_CHAIN: in IsQRMVEInstruction()
7959 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in LowerBUILD_VECTOR()
7969 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in LowerBUILD_VECTOR()
8040 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode())) in LowerBUILD_VECTOR()
8041 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
8057 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in LowerBUILD_VECTOR()
8067 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT), in LowerBUILD_VECTOR()
8086 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops); in LowerBUILD_VECTOR()
8097 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, IVT, in LowerBUILD_VECTOR()
8103 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
8139 if (Lower.getOpcode() == ISD::BUILD_VECTOR) in LowerBUILD_VECTOR()
8143 if (Upper.getOpcode() == ISD::BUILD_VECTOR) in LowerBUILD_VECTOR()
8146 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper); in LowerBUILD_VECTOR()
8159 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i))); in LowerBUILD_VECTOR()
8161 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
8177 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
8189 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in ReconstructShuffle()
8221 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) { in ReconstructShuffle()
8284 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
8300 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
8306 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
8311 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
8314 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
8617 DAG.getNode(ISD::VSELECT, dl, MVT::v16i8, RecastV1, AllOnes, AllZeroes); in PromoteMVEPredVector()
8621 return DAG.getNode(ISD::BITCAST, dl, NewVT, PredAsVector); in PromoteMVEPredVector()
8638 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast); in LowerVECTOR_SHUFFLE_i1()
8639 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit, in LowerVECTOR_SHUFFLE_i1()
8730 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast, in LowerVECTOR_SHUFFLEUsingMovs()
8753 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, in LowerVECTOR_SHUFFLEUsingMovs()
8806 ISD::EXTRACT_VECTOR_ELT, dl, SVT, in LowerVECTOR_SHUFFLEUsingOneOff()
8809 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt, in LowerVECTOR_SHUFFLEUsingOneOff()
8840 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) { in LowerVECTOR_SHUFFLE()
8846 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR && in LowerVECTOR_SHUFFLE()
8925 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) { in LowerVECTOR_SHUFFLE()
8943 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0), in LowerVECTOR_SHUFFLE()
8963 Lo = DAG.getNode(ISD::SRL, dl, FromVT, Lo, Amt); in LowerVECTOR_SHUFFLE()
8964 Hi = DAG.getNode(ISD::SRL, dl, FromVT, Hi, Amt); in LowerVECTOR_SHUFFLE()
9010 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1); in LowerVECTOR_SHUFFLE()
9011 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2); in LowerVECTOR_SHUFFLE()
9017 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, in LowerVECTOR_SHUFFLE()
9023 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerVECTOR_SHUFFLE()
9055 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, in LowerINSERT_VECTOR_ELT_i1()
9094 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt); in LowerINSERT_VECTOR_ELT()
9095 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn); in LowerINSERT_VECTOR_ELT()
9096 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT, in LowerINSERT_VECTOR_ELT()
9098 return DAG.getNode(ISD::BITCAST, dl, VecVT, IVecOut); in LowerINSERT_VECTOR_ELT()
9117 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv, in LowerEXTRACT_VECTOR_ELT_i1()
9194 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV, in LowerCONCAT_VECTORS_i1()
9196 ConVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ConcatVT, ConVec, Elt, in LowerCONCAT_VECTORS_i1()
9202 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT); in LowerCONCAT_VECTORS_i1()
9240 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val, in LowerCONCAT_VECTORS()
9241 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0), in LowerCONCAT_VECTORS()
9244 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val, in LowerCONCAT_VECTORS()
9245 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1), in LowerCONCAT_VECTORS()
9247 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val); in LowerCONCAT_VECTORS()
9274 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT); in LowerEXTRACT_SUBVECTOR()
9276 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1, in LowerEXTRACT_SUBVECTOR()
9278 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, in LowerEXTRACT_SUBVECTOR()
9280 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, in LowerEXTRACT_SUBVECTOR()
9289 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT); in LowerEXTRACT_SUBVECTOR()
9291 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1, in LowerEXTRACT_SUBVECTOR()
9293 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, in LowerEXTRACT_SUBVECTOR()
9315 DAG.getNode(ISD::AND, DL, FromVT, Op, DAG.getConstant(1, DL, FromVT)); in LowerTruncatei1()
9316 return DAG.getNode(ISD::SETCC, DL, VT, And, DAG.getConstant(0, DL, FromVT), in LowerTruncatei1()
9317 DAG.getCondCode(ISD::SETNE)); in LowerTruncatei1()
9401 N->getOpcode() == ISD::SIGN_EXTEND ? ARMISD::MVESEXT : ARMISD::MVEZEXT; in LowerVectorExtend()
9410 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Ext, Ext1); in LowerVectorExtend()
9420 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { in isExtendedBUILD_VECTOR()
9423 BVN->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
9444 if (N->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
9470 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N)) in isSignExtended()
9480 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND || in isZeroExtended()
9481 ISD::isZEXTLoad(N)) in isZeroExtended()
9555 if (N->getOpcode() == ISD::SIGN_EXTEND || in SkipExtensionForVMULL()
9556 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND) in SkipExtensionForVMULL()
9563 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) && in SkipExtensionForVMULL()
9568 unsigned Opcode = ISD::isSEXTLoad(LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in SkipExtensionForVMULL()
9578 if (N->getOpcode() == ISD::BITCAST) { in SkipExtensionForVMULL()
9580 assert(BVN->getOpcode() == ISD::BUILD_VECTOR && in SkipExtensionForVMULL()
9588 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); in SkipExtensionForVMULL()
9606 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubSExt()
9617 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubZExt()
9696 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), in LowerMUL()
9698 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); in LowerMUL()
9708 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X); in LowerSDIV_v4i8()
9709 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y); in LowerSDIV_v4i8()
9710 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X); in LowerSDIV_v4i8()
9711 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y); in LowerSDIV_v4i8()
9714 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, in LowerSDIV_v4i8()
9721 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y); in LowerSDIV_v4i8()
9722 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X); in LowerSDIV_v4i8()
9724 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y); in LowerSDIV_v4i8()
9725 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X); in LowerSDIV_v4i8()
9727 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X); in LowerSDIV_v4i8()
9728 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X); in LowerSDIV_v4i8()
9740 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0); in LowerSDIV_v4i16()
9741 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1); in LowerSDIV_v4i16()
9742 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0); in LowerSDIV_v4i16()
9743 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1); in LowerSDIV_v4i16()
9748 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, in LowerSDIV_v4i16()
9751 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, in LowerSDIV_v4i16()
9754 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); in LowerSDIV_v4i16()
9759 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2); in LowerSDIV_v4i16()
9760 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0); in LowerSDIV_v4i16()
9762 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1); in LowerSDIV_v4i16()
9763 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0); in LowerSDIV_v4i16()
9766 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0); in LowerSDIV_v4i16()
9767 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0); in LowerSDIV_v4i16()
9783 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0); in LowerSDIV()
9784 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1); in LowerSDIV()
9786 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, in LowerSDIV()
9788 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, in LowerSDIV()
9790 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, in LowerSDIV()
9792 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, in LowerSDIV()
9798 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2); in LowerSDIV()
9801 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0); in LowerSDIV()
9820 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0); in LowerUDIV()
9821 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1); in LowerUDIV()
9823 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, in LowerUDIV()
9825 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, in LowerUDIV()
9827 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0, in LowerUDIV()
9829 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1, in LowerUDIV()
9835 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2); in LowerUDIV()
9838 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8, in LowerUDIV()
9848 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0); in LowerUDIV()
9849 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1); in LowerUDIV()
9850 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0); in LowerUDIV()
9851 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1); in LowerUDIV()
9857 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, in LowerUDIV()
9860 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, in LowerUDIV()
9863 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); in LowerUDIV()
9864 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32, in LowerUDIV()
9867 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2); in LowerUDIV()
9872 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2); in LowerUDIV()
9873 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0); in LowerUDIV()
9875 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1); in LowerUDIV()
9876 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0); in LowerUDIV()
9879 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0); in LowerUDIV()
9880 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0); in LowerUDIV()
9894 if (Op.getOpcode() == ISD::UADDO_CARRY) { in LowerUADDSUBO_CARRY()
9907 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerUADDSUBO_CARRY()
9920 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerUADDSUBO_CARRY()
9925 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry); in LowerUADDSUBO_CARRY()
9993 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet, in LowerFSINCOS()
9999 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, in LowerFSINCOS()
10049 if (N->getOpcode() != ISD::SDIV) in BuildSDIVPow2()
10103 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi)); in WinDBZCheckDenominator()
10120 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result); in ExpandDIV_Windows()
10121 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result, in ExpandDIV_Windows()
10123 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper); in ExpandDIV_Windows()
10125 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lower, Upper)); in ExpandDIV_Windows()
10135 assert(LD->getExtensionType() == ISD::NON_EXTLOAD && in LowerPredicateLoad()
10151 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(), in LowerPredicateLoad()
10156 Val = DAG.getNode(ISD::SRL, dl, MVT::i32, in LowerPredicateLoad()
10157 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Load), in LowerPredicateLoad()
10161 Pred = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MemVT, Pred, in LowerPredicateLoad()
10181 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); in LowerLOAD()
10206 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Build, in LowerPredicateStore()
10211 Build = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i1, Ops); in LowerPredicateStore()
10215 GRP = DAG.getNode(ISD::SRL, dl, MVT::i32, in LowerPredicateStore()
10216 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, GRP), in LowerPredicateStore()
10237 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(), in LowerSTORE()
10241 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(), in LowerSTORE()
10258 return (ISD::isBuildVectorAllZeros(N.getNode()) || in isZeroVector()
10282 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST || in LowerMLOAD()
10286 Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru); in LowerMLOAD()
10299 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break; in LowerVecReduce()
10300 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break; in LowerVecReduce()
10301 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break; in LowerVecReduce()
10302 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break; in LowerVecReduce()
10303 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break; in LowerVecReduce()
10304 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break; in LowerVecReduce()
10305 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break; in LowerVecReduce()
10306 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break; in LowerVecReduce()
10331 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduce()
10333 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduce()
10335 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduce()
10337 SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduce()
10343 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduce()
10345 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduce()
10352 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res); in LowerVecReduce()
10377 case ISD::VECREDUCE_UMIN: in LowerVecReduceMinMax()
10380 case ISD::VECREDUCE_UMAX: in LowerVecReduceMinMax()
10383 case ISD::VECREDUCE_SMIN: in LowerVecReduceMinMax()
10386 case ISD::VECREDUCE_SMAX: in LowerVecReduceMinMax()
10404 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Lo, Hi}); in LowerVecReduceMinMax()
10410 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Op0, Op0}); in LowerVecReduceMinMax()
10414 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in LowerVecReduceMinMax()
10423 case ISD::VECREDUCE_UMIN: in LowerVecReduceMinMax()
10424 case ISD::VECREDUCE_UMAX: in LowerVecReduceMinMax()
10425 Extend = ISD::ZERO_EXTEND; in LowerVecReduceMinMax()
10427 case ISD::VECREDUCE_SMIN: in LowerVecReduceMinMax()
10428 case ISD::VECREDUCE_SMAX: in LowerVecReduceMinMax()
10429 Extend = ISD::SIGN_EXTEND; in LowerVecReduceMinMax()
10463 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, in ReplaceREADCYCLECOUNTER()
10465 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32, in ReplaceREADCYCLECOUNTER()
10509 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i64, Lo, Hi)); in ReplaceCMP_SWAP_64Results()
10519 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get(); in LowerFSETCC()
10520 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS; in LowerFSETCC()
10529 CC = ISD::SETNE; in LowerFSETCC()
10531 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS, in LowerFSETCC()
10571 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG); in LowerOperation()
10572 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); in LowerOperation()
10573 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); in LowerOperation()
10574 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); in LowerOperation()
10575 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); in LowerOperation()
10576 case ISD::SELECT: return LowerSELECT(Op, DAG); in LowerOperation()
10577 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); in LowerOperation()
10578 case ISD::BRCOND: return LowerBRCOND(Op, DAG); in LowerOperation()
10579 case ISD::BR_CC: return LowerBR_CC(Op, DAG); in LowerOperation()
10580 case ISD::BR_JT: return LowerBR_JT(Op, DAG); in LowerOperation()
10581 case ISD::VASTART: return LowerVASTART(Op, DAG); in LowerOperation()
10582 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget); in LowerOperation()
10583 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget); in LowerOperation()
10584 case ISD::SINT_TO_FP: in LowerOperation()
10585 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG); in LowerOperation()
10586 case ISD::STRICT_FP_TO_SINT: in LowerOperation()
10587 case ISD::STRICT_FP_TO_UINT: in LowerOperation()
10588 case ISD::FP_TO_SINT: in LowerOperation()
10589 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); in LowerOperation()
10590 case ISD::FP_TO_SINT_SAT: in LowerOperation()
10591 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget); in LowerOperation()
10592 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); in LowerOperation()
10593 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); in LowerOperation()
10594 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); in LowerOperation()
10595 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG); in LowerOperation()
10596 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG); in LowerOperation()
10597 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG); in LowerOperation()
10598 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget); in LowerOperation()
10599 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG, in LowerOperation()
10601 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget); in LowerOperation()
10602 case ISD::SHL: in LowerOperation()
10603 case ISD::SRL: in LowerOperation()
10604 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget); in LowerOperation()
10605 case ISD::SREM: return LowerREM(Op.getNode(), DAG); in LowerOperation()
10606 case ISD::UREM: return LowerREM(Op.getNode(), DAG); in LowerOperation()
10607 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG); in LowerOperation()
10608 case ISD::SRL_PARTS: in LowerOperation()
10609 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG); in LowerOperation()
10610 case ISD::CTTZ: in LowerOperation()
10611 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget); in LowerOperation()
10612 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget); in LowerOperation()
10613 case ISD::SETCC: return LowerVSETCC(Op, DAG, Subtarget); in LowerOperation()
10614 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG); in LowerOperation()
10615 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget); in LowerOperation()
10616 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget); in LowerOperation()
10617 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, Subtarget); in LowerOperation()
10618 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, Subtarget); in LowerOperation()
10619 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); in LowerOperation()
10620 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, Subtarget); in LowerOperation()
10621 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, Subtarget); in LowerOperation()
10622 case ISD::TRUNCATE: return LowerTruncate(Op.getNode(), DAG, Subtarget); in LowerOperation()
10623 case ISD::SIGN_EXTEND: in LowerOperation()
10624 case ISD::ZERO_EXTEND: return LowerVectorExtend(Op.getNode(), DAG, Subtarget); in LowerOperation()
10625 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG); in LowerOperation()
10626 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG); in LowerOperation()
10627 case ISD::SET_FPMODE: in LowerOperation()
10629 case ISD::RESET_FPMODE: in LowerOperation()
10631 case ISD::MUL: return LowerMUL(Op, DAG); in LowerOperation()
10632 case ISD::SDIV: in LowerOperation()
10636 case ISD::UDIV: in LowerOperation()
10640 case ISD::UADDO_CARRY: in LowerOperation()
10641 case ISD::USUBO_CARRY: in LowerOperation()
10643 case ISD::SADDO: in LowerOperation()
10644 case ISD::SSUBO: in LowerOperation()
10646 case ISD::UADDO: in LowerOperation()
10647 case ISD::USUBO: in LowerOperation()
10649 case ISD::SADDSAT: in LowerOperation()
10650 case ISD::SSUBSAT: in LowerOperation()
10651 case ISD::UADDSAT: in LowerOperation()
10652 case ISD::USUBSAT: in LowerOperation()
10654 case ISD::LOAD: in LowerOperation()
10656 case ISD::STORE: in LowerOperation()
10658 case ISD::MLOAD: in LowerOperation()
10660 case ISD::VECREDUCE_MUL: in LowerOperation()
10661 case ISD::VECREDUCE_AND: in LowerOperation()
10662 case ISD::VECREDUCE_OR: in LowerOperation()
10663 case ISD::VECREDUCE_XOR: in LowerOperation()
10665 case ISD::VECREDUCE_FADD: in LowerOperation()
10666 case ISD::VECREDUCE_FMUL: in LowerOperation()
10667 case ISD::VECREDUCE_FMIN: in LowerOperation()
10668 case ISD::VECREDUCE_FMAX: in LowerOperation()
10670 case ISD::VECREDUCE_UMIN: in LowerOperation()
10671 case ISD::VECREDUCE_UMAX: in LowerOperation()
10672 case ISD::VECREDUCE_SMIN: in LowerOperation()
10673 case ISD::VECREDUCE_SMAX: in LowerOperation()
10675 case ISD::ATOMIC_LOAD: in LowerOperation()
10676 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG); in LowerOperation()
10677 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG); in LowerOperation()
10678 case ISD::SDIVREM: in LowerOperation()
10679 case ISD::UDIVREM: return LowerDivRem(Op, DAG); in LowerOperation()
10680 case ISD::DYNAMIC_STACKALLOC: in LowerOperation()
10684 case ISD::STRICT_FP_ROUND: in LowerOperation()
10685 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG); in LowerOperation()
10686 case ISD::STRICT_FP_EXTEND: in LowerOperation()
10687 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); in LowerOperation()
10688 case ISD::STRICT_FSETCC: in LowerOperation()
10689 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG); in LowerOperation()
10690 case ISD::SPONENTRY: in LowerOperation()
10719 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, in ReplaceLongIntrinsic()
10732 case ISD::READ_REGISTER: in ReplaceNodeResults()
10735 case ISD::BITCAST: in ReplaceNodeResults()
10738 case ISD::SRL: in ReplaceNodeResults()
10739 case ISD::SRA: in ReplaceNodeResults()
10740 case ISD::SHL: in ReplaceNodeResults()
10743 case ISD::SREM: in ReplaceNodeResults()
10744 case ISD::UREM: in ReplaceNodeResults()
10747 case ISD::SDIVREM: in ReplaceNodeResults()
10748 case ISD::UDIVREM: in ReplaceNodeResults()
10754 case ISD::SADDSAT: in ReplaceNodeResults()
10755 case ISD::SSUBSAT: in ReplaceNodeResults()
10756 case ISD::UADDSAT: in ReplaceNodeResults()
10757 case ISD::USUBSAT: in ReplaceNodeResults()
10760 case ISD::READCYCLECOUNTER: in ReplaceNodeResults()
10763 case ISD::UDIV: in ReplaceNodeResults()
10764 case ISD::SDIV: in ReplaceNodeResults()
10766 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV, in ReplaceNodeResults()
10768 case ISD::ATOMIC_CMP_SWAP: in ReplaceNodeResults()
10771 case ISD::INTRINSIC_WO_CHAIN: in ReplaceNodeResults()
10773 case ISD::LOAD: in ReplaceNodeResults()
10776 case ISD::TRUNCATE: in ReplaceNodeResults()
10779 case ISD::SIGN_EXTEND: in ReplaceNodeResults()
10780 case ISD::ZERO_EXTEND: in ReplaceNodeResults()
10783 case ISD::FP_TO_SINT_SAT: in ReplaceNodeResults()
10784 case ISD::FP_TO_UINT_SAT: in ReplaceNodeResults()
12524 case ISD::SELECT: { in isConditionalZeroOrAllOnes()
12540 case ISD::ZERO_EXTEND: in isConditionalZeroOrAllOnes()
12545 case ISD::SIGN_EXTEND: { in isConditionalZeroOrAllOnes()
12549 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC) in isConditionalZeroOrAllOnes()
12556 else if (N->getOpcode() == ISD::ZERO_EXTEND) in isConditionalZeroOrAllOnes()
12611 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, in combineSelectAndUse()
12667 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); in AddCombineToVPADD()
12674 if (!(N0.getOpcode() == ISD::SIGN_EXTEND && in AddCombineVUZPToVPADDL()
12675 N1.getOpcode() == ISD::SIGN_EXTEND) && in AddCombineVUZPToVPADDL()
12676 !(N0.getOpcode() == ISD::ZERO_EXTEND && in AddCombineVUZPToVPADDL()
12677 N1.getOpcode() == ISD::ZERO_EXTEND)) in AddCombineVUZPToVPADDL()
12703 if (N0.getOpcode() == ISD::SIGN_EXTEND) in AddCombineVUZPToVPADDL()
12712 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT, in AddCombineVUZPToVPADDL()
12716 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); in AddCombineVUZPToVPADDL()
12729 || N0.getOpcode() != ISD::BUILD_VECTOR in AddCombineBUILD_VECTORToVPADDL()
12730 || N1.getOpcode() != ISD::BUILD_VECTOR) in AddCombineBUILD_VECTORToVPADDL()
12745 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in AddCombineBUILD_VECTORToVPADDL()
12755 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT in AddCombineBUILD_VECTORToVPADDL()
12756 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in AddCombineBUILD_VECTORToVPADDL()
12815 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops); in AddCombineBUILD_VECTORToVPADDL()
12816 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE; in AddCombineBUILD_VECTORToVPADDL()
12821 if (V->getOpcode() == ISD::UMUL_LOHI || in findMUL_LOHI()
12822 V->getOpcode() == ISD::SMUL_LOHI) in findMUL_LOHI()
12839 if (Mul.getOpcode() != ISD::MUL) { in AddCombineTo64BitSMLAL16()
12842 if (Mul.getOpcode() != ISD::MUL) in AddCombineTo64BitSMLAL16()
12848 if (SRA.getOpcode() != ISD::SRA) { in AddCombineTo64BitSMLAL16()
12851 if (SRA.getOpcode() != ISD::SRA) in AddCombineTo64BitSMLAL16()
12957 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI && in AddCombineTo64bitMLAL()
12958 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI && in AddCombineTo64bitMLAL()
12959 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI && in AddCombineTo64bitMLAL()
12960 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI) in AddCombineTo64bitMLAL()
12983 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL; in AddCombineTo64bitMLAL()
13034 LowAddSub->getNode()->getOpcode() == ISD::Constant && in AddCombineTo64bitMLAL()
13202 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) { in PerformAddeSubeCombine()
13218 ISD::CondCode CC; in PerformSELECTCombine()
13222 if (N->getOpcode() == ISD::SELECT && in PerformSELECTCombine()
13223 N->getOperand(0)->getOpcode() == ISD::SETCC) { in PerformSELECTCombine()
13230 } else if (N->getOpcode() == ISD::SELECT_CC) { in PerformSELECTCombine()
13241 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN || in PerformSELECTCombine()
13242 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) && in PerformSELECTCombine()
13243 (CC == ISD::SETULT || CC == ISD::SETUGT)) { in PerformSELECTCombine()
13245 if (CC == ISD::SETUGT) in PerformSELECTCombine()
13247 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN || in PerformSELECTCombine()
13248 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) && in PerformSELECTCombine()
13249 (CC == ISD::SETLT || CC == ISD::SETGT)) { in PerformSELECTCombine()
13251 if (CC == ISD::SETGT) in PerformSELECTCombine()
13253 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX || in PerformSELECTCombine()
13254 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) && in PerformSELECTCombine()
13255 (CC == ISD::SETUGT || CC == ISD::SETULT)) { in PerformSELECTCombine()
13257 if (CC == ISD::SETULT) in PerformSELECTCombine()
13259 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX || in PerformSELECTCombine()
13260 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) && in PerformSELECTCombine()
13261 (CC == ISD::SETGT || CC == ISD::SETLT)) { in PerformSELECTCombine()
13263 if (CC == ISD::SETLT) in PerformSELECTCombine()
13270 case ISD::VECREDUCE_UMIN: in PerformSELECTCombine()
13271 case ISD::VECREDUCE_SMIN: in PerformSELECTCombine()
13272 case ISD::VECREDUCE_UMAX: in PerformSELECTCombine()
13273 case ISD::VECREDUCE_SMAX: in PerformSELECTCombine()
13300 LHS = DCI.DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); in PerformSELECTCombine()
13308 Reduction = DCI.DAG.getNode(ISD::TRUNCATE, dl, VectorScalarType, Reduction); in PerformSELECTCombine()
13327 if (N->getOpcode() == ISD::SMIN) { in PerformVQDMULHCombine()
13330 } else if (N->getOpcode() == ISD::VSELECT) { in PerformVQDMULHCombine()
13333 if (Cmp.getOpcode() != ISD::SETCC || in PerformVQDMULHCombine()
13334 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() != ISD::SETLT || in PerformVQDMULHCombine()
13365 if (Shft.getOpcode() != ISD::SRA) in PerformVQDMULHCombine()
13372 if (Mul.getOpcode() != ISD::MUL) in PerformVQDMULHCombine()
13377 if (Ext0.getOpcode() != ISD::SIGN_EXTEND || in PerformVQDMULHCombine()
13378 Ext1.getOpcode() != ISD::SIGN_EXTEND) in PerformVQDMULHCombine()
13398 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext0.getOperand(0)); in PerformVQDMULHCombine()
13400 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext1.getOperand(0)); in PerformVQDMULHCombine()
13405 Trunc = DAG.getNode(ISD::TRUNCATE, DL, VecVT, Trunc); in PerformVQDMULHCombine()
13406 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Trunc); in PerformVQDMULHCombine()
13415 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext0.getOperand(0), in PerformVQDMULHCombine()
13418 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext1.getOperand(0), in PerformVQDMULHCombine()
13423 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, in PerformVQDMULHCombine()
13424 DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Parts)); in PerformVQDMULHCombine()
13445 if (N->getOperand(0).getOpcode() != ISD::XOR) in PerformVSELECTCombine()
13463 return DCI.DAG.getNode(ISD::VSELECT, SDLoc(N), Type, Cond, RHS, LHS); in PerformVSELECTCombine()
13472 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); in PerformVSetCCToVCTPCombine()
13479 if (CC == ISD::SETUGE) { in PerformVSetCCToVCTPCombine()
13481 CC = ISD::SETULT; in PerformVSetCCToVCTPCombine()
13484 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 || in PerformVSetCCToVCTPCombine()
13485 Op0.getOpcode() != ISD::BUILD_VECTOR) in PerformVSetCCToVCTPCombine()
13520 return DCI.DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformVSetCCToVCTPCombine()
13574 case ISD::VECREDUCE_ADD: in TryDistrubutionADDVecReduce()
13588 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) && in TryDistrubutionADDVecReduce()
13591 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0, N1.getOperand(0)); in TryDistrubutionADDVecReduce()
13592 return DAG.getNode(ISD::ADD, dl, VT, Add0, N1.getOperand(1)); in TryDistrubutionADDVecReduce()
13596 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD && in TryDistrubutionADDVecReduce()
13597 N1.getOpcode() == ISD::ADD && N0->hasOneUse() && N1->hasOneUse()) { in TryDistrubutionADDVecReduce()
13611 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp), in TryDistrubutionADDVecReduce()
13614 DAG.getNode(ISD::ADD, dl, VT, Add0, N0.getOperand(N0RedOp)); in TryDistrubutionADDVecReduce()
13615 return DAG.getNode(ISD::ADD, dl, VT, Add1, N1.getOperand(N1RedOp)); in TryDistrubutionADDVecReduce()
13635 if (N0.getOpcode() == ISD::MUL) in TryDistrubutionADDVecReduce()
13637 if (N1.getOpcode() == ISD::MUL) in TryDistrubutionADDVecReduce()
13664 if (N0.getOpcode() == ISD::ADD && N0->hasOneUse()) { in TryDistrubutionADDVecReduce()
13690 return DAG.getNode(ISD::ADD, dl, VT, N1, N0); in TryDistrubutionADDVecReduce()
13701 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, X, N1); in TryDistrubutionADDVecReduce()
13702 return DAG.getNode(ISD::ADD, dl, VT, Add0, N0); in TryDistrubutionADDVecReduce()
13737 if (NB->getOpcode() != ISD::BUILD_PAIR) in PerformADDVecReduce()
13747 SDValue Inp = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, in PerformADDVecReduce()
13749 NA = DAG.getNode(ISD::ADD, dl, MVT::i64, Inp, NA); in PerformADDVecReduce()
13760 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red, in PerformADDVecReduce()
13802 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
13803 N->getOpcode() == ISD::SRL) && in isDesirableToCommuteWithShift()
13809 if (N->getOpcode() != ISD::SHL) in isDesirableToCommuteWithShift()
13816 if (N->getOpcode() != ISD::SHL) in isDesirableToCommuteWithShift()
13819 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND && in isDesirableToCommuteWithShift()
13820 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR) in isDesirableToCommuteWithShift()
13825 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(0) && in isDesirableToCommuteWithShift()
13841 assert(N->getOpcode() == ISD::XOR && in isDesirableToCommuteXorWithShift()
13842 (N->getOperand(0).getOpcode() == ISD::SHL || in isDesirableToCommuteXorWithShift()
13843 N->getOperand(0).getOpcode() == ISD::SRL) && in isDesirableToCommuteXorWithShift()
13854 if (N->getOperand(0).getOpcode() == ISD::SHL) in isDesirableToCommuteXorWithShift()
13865 assert(((N->getOpcode() == ISD::SHL && in shouldFoldConstantShiftPairToMask()
13866 N->getOperand(0).getOpcode() == ISD::SRL) || in shouldFoldConstantShiftPairToMask()
13867 (N->getOpcode() == ISD::SRL && in shouldFoldConstantShiftPairToMask()
13868 N->getOperand(0).getOpcode() == ISD::SHL)) && in shouldFoldConstantShiftPairToMask()
13945 case ISD::SUB: in PerformSHLSimplify()
13946 case ISD::ADD: in PerformSHLSimplify()
13947 case ISD::AND: in PerformSHLSimplify()
13948 case ISD::OR: in PerformSHLSimplify()
13949 case ISD::XOR: in PerformSHLSimplify()
13950 case ISD::SETCC: in PerformSHLSimplify()
13960 if (U->getOperand(0).getOpcode() == ISD::SHL || in PerformSHLSimplify()
13961 U->getOperand(1).getOpcode() == ISD::SHL) in PerformSHLSimplify()
13967 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR && in PerformSHLSimplify()
13968 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND) in PerformSHLSimplify()
13971 if (N->getOperand(0).getOpcode() != ISD::SHL) in PerformSHLSimplify()
14011 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1)); in PerformSHLSimplify()
14057 DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32, N->getOperand(0), in PerformSubCSINCCombine()
14090 if (VMov->getOpcode() == ISD::BITCAST) in PerformSUBCombine()
14097 SDValue Negate = DCI.DAG.getNode(ISD::SUB, dl, MVT::i32, in PerformSUBCombine()
14128 if (Opcode != ISD::ADD && Opcode != ISD::SUB && in PerformVMULCombine()
14129 Opcode != ISD::FADD && Opcode != ISD::FSUB) { in PerformVMULCombine()
14131 if (Opcode != ISD::ADD && Opcode != ISD::SUB && in PerformVMULCombine()
14132 Opcode != ISD::FADD && Opcode != ISD::FSUB) in PerformVMULCombine()
14145 DAG.getNode(ISD::MUL, DL, VT, N00, N1), in PerformVMULCombine()
14146 DAG.getNode(ISD::MUL, DL, VT, N01, N1)); in PerformVMULCombine()
14159 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG) in PerformMVEVMULLCombine()
14176 if (And->getOpcode() == ISD::BITCAST) in PerformMVEVMULLCombine()
14178 if (And->getOpcode() != ISD::AND) in PerformMVEVMULLCombine()
14181 if (Mask->getOpcode() == ISD::BITCAST) in PerformMVEVMULLCombine()
14184 if (Mask->getOpcode() != ISD::BUILD_VECTOR || in PerformMVEVMULLCombine()
14251 Res = DAG.getNode(ISD::ADD, DL, VT, in PerformMULCombine()
14253 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14259 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
14260 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14271 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
14273 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14279 Res = DAG.getNode(ISD::ADD, DL, VT, in PerformMULCombine()
14281 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14285 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
14292 Res = DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14323 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL) in CombineANDShift()
14326 bool LeftShift = N0->getOpcode() == ISD::SHL; in CombineANDShift()
14354 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14356 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL, in CombineANDShift()
14365 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14367 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL, in CombineANDShift()
14378 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14380 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL, in CombineANDShift()
14391 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14393 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL, in CombineANDShift()
14403 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i32, N0->getOperand(0), in CombineANDShift()
14405 return DAG.getNode(ISD::SHL, DL, MVT::i32, And, in CombineANDShift()
14438 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0)); in PerformANDCombine()
14440 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic); in PerformANDCombine()
14473 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) { in PerformORCombineToSMULWBT()
14483 SRL.getOperand(0).getOpcode() != ISD::SMUL_LOHI) in PerformORCombineToSMULWBT()
14579 } else if (N1.getOpcode() == ISD::AND) { in PerformORCombineToBFI()
14597 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0), in PerformORCombineToBFI()
14614 Res = DAG.getNode(ISD::SRL, DL, VT, N00, in PerformORCombineToBFI()
14626 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) && in PerformORCombineToBFI()
14700 SDValue And = DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1); in PerformORCombine_i1()
14734 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0)); in PerformORCombine()
14736 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr); in PerformORCombine()
14753 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() && in PerformORCombine()
14759 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse()) in PerformORCombine()
14794 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) { in PerformORCombine()
14859 if (From->getOpcode() == ISD::SRL && in ParseBFI()
14914 if (N1.getOpcode() == ISD::AND) { in PerformBFICombine()
14953 From1 = DAG.getNode(ISD::SRL, dl, VT, From1, in PerformBFICombine()
14993 while (CSInc.getOpcode() == ISD::AND && in IsCMPZCSINC()
15063 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() && in PerformVMOVRRDCombine()
15065 InNode->getOperand(1).getOpcode() == ISD::FrameIndex && in PerformVMOVRRDCombine()
15077 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, in PerformVMOVRRDCombine()
15094 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformVMOVRRDCombine()
15099 bool BVSwap = BV.getOpcode() == ISD::BITCAST; in PerformVMOVRRDCombine()
15101 (BV.getOpcode() == ISD::BITCAST || in PerformVMOVRRDCombine()
15104 BVSwap = BV.getOpcode() == ISD::BITCAST; in PerformVMOVRRDCombine()
15113 if (BV.getOpcode() == ISD::BUILD_VECTOR) { in PerformVMOVRRDCombine()
15125 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) { in PerformVMOVRRDCombine()
15149 if (Op0.getOpcode() == ISD::BITCAST) in PerformVMOVDRRCombine()
15151 if (Op1.getOpcode() == ISD::BITCAST) in PerformVMOVDRRCombine()
15156 return DAG.getNode(ISD::BITCAST, SDLoc(N), in PerformVMOVDRRCombine()
15177 if (Op0->getOpcode() == ISD::BITCAST) { in PerformVMOVhrCombine()
15180 Copy->getOpcode() == ISD::CopyFromReg) { in PerformVMOVhrCombine()
15186 DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N), in PerformVMOVhrCombine()
15234 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) { in PerformVMOVrhCombine()
15238 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(), in PerformVMOVrhCombine()
15246 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformVMOVrhCombine()
15262 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile()) in hasNormalLoadOperand()
15291 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i)); in PerformBUILD_VECTORCombine()
15298 return DAG.getNode(ISD::BITCAST, dl, VT, BV); in PerformBUILD_VECTORCombine()
15328 if (Use->getOpcode() != ISD::BITCAST || in PerformARMBUILD_VECTORCombine()
15342 if (Elt->getOpcode() == ISD::BITCAST) { in PerformARMBUILD_VECTORCombine()
15375 if (V.getOpcode() == ISD::BITCAST && in PerformARMBUILD_VECTORCombine()
15380 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V); in PerformARMBUILD_VECTORCombine()
15385 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx); in PerformARMBUILD_VECTORCombine()
15387 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec); in PerformARMBUILD_VECTORCombine()
15414 return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C); in PerformPREDICATE_CASTCombine()
15435 return DAG.getNode(ISD::BITCAST, dl, VT, Op); in PerformVECTOR_REG_CASTCombine()
15491 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile()) in PerformInsertEltCombine()
15498 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0)); in PerformInsertEltCombine()
15499 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1)); in PerformInsertEltCombine()
15503 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT, in PerformInsertEltCombine()
15505 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt); in PerformInsertEltCombine()
15522 if (Ext.getOpcode() == ISD::BITCAST && in PerformExtractEltToVMOVRRD()
15525 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in PerformExtractEltToVMOVRRD()
15530 (Ext->use_begin()->getOpcode() == ISD::SINT_TO_FP || in PerformExtractEltToVMOVRRD()
15531 Ext->use_begin()->getOpcode() == ISD::UINT_TO_FP)) in PerformExtractEltToVMOVRRD()
15543 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformExtractEltToVMOVRRD()
15556 OtherExt->use_begin()->getOpcode() != ISD::BITCAST || in PerformExtractEltToVMOVRRD()
15564 ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in PerformExtractEltToVMOVRRD()
15589 return DCI.DAG.getNode(ISD::BITCAST, dl, VT, X); in PerformExtractEltCombine()
15591 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST) in PerformExtractEltCombine()
15607 Op0.getOpcode() == ISD::BITCAST && in PerformExtractEltCombine()
15608 Op0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && in PerformExtractEltCombine()
15628 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Op0.getOperand(Vec), in PerformExtractEltCombine()
15680 Hi = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec, in PerformInsertSubvectorCombine()
15683 Lo = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec, in PerformInsertSubvectorCombine()
15687 return DCI.DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, Hi); in PerformInsertSubvectorCombine()
15732 if (Op0.getOpcode() != ISD::CONCAT_VECTORS || in PerformVECTOR_SHUFFLECombine()
15733 Op1.getOpcode() != ISD::CONCAT_VECTORS || in PerformVECTOR_SHUFFLECombine()
15749 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in PerformVECTOR_SHUFFLECombine()
15945 case ISD::LOAD: in TryCombineBaseUpdate()
15950 case ISD::STORE: in TryCombineBaseUpdate()
16060 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) { in TryCombineBaseUpdate()
16062 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal); in TryCombineBaseUpdate()
16076 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) { in TryCombineBaseUpdate()
16078 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal); in TryCombineBaseUpdate()
16098 case ISD::ADD: in getPointerConstIncrement()
16100 case ISD::OR: { in getPointerConstIncrement()
16114 case ISD::ADD: in findPointerConstIncrement()
16115 case ISD::OR: { in findPointerConstIncrement()
16158 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID || in CombineBaseUpdate()
16159 N->getOpcode() == ISD::INTRINSIC_W_CHAIN); in CombineBaseUpdate()
16160 const bool isStore = N->getOpcode() == ISD::STORE; in CombineBaseUpdate()
16180 if (ConstInc || User->getOpcode() == ISD::ADD) in CombineBaseUpdate()
16276 if (User->getOpcode() != ISD::ADD || in PerformMVEVLDCombine()
16386 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN) in CombineVLDDUP()
16467 SDValue Extract = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), ExtractVT, in PerformVDUPLANECombine()
16479 while (Op.getOpcode() == ISD::BITCAST) in PerformVDUPLANECombine()
16494 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op); in PerformVDUPLANECombine()
16508 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op)); in PerformVDUPCombine()
16542 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() && in PerformLOADCombine()
16583 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal); in PerformTruncatingStoreCombine()
16613 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff); in PerformTruncatingStoreCombine()
16622 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType, in PerformTruncatingStoreCombine()
16628 DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, Increment); in PerformTruncatingStoreCombine()
16631 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in PerformTruncatingStoreCombine()
16642 if (Trunc->getOpcode() != ISD::FP_ROUND) in PerformSplittingToNarrowingStores()
16711 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0), in PerformSplittingToNarrowingStores()
16724 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); in PerformSplittingToNarrowingStores()
16765 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); in PerformSplittingMVETruncToNarrowingStores()
16779 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in PerformExtractFpToIntStores()
16831 if (!ISD::isNormalStore(St)) in PerformSTORECombine()
16847 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, in PerformSTORECombine()
16857 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in PerformSTORECombine()
16866 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec); in PerformSTORECombine()
16867 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in PerformSTORECombine()
16870 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt); in PerformSTORECombine()
16881 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() && in PerformSTORECombine()
16904 Op.getOpcode() != ISD::FMUL) in PerformVCVTCombine()
16931 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT; in PerformVCVTCombine()
16935 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, in PerformVCVTCombine()
16940 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv); in PerformVCVTCombine()
16961 if (Op.getOpcode() != ISD::BITCAST || in PerformFAddVSelectCombine()
16972 if (Op0.getOpcode() == ISD::VSELECT && Op1.getOpcode() != ISD::VSELECT) in PerformFAddVSelectCombine()
16975 if (Op1.getOpcode() != ISD::VSELECT) in PerformFAddVSelectCombine()
16984 DAG.getNode(ISD::FADD, DL, VT, Op0, Op1.getOperand(1), FaddFlags); in PerformFAddVSelectCombine()
16985 return DAG.getNode(ISD::VSELECT, DL, VT, Op1.getOperand(0), FAdd, Op0, FaddFlags); in PerformFAddVSelectCombine()
16999 if (A.getOpcode() != ISD::INTRINSIC_WO_CHAIN) in PerformFADDVCMLACombine()
17005 ISD::INTRINSIC_WO_CHAIN, DL, VT, A.getOperand(0), A.getOperand(1), in PerformFADDVCMLACombine()
17006 DAG.getNode(ISD::FADD, DL, VT, A.getOperand(2), B, N->getFlags()), in PerformFADDVCMLACombine()
17045 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP)) in PerformVMulVCTPCombine()
17082 bool isSigned = OpOpcode == ISD::SINT_TO_FP; in PerformVMulVCTPCombine()
17085 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in PerformVMulVCTPCombine()
17090 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in PerformVMulVCTPCombine()
17100 assert(N->getOpcode() == ISD::VECREDUCE_ADD); in PerformVECREDUCE_ADDCombine()
17106 if (ResVT == MVT::i32 && N0.getOpcode() == ISD::ADD && in PerformVECREDUCE_ADDCombine()
17109 SDValue Red0 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(0)); in PerformVECREDUCE_ADDCombine()
17110 SDValue Red1 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(1)); in PerformVECREDUCE_ADDCombine()
17111 return DAG.getNode(ISD::ADD, dl, ResVT, Red0, Red1); in PerformVECREDUCE_ADDCombine()
17154 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT || in PerformVECREDUCE_ADDCombine()
17155 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode())) in PerformVECREDUCE_ADDCombine()
17183 if (Mul->getOpcode() != ISD::MUL) in PerformVECREDUCE_ADDCombine()
17206 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT || in PerformVECREDUCE_ADDCombine()
17207 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode())) in PerformVECREDUCE_ADDCombine()
17215 if (Mul->getOpcode() != ISD::MUL) in PerformVECREDUCE_ADDCombine()
17255 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1)); in PerformVECREDUCE_ADDCombine()
17258 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node, in PerformVECREDUCE_ADDCombine()
17264 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B)) in PerformVECREDUCE_ADDCombine()
17266 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B)) in PerformVECREDUCE_ADDCombine()
17268 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32}, in PerformVECREDUCE_ADDCombine()
17271 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32}, in PerformVECREDUCE_ADDCombine()
17274 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B)) in PerformVECREDUCE_ADDCombine()
17275 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17277 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B)) in PerformVECREDUCE_ADDCombine()
17278 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17281 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, in PerformVECREDUCE_ADDCombine()
17284 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, in PerformVECREDUCE_ADDCombine()
17287 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, in PerformVECREDUCE_ADDCombine()
17290 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, in PerformVECREDUCE_ADDCombine()
17293 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask)) in PerformVECREDUCE_ADDCombine()
17294 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17296 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask)) in PerformVECREDUCE_ADDCombine()
17297 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17300 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8})) in PerformVECREDUCE_ADDCombine()
17302 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8})) in PerformVECREDUCE_ADDCombine()
17304 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32})) in PerformVECREDUCE_ADDCombine()
17306 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32})) in PerformVECREDUCE_ADDCombine()
17308 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8})) in PerformVECREDUCE_ADDCombine()
17309 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17311 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8})) in PerformVECREDUCE_ADDCombine()
17312 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17315 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask)) in PerformVECREDUCE_ADDCombine()
17317 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask)) in PerformVECREDUCE_ADDCombine()
17319 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask)) in PerformVECREDUCE_ADDCombine()
17321 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask)) in PerformVECREDUCE_ADDCombine()
17323 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask)) in PerformVECREDUCE_ADDCombine()
17324 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17326 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask)) in PerformVECREDUCE_ADDCombine()
17327 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, in PerformVECREDUCE_ADDCombine()
17334 if (Op->getOpcode() == ISD::VSELECT) in PerformVECREDUCE_ADDCombine()
17336 if (Op->getOpcode() == ISD::ZERO_EXTEND && in PerformVECREDUCE_ADDCombine()
17337 Op->getOperand(0)->getOpcode() == ISD::MUL) { in PerformVECREDUCE_ADDCombine()
17340 Mul->getOperand(0)->getOpcode() == ISD::SIGN_EXTEND) { in PerformVECREDUCE_ADDCombine()
17341 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, N0->getValueType(0), Mul); in PerformVECREDUCE_ADDCombine()
17343 Ext = DAG.getNode(ISD::VSELECT, dl, N0->getValueType(0), in PerformVECREDUCE_ADDCombine()
17345 return DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, Ext); in PerformVECREDUCE_ADDCombine()
17708 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, val.getValue(0), in PerformIntrinsicCombine()
17727 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 && in PerformShiftCombine()
17728 N->getOperand(0)->getOpcode() == ISD::AND && in PerformShiftCombine()
17752 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0), in PerformShiftCombine()
17755 ISD::SRL, DL, MVT::i32, SHL, in PerformShiftCombine()
17773 case ISD::SHL: in PerformShiftCombine()
17781 case ISD::SRA: in PerformShiftCombine()
17782 case ISD::SRL: in PerformShiftCombine()
17785 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM); in PerformShiftCombine()
17800 if (N0.getOpcode() != ISD::LOAD) in PerformSplittingToWideningLoad()
17804 LD->getExtensionType() != ISD::NON_EXTLOAD) in PerformSplittingToWideningLoad()
17834 ISD::LoadExtType NewExtType = in PerformSplittingToWideningLoad()
17835 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in PerformSplittingToWideningLoad()
17850 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset, in PerformSplittingToWideningLoad()
17872 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in PerformSplittingToWideningLoad()
17874 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Loads); in PerformSplittingToWideningLoad()
17888 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in PerformExtendCombine()
17903 case ISD::SIGN_EXTEND: in PerformExtendCombine()
17906 case ISD::ZERO_EXTEND: in PerformExtendCombine()
17907 case ISD::ANY_EXTEND: in PerformExtendCombine()
17943 (Op0.getOpcode() != ISD::SMIN && Op0.getOpcode() != ISD::SMAX) || in PerformMinMaxToSatCombine()
17951 if (Min.getOpcode() == ISD::SMAX) in PerformMinMaxToSatCombine()
17957 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX || in PerformMinMaxToSatCombine()
17993 if (Min->getOpcode() != ISD::SMIN) in PerformMinMaxCombine()
17995 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX) in PerformMinMaxCombine()
18005 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) || in PerformMinMaxCombine()
18008 if (!ISD::isConstantSplatVector(Max->getOperand(1).getNode(), MaxC) || in PerformMinMaxCombine()
18032 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Bitcast, in PerformMinMaxCombine()
18038 if (Min->getOpcode() != ISD::UMIN) in PerformMinMaxCombine()
18048 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) || in PerformMinMaxCombine()
18073 return DAG.getNode(ISD::AND, DL, VT, Bitcast, in PerformMinMaxCombine()
18113 if (And->getOpcode() != ISD::AND) in PerformCMOVToBFICombine()
18128 if (Op1->getOpcode() != ISD::OR) in PerformCMOVToBFICombine()
18159 X = DAG.getNode(ISD::SRL, dl, VT, X, in PerformCMOVToBFICombine()
18183 static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, in SearchLoopIntrinsic()
18188 case ISD::XOR: { in SearchLoopIntrinsic()
18196 case ISD::SETCC: { in SearchLoopIntrinsic()
18209 case ISD::INTRINSIC_W_CHAIN: { in SearchLoopIntrinsic()
18235 ISD::CondCode CC; in PerformHWLoopCombine()
18242 if (N->getOpcode() == ISD::BRCOND) { in PerformHWLoopCombine()
18243 CC = ISD::SETEQ; in PerformHWLoopCombine()
18247 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!"); in PerformHWLoopCombine()
18264 CC = ISD::getSetCCInverse(CC, /* Integer inverse */ MVT::i32); in PerformHWLoopCombine()
18266 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) { in PerformHWLoopCombine()
18267 return (CC == ISD::SETEQ && Imm == 0) || in PerformHWLoopCombine()
18268 (CC == ISD::SETNE && Imm == 1) || in PerformHWLoopCombine()
18269 (CC == ISD::SETLT && Imm == 1) || in PerformHWLoopCombine()
18270 (CC == ISD::SETULT && Imm == 1); in PerformHWLoopCombine()
18273 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) { in PerformHWLoopCombine()
18274 return (CC == ISD::SETEQ && Imm == 1) || in PerformHWLoopCombine()
18275 (CC == ISD::SETNE && Imm == 0) || in PerformHWLoopCombine()
18276 (CC == ISD::SETGT && Imm == 0) || in PerformHWLoopCombine()
18277 (CC == ISD::SETUGT && Imm == 0) || in PerformHWLoopCombine()
18278 (CC == ISD::SETGE && Imm == 1) || in PerformHWLoopCombine()
18279 (CC == ISD::SETUGE && Imm == 1); in PerformHWLoopCombine()
18289 assert((N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BR) in PerformHWLoopCombine()
18297 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps); in PerformHWLoopCombine()
18337 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformHWLoopCombine()
18365 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() && in PerformBRCONDCombine()
18426 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl); in PerformCMOVCombine()
18468 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in PerformCMOVCombine()
18469 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub), in PerformCMOVCombine()
18481 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in PerformCMOVCombine()
18483 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub); in PerformCMOVCombine()
18487 DAG.getNode(ISD::SUB, dl, MVT::i32, in PerformCMOVCombine()
18489 Res = DAG.getNode(ISD::UADDO_CARRY, dl, VTs, Sub, Neg, Carry); in PerformCMOVCombine()
18543 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal); in PerformCMOVCombine()
18544 Res = DAG.getNode(ISD::USUBO_CARRY, dl, VTs, FalseVal, Subc, in PerformCMOVCombine()
18548 Res = DAG.getNode(ISD::SHL, dl, VT, Res, in PerformCMOVCombine()
18556 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res, in PerformCMOVCombine()
18559 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res, in PerformCMOVCombine()
18562 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res, in PerformCMOVCombine()
18628 N->getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE && in PerformMVETruncCombine()
18629 N->getOperand(1).getOpcode() == ISD::VECTOR_SHUFFLE) { in PerformMVETruncCombine()
18657 return Op.getOpcode() == ISD::BUILD_VECTOR || in PerformMVETruncCombine()
18658 Op.getOpcode() == ISD::VECTOR_SHUFFLE || in PerformMVETruncCombine()
18659 (Op.getOpcode() == ISD::BITCAST && in PerformMVETruncCombine()
18660 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR); in PerformMVETruncCombine()
18666 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, O, in PerformMVETruncCombine()
18693 ISD::ADD, DL, StackPtr.getValueType(), StackPtr, in PerformMVETruncCombine()
18702 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in PerformMVETruncCombine()
18731 ISD::LoadExtType NewExtType = in PerformSplittingMVEEXTToWideningLoad()
18732 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in PerformSplittingMVEEXTToWideningLoad()
18733 if (LD->getExtensionType() != ISD::NON_EXTLOAD && in PerformSplittingMVEEXTToWideningLoad()
18734 LD->getExtensionType() != ISD::EXTLOAD && in PerformSplittingMVEEXTToWideningLoad()
18761 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset, in PerformSplittingMVEEXTToWideningLoad()
18768 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in PerformSplittingMVEEXTToWideningLoad()
18789 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, VVT, in PerformMVEExtCombine()
18840 if (N->getOperand(0)->getOpcode() == ISD::LOAD) in PerformMVEExtCombine()
18867 ISD::ADD, DL, StackPtr.getValueType(), StackPtr, in PerformMVEExtCombine()
18872 N->getOpcode() == ARMISD::MVESEXT ? ISD::SEXTLOAD : ISD::ZEXTLOAD, DL, in PerformMVEExtCombine()
18884 case ISD::SELECT_CC: in PerformDAGCombine()
18885 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget); in PerformDAGCombine()
18886 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget); in PerformDAGCombine()
18887 case ISD::SETCC: return PerformVSetCCToVCTPCombine(N, DCI, Subtarget); in PerformDAGCombine()
18890 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget); in PerformDAGCombine()
18891 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget); in PerformDAGCombine()
18892 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget); in PerformDAGCombine()
18893 case ISD::OR: return PerformORCombine(N, DCI, Subtarget); in PerformDAGCombine()
18894 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget); in PerformDAGCombine()
18895 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget); in PerformDAGCombine()
18896 case ISD::BRCOND: in PerformDAGCombine()
18897 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, Subtarget); in PerformDAGCombine()
18906 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget); in PerformDAGCombine()
18907 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget); in PerformDAGCombine()
18908 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI); in PerformDAGCombine()
18909 case ISD::EXTRACT_VECTOR_ELT: in PerformDAGCombine()
18911 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DCI.DAG); in PerformDAGCombine()
18912 case ISD::INSERT_SUBVECTOR: return PerformInsertSubvectorCombine(N, DCI); in PerformDAGCombine()
18913 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG); in PerformDAGCombine()
18916 case ISD::FP_TO_SINT: in PerformDAGCombine()
18917 case ISD::FP_TO_UINT: in PerformDAGCombine()
18919 case ISD::FADD: in PerformDAGCombine()
18921 case ISD::FMUL: in PerformDAGCombine()
18923 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
18925 case ISD::SHL: in PerformDAGCombine()
18926 case ISD::SRA: in PerformDAGCombine()
18927 case ISD::SRL: in PerformDAGCombine()
18929 case ISD::SIGN_EXTEND: in PerformDAGCombine()
18930 case ISD::ZERO_EXTEND: in PerformDAGCombine()
18931 case ISD::ANY_EXTEND: in PerformDAGCombine()
18933 case ISD::FP_EXTEND: in PerformDAGCombine()
18935 case ISD::SMIN: in PerformDAGCombine()
18936 case ISD::UMIN: in PerformDAGCombine()
18937 case ISD::SMAX: in PerformDAGCombine()
18938 case ISD::UMAX: in PerformDAGCombine()
18950 case ISD::LOAD: in PerformDAGCombine()
18959 case ISD::BITCAST: in PerformDAGCombine()
18972 case ISD::VECREDUCE_ADD: in PerformDAGCombine()
19063 case ISD::INTRINSIC_VOID: in PerformDAGCombine()
19064 case ISD::INTRINSIC_W_CHAIN: in PerformDAGCombine()
19104 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE); in isDesirableToTransformToIntegerOp()
19226 if (Val.getOpcode() != ISD::LOAD) in isZExtFree()
19443 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB || in isVectorLoadExtDesirable()
19444 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM)) in isVectorLoadExtDesirable()
19789 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getARMIndexedAddressParts()
19798 assert(Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19804 isInc = (Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19812 assert(Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19820 if (Ptr->getOpcode() == ISD::ADD) { in getARMIndexedAddressParts()
19834 isInc = (Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
19848 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getT2IndexedAddressParts()
19855 assert(Ptr->getOpcode() == ISD::ADD); in getT2IndexedAddressParts()
19860 isInc = Ptr->getOpcode() == ISD::ADD; in getT2IndexedAddressParts()
19873 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getMVEIndexedAddressParts()
19888 assert(Ptr->getOpcode() == ISD::ADD); in getMVEIndexedAddressParts()
19893 isInc = Ptr->getOpcode() == ISD::ADD; in getMVEIndexedAddressParts()
19928 ISD::MemIndexedMode &AM, in getPreIndexedAddressParts()
19942 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPreIndexedAddressParts()
19951 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPreIndexedAddressParts()
19979 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC; in getPreIndexedAddressParts()
19989 ISD::MemIndexedMode &AM, in getPostIndexedAddressParts()
20000 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPostIndexedAddressParts()
20001 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD; in getPostIndexedAddressParts()
20011 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPostIndexedAddressParts()
20012 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD; in getPostIndexedAddressParts()
20027 if (Op->getOpcode() != ISD::ADD || !isNonExt) in getPostIndexedAddressParts()
20037 AM = ISD::POST_INC; in getPostIndexedAddressParts()
20062 if (Ptr == Offset && Op->getOpcode() == ISD::ADD && in getPostIndexedAddressParts()
20071 AM = isInc ? ISD::POST_INC : ISD::POST_DEC; in getPostIndexedAddressParts()
20110 case ISD::INTRINSIC_W_CHAIN: { in computeKnownBitsForTargetNode()
20204 if (Op.getOpcode() != ISD::AND) in targetShrinkDemandedConstant()
20245 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
20296 ISD::SHL, SDLoc(Op), MVT::i32, Op.getOperand(1), in SimplifyDemandedBitsForTargetNode()
20679 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || in getDivRemLibcall()
20680 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && in getDivRemLibcall()
20682 bool isSigned = N->getOpcode() == ISD::SDIVREM || in getDivRemLibcall()
20683 N->getOpcode() == ISD::SREM; in getDivRemLibcall()
20697 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || in getDivRemArgList()
20698 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && in getDivRemArgList()
20700 bool isSigned = N->getOpcode() == ISD::SDIVREM || in getDivRemArgList()
20701 N->getOpcode() == ISD::SREM; in getDivRemArgList()
20724 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) && in LowerDivRem()
20726 bool isSigned = (Opcode == ISD::SDIVREM); in LowerDivRem()
20734 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[0], Result[1]); in LowerDivRem()
20736 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[2], Result[3]); in LowerDivRem()
20737 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), in LowerDivRem()
20753 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in LowerDivRem()
20757 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor); in LowerDivRem()
20758 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in LowerDivRem()
20761 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values); in LowerDivRem()
20797 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), N->getValueType(0), in LowerREM()
20823 bool isSigned = N->getOpcode() == ISD::SREM; in LowerREM()
20858 SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size); in LowerDYNAMIC_STACKALLOC()
20861 DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
20868 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size, in LowerDYNAMIC_STACKALLOC()
20904 SDValue Result = DAG.getNode(ISD::FP_EXTEND, in LowerFP_EXTEND()
20926 SrcVal = DAG.getNode(ISD::STRICT_FP_EXTEND, Loc, in LowerFP_EXTEND()
20930 SrcVal = DAG.getNode(ISD::FP_EXTEND, Loc, DstVT, SrcVal); in LowerFP_EXTEND()
21029 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21045 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21064 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
21086 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
21106 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21120 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
21134 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21143 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21152 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21164 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
21173 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21182 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
21196 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21208 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21218 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21228 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
21257 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) in isExtractSubvectorCheap()