Lines Matching refs:ISD
287 case ISD::LOAD: { in LegalizeOp()
289 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeOp()
291 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD) in LegalizeOp()
295 case ISD::STORE: { in LegalizeOp()
303 case ISD::MERGE_VALUES: in LegalizeOp()
311 case ISD::STRICT_##DAGN: in LegalizeOp()
314 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP || in LegalizeOp()
315 Op.getOpcode() == ISD::STRICT_UINT_TO_FP) in LegalizeOp()
317 if (Op.getOpcode() == ISD::STRICT_FSETCC || in LegalizeOp()
318 Op.getOpcode() == ISD::STRICT_FSETCCS) { in LegalizeOp()
320 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get(); in LegalizeOp()
343 case ISD::ADD: in LegalizeOp()
344 case ISD::SUB: in LegalizeOp()
345 case ISD::MUL: in LegalizeOp()
346 case ISD::MULHS: in LegalizeOp()
347 case ISD::MULHU: in LegalizeOp()
348 case ISD::SDIV: in LegalizeOp()
349 case ISD::UDIV: in LegalizeOp()
350 case ISD::SREM: in LegalizeOp()
351 case ISD::UREM: in LegalizeOp()
352 case ISD::SDIVREM: in LegalizeOp()
353 case ISD::UDIVREM: in LegalizeOp()
354 case ISD::FADD: in LegalizeOp()
355 case ISD::FSUB: in LegalizeOp()
356 case ISD::FMUL: in LegalizeOp()
357 case ISD::FDIV: in LegalizeOp()
358 case ISD::FREM: in LegalizeOp()
359 case ISD::AND: in LegalizeOp()
360 case ISD::OR: in LegalizeOp()
361 case ISD::XOR: in LegalizeOp()
362 case ISD::SHL: in LegalizeOp()
363 case ISD::SRA: in LegalizeOp()
364 case ISD::SRL: in LegalizeOp()
365 case ISD::FSHL: in LegalizeOp()
366 case ISD::FSHR: in LegalizeOp()
367 case ISD::ROTL: in LegalizeOp()
368 case ISD::ROTR: in LegalizeOp()
369 case ISD::ABS: in LegalizeOp()
370 case ISD::ABDS: in LegalizeOp()
371 case ISD::ABDU: in LegalizeOp()
372 case ISD::AVGCEILS: in LegalizeOp()
373 case ISD::AVGCEILU: in LegalizeOp()
374 case ISD::AVGFLOORS: in LegalizeOp()
375 case ISD::AVGFLOORU: in LegalizeOp()
376 case ISD::BSWAP: in LegalizeOp()
377 case ISD::BITREVERSE: in LegalizeOp()
378 case ISD::CTLZ: in LegalizeOp()
379 case ISD::CTTZ: in LegalizeOp()
380 case ISD::CTLZ_ZERO_UNDEF: in LegalizeOp()
381 case ISD::CTTZ_ZERO_UNDEF: in LegalizeOp()
382 case ISD::CTPOP: in LegalizeOp()
383 case ISD::SELECT: in LegalizeOp()
384 case ISD::VSELECT: in LegalizeOp()
385 case ISD::SELECT_CC: in LegalizeOp()
386 case ISD::ZERO_EXTEND: in LegalizeOp()
387 case ISD::ANY_EXTEND: in LegalizeOp()
388 case ISD::TRUNCATE: in LegalizeOp()
389 case ISD::SIGN_EXTEND: in LegalizeOp()
390 case ISD::FP_TO_SINT: in LegalizeOp()
391 case ISD::FP_TO_UINT: in LegalizeOp()
392 case ISD::FNEG: in LegalizeOp()
393 case ISD::FABS: in LegalizeOp()
394 case ISD::FMINNUM: in LegalizeOp()
395 case ISD::FMAXNUM: in LegalizeOp()
396 case ISD::FMINNUM_IEEE: in LegalizeOp()
397 case ISD::FMAXNUM_IEEE: in LegalizeOp()
398 case ISD::FMINIMUM: in LegalizeOp()
399 case ISD::FMAXIMUM: in LegalizeOp()
400 case ISD::FCOPYSIGN: in LegalizeOp()
401 case ISD::FSQRT: in LegalizeOp()
402 case ISD::FSIN: in LegalizeOp()
403 case ISD::FCOS: in LegalizeOp()
404 case ISD::FTAN: in LegalizeOp()
405 case ISD::FASIN: in LegalizeOp()
406 case ISD::FACOS: in LegalizeOp()
407 case ISD::FATAN: in LegalizeOp()
408 case ISD::FSINH: in LegalizeOp()
409 case ISD::FCOSH: in LegalizeOp()
410 case ISD::FTANH: in LegalizeOp()
411 case ISD::FLDEXP: in LegalizeOp()
412 case ISD::FPOWI: in LegalizeOp()
413 case ISD::FPOW: in LegalizeOp()
414 case ISD::FLOG: in LegalizeOp()
415 case ISD::FLOG2: in LegalizeOp()
416 case ISD::FLOG10: in LegalizeOp()
417 case ISD::FEXP: in LegalizeOp()
418 case ISD::FEXP2: in LegalizeOp()
419 case ISD::FEXP10: in LegalizeOp()
420 case ISD::FCEIL: in LegalizeOp()
421 case ISD::FTRUNC: in LegalizeOp()
422 case ISD::FRINT: in LegalizeOp()
423 case ISD::FNEARBYINT: in LegalizeOp()
424 case ISD::FROUND: in LegalizeOp()
425 case ISD::FROUNDEVEN: in LegalizeOp()
426 case ISD::FFLOOR: in LegalizeOp()
427 case ISD::FP_ROUND: in LegalizeOp()
428 case ISD::FP_EXTEND: in LegalizeOp()
429 case ISD::FPTRUNC_ROUND: in LegalizeOp()
430 case ISD::FMA: in LegalizeOp()
431 case ISD::SIGN_EXTEND_INREG: in LegalizeOp()
432 case ISD::ANY_EXTEND_VECTOR_INREG: in LegalizeOp()
433 case ISD::SIGN_EXTEND_VECTOR_INREG: in LegalizeOp()
434 case ISD::ZERO_EXTEND_VECTOR_INREG: in LegalizeOp()
435 case ISD::SMIN: in LegalizeOp()
436 case ISD::SMAX: in LegalizeOp()
437 case ISD::UMIN: in LegalizeOp()
438 case ISD::UMAX: in LegalizeOp()
439 case ISD::SMUL_LOHI: in LegalizeOp()
440 case ISD::UMUL_LOHI: in LegalizeOp()
441 case ISD::SADDO: in LegalizeOp()
442 case ISD::UADDO: in LegalizeOp()
443 case ISD::SSUBO: in LegalizeOp()
444 case ISD::USUBO: in LegalizeOp()
445 case ISD::SMULO: in LegalizeOp()
446 case ISD::UMULO: in LegalizeOp()
447 case ISD::FCANONICALIZE: in LegalizeOp()
448 case ISD::FFREXP: in LegalizeOp()
449 case ISD::SADDSAT: in LegalizeOp()
450 case ISD::UADDSAT: in LegalizeOp()
451 case ISD::SSUBSAT: in LegalizeOp()
452 case ISD::USUBSAT: in LegalizeOp()
453 case ISD::SSHLSAT: in LegalizeOp()
454 case ISD::USHLSAT: in LegalizeOp()
455 case ISD::FP_TO_SINT_SAT: in LegalizeOp()
456 case ISD::FP_TO_UINT_SAT: in LegalizeOp()
457 case ISD::MGATHER: in LegalizeOp()
458 case ISD::VECTOR_COMPRESS: in LegalizeOp()
459 case ISD::SCMP: in LegalizeOp()
460 case ISD::UCMP: in LegalizeOp()
463 case ISD::SMULFIX: in LegalizeOp()
464 case ISD::SMULFIXSAT: in LegalizeOp()
465 case ISD::UMULFIX: in LegalizeOp()
466 case ISD::UMULFIXSAT: in LegalizeOp()
467 case ISD::SDIVFIX: in LegalizeOp()
468 case ISD::SDIVFIXSAT: in LegalizeOp()
469 case ISD::UDIVFIX: in LegalizeOp()
470 case ISD::UDIVFIXSAT: { in LegalizeOp()
476 case ISD::LRINT: in LegalizeOp()
477 case ISD::LLRINT: in LegalizeOp()
478 case ISD::SINT_TO_FP: in LegalizeOp()
479 case ISD::UINT_TO_FP: in LegalizeOp()
480 case ISD::VECREDUCE_ADD: in LegalizeOp()
481 case ISD::VECREDUCE_MUL: in LegalizeOp()
482 case ISD::VECREDUCE_AND: in LegalizeOp()
483 case ISD::VECREDUCE_OR: in LegalizeOp()
484 case ISD::VECREDUCE_XOR: in LegalizeOp()
485 case ISD::VECREDUCE_SMAX: in LegalizeOp()
486 case ISD::VECREDUCE_SMIN: in LegalizeOp()
487 case ISD::VECREDUCE_UMAX: in LegalizeOp()
488 case ISD::VECREDUCE_UMIN: in LegalizeOp()
489 case ISD::VECREDUCE_FADD: in LegalizeOp()
490 case ISD::VECREDUCE_FMUL: in LegalizeOp()
491 case ISD::VECREDUCE_FMAX: in LegalizeOp()
492 case ISD::VECREDUCE_FMIN: in LegalizeOp()
493 case ISD::VECREDUCE_FMAXIMUM: in LegalizeOp()
494 case ISD::VECREDUCE_FMINIMUM: in LegalizeOp()
498 case ISD::VECREDUCE_SEQ_FADD: in LegalizeOp()
499 case ISD::VECREDUCE_SEQ_FMUL: in LegalizeOp()
503 case ISD::SETCC: { in LegalizeOp()
505 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in LegalizeOp()
513 case ISD::VPID: { \ in LegalizeOp()
516 if (ISD::VPID == ISD::VP_SETCC) { \ in LegalizeOp()
517 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \ in LegalizeOp()
539 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) && in LegalizeOp()
603 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND; in PromoteSETCC()
612 if (Node->getOpcode() == ISD::VP_SETCC) { in PromoteSETCC()
636 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteSTRICT()
637 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand. in PromoteSTRICT()
641 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVecVT, MVT::Other}, in PromoteSTRICT()
650 Operands[0] = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in PromoteSTRICT()
656 DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VecVT, MVT::Other}, in PromoteSTRICT()
668 case ISD::SINT_TO_FP: in Promote()
669 case ISD::UINT_TO_FP: in Promote()
670 case ISD::STRICT_SINT_TO_FP: in Promote()
671 case ISD::STRICT_UINT_TO_FP: in Promote()
675 case ISD::FP_TO_UINT: in Promote()
676 case ISD::FP_TO_SINT: in Promote()
677 case ISD::STRICT_FP_TO_UINT: in Promote()
678 case ISD::STRICT_FP_TO_SINT: in Promote()
682 case ISD::VP_SETCC: in Promote()
683 case ISD::SETCC: in Promote()
687 case ISD::STRICT_FADD: in Promote()
688 case ISD::STRICT_FSUB: in Promote()
689 case ISD::STRICT_FMUL: in Promote()
690 case ISD::STRICT_FDIV: in Promote()
691 case ISD::STRICT_FSQRT: in Promote()
692 case ISD::STRICT_FMA: in Promote()
695 case ISD::FP_ROUND: in Promote()
696 case ISD::FP_EXTEND: in Promote()
716 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) && in Promote()
717 ISD::getVPMaskIdx(Node->getOpcode()) == j; in Promote()
724 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j)); in Promote()
726 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j)); in Promote()
737 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res, in Promote()
740 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res); in Promote()
758 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteINT_TO_FP()
759 Node->getOpcode() == ISD::STRICT_UINT_TO_FP) in PromoteINT_TO_FP()
760 ? ISD::ZERO_EXTEND in PromoteINT_TO_FP()
761 : ISD::SIGN_EXTEND; in PromoteINT_TO_FP()
797 if (NewOpc == ISD::FP_TO_UINT && in PromoteFP_TO_INT()
798 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) in PromoteFP_TO_INT()
799 NewOpc = ISD::FP_TO_SINT; in PromoteFP_TO_INT()
801 if (NewOpc == ISD::STRICT_FP_TO_UINT && in PromoteFP_TO_INT()
802 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) in PromoteFP_TO_INT()
803 NewOpc = ISD::STRICT_FP_TO_SINT; in PromoteFP_TO_INT()
817 if (Node->getOpcode() == ISD::FP_TO_UINT || in PromoteFP_TO_INT()
818 Node->getOpcode() == ISD::STRICT_FP_TO_UINT) in PromoteFP_TO_INT()
819 NewOpc = ISD::AssertZext; in PromoteFP_TO_INT()
821 NewOpc = ISD::AssertSext; in PromoteFP_TO_INT()
825 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted); in PromoteFP_TO_INT()
844 case ISD::LOAD: { in Expand()
850 case ISD::STORE: in Expand()
853 case ISD::MERGE_VALUES: in Expand()
857 case ISD::SIGN_EXTEND_INREG: in Expand()
860 case ISD::ANY_EXTEND_VECTOR_INREG: in Expand()
863 case ISD::SIGN_EXTEND_VECTOR_INREG: in Expand()
866 case ISD::ZERO_EXTEND_VECTOR_INREG: in Expand()
869 case ISD::BSWAP: in Expand()
872 case ISD::VP_BSWAP: in Expand()
875 case ISD::VSELECT: in Expand()
878 case ISD::VP_SELECT: in Expand()
881 case ISD::VP_SREM: in Expand()
882 case ISD::VP_UREM: in Expand()
888 case ISD::SELECT: in Expand()
891 case ISD::SELECT_CC: { in Expand()
896 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0), in Expand()
905 case ISD::FP_TO_UINT: in Expand()
908 case ISD::UINT_TO_FP: in Expand()
911 case ISD::FNEG: in Expand()
914 case ISD::FSUB: in Expand()
917 case ISD::SETCC: in Expand()
918 case ISD::VP_SETCC: in Expand()
921 case ISD::ABS: in Expand()
927 case ISD::ABDS: in Expand()
928 case ISD::ABDU: in Expand()
934 case ISD::AVGCEILS: in Expand()
935 case ISD::AVGCEILU: in Expand()
936 case ISD::AVGFLOORS: in Expand()
937 case ISD::AVGFLOORU: in Expand()
943 case ISD::BITREVERSE: in Expand()
946 case ISD::VP_BITREVERSE: in Expand()
952 case ISD::CTPOP: in Expand()
958 case ISD::VP_CTPOP: in Expand()
964 case ISD::CTLZ: in Expand()
965 case ISD::CTLZ_ZERO_UNDEF: in Expand()
971 case ISD::VP_CTLZ: in Expand()
972 case ISD::VP_CTLZ_ZERO_UNDEF: in Expand()
978 case ISD::CTTZ: in Expand()
979 case ISD::CTTZ_ZERO_UNDEF: in Expand()
985 case ISD::VP_CTTZ: in Expand()
986 case ISD::VP_CTTZ_ZERO_UNDEF: in Expand()
992 case ISD::FSHL: in Expand()
993 case ISD::VP_FSHL: in Expand()
994 case ISD::FSHR: in Expand()
995 case ISD::VP_FSHR: in Expand()
1001 case ISD::ROTL: in Expand()
1002 case ISD::ROTR: in Expand()
1008 case ISD::FMINNUM: in Expand()
1009 case ISD::FMAXNUM: in Expand()
1015 case ISD::FMINIMUM: in Expand()
1016 case ISD::FMAXIMUM: in Expand()
1019 case ISD::SMIN: in Expand()
1020 case ISD::SMAX: in Expand()
1021 case ISD::UMIN: in Expand()
1022 case ISD::UMAX: in Expand()
1028 case ISD::UADDO: in Expand()
1029 case ISD::USUBO: in Expand()
1032 case ISD::SADDO: in Expand()
1033 case ISD::SSUBO: in Expand()
1036 case ISD::UMULO: in Expand()
1037 case ISD::SMULO: in Expand()
1040 case ISD::USUBSAT: in Expand()
1041 case ISD::SSUBSAT: in Expand()
1042 case ISD::UADDSAT: in Expand()
1043 case ISD::SADDSAT: in Expand()
1049 case ISD::USHLSAT: in Expand()
1050 case ISD::SSHLSAT: in Expand()
1056 case ISD::FP_TO_SINT_SAT: in Expand()
1057 case ISD::FP_TO_UINT_SAT: in Expand()
1066 case ISD::SMULFIX: in Expand()
1067 case ISD::UMULFIX: in Expand()
1073 case ISD::SMULFIXSAT: in Expand()
1074 case ISD::UMULFIXSAT: in Expand()
1080 case ISD::SDIVFIX: in Expand()
1081 case ISD::UDIVFIX: in Expand()
1084 case ISD::SDIVFIXSAT: in Expand()
1085 case ISD::UDIVFIXSAT: in Expand()
1088 case ISD::STRICT_##DAGN: in Expand()
1092 case ISD::VECREDUCE_ADD: in Expand()
1093 case ISD::VECREDUCE_MUL: in Expand()
1094 case ISD::VECREDUCE_AND: in Expand()
1095 case ISD::VECREDUCE_OR: in Expand()
1096 case ISD::VECREDUCE_XOR: in Expand()
1097 case ISD::VECREDUCE_SMAX: in Expand()
1098 case ISD::VECREDUCE_SMIN: in Expand()
1099 case ISD::VECREDUCE_UMAX: in Expand()
1100 case ISD::VECREDUCE_UMIN: in Expand()
1101 case ISD::VECREDUCE_FADD: in Expand()
1102 case ISD::VECREDUCE_FMUL: in Expand()
1103 case ISD::VECREDUCE_FMAX: in Expand()
1104 case ISD::VECREDUCE_FMIN: in Expand()
1105 case ISD::VECREDUCE_FMAXIMUM: in Expand()
1106 case ISD::VECREDUCE_FMINIMUM: in Expand()
1109 case ISD::VECREDUCE_SEQ_FADD: in Expand()
1110 case ISD::VECREDUCE_SEQ_FMUL: in Expand()
1113 case ISD::SREM: in Expand()
1114 case ISD::UREM: in Expand()
1117 case ISD::VP_MERGE: in Expand()
1120 case ISD::FREM: in Expand()
1127 case ISD::VECTOR_COMPRESS: in Expand()
1165 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || in ExpandSELECT()
1166 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || in ExpandSELECT()
1167 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || in ExpandSELECT()
1168 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR in ExpandSELECT()
1169 : ISD::SPLAT_VECTOR, in ExpandSELECT()
1188 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1); in ExpandSELECT()
1189 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2); in ExpandSELECT()
1193 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask); in ExpandSELECT()
1194 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask); in ExpandSELECT()
1195 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2); in ExpandSELECT()
1196 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandSELECT()
1203 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand || in ExpandSEXTINREG()
1204 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand) in ExpandSEXTINREG()
1214 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz); in ExpandSEXTINREG()
1215 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz); in ExpandSEXTINREG()
1236 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), in ExpandANY_EXTEND_VECTOR_INREG()
1251 ISD::BITCAST, DL, VT, in ExpandANY_EXTEND_VECTOR_INREG()
1263 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src); in ExpandSIGN_EXTEND_VECTOR_INREG()
1271 return DAG.getNode(ISD::SRA, DL, VT, in ExpandSIGN_EXTEND_VECTOR_INREG()
1272 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount), in ExpandSIGN_EXTEND_VECTOR_INREG()
1295 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), in ExpandZERO_EXTEND_VECTOR_INREG()
1311 return DAG.getNode(ISD::BITCAST, DL, VT, in ExpandZERO_EXTEND_VECTOR_INREG()
1337 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBSWAP()
1339 return DAG.getNode(ISD::BITCAST, DL, VT, Op); in ExpandBSWAP()
1344 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandBSWAP()
1345 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in ExpandBSWAP()
1346 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) && in ExpandBSWAP()
1347 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) in ExpandBSWAP()
1365 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) { in ExpandBITREVERSE()
1381 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) || in ExpandBITREVERSE()
1382 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) && in ExpandBITREVERSE()
1383 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) && in ExpandBITREVERSE()
1384 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) && in ExpandBITREVERSE()
1385 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) { in ExpandBITREVERSE()
1387 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBITREVERSE()
1390 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op); in ExpandBITREVERSE()
1391 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op); in ExpandBITREVERSE()
1399 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandBITREVERSE()
1400 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in ExpandBITREVERSE()
1401 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) && in ExpandBITREVERSE()
1402 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) { in ExpandBITREVERSE()
1427 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || in ExpandVSELECT()
1428 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || in ExpandVSELECT()
1429 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand) in ExpandVSELECT()
1451 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1); in ExpandVSELECT()
1452 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2); in ExpandVSELECT()
1456 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask); in ExpandVSELECT()
1457 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask); in ExpandVSELECT()
1458 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2); in ExpandVSELECT()
1459 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandVSELECT()
1476 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand || in ExpandVP_SELECT()
1477 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand || in ExpandVP_SELECT()
1478 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand) in ExpandVP_SELECT()
1486 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL); in ExpandVP_SELECT()
1488 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL); in ExpandVP_SELECT()
1489 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL); in ExpandVP_SELECT()
1490 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL); in ExpandVP_SELECT()
1513 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) || in ExpandVP_MERGE()
1515 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) || in ExpandVP_MERGE()
1516 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT)))) in ExpandVP_MERGE()
1528 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT); in ExpandVP_MERGE()
1530 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask); in ExpandVP_MERGE()
1538 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV; in ExpandVP_REM()
1541 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) || in ExpandVP_REM()
1542 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT)) in ExpandVP_REM()
1554 SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL); in ExpandVP_REM()
1555 return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL); in ExpandVP_REM()
1597 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) == in ExpandUINT_TO_FLOAT()
1599 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) == in ExpandUINT_TO_FLOAT()
1601 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) { in ExpandUINT_TO_FLOAT()
1628 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord); in ExpandUINT_TO_FLOAT()
1629 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask); in ExpandUINT_TO_FLOAT()
1635 SDValue fHI = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, in ExpandUINT_TO_FLOAT()
1638 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1640 SDValue fLO = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, in ExpandUINT_TO_FLOAT()
1644 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1), in ExpandUINT_TO_FLOAT()
1649 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1660 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI); in ExpandUINT_TO_FLOAT()
1661 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW); in ExpandUINT_TO_FLOAT()
1662 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO); in ExpandUINT_TO_FLOAT()
1666 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO)); in ExpandUINT_TO_FLOAT()
1670 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) { in ExpandFNEG()
1674 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero, in ExpandFNEG()
1686 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && in ExpandFSUB()
1687 TLI.isOperationLegalOrCustom(ISD::FADD, VT)) in ExpandFSUB()
1697 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandSETCC()
1698 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandSETCC()
1699 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1700 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1709 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); in ExpandSETCC()
1740 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0), in ExpandSETCC()
1743 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC, in ExpandSETCC()
1763 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS, in ExpandSETCC()
1810 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) { in ExpandStrictFPOp()
1814 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) { in ExpandStrictFPOp()
1819 if (Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandStrictFPOp()
1820 Node->getOpcode() == ISD::STRICT_FSETCCS) { in ExpandStrictFPOp()
1830 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) && in ExpandREM()
1959 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1960 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
1983 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in UnrollStrictFPOp()
1993 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1994 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
2004 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); in UnrollStrictFPOp()
2021 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, in UnrollVSETCC()
2023 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, in UnrollVSETCC()
2025 Ops[i] = DAG.getNode(ISD::SETCC, dl, in UnrollVSETCC()