Lines Matching refs:TargetOpcode
227 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); in getUnmergeResults()
272 if (PadStrategy == TargetOpcode::G_ZEXT) in buildLCMMergePieces()
274 else if (PadStrategy == TargetOpcode::G_ANYEXT) in buildLCMMergePieces()
277 assert(PadStrategy == TargetOpcode::G_SEXT); in buildLCMMergePieces()
319 if (PadStrategy == TargetOpcode::G_ANYEXT) in buildLCMMergePieces()
321 else if (PadStrategy == TargetOpcode::G_ZEXT) in buildLCMMergePieces()
414 case TargetOpcode::G_MUL: in getRTLibDesc()
416 case TargetOpcode::G_SDIV: in getRTLibDesc()
418 case TargetOpcode::G_UDIV: in getRTLibDesc()
420 case TargetOpcode::G_SREM: in getRTLibDesc()
422 case TargetOpcode::G_UREM: in getRTLibDesc()
424 case TargetOpcode::G_CTLZ_ZERO_UNDEF: in getRTLibDesc()
426 case TargetOpcode::G_FADD: in getRTLibDesc()
428 case TargetOpcode::G_FSUB: in getRTLibDesc()
430 case TargetOpcode::G_FMUL: in getRTLibDesc()
432 case TargetOpcode::G_FDIV: in getRTLibDesc()
434 case TargetOpcode::G_FEXP: in getRTLibDesc()
436 case TargetOpcode::G_FEXP2: in getRTLibDesc()
438 case TargetOpcode::G_FEXP10: in getRTLibDesc()
440 case TargetOpcode::G_FREM: in getRTLibDesc()
442 case TargetOpcode::G_FPOW: in getRTLibDesc()
444 case TargetOpcode::G_FPOWI: in getRTLibDesc()
446 case TargetOpcode::G_FMA: in getRTLibDesc()
448 case TargetOpcode::G_FSIN: in getRTLibDesc()
450 case TargetOpcode::G_FCOS: in getRTLibDesc()
452 case TargetOpcode::G_FTAN: in getRTLibDesc()
454 case TargetOpcode::G_FASIN: in getRTLibDesc()
456 case TargetOpcode::G_FACOS: in getRTLibDesc()
458 case TargetOpcode::G_FATAN: in getRTLibDesc()
460 case TargetOpcode::G_FSINH: in getRTLibDesc()
462 case TargetOpcode::G_FCOSH: in getRTLibDesc()
464 case TargetOpcode::G_FTANH: in getRTLibDesc()
466 case TargetOpcode::G_FLOG10: in getRTLibDesc()
468 case TargetOpcode::G_FLOG: in getRTLibDesc()
470 case TargetOpcode::G_FLOG2: in getRTLibDesc()
472 case TargetOpcode::G_FLDEXP: in getRTLibDesc()
474 case TargetOpcode::G_FCEIL: in getRTLibDesc()
476 case TargetOpcode::G_FFLOOR: in getRTLibDesc()
478 case TargetOpcode::G_FMINNUM: in getRTLibDesc()
480 case TargetOpcode::G_FMAXNUM: in getRTLibDesc()
482 case TargetOpcode::G_FSQRT: in getRTLibDesc()
484 case TargetOpcode::G_FRINT: in getRTLibDesc()
486 case TargetOpcode::G_FNEARBYINT: in getRTLibDesc()
488 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: in getRTLibDesc()
490 case TargetOpcode::G_INTRINSIC_LRINT: in getRTLibDesc()
492 case TargetOpcode::G_INTRINSIC_LLRINT: in getRTLibDesc()
530 if (MI.getOpcode() == TargetOpcode::G_BZERO) in isLibCallInTailPosition()
664 case TargetOpcode::G_BZERO: in createMemLibcall()
667 case TargetOpcode::G_MEMCPY: in createMemLibcall()
671 case TargetOpcode::G_MEMMOVE: in createMemLibcall()
675 case TargetOpcode::G_MEMSET: in createMemLibcall()
743 case TargetOpcode::G_ATOMIC_CMPXCHG: in getOutlineAtomicLibcall()
744 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { in getOutlineAtomicLibcall()
748 case TargetOpcode::G_ATOMICRMW_XCHG: { in getOutlineAtomicLibcall()
752 case TargetOpcode::G_ATOMICRMW_ADD: in getOutlineAtomicLibcall()
753 case TargetOpcode::G_ATOMICRMW_SUB: { in getOutlineAtomicLibcall()
757 case TargetOpcode::G_ATOMICRMW_AND: { in getOutlineAtomicLibcall()
761 case TargetOpcode::G_ATOMICRMW_OR: { in getOutlineAtomicLibcall()
765 case TargetOpcode::G_ATOMICRMW_XOR: { in getOutlineAtomicLibcall()
785 case TargetOpcode::G_ATOMIC_CMPXCHG: in createAtomicLibcall()
786 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { in createAtomicLibcall()
793 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) { in createAtomicLibcall()
805 case TargetOpcode::G_ATOMICRMW_XCHG: in createAtomicLibcall()
806 case TargetOpcode::G_ATOMICRMW_ADD: in createAtomicLibcall()
807 case TargetOpcode::G_ATOMICRMW_SUB: in createAtomicLibcall()
808 case TargetOpcode::G_ATOMICRMW_AND: in createAtomicLibcall()
809 case TargetOpcode::G_ATOMICRMW_OR: in createAtomicLibcall()
810 case TargetOpcode::G_ATOMICRMW_XOR: { in createAtomicLibcall()
814 if (Opc == TargetOpcode::G_ATOMICRMW_AND) in createAtomicLibcall()
818 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB) in createAtomicLibcall()
860 case TargetOpcode::G_FPEXT: in getConvRTLibDesc()
862 case TargetOpcode::G_FPTRUNC: in getConvRTLibDesc()
864 case TargetOpcode::G_FPTOSI: in getConvRTLibDesc()
866 case TargetOpcode::G_FPTOUI: in getConvRTLibDesc()
868 case TargetOpcode::G_SITOFP: in getConvRTLibDesc()
870 case TargetOpcode::G_UITOFP: in getConvRTLibDesc()
889 case TargetOpcode::G_GET_FPENV: in getStateLibraryFunctionFor()
892 case TargetOpcode::G_SET_FPENV: in getStateLibraryFunctionFor()
893 case TargetOpcode::G_RESET_FPENV: in getStateLibraryFunctionFor()
896 case TargetOpcode::G_GET_FPMODE: in getStateLibraryFunctionFor()
899 case TargetOpcode::G_SET_FPMODE: in getStateLibraryFunctionFor()
900 case TargetOpcode::G_RESET_FPMODE: in getStateLibraryFunctionFor()
955 MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, Dst, Temp, *MMO); in createGetStateLibcall()
1031 case TargetOpcode::G_MUL: in libcall()
1032 case TargetOpcode::G_SDIV: in libcall()
1033 case TargetOpcode::G_UDIV: in libcall()
1034 case TargetOpcode::G_SREM: in libcall()
1035 case TargetOpcode::G_UREM: in libcall()
1036 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { in libcall()
1045 case TargetOpcode::G_FADD: in libcall()
1046 case TargetOpcode::G_FSUB: in libcall()
1047 case TargetOpcode::G_FMUL: in libcall()
1048 case TargetOpcode::G_FDIV: in libcall()
1049 case TargetOpcode::G_FMA: in libcall()
1050 case TargetOpcode::G_FPOW: in libcall()
1051 case TargetOpcode::G_FREM: in libcall()
1052 case TargetOpcode::G_FCOS: in libcall()
1053 case TargetOpcode::G_FSIN: in libcall()
1054 case TargetOpcode::G_FTAN: in libcall()
1055 case TargetOpcode::G_FACOS: in libcall()
1056 case TargetOpcode::G_FASIN: in libcall()
1057 case TargetOpcode::G_FATAN: in libcall()
1058 case TargetOpcode::G_FCOSH: in libcall()
1059 case TargetOpcode::G_FSINH: in libcall()
1060 case TargetOpcode::G_FTANH: in libcall()
1061 case TargetOpcode::G_FLOG10: in libcall()
1062 case TargetOpcode::G_FLOG: in libcall()
1063 case TargetOpcode::G_FLOG2: in libcall()
1064 case TargetOpcode::G_FLDEXP: in libcall()
1065 case TargetOpcode::G_FEXP: in libcall()
1066 case TargetOpcode::G_FEXP2: in libcall()
1067 case TargetOpcode::G_FEXP10: in libcall()
1068 case TargetOpcode::G_FCEIL: in libcall()
1069 case TargetOpcode::G_FFLOOR: in libcall()
1070 case TargetOpcode::G_FMINNUM: in libcall()
1071 case TargetOpcode::G_FMAXNUM: in libcall()
1072 case TargetOpcode::G_FSQRT: in libcall()
1073 case TargetOpcode::G_FRINT: in libcall()
1074 case TargetOpcode::G_FNEARBYINT: in libcall()
1075 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: { in libcall()
1088 case TargetOpcode::G_INTRINSIC_LRINT: in libcall()
1089 case TargetOpcode::G_INTRINSIC_LLRINT: { in libcall()
1108 case TargetOpcode::G_FPOWI: { in libcall()
1129 case TargetOpcode::G_FPEXT: in libcall()
1130 case TargetOpcode::G_FPTRUNC: { in libcall()
1141 case TargetOpcode::G_FPTOSI: in libcall()
1142 case TargetOpcode::G_FPTOUI: { in libcall()
1155 case TargetOpcode::G_SITOFP: in libcall()
1156 case TargetOpcode::G_UITOFP: { in libcall()
1168 case TargetOpcode::G_ATOMICRMW_XCHG: in libcall()
1169 case TargetOpcode::G_ATOMICRMW_ADD: in libcall()
1170 case TargetOpcode::G_ATOMICRMW_SUB: in libcall()
1171 case TargetOpcode::G_ATOMICRMW_AND: in libcall()
1172 case TargetOpcode::G_ATOMICRMW_OR: in libcall()
1173 case TargetOpcode::G_ATOMICRMW_XOR: in libcall()
1174 case TargetOpcode::G_ATOMIC_CMPXCHG: in libcall()
1175 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { in libcall()
1181 case TargetOpcode::G_BZERO: in libcall()
1182 case TargetOpcode::G_MEMCPY: in libcall()
1183 case TargetOpcode::G_MEMMOVE: in libcall()
1184 case TargetOpcode::G_MEMSET: { in libcall()
1192 case TargetOpcode::G_GET_FPENV: in libcall()
1193 case TargetOpcode::G_GET_FPMODE: { in libcall()
1199 case TargetOpcode::G_SET_FPENV: in libcall()
1200 case TargetOpcode::G_SET_FPMODE: { in libcall()
1206 case TargetOpcode::G_RESET_FPENV: in libcall()
1207 case TargetOpcode::G_RESET_FPMODE: { in libcall()
1229 case TargetOpcode::G_IMPLICIT_DEF: { in narrowScalar()
1264 case TargetOpcode::G_CONSTANT: { in narrowScalar()
1296 case TargetOpcode::G_SEXT: in narrowScalar()
1297 case TargetOpcode::G_ZEXT: in narrowScalar()
1298 case TargetOpcode::G_ANYEXT: in narrowScalar()
1300 case TargetOpcode::G_TRUNC: { in narrowScalar()
1315 case TargetOpcode::G_CONSTANT_FOLD_BARRIER: in narrowScalar()
1316 case TargetOpcode::G_FREEZE: { in narrowScalar()
1337 case TargetOpcode::G_ADD: in narrowScalar()
1338 case TargetOpcode::G_SUB: in narrowScalar()
1339 case TargetOpcode::G_SADDO: in narrowScalar()
1340 case TargetOpcode::G_SSUBO: in narrowScalar()
1341 case TargetOpcode::G_SADDE: in narrowScalar()
1342 case TargetOpcode::G_SSUBE: in narrowScalar()
1343 case TargetOpcode::G_UADDO: in narrowScalar()
1344 case TargetOpcode::G_USUBO: in narrowScalar()
1345 case TargetOpcode::G_UADDE: in narrowScalar()
1346 case TargetOpcode::G_USUBE: in narrowScalar()
1348 case TargetOpcode::G_MUL: in narrowScalar()
1349 case TargetOpcode::G_UMULH: in narrowScalar()
1351 case TargetOpcode::G_EXTRACT: in narrowScalar()
1353 case TargetOpcode::G_INSERT: in narrowScalar()
1355 case TargetOpcode::G_LOAD: { in narrowScalar()
1372 case TargetOpcode::G_ZEXTLOAD: in narrowScalar()
1373 case TargetOpcode::G_SEXTLOAD: { in narrowScalar()
1399 case TargetOpcode::G_STORE: { in narrowScalar()
1423 case TargetOpcode::G_SELECT: in narrowScalar()
1425 case TargetOpcode::G_AND: in narrowScalar()
1426 case TargetOpcode::G_OR: in narrowScalar()
1427 case TargetOpcode::G_XOR: { in narrowScalar()
1439 case TargetOpcode::G_SHL: in narrowScalar()
1440 case TargetOpcode::G_LSHR: in narrowScalar()
1441 case TargetOpcode::G_ASHR: in narrowScalar()
1443 case TargetOpcode::G_CTLZ: in narrowScalar()
1444 case TargetOpcode::G_CTLZ_ZERO_UNDEF: in narrowScalar()
1445 case TargetOpcode::G_CTTZ: in narrowScalar()
1446 case TargetOpcode::G_CTTZ_ZERO_UNDEF: in narrowScalar()
1447 case TargetOpcode::G_CTPOP: in narrowScalar()
1450 case TargetOpcode::G_CTLZ: in narrowScalar()
1451 case TargetOpcode::G_CTLZ_ZERO_UNDEF: in narrowScalar()
1453 case TargetOpcode::G_CTTZ: in narrowScalar()
1454 case TargetOpcode::G_CTTZ_ZERO_UNDEF: in narrowScalar()
1456 case TargetOpcode::G_CTPOP: in narrowScalar()
1463 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1466 case TargetOpcode::G_INTTOPTR: in narrowScalar()
1474 case TargetOpcode::G_PTRTOINT: in narrowScalar()
1479 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1482 case TargetOpcode::G_PHI: { in narrowScalar()
1503 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); in narrowScalar()
1513 case TargetOpcode::G_EXTRACT_VECTOR_ELT: in narrowScalar()
1514 case TargetOpcode::G_INSERT_VECTOR_ELT: { in narrowScalar()
1518 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3; in narrowScalar()
1524 case TargetOpcode::G_ICMP: { in narrowScalar()
1573 /* PadStrategy = */ TargetOpcode::G_ZEXT); in narrowScalar()
1603 case TargetOpcode::G_FCMP: in narrowScalar()
1608 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); in narrowScalar()
1612 case TargetOpcode::G_SEXT_INREG: { in narrowScalar()
1686 TargetOpcode::G_SEXT_INREG, {NarrowTy}, in narrowScalar()
1699 case TargetOpcode::G_BSWAP: in narrowScalar()
1700 case TargetOpcode::G_BITREVERSE: { in narrowScalar()
1722 case TargetOpcode::G_PTR_ADD: in narrowScalar()
1723 case TargetOpcode::G_PTRMASK: { in narrowScalar()
1731 case TargetOpcode::G_FPTOUI: in narrowScalar()
1732 case TargetOpcode::G_FPTOSI: in narrowScalar()
1734 case TargetOpcode::G_FPEXT: in narrowScalar()
1738 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT); in narrowScalar()
1741 case TargetOpcode::G_FLDEXP: in narrowScalar()
1742 case TargetOpcode::G_STRICT_FLDEXP: in narrowScalar()
1744 case TargetOpcode::G_VSCALE: { in narrowScalar()
2059 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in widenScalarUnmergeValues()
2144 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalarExtract()
2159 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalarExtract()
2174 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalarInsert()
2189 case TargetOpcode::G_SADDO: in widenScalarAddSubOverflow()
2190 Opcode = TargetOpcode::G_ADD; in widenScalarAddSubOverflow()
2191 ExtOpcode = TargetOpcode::G_SEXT; in widenScalarAddSubOverflow()
2193 case TargetOpcode::G_SSUBO: in widenScalarAddSubOverflow()
2194 Opcode = TargetOpcode::G_SUB; in widenScalarAddSubOverflow()
2195 ExtOpcode = TargetOpcode::G_SEXT; in widenScalarAddSubOverflow()
2197 case TargetOpcode::G_UADDO: in widenScalarAddSubOverflow()
2198 Opcode = TargetOpcode::G_ADD; in widenScalarAddSubOverflow()
2199 ExtOpcode = TargetOpcode::G_ZEXT; in widenScalarAddSubOverflow()
2201 case TargetOpcode::G_USUBO: in widenScalarAddSubOverflow()
2202 Opcode = TargetOpcode::G_SUB; in widenScalarAddSubOverflow()
2203 ExtOpcode = TargetOpcode::G_ZEXT; in widenScalarAddSubOverflow()
2205 case TargetOpcode::G_SADDE: in widenScalarAddSubOverflow()
2206 Opcode = TargetOpcode::G_UADDE; in widenScalarAddSubOverflow()
2207 ExtOpcode = TargetOpcode::G_SEXT; in widenScalarAddSubOverflow()
2210 case TargetOpcode::G_SSUBE: in widenScalarAddSubOverflow()
2211 Opcode = TargetOpcode::G_USUBE; in widenScalarAddSubOverflow()
2212 ExtOpcode = TargetOpcode::G_SEXT; in widenScalarAddSubOverflow()
2215 case TargetOpcode::G_UADDE: in widenScalarAddSubOverflow()
2216 Opcode = TargetOpcode::G_UADDE; in widenScalarAddSubOverflow()
2217 ExtOpcode = TargetOpcode::G_ZEXT; in widenScalarAddSubOverflow()
2220 case TargetOpcode::G_USUBE: in widenScalarAddSubOverflow()
2221 Opcode = TargetOpcode::G_USUBE; in widenScalarAddSubOverflow()
2222 ExtOpcode = TargetOpcode::G_ZEXT; in widenScalarAddSubOverflow()
2266 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT || in widenScalarAddSubShlSat()
2267 MI.getOpcode() == TargetOpcode::G_SSUBSAT || in widenScalarAddSubShlSat()
2268 MI.getOpcode() == TargetOpcode::G_SSHLSAT; in widenScalarAddSubShlSat()
2269 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT || in widenScalarAddSubShlSat()
2270 MI.getOpcode() == TargetOpcode::G_USHLSAT; in widenScalarAddSubShlSat()
2317 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO; in widenScalarMulo()
2327 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; in widenScalarMulo()
2336 WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL; in widenScalarMulo()
2379 case TargetOpcode::G_ATOMICRMW_XCHG: in widenScalar()
2380 case TargetOpcode::G_ATOMICRMW_ADD: in widenScalar()
2381 case TargetOpcode::G_ATOMICRMW_SUB: in widenScalar()
2382 case TargetOpcode::G_ATOMICRMW_AND: in widenScalar()
2383 case TargetOpcode::G_ATOMICRMW_OR: in widenScalar()
2384 case TargetOpcode::G_ATOMICRMW_XOR: in widenScalar()
2385 case TargetOpcode::G_ATOMICRMW_MIN: in widenScalar()
2386 case TargetOpcode::G_ATOMICRMW_MAX: in widenScalar()
2387 case TargetOpcode::G_ATOMICRMW_UMIN: in widenScalar()
2388 case TargetOpcode::G_ATOMICRMW_UMAX: in widenScalar()
2391 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); in widenScalar()
2395 case TargetOpcode::G_ATOMIC_CMPXCHG: in widenScalar()
2398 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); in widenScalar()
2399 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); in widenScalar()
2403 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: in widenScalar()
2406 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); in widenScalar()
2407 widenScalarSrc(MI, WideTy, 4, TargetOpcode::G_ANYEXT); in widenScalar()
2418 case TargetOpcode::G_EXTRACT: in widenScalar()
2420 case TargetOpcode::G_INSERT: in widenScalar()
2422 case TargetOpcode::G_MERGE_VALUES: in widenScalar()
2424 case TargetOpcode::G_UNMERGE_VALUES: in widenScalar()
2426 case TargetOpcode::G_SADDO: in widenScalar()
2427 case TargetOpcode::G_SSUBO: in widenScalar()
2428 case TargetOpcode::G_UADDO: in widenScalar()
2429 case TargetOpcode::G_USUBO: in widenScalar()
2430 case TargetOpcode::G_SADDE: in widenScalar()
2431 case TargetOpcode::G_SSUBE: in widenScalar()
2432 case TargetOpcode::G_UADDE: in widenScalar()
2433 case TargetOpcode::G_USUBE: in widenScalar()
2435 case TargetOpcode::G_UMULO: in widenScalar()
2436 case TargetOpcode::G_SMULO: in widenScalar()
2438 case TargetOpcode::G_SADDSAT: in widenScalar()
2439 case TargetOpcode::G_SSUBSAT: in widenScalar()
2440 case TargetOpcode::G_SSHLSAT: in widenScalar()
2441 case TargetOpcode::G_UADDSAT: in widenScalar()
2442 case TargetOpcode::G_USUBSAT: in widenScalar()
2443 case TargetOpcode::G_USHLSAT: in widenScalar()
2445 case TargetOpcode::G_CTTZ: in widenScalar()
2446 case TargetOpcode::G_CTTZ_ZERO_UNDEF: in widenScalar()
2447 case TargetOpcode::G_CTLZ: in widenScalar()
2448 case TargetOpcode::G_CTLZ_ZERO_UNDEF: in widenScalar()
2449 case TargetOpcode::G_CTPOP: { in widenScalar()
2460 unsigned ExtOpc = MI.getOpcode() == TargetOpcode::G_CTTZ || in widenScalar()
2461 MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF in widenScalar()
2462 ? TargetOpcode::G_ANYEXT in widenScalar()
2463 : TargetOpcode::G_ZEXT; in widenScalar()
2467 if (NewOpc == TargetOpcode::G_CTTZ) { in widenScalar()
2476 NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF; in widenScalar()
2481 if (MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) { in widenScalar()
2493 if (MI.getOpcode() == TargetOpcode::G_CTLZ) { in widenScalar()
2503 case TargetOpcode::G_BSWAP: { in widenScalar()
2510 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2525 case TargetOpcode::G_BITREVERSE: { in widenScalar()
2533 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2543 case TargetOpcode::G_FREEZE: in widenScalar()
2544 case TargetOpcode::G_CONSTANT_FOLD_BARRIER: in widenScalar()
2546 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2551 case TargetOpcode::G_ABS: in widenScalar()
2553 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); in widenScalar()
2558 case TargetOpcode::G_ADD: in widenScalar()
2559 case TargetOpcode::G_AND: in widenScalar()
2560 case TargetOpcode::G_MUL: in widenScalar()
2561 case TargetOpcode::G_OR: in widenScalar()
2562 case TargetOpcode::G_XOR: in widenScalar()
2563 case TargetOpcode::G_SUB: in widenScalar()
2564 case TargetOpcode::G_SHUFFLE_VECTOR: in widenScalar()
2569 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2570 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); in widenScalar()
2575 case TargetOpcode::G_SBFX: in widenScalar()
2576 case TargetOpcode::G_UBFX: in widenScalar()
2580 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2583 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
2584 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT); in widenScalar()
2590 case TargetOpcode::G_SHL: in widenScalar()
2594 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2600 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
2606 case TargetOpcode::G_ROTR: in widenScalar()
2607 case TargetOpcode::G_ROTL: in widenScalar()
2612 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
2616 case TargetOpcode::G_SDIV: in widenScalar()
2617 case TargetOpcode::G_SREM: in widenScalar()
2618 case TargetOpcode::G_SMIN: in widenScalar()
2619 case TargetOpcode::G_SMAX: in widenScalar()
2621 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); in widenScalar()
2622 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); in widenScalar()
2627 case TargetOpcode::G_SDIVREM: in widenScalar()
2629 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); in widenScalar()
2630 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); in widenScalar()
2636 case TargetOpcode::G_ASHR: in widenScalar()
2637 case TargetOpcode::G_LSHR: in widenScalar()
2641 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ? in widenScalar()
2642 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; in widenScalar()
2650 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
2655 case TargetOpcode::G_UDIV: in widenScalar()
2656 case TargetOpcode::G_UREM: in widenScalar()
2657 case TargetOpcode::G_UMIN: in widenScalar()
2658 case TargetOpcode::G_UMAX: in widenScalar()
2660 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); in widenScalar()
2661 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
2666 case TargetOpcode::G_UDIVREM: in widenScalar()
2668 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
2669 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT); in widenScalar()
2675 case TargetOpcode::G_SELECT: in widenScalar()
2681 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); in widenScalar()
2682 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); in widenScalar()
2692 case TargetOpcode::G_FPTOSI: in widenScalar()
2693 case TargetOpcode::G_FPTOUI: in widenScalar()
2694 case TargetOpcode::G_INTRINSIC_LRINT: in widenScalar()
2695 case TargetOpcode::G_INTRINSIC_LLRINT: in widenScalar()
2696 case TargetOpcode::G_IS_FPCLASS: in widenScalar()
2702 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); in widenScalar()
2706 case TargetOpcode::G_SITOFP: in widenScalar()
2710 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); in widenScalar()
2712 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); in widenScalar()
2716 case TargetOpcode::G_UITOFP: in widenScalar()
2720 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); in widenScalar()
2722 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); in widenScalar()
2726 case TargetOpcode::G_LOAD: in widenScalar()
2727 case TargetOpcode::G_SEXTLOAD: in widenScalar()
2728 case TargetOpcode::G_ZEXTLOAD: in widenScalar()
2734 case TargetOpcode::G_STORE: { in widenScalar()
2745 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT; in widenScalar()
2751 case TargetOpcode::G_CONSTANT: { in widenScalar()
2756 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || in widenScalar()
2757 ExtOpc == TargetOpcode::G_ANYEXT) && in widenScalar()
2760 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT) in widenScalar()
2770 case TargetOpcode::G_FCONSTANT: { in widenScalar()
2777 widenScalarDst(*IntCst, WideTy, 0, TargetOpcode::G_TRUNC); in widenScalar()
2781 case TargetOpcode::G_IMPLICIT_DEF: { in widenScalar()
2787 case TargetOpcode::G_BRCOND: in widenScalar()
2793 case TargetOpcode::G_FCMP: in widenScalar()
2798 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); in widenScalar()
2799 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT); in widenScalar()
2804 case TargetOpcode::G_ICMP: in widenScalar()
2811 ? TargetOpcode::G_SEXT in widenScalar()
2812 : TargetOpcode::G_ZEXT; in widenScalar()
2819 case TargetOpcode::G_PTR_ADD: in widenScalar()
2822 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); in widenScalar()
2826 case TargetOpcode::G_PHI: { in widenScalar()
2833 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT); in widenScalar()
2842 case TargetOpcode::G_EXTRACT_VECTOR_ELT: { in widenScalar()
2850 TargetOpcode::G_ANYEXT); in widenScalar()
2861 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); in widenScalar()
2865 case TargetOpcode::G_INSERT_VECTOR_ELT: { in widenScalar()
2870 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2871 widenScalarSrc(MI, WideEltTy, 2, TargetOpcode::G_ANYEXT); in widenScalar()
2884 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
2885 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); in widenScalar()
2894 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); in widenScalar()
2901 case TargetOpcode::G_FADD: in widenScalar()
2902 case TargetOpcode::G_FMUL: in widenScalar()
2903 case TargetOpcode::G_FSUB: in widenScalar()
2904 case TargetOpcode::G_FMA: in widenScalar()
2905 case TargetOpcode::G_FMAD: in widenScalar()
2906 case TargetOpcode::G_FNEG: in widenScalar()
2907 case TargetOpcode::G_FABS: in widenScalar()
2908 case TargetOpcode::G_FCANONICALIZE: in widenScalar()
2909 case TargetOpcode::G_FMINNUM: in widenScalar()
2910 case TargetOpcode::G_FMAXNUM: in widenScalar()
2911 case TargetOpcode::G_FMINNUM_IEEE: in widenScalar()
2912 case TargetOpcode::G_FMAXNUM_IEEE: in widenScalar()
2913 case TargetOpcode::G_FMINIMUM: in widenScalar()
2914 case TargetOpcode::G_FMAXIMUM: in widenScalar()
2915 case TargetOpcode::G_FDIV: in widenScalar()
2916 case TargetOpcode::G_FREM: in widenScalar()
2917 case TargetOpcode::G_FCEIL: in widenScalar()
2918 case TargetOpcode::G_FFLOOR: in widenScalar()
2919 case TargetOpcode::G_FCOS: in widenScalar()
2920 case TargetOpcode::G_FSIN: in widenScalar()
2921 case TargetOpcode::G_FTAN: in widenScalar()
2922 case TargetOpcode::G_FACOS: in widenScalar()
2923 case TargetOpcode::G_FASIN: in widenScalar()
2924 case TargetOpcode::G_FATAN: in widenScalar()
2925 case TargetOpcode::G_FCOSH: in widenScalar()
2926 case TargetOpcode::G_FSINH: in widenScalar()
2927 case TargetOpcode::G_FTANH: in widenScalar()
2928 case TargetOpcode::G_FLOG10: in widenScalar()
2929 case TargetOpcode::G_FLOG: in widenScalar()
2930 case TargetOpcode::G_FLOG2: in widenScalar()
2931 case TargetOpcode::G_FRINT: in widenScalar()
2932 case TargetOpcode::G_FNEARBYINT: in widenScalar()
2933 case TargetOpcode::G_FSQRT: in widenScalar()
2934 case TargetOpcode::G_FEXP: in widenScalar()
2935 case TargetOpcode::G_FEXP2: in widenScalar()
2936 case TargetOpcode::G_FEXP10: in widenScalar()
2937 case TargetOpcode::G_FPOW: in widenScalar()
2938 case TargetOpcode::G_INTRINSIC_TRUNC: in widenScalar()
2939 case TargetOpcode::G_INTRINSIC_ROUND: in widenScalar()
2940 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: in widenScalar()
2945 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT); in widenScalar()
2947 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); in widenScalar()
2950 case TargetOpcode::G_FPOWI: in widenScalar()
2951 case TargetOpcode::G_FLDEXP: in widenScalar()
2952 case TargetOpcode::G_STRICT_FLDEXP: { in widenScalar()
2954 if (MI.getOpcode() == TargetOpcode::G_STRICT_FLDEXP) in widenScalar()
2958 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); in widenScalar()
2959 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); in widenScalar()
2968 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); in widenScalar()
2975 case TargetOpcode::G_FFREXP: { in widenScalar()
2979 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); in widenScalar()
2980 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); in widenScalar()
2988 case TargetOpcode::G_INTTOPTR: in widenScalar()
2993 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); in widenScalar()
2996 case TargetOpcode::G_PTRTOINT: in widenScalar()
3004 case TargetOpcode::G_BUILD_VECTOR: { in widenScalar()
3009 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT); in widenScalar()
3014 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); in widenScalar()
3022 case TargetOpcode::G_SEXT_INREG: in widenScalar()
3027 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
3028 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC); in widenScalar()
3031 case TargetOpcode::G_PTRMASK: { in widenScalar()
3035 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); in widenScalar()
3039 case TargetOpcode::G_VECREDUCE_FADD: in widenScalar()
3040 case TargetOpcode::G_VECREDUCE_FMUL: in widenScalar()
3041 case TargetOpcode::G_VECREDUCE_FMIN: in widenScalar()
3042 case TargetOpcode::G_VECREDUCE_FMAX: in widenScalar()
3043 case TargetOpcode::G_VECREDUCE_FMINIMUM: in widenScalar()
3044 case TargetOpcode::G_VECREDUCE_FMAXIMUM: { in widenScalar()
3053 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT); in widenScalar()
3054 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); in widenScalar()
3058 case TargetOpcode::G_VSCALE: { in widenScalar()
3070 case TargetOpcode::G_SPLAT_VECTOR: { in widenScalar()
3075 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); in widenScalar()
3108 MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, DstReg, Addr, *MMO); in emitLoadFromConstantPool()
3448 if (!LI.isLegal({TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) { in bitcastConcatVector()
3575 auto LargeLoad = MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, AnyExtTy, in lowerLoad()
3708 case TargetOpcode::G_LOAD: { in bitcast()
3726 case TargetOpcode::G_STORE: { in bitcast()
3742 case TargetOpcode::G_SELECT: { in bitcast()
3759 case TargetOpcode::G_AND: in bitcast()
3760 case TargetOpcode::G_OR: in bitcast()
3761 case TargetOpcode::G_XOR: { in bitcast()
3769 case TargetOpcode::G_EXTRACT_VECTOR_ELT: in bitcast()
3771 case TargetOpcode::G_INSERT_VECTOR_ELT: in bitcast()
3773 case TargetOpcode::G_CONCAT_VECTORS: in bitcast()
3789 using namespace TargetOpcode; in lower()
3794 case TargetOpcode::G_FCONSTANT: in lower()
3796 case TargetOpcode::G_BITCAST: in lower()
3798 case TargetOpcode::G_SREM: in lower()
3799 case TargetOpcode::G_UREM: { in lower()
3810 case TargetOpcode::G_SADDO: in lower()
3811 case TargetOpcode::G_SSUBO: in lower()
3813 case TargetOpcode::G_UMULH: in lower()
3814 case TargetOpcode::G_SMULH: in lower()
3816 case TargetOpcode::G_SMULO: in lower()
3817 case TargetOpcode::G_UMULO: { in lower()
3823 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO in lower()
3824 ? TargetOpcode::G_SMULH in lower()
3825 : TargetOpcode::G_UMULH; in lower()
3829 MI.setDesc(TII.get(TargetOpcode::G_MUL)); in lower()
3841 if (Opcode == TargetOpcode::G_SMULH) { in lower()
3850 case TargetOpcode::G_FNEG: { in lower()
3864 case TargetOpcode::G_FSUB: in lower()
3865 case TargetOpcode::G_STRICT_FSUB: { in lower()
3872 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB) in lower()
3880 case TargetOpcode::G_FMAD: in lower()
3882 case TargetOpcode::G_FFLOOR: in lower()
3884 case TargetOpcode::G_INTRINSIC_ROUND: in lower()
3886 case TargetOpcode::G_FRINT: { in lower()
3889 changeOpcode(MI, TargetOpcode::G_INTRINSIC_ROUNDEVEN); in lower()
3892 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { in lower()
3902 case TargetOpcode::G_LOAD: in lower()
3903 case TargetOpcode::G_SEXTLOAD: in lower()
3904 case TargetOpcode::G_ZEXTLOAD: in lower()
3906 case TargetOpcode::G_STORE: in lower()
3908 case TargetOpcode::G_CTLZ_ZERO_UNDEF: in lower()
3909 case TargetOpcode::G_CTTZ_ZERO_UNDEF: in lower()
3910 case TargetOpcode::G_CTLZ: in lower()
3911 case TargetOpcode::G_CTTZ: in lower()
3912 case TargetOpcode::G_CTPOP: in lower()
4021 case TargetOpcode::G_SEXT_INREG: { in lower()
4376 auto Phi = MIRBuilder.buildInstr(TargetOpcode::G_PHI); in fewerElementsVectorPhi()
4440 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in fewerElementsVectorUnmergeValues()
4543 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT; in fewerElementsVectorExtractInsertVectorElt()
4575 TargetOpcode::G_ANYEXT); in fewerElementsVectorExtractInsertVectorElt()
4709 using namespace TargetOpcode; in fewerElementsVector()
4858 case TargetOpcode::G_VECREDUCE_SEQ_FADD: in fewerElementsVector()
4859 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: in fewerElementsVector()
4877 assert(MI.getOpcode() == TargetOpcode::G_BITCAST && in fewerElementsBitcast()
4907 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); in fewerElementsVectorShuffle()
5139 assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD || in fewerElementsVectorSeqReductions()
5140 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) && in fewerElementsVectorSeqReductions()
5142 unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD in fewerElementsVectorSeqReductions()
5143 ? TargetOpcode::G_FADD in fewerElementsVectorSeqReductions()
5144 : TargetOpcode::G_FMUL; in fewerElementsVectorSeqReductions()
5208 if (MI.getOpcode() == TargetOpcode::G_SHL) { in narrowScalarShiftByConstant()
5226 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) { in narrowScalarShiftByConstant()
5329 case TargetOpcode::G_SHL: { in narrowScalarShift()
5349 case TargetOpcode::G_LSHR: in narrowScalarShift()
5350 case TargetOpcode::G_ASHR: { in narrowScalarShift()
5360 if (MI.getOpcode() == TargetOpcode::G_LSHR) { in narrowScalarShift()
5414 case TargetOpcode::G_VECREDUCE_ADD: in getNeutralElementForVecReduce()
5415 case TargetOpcode::G_VECREDUCE_OR: in getNeutralElementForVecReduce()
5416 case TargetOpcode::G_VECREDUCE_XOR: in getNeutralElementForVecReduce()
5417 case TargetOpcode::G_VECREDUCE_UMAX: in getNeutralElementForVecReduce()
5419 case TargetOpcode::G_VECREDUCE_MUL: in getNeutralElementForVecReduce()
5421 case TargetOpcode::G_VECREDUCE_AND: in getNeutralElementForVecReduce()
5422 case TargetOpcode::G_VECREDUCE_UMIN: in getNeutralElementForVecReduce()
5425 case TargetOpcode::G_VECREDUCE_SMAX: in getNeutralElementForVecReduce()
5428 case TargetOpcode::G_VECREDUCE_SMIN: in getNeutralElementForVecReduce()
5431 case TargetOpcode::G_VECREDUCE_FADD: in getNeutralElementForVecReduce()
5433 case TargetOpcode::G_VECREDUCE_FMUL: in getNeutralElementForVecReduce()
5435 case TargetOpcode::G_VECREDUCE_FMINIMUM: in getNeutralElementForVecReduce()
5436 case TargetOpcode::G_VECREDUCE_FMAXIMUM: in getNeutralElementForVecReduce()
5448 case TargetOpcode::G_IMPLICIT_DEF: in moreElementsVector()
5449 case TargetOpcode::G_LOAD: { in moreElementsVector()
5457 case TargetOpcode::G_STORE: in moreElementsVector()
5464 case TargetOpcode::G_AND: in moreElementsVector()
5465 case TargetOpcode::G_OR: in moreElementsVector()
5466 case TargetOpcode::G_XOR: in moreElementsVector()
5467 case TargetOpcode::G_ADD: in moreElementsVector()
5468 case TargetOpcode::G_SUB: in moreElementsVector()
5469 case TargetOpcode::G_MUL: in moreElementsVector()
5470 case TargetOpcode::G_FADD: in moreElementsVector()
5471 case TargetOpcode::G_FSUB: in moreElementsVector()
5472 case TargetOpcode::G_FMUL: in moreElementsVector()
5473 case TargetOpcode::G_FDIV: in moreElementsVector()
5474 case TargetOpcode::G_FCOPYSIGN: in moreElementsVector()
5475 case TargetOpcode::G_UADDSAT: in moreElementsVector()
5476 case TargetOpcode::G_USUBSAT: in moreElementsVector()
5477 case TargetOpcode::G_SADDSAT: in moreElementsVector()
5478 case TargetOpcode::G_SSUBSAT: in moreElementsVector()
5479 case TargetOpcode::G_SMIN: in moreElementsVector()
5480 case TargetOpcode::G_SMAX: in moreElementsVector()
5481 case TargetOpcode::G_UMIN: in moreElementsVector()
5482 case TargetOpcode::G_UMAX: in moreElementsVector()
5483 case TargetOpcode::G_FMINNUM: in moreElementsVector()
5484 case TargetOpcode::G_FMAXNUM: in moreElementsVector()
5485 case TargetOpcode::G_FMINNUM_IEEE: in moreElementsVector()
5486 case TargetOpcode::G_FMAXNUM_IEEE: in moreElementsVector()
5487 case TargetOpcode::G_FMINIMUM: in moreElementsVector()
5488 case TargetOpcode::G_FMAXIMUM: in moreElementsVector()
5489 case TargetOpcode::G_STRICT_FADD: in moreElementsVector()
5490 case TargetOpcode::G_STRICT_FSUB: in moreElementsVector()
5491 case TargetOpcode::G_STRICT_FMUL: in moreElementsVector()
5492 case TargetOpcode::G_SHL: in moreElementsVector()
5493 case TargetOpcode::G_ASHR: in moreElementsVector()
5494 case TargetOpcode::G_LSHR: { in moreElementsVector()
5502 case TargetOpcode::G_FMA: in moreElementsVector()
5503 case TargetOpcode::G_STRICT_FMA: in moreElementsVector()
5504 case TargetOpcode::G_FSHR: in moreElementsVector()
5505 case TargetOpcode::G_FSHL: { in moreElementsVector()
5514 case TargetOpcode::G_EXTRACT_VECTOR_ELT: in moreElementsVector()
5515 case TargetOpcode::G_EXTRACT: in moreElementsVector()
5522 case TargetOpcode::G_INSERT: in moreElementsVector()
5523 case TargetOpcode::G_INSERT_VECTOR_ELT: in moreElementsVector()
5524 case TargetOpcode::G_FREEZE: in moreElementsVector()
5525 case TargetOpcode::G_FNEG: in moreElementsVector()
5526 case TargetOpcode::G_FABS: in moreElementsVector()
5527 case TargetOpcode::G_FSQRT: in moreElementsVector()
5528 case TargetOpcode::G_FCEIL: in moreElementsVector()
5529 case TargetOpcode::G_FFLOOR: in moreElementsVector()
5530 case TargetOpcode::G_FNEARBYINT: in moreElementsVector()
5531 case TargetOpcode::G_FRINT: in moreElementsVector()
5532 case TargetOpcode::G_INTRINSIC_ROUND: in moreElementsVector()
5533 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: in moreElementsVector()
5534 case TargetOpcode::G_INTRINSIC_TRUNC: in moreElementsVector()
5535 case TargetOpcode::G_BSWAP: in moreElementsVector()
5536 case TargetOpcode::G_FCANONICALIZE: in moreElementsVector()
5537 case TargetOpcode::G_SEXT_INREG: in moreElementsVector()
5538 case TargetOpcode::G_ABS: in moreElementsVector()
5546 case TargetOpcode::G_SELECT: { in moreElementsVector()
5572 case TargetOpcode::G_UNMERGE_VALUES: in moreElementsVector()
5574 case TargetOpcode::G_PHI: in moreElementsVector()
5576 case TargetOpcode::G_SHUFFLE_VECTOR: in moreElementsVector()
5578 case TargetOpcode::G_BUILD_VECTOR: { in moreElementsVector()
5593 case TargetOpcode::G_SEXT: in moreElementsVector()
5594 case TargetOpcode::G_ZEXT: in moreElementsVector()
5595 case TargetOpcode::G_ANYEXT: in moreElementsVector()
5596 case TargetOpcode::G_TRUNC: in moreElementsVector()
5597 case TargetOpcode::G_FPTRUNC: in moreElementsVector()
5598 case TargetOpcode::G_FPEXT: in moreElementsVector()
5599 case TargetOpcode::G_FPTOSI: in moreElementsVector()
5600 case TargetOpcode::G_FPTOUI: in moreElementsVector()
5601 case TargetOpcode::G_SITOFP: in moreElementsVector()
5602 case TargetOpcode::G_UITOFP: { in moreElementsVector()
5622 case TargetOpcode::G_ICMP: in moreElementsVector()
5623 case TargetOpcode::G_FCMP: { in moreElementsVector()
5637 case TargetOpcode::G_BITCAST: { in moreElementsVector()
5658 case TargetOpcode::G_VECREDUCE_FADD: in moreElementsVector()
5659 case TargetOpcode::G_VECREDUCE_FMUL: in moreElementsVector()
5660 case TargetOpcode::G_VECREDUCE_ADD: in moreElementsVector()
5661 case TargetOpcode::G_VECREDUCE_MUL: in moreElementsVector()
5662 case TargetOpcode::G_VECREDUCE_AND: in moreElementsVector()
5663 case TargetOpcode::G_VECREDUCE_OR: in moreElementsVector()
5664 case TargetOpcode::G_VECREDUCE_XOR: in moreElementsVector()
5665 case TargetOpcode::G_VECREDUCE_SMAX: in moreElementsVector()
5666 case TargetOpcode::G_VECREDUCE_SMIN: in moreElementsVector()
5667 case TargetOpcode::G_VECREDUCE_UMAX: in moreElementsVector()
5668 case TargetOpcode::G_VECREDUCE_UMIN: { in moreElementsVector()
5887 case TargetOpcode::G_SADDO: in narrowScalarAddSub()
5888 case TargetOpcode::G_SADDE: in narrowScalarAddSub()
5889 case TargetOpcode::G_UADDO: in narrowScalarAddSub()
5890 case TargetOpcode::G_UADDE: in narrowScalarAddSub()
5891 case TargetOpcode::G_ADD: in narrowScalarAddSub()
5892 OpO = TargetOpcode::G_UADDO; in narrowScalarAddSub()
5893 OpE = TargetOpcode::G_UADDE; in narrowScalarAddSub()
5894 OpF = TargetOpcode::G_UADDE; in narrowScalarAddSub()
5895 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE) in narrowScalarAddSub()
5896 OpF = TargetOpcode::G_SADDE; in narrowScalarAddSub()
5898 case TargetOpcode::G_SSUBO: in narrowScalarAddSub()
5899 case TargetOpcode::G_SSUBE: in narrowScalarAddSub()
5900 case TargetOpcode::G_USUBO: in narrowScalarAddSub()
5901 case TargetOpcode::G_USUBE: in narrowScalarAddSub()
5902 case TargetOpcode::G_SUB: in narrowScalarAddSub()
5903 OpO = TargetOpcode::G_USUBO; in narrowScalarAddSub()
5904 OpE = TargetOpcode::G_USUBE; in narrowScalarAddSub()
5905 OpF = TargetOpcode::G_USUBE; in narrowScalarAddSub()
5906 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE) in narrowScalarAddSub()
5907 OpF = TargetOpcode::G_SSUBE; in narrowScalarAddSub()
5982 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH; in narrowScalarMul()
6004 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI; in narrowScalarFPTOI()
6018 IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT); in narrowScalarFPTOI()
6293 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF; in narrowScalarCTLZ()
6326 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF; in narrowScalarCTTZ()
6408 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { in lowerBitCount()
6411 MI.setDesc(TII.get(TargetOpcode::G_CTLZ)); in lowerBitCount()
6415 case TargetOpcode::G_CTLZ: { in lowerBitCount()
6419 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) { in lowerBitCount()
6455 case TargetOpcode::G_CTTZ_ZERO_UNDEF: { in lowerBitCount()
6458 MI.setDesc(TII.get(TargetOpcode::G_CTTZ)); in lowerBitCount()
6462 case TargetOpcode::G_CTTZ: { in lowerBitCount()
6466 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) { in lowerBitCount()
6486 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) && in lowerBitCount()
6487 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) { in lowerBitCount()
6495 MI.setDesc(TII.get(TargetOpcode::G_CTPOP)); in lowerBitCount()
6500 case TargetOpcode::G_CTPOP: { in lowerBitCount()
6549 auto Action = LI.getAction({TargetOpcode::G_MUL, {Ty}}).Action; in lowerBitCount()
6594 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL; in lowerFunnelShiftWithInverse()
6595 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL; in lowerFunnelShiftWithInverse()
6629 const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL; in lowerFunnelShiftAsShifts()
6687 bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL; in lowerFunnelShift()
6688 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL; in lowerFunnelShift()
6754 assert(MI.getOpcode() == TargetOpcode::G_TRUNC); in lowerTRUNC()
6804 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL; in lowerRotateWithReverseRotate()
6805 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL; in lowerRotateWithReverseRotate()
6816 bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL; in lowerRotate()
6821 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL; in lowerRotate()
6827 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR; in lowerRotate()
6828 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR; in lowerRotate()
6848 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR; in lowerRotate()
6849 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL; in lowerRotate()
7249 case TargetOpcode::G_SMIN: in minMaxToCompare()
7251 case TargetOpcode::G_SMAX: in minMaxToCompare()
7253 case TargetOpcode::G_UMIN: in minMaxToCompare()
7255 case TargetOpcode::G_UMAX: in minMaxToCompare()
7349 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ? in lowerFMinNumMaxNum()
7350 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE; in lowerFMinNumMaxNum()
7531 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) in lowerExtractInsertVectorElt()
7661 MRI.getVRegDef(Passthru)->getOpcode() != TargetOpcode::G_IMPLICIT_DEF; in lowerVECTOR_COMPRESS()
7675 Popcount = MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_ADD, in lowerVECTOR_COMPRESS()
7706 OutPos = MIRBuilder.buildInstr(TargetOpcode::G_UMIN, {IdxTy}, in lowerVECTOR_COMPRESS()
7940 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO; in lowerSADDO_SSUBO()
7985 case TargetOpcode::G_UADDSAT: in lowerAddSubSatToMinMax()
7988 BaseOp = TargetOpcode::G_ADD; in lowerAddSubSatToMinMax()
7990 case TargetOpcode::G_SADDSAT: in lowerAddSubSatToMinMax()
7993 BaseOp = TargetOpcode::G_ADD; in lowerAddSubSatToMinMax()
7995 case TargetOpcode::G_USUBSAT: in lowerAddSubSatToMinMax()
7998 BaseOp = TargetOpcode::G_SUB; in lowerAddSubSatToMinMax()
8000 case TargetOpcode::G_SSUBSAT: in lowerAddSubSatToMinMax()
8003 BaseOp = TargetOpcode::G_SUB; in lowerAddSubSatToMinMax()
8061 case TargetOpcode::G_UADDSAT: in lowerAddSubSatToAddoSubo()
8064 OverflowOp = TargetOpcode::G_UADDO; in lowerAddSubSatToAddoSubo()
8066 case TargetOpcode::G_SADDSAT: in lowerAddSubSatToAddoSubo()
8069 OverflowOp = TargetOpcode::G_SADDO; in lowerAddSubSatToAddoSubo()
8071 case TargetOpcode::G_USUBSAT: in lowerAddSubSatToAddoSubo()
8074 OverflowOp = TargetOpcode::G_USUBO; in lowerAddSubSatToAddoSubo()
8076 case TargetOpcode::G_SSUBSAT: in lowerAddSubSatToAddoSubo()
8079 OverflowOp = TargetOpcode::G_SSUBO; in lowerAddSubSatToAddoSubo()
8118 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT || in lowerShlSat()
8119 MI.getOpcode() == TargetOpcode::G_USHLSAT) && in lowerShlSat()
8121 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT; in lowerShlSat()
8200 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); in lowerBitreverse()
8250 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER; in lowerReadWriteRegister()
8274 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH; in lowerSMULH_UMULH()
8275 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; in lowerSMULH_UMULH()
8284 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR; in lowerSMULH_UMULH()
8531 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV in lowerDIVREM()
8532 : TargetOpcode::G_UDIV, in lowerDIVREM()
8535 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM in lowerDIVREM()
8536 : TargetOpcode::G_UREM, in lowerDIVREM()
8599 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::COPY)); in lowerVectorReduction()
8777 MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); in lowerMemset()
8877 assert(MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE); in lowerMemcpyInline()
8909 assert(MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE); in lowerMemcpyInline()
8930 MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); in lowerMemcpy()
9038 MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); in lowerMemmove()
9133 assert((Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMMOVE || in lowerMemCpyFamily()
9134 Opc == TargetOpcode::G_MEMSET) && in lowerMemCpyFamily()
9144 if (Opc != TargetOpcode::G_MEMSET) { in lowerMemCpyFamily()
9162 if (Opc == TargetOpcode::G_MEMCPY_INLINE) in lowerMemCpyFamily()
9173 if (Opc == TargetOpcode::G_MEMCPY) { in lowerMemCpyFamily()
9181 if (Opc == TargetOpcode::G_MEMMOVE) in lowerMemCpyFamily()
9183 if (Opc == TargetOpcode::G_MEMSET) in lowerMemCpyFamily()