10b57cec5SDimitry Andric //==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file defines the interfaces that AArch64 uses to lower LLVM code into a 100b57cec5SDimitry Andric // selection DAG. 110b57cec5SDimitry Andric // 120b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andric #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H 150b57cec5SDimitry Andric #define LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H 160b57cec5SDimitry Andric 170b57cec5SDimitry Andric #include "AArch64.h" 18*bdd1243dSDimitry Andric #include "Utils/AArch64SMEAttributes.h" 190b57cec5SDimitry Andric #include "llvm/CodeGen/CallingConvLower.h" 20fe6060f1SDimitry Andric #include "llvm/CodeGen/MachineFunction.h" 210b57cec5SDimitry Andric #include "llvm/CodeGen/SelectionDAG.h" 220b57cec5SDimitry Andric #include "llvm/CodeGen/TargetLowering.h" 230b57cec5SDimitry Andric #include "llvm/IR/CallingConv.h" 240b57cec5SDimitry Andric #include "llvm/IR/Instruction.h" 250b57cec5SDimitry Andric 260b57cec5SDimitry Andric namespace llvm { 270b57cec5SDimitry Andric 280b57cec5SDimitry Andric namespace AArch64ISD { 290b57cec5SDimitry Andric 305ffd83dbSDimitry Andric // For predicated nodes where the result is a vector, the operation is 315ffd83dbSDimitry Andric // controlled by a governing predicate and the inactive lanes are explicitly 325ffd83dbSDimitry Andric // defined with a value, please stick the following naming convention: 335ffd83dbSDimitry Andric // 345ffd83dbSDimitry Andric // _MERGE_OP<n> The result value is a vector with inactive lanes equal 355ffd83dbSDimitry Andric // to source operand OP<n>. 365ffd83dbSDimitry Andric // 375ffd83dbSDimitry Andric // _MERGE_ZERO The result value is a vector with inactive lanes 385ffd83dbSDimitry Andric // actively zeroed. 395ffd83dbSDimitry Andric // 405ffd83dbSDimitry Andric // _MERGE_PASSTHRU The result value is a vector with inactive lanes equal 415ffd83dbSDimitry Andric // to the last source operand which only purpose is being 425ffd83dbSDimitry Andric // a passthru value. 435ffd83dbSDimitry Andric // 445ffd83dbSDimitry Andric // For other cases where no explicit action is needed to set the inactive lanes, 455ffd83dbSDimitry Andric // or when the result is not a vector and it is needed or helpful to 465ffd83dbSDimitry Andric // distinguish a node from similar unpredicated nodes, use: 475ffd83dbSDimitry Andric // 485ffd83dbSDimitry Andric // _PRED 495ffd83dbSDimitry Andric // 500b57cec5SDimitry Andric enum NodeType : unsigned { 510b57cec5SDimitry Andric FIRST_NUMBER = ISD::BUILTIN_OP_END, 520b57cec5SDimitry Andric WrapperLarge, // 4-instruction MOVZ/MOVK sequence for 64-bit addresses. 530b57cec5SDimitry Andric CALL, // Function call. 540b57cec5SDimitry Andric 55fe6060f1SDimitry Andric // Pseudo for a OBJC call that gets emitted together with a special `mov 56fe6060f1SDimitry Andric // x29, x29` marker instruction. 57fe6060f1SDimitry Andric CALL_RVMARKER, 58fe6060f1SDimitry Andric 593a9a9c0cSDimitry Andric CALL_BTI, // Function call followed by a BTI instruction. 603a9a9c0cSDimitry Andric 61*bdd1243dSDimitry Andric // Essentially like a normal COPY that works on GPRs, but cannot be 62*bdd1243dSDimitry Andric // rematerialised by passes like the simple register coalescer. It's 63*bdd1243dSDimitry Andric // required for SME when lowering calls because we cannot allow frame 64*bdd1243dSDimitry Andric // index calculations using addvl to slip in between the smstart/smstop 65*bdd1243dSDimitry Andric // and the bl instruction. The scalable vector length may change across 66*bdd1243dSDimitry Andric // the smstart/smstop boundary. 67*bdd1243dSDimitry Andric OBSCURE_COPY, 68*bdd1243dSDimitry Andric SMSTART, 69*bdd1243dSDimitry Andric SMSTOP, 70*bdd1243dSDimitry Andric RESTORE_ZA, 71*bdd1243dSDimitry Andric 720b57cec5SDimitry Andric // Produces the full sequence of instructions for getting the thread pointer 730b57cec5SDimitry Andric // offset of a variable into X0, using the TLSDesc model. 740b57cec5SDimitry Andric TLSDESC_CALLSEQ, 750b57cec5SDimitry Andric ADRP, // Page address of a TargetGlobalAddress operand. 760b57cec5SDimitry Andric ADR, // ADR 770b57cec5SDimitry Andric ADDlow, // Add the low 12 bits of a TargetGlobalAddress operand. 780b57cec5SDimitry Andric LOADgot, // Load from automatically generated descriptor (e.g. Global 790b57cec5SDimitry Andric // Offset Table, TLS record). 800b57cec5SDimitry Andric RET_FLAG, // Return with a flag operand. Operand 0 is the chain operand. 810b57cec5SDimitry Andric BRCOND, // Conditional branch instruction; "b.cond". 820b57cec5SDimitry Andric CSEL, 830b57cec5SDimitry Andric CSINV, // Conditional select invert. 840b57cec5SDimitry Andric CSNEG, // Conditional select negate. 850b57cec5SDimitry Andric CSINC, // Conditional select increment. 860b57cec5SDimitry Andric 870b57cec5SDimitry Andric // Pointer to the thread's local storage area. Materialised from TPIDR_EL0 on 880b57cec5SDimitry Andric // ELF. 890b57cec5SDimitry Andric THREAD_POINTER, 900b57cec5SDimitry Andric ADC, 910b57cec5SDimitry Andric SBC, // adc, sbc instructions 920b57cec5SDimitry Andric 93e8d8bef9SDimitry Andric // Predicated instructions where inactive lanes produce undefined results. 9404eeddc0SDimitry Andric ABDS_PRED, 9504eeddc0SDimitry Andric ABDU_PRED, 965ffd83dbSDimitry Andric FADD_PRED, 97e8d8bef9SDimitry Andric FDIV_PRED, 985ffd83dbSDimitry Andric FMA_PRED, 99fe6060f1SDimitry Andric FMAX_PRED, 10004eeddc0SDimitry Andric FMAXNM_PRED, 101fe6060f1SDimitry Andric FMIN_PRED, 10204eeddc0SDimitry Andric FMINNM_PRED, 103e8d8bef9SDimitry Andric FMUL_PRED, 104e8d8bef9SDimitry Andric FSUB_PRED, 105*bdd1243dSDimitry Andric HADDS_PRED, 106*bdd1243dSDimitry Andric HADDU_PRED, 107e8d8bef9SDimitry Andric MUL_PRED, 108fe6060f1SDimitry Andric MULHS_PRED, 109fe6060f1SDimitry Andric MULHU_PRED, 110*bdd1243dSDimitry Andric RHADDS_PRED, 111*bdd1243dSDimitry Andric RHADDU_PRED, 112e8d8bef9SDimitry Andric SDIV_PRED, 113e8d8bef9SDimitry Andric SHL_PRED, 114e8d8bef9SDimitry Andric SMAX_PRED, 115e8d8bef9SDimitry Andric SMIN_PRED, 116e8d8bef9SDimitry Andric SRA_PRED, 117e8d8bef9SDimitry Andric SRL_PRED, 118e8d8bef9SDimitry Andric UDIV_PRED, 119e8d8bef9SDimitry Andric UMAX_PRED, 120e8d8bef9SDimitry Andric UMIN_PRED, 121e8d8bef9SDimitry Andric 122fe6060f1SDimitry Andric // Unpredicated vector instructions 123fe6060f1SDimitry Andric BIC, 124fe6060f1SDimitry Andric 1254824e7fdSDimitry Andric SRAD_MERGE_OP1, 1264824e7fdSDimitry Andric 127e8d8bef9SDimitry Andric // Predicated instructions with the result of inactive lanes provided by the 128e8d8bef9SDimitry Andric // last operand. 129e8d8bef9SDimitry Andric FABS_MERGE_PASSTHRU, 130e8d8bef9SDimitry Andric FCEIL_MERGE_PASSTHRU, 131e8d8bef9SDimitry Andric FFLOOR_MERGE_PASSTHRU, 132e8d8bef9SDimitry Andric FNEARBYINT_MERGE_PASSTHRU, 133e8d8bef9SDimitry Andric FNEG_MERGE_PASSTHRU, 134e8d8bef9SDimitry Andric FRECPX_MERGE_PASSTHRU, 135e8d8bef9SDimitry Andric FRINT_MERGE_PASSTHRU, 136e8d8bef9SDimitry Andric FROUND_MERGE_PASSTHRU, 137e8d8bef9SDimitry Andric FROUNDEVEN_MERGE_PASSTHRU, 138e8d8bef9SDimitry Andric FSQRT_MERGE_PASSTHRU, 139e8d8bef9SDimitry Andric FTRUNC_MERGE_PASSTHRU, 140e8d8bef9SDimitry Andric FP_ROUND_MERGE_PASSTHRU, 141e8d8bef9SDimitry Andric FP_EXTEND_MERGE_PASSTHRU, 142e8d8bef9SDimitry Andric UINT_TO_FP_MERGE_PASSTHRU, 143e8d8bef9SDimitry Andric SINT_TO_FP_MERGE_PASSTHRU, 144e8d8bef9SDimitry Andric FCVTZU_MERGE_PASSTHRU, 145e8d8bef9SDimitry Andric FCVTZS_MERGE_PASSTHRU, 146e8d8bef9SDimitry Andric SIGN_EXTEND_INREG_MERGE_PASSTHRU, 147e8d8bef9SDimitry Andric ZERO_EXTEND_INREG_MERGE_PASSTHRU, 148e8d8bef9SDimitry Andric ABS_MERGE_PASSTHRU, 149e8d8bef9SDimitry Andric NEG_MERGE_PASSTHRU, 1505ffd83dbSDimitry Andric 1515ffd83dbSDimitry Andric SETCC_MERGE_ZERO, 1525ffd83dbSDimitry Andric 1530b57cec5SDimitry Andric // Arithmetic instructions which write flags. 1540b57cec5SDimitry Andric ADDS, 1550b57cec5SDimitry Andric SUBS, 1560b57cec5SDimitry Andric ADCS, 1570b57cec5SDimitry Andric SBCS, 1580b57cec5SDimitry Andric ANDS, 1590b57cec5SDimitry Andric 1600b57cec5SDimitry Andric // Conditional compares. Operands: left,right,falsecc,cc,flags 1610b57cec5SDimitry Andric CCMP, 1620b57cec5SDimitry Andric CCMN, 1630b57cec5SDimitry Andric FCCMP, 1640b57cec5SDimitry Andric 1650b57cec5SDimitry Andric // Floating point comparison 1660b57cec5SDimitry Andric FCMP, 1670b57cec5SDimitry Andric 1680b57cec5SDimitry Andric // Scalar extract 1690b57cec5SDimitry Andric EXTR, 1700b57cec5SDimitry Andric 1710b57cec5SDimitry Andric // Scalar-to-vector duplication 1720b57cec5SDimitry Andric DUP, 1730b57cec5SDimitry Andric DUPLANE8, 1740b57cec5SDimitry Andric DUPLANE16, 1750b57cec5SDimitry Andric DUPLANE32, 1760b57cec5SDimitry Andric DUPLANE64, 17781ad6265SDimitry Andric DUPLANE128, 1780b57cec5SDimitry Andric 1790b57cec5SDimitry Andric // Vector immedate moves 1800b57cec5SDimitry Andric MOVI, 1810b57cec5SDimitry Andric MOVIshift, 1820b57cec5SDimitry Andric MOVIedit, 1830b57cec5SDimitry Andric MOVImsl, 1840b57cec5SDimitry Andric FMOV, 1850b57cec5SDimitry Andric MVNIshift, 1860b57cec5SDimitry Andric MVNImsl, 1870b57cec5SDimitry Andric 1880b57cec5SDimitry Andric // Vector immediate ops 1890b57cec5SDimitry Andric BICi, 1900b57cec5SDimitry Andric ORRi, 1910b57cec5SDimitry Andric 1925ffd83dbSDimitry Andric // Vector bitwise select: similar to ISD::VSELECT but not all bits within an 1930b57cec5SDimitry Andric // element must be identical. 1945ffd83dbSDimitry Andric BSP, 1950b57cec5SDimitry Andric 1960b57cec5SDimitry Andric // Vector shuffles 1970b57cec5SDimitry Andric ZIP1, 1980b57cec5SDimitry Andric ZIP2, 1990b57cec5SDimitry Andric UZP1, 2000b57cec5SDimitry Andric UZP2, 2010b57cec5SDimitry Andric TRN1, 2020b57cec5SDimitry Andric TRN2, 2030b57cec5SDimitry Andric REV16, 2040b57cec5SDimitry Andric REV32, 2050b57cec5SDimitry Andric REV64, 2060b57cec5SDimitry Andric EXT, 207fe6060f1SDimitry Andric SPLICE, 2080b57cec5SDimitry Andric 2090b57cec5SDimitry Andric // Vector shift by scalar 2100b57cec5SDimitry Andric VSHL, 2110b57cec5SDimitry Andric VLSHR, 2120b57cec5SDimitry Andric VASHR, 2130b57cec5SDimitry Andric 2140b57cec5SDimitry Andric // Vector shift by scalar (again) 2150b57cec5SDimitry Andric SQSHL_I, 2160b57cec5SDimitry Andric UQSHL_I, 2170b57cec5SDimitry Andric SQSHLU_I, 2180b57cec5SDimitry Andric SRSHR_I, 2190b57cec5SDimitry Andric URSHR_I, 2200b57cec5SDimitry Andric 2215ffd83dbSDimitry Andric // Vector shift by constant and insert 2225ffd83dbSDimitry Andric VSLI, 2235ffd83dbSDimitry Andric VSRI, 2245ffd83dbSDimitry Andric 2250b57cec5SDimitry Andric // Vector comparisons 2260b57cec5SDimitry Andric CMEQ, 2270b57cec5SDimitry Andric CMGE, 2280b57cec5SDimitry Andric CMGT, 2290b57cec5SDimitry Andric CMHI, 2300b57cec5SDimitry Andric CMHS, 2310b57cec5SDimitry Andric FCMEQ, 2320b57cec5SDimitry Andric FCMGE, 2330b57cec5SDimitry Andric FCMGT, 2340b57cec5SDimitry Andric 2350b57cec5SDimitry Andric // Vector zero comparisons 2360b57cec5SDimitry Andric CMEQz, 2370b57cec5SDimitry Andric CMGEz, 2380b57cec5SDimitry Andric CMGTz, 2390b57cec5SDimitry Andric CMLEz, 2400b57cec5SDimitry Andric CMLTz, 2410b57cec5SDimitry Andric FCMEQz, 2420b57cec5SDimitry Andric FCMGEz, 2430b57cec5SDimitry Andric FCMGTz, 2440b57cec5SDimitry Andric FCMLEz, 2450b57cec5SDimitry Andric FCMLTz, 2460b57cec5SDimitry Andric 2470b57cec5SDimitry Andric // Vector across-lanes addition 2480b57cec5SDimitry Andric // Only the lower result lane is defined. 2490b57cec5SDimitry Andric SADDV, 2500b57cec5SDimitry Andric UADDV, 2510b57cec5SDimitry Andric 25281ad6265SDimitry Andric // Add Pairwise of two vectors 25381ad6265SDimitry Andric ADDP, 25481ad6265SDimitry Andric // Add Long Pairwise 25581ad6265SDimitry Andric SADDLP, 256fe6060f1SDimitry Andric UADDLP, 257fe6060f1SDimitry Andric 258fe6060f1SDimitry Andric // udot/sdot instructions 259fe6060f1SDimitry Andric UDOT, 260fe6060f1SDimitry Andric SDOT, 261e8d8bef9SDimitry Andric 2620b57cec5SDimitry Andric // Vector across-lanes min/max 2630b57cec5SDimitry Andric // Only the lower result lane is defined. 2640b57cec5SDimitry Andric SMINV, 2650b57cec5SDimitry Andric UMINV, 2660b57cec5SDimitry Andric SMAXV, 2670b57cec5SDimitry Andric UMAXV, 2680b57cec5SDimitry Andric 269e8d8bef9SDimitry Andric SADDV_PRED, 270e8d8bef9SDimitry Andric UADDV_PRED, 271480093f4SDimitry Andric SMAXV_PRED, 272480093f4SDimitry Andric UMAXV_PRED, 273480093f4SDimitry Andric SMINV_PRED, 274480093f4SDimitry Andric UMINV_PRED, 275480093f4SDimitry Andric ORV_PRED, 276480093f4SDimitry Andric EORV_PRED, 277480093f4SDimitry Andric ANDV_PRED, 278480093f4SDimitry Andric 2795ffd83dbSDimitry Andric // Vector bitwise insertion 2800b57cec5SDimitry Andric BIT, 2810b57cec5SDimitry Andric 2820b57cec5SDimitry Andric // Compare-and-branch 2830b57cec5SDimitry Andric CBZ, 2840b57cec5SDimitry Andric CBNZ, 2850b57cec5SDimitry Andric TBZ, 2860b57cec5SDimitry Andric TBNZ, 2870b57cec5SDimitry Andric 2880b57cec5SDimitry Andric // Tail calls 2890b57cec5SDimitry Andric TC_RETURN, 2900b57cec5SDimitry Andric 2910b57cec5SDimitry Andric // Custom prefetch handling 2920b57cec5SDimitry Andric PREFETCH, 2930b57cec5SDimitry Andric 2940b57cec5SDimitry Andric // {s|u}int to FP within a FP register. 2950b57cec5SDimitry Andric SITOF, 2960b57cec5SDimitry Andric UITOF, 2970b57cec5SDimitry Andric 2980b57cec5SDimitry Andric /// Natural vector cast. ISD::BITCAST is not natural in the big-endian 2990b57cec5SDimitry Andric /// world w.r.t vectors; which causes additional REV instructions to be 3000b57cec5SDimitry Andric /// generated to compensate for the byte-swapping. But sometimes we do 3010b57cec5SDimitry Andric /// need to re-interpret the data in SIMD vector registers in big-endian 3020b57cec5SDimitry Andric /// mode without emitting such REV instructions. 3030b57cec5SDimitry Andric NVCAST, 3040b57cec5SDimitry Andric 305fe6060f1SDimitry Andric MRS, // MRS, also sets the flags via a glue. 306fe6060f1SDimitry Andric 3070b57cec5SDimitry Andric SMULL, 3080b57cec5SDimitry Andric UMULL, 3090b57cec5SDimitry Andric 310*bdd1243dSDimitry Andric PMULL, 311*bdd1243dSDimitry Andric 3120b57cec5SDimitry Andric // Reciprocal estimates and steps. 3135ffd83dbSDimitry Andric FRECPE, 3145ffd83dbSDimitry Andric FRECPS, 3155ffd83dbSDimitry Andric FRSQRTE, 3165ffd83dbSDimitry Andric FRSQRTS, 3170b57cec5SDimitry Andric 3188bcb0991SDimitry Andric SUNPKHI, 3198bcb0991SDimitry Andric SUNPKLO, 3208bcb0991SDimitry Andric UUNPKHI, 3218bcb0991SDimitry Andric UUNPKLO, 3228bcb0991SDimitry Andric 323480093f4SDimitry Andric CLASTA_N, 324480093f4SDimitry Andric CLASTB_N, 325480093f4SDimitry Andric LASTA, 326480093f4SDimitry Andric LASTB, 327480093f4SDimitry Andric TBL, 328480093f4SDimitry Andric 3295ffd83dbSDimitry Andric // Floating-point reductions. 3305ffd83dbSDimitry Andric FADDA_PRED, 3315ffd83dbSDimitry Andric FADDV_PRED, 3325ffd83dbSDimitry Andric FMAXV_PRED, 3335ffd83dbSDimitry Andric FMAXNMV_PRED, 3345ffd83dbSDimitry Andric FMINV_PRED, 3355ffd83dbSDimitry Andric FMINNMV_PRED, 3365ffd83dbSDimitry Andric 337480093f4SDimitry Andric INSR, 338480093f4SDimitry Andric PTEST, 339*bdd1243dSDimitry Andric PTEST_ANY, 340480093f4SDimitry Andric PTRUE, 341480093f4SDimitry Andric 342e8d8bef9SDimitry Andric BITREVERSE_MERGE_PASSTHRU, 343e8d8bef9SDimitry Andric BSWAP_MERGE_PASSTHRU, 3440eae32dcSDimitry Andric REVH_MERGE_PASSTHRU, 3450eae32dcSDimitry Andric REVW_MERGE_PASSTHRU, 346e8d8bef9SDimitry Andric CTLZ_MERGE_PASSTHRU, 347e8d8bef9SDimitry Andric CTPOP_MERGE_PASSTHRU, 3485ffd83dbSDimitry Andric DUP_MERGE_PASSTHRU, 3495ffd83dbSDimitry Andric INDEX_VECTOR, 3505ffd83dbSDimitry Andric 351e8d8bef9SDimitry Andric // Cast between vectors of the same element type but differ in length. 3525ffd83dbSDimitry Andric REINTERPRET_CAST, 3535ffd83dbSDimitry Andric 3546e75b2fbSDimitry Andric // Nodes to build an LD64B / ST64B 64-bit quantity out of i64, and vice versa 3556e75b2fbSDimitry Andric LS64_BUILD, 3566e75b2fbSDimitry Andric LS64_EXTRACT, 3576e75b2fbSDimitry Andric 3585ffd83dbSDimitry Andric LD1_MERGE_ZERO, 3595ffd83dbSDimitry Andric LD1S_MERGE_ZERO, 3605ffd83dbSDimitry Andric LDNF1_MERGE_ZERO, 3615ffd83dbSDimitry Andric LDNF1S_MERGE_ZERO, 3625ffd83dbSDimitry Andric LDFF1_MERGE_ZERO, 3635ffd83dbSDimitry Andric LDFF1S_MERGE_ZERO, 3645ffd83dbSDimitry Andric LD1RQ_MERGE_ZERO, 3655ffd83dbSDimitry Andric LD1RO_MERGE_ZERO, 3665ffd83dbSDimitry Andric 3675ffd83dbSDimitry Andric // Structured loads. 3685ffd83dbSDimitry Andric SVE_LD2_MERGE_ZERO, 3695ffd83dbSDimitry Andric SVE_LD3_MERGE_ZERO, 3705ffd83dbSDimitry Andric SVE_LD4_MERGE_ZERO, 3715ffd83dbSDimitry Andric 372480093f4SDimitry Andric // Unsigned gather loads. 3735ffd83dbSDimitry Andric GLD1_MERGE_ZERO, 3745ffd83dbSDimitry Andric GLD1_SCALED_MERGE_ZERO, 3755ffd83dbSDimitry Andric GLD1_UXTW_MERGE_ZERO, 3765ffd83dbSDimitry Andric GLD1_SXTW_MERGE_ZERO, 3775ffd83dbSDimitry Andric GLD1_UXTW_SCALED_MERGE_ZERO, 3785ffd83dbSDimitry Andric GLD1_SXTW_SCALED_MERGE_ZERO, 3795ffd83dbSDimitry Andric GLD1_IMM_MERGE_ZERO, 380480093f4SDimitry Andric 381480093f4SDimitry Andric // Signed gather loads 3825ffd83dbSDimitry Andric GLD1S_MERGE_ZERO, 3835ffd83dbSDimitry Andric GLD1S_SCALED_MERGE_ZERO, 3845ffd83dbSDimitry Andric GLD1S_UXTW_MERGE_ZERO, 3855ffd83dbSDimitry Andric GLD1S_SXTW_MERGE_ZERO, 3865ffd83dbSDimitry Andric GLD1S_UXTW_SCALED_MERGE_ZERO, 3875ffd83dbSDimitry Andric GLD1S_SXTW_SCALED_MERGE_ZERO, 3885ffd83dbSDimitry Andric GLD1S_IMM_MERGE_ZERO, 3895ffd83dbSDimitry Andric 3905ffd83dbSDimitry Andric // Unsigned gather loads. 3915ffd83dbSDimitry Andric GLDFF1_MERGE_ZERO, 3925ffd83dbSDimitry Andric GLDFF1_SCALED_MERGE_ZERO, 3935ffd83dbSDimitry Andric GLDFF1_UXTW_MERGE_ZERO, 3945ffd83dbSDimitry Andric GLDFF1_SXTW_MERGE_ZERO, 3955ffd83dbSDimitry Andric GLDFF1_UXTW_SCALED_MERGE_ZERO, 3965ffd83dbSDimitry Andric GLDFF1_SXTW_SCALED_MERGE_ZERO, 3975ffd83dbSDimitry Andric GLDFF1_IMM_MERGE_ZERO, 3985ffd83dbSDimitry Andric 3995ffd83dbSDimitry Andric // Signed gather loads. 4005ffd83dbSDimitry Andric GLDFF1S_MERGE_ZERO, 4015ffd83dbSDimitry Andric GLDFF1S_SCALED_MERGE_ZERO, 4025ffd83dbSDimitry Andric GLDFF1S_UXTW_MERGE_ZERO, 4035ffd83dbSDimitry Andric GLDFF1S_SXTW_MERGE_ZERO, 4045ffd83dbSDimitry Andric GLDFF1S_UXTW_SCALED_MERGE_ZERO, 4055ffd83dbSDimitry Andric GLDFF1S_SXTW_SCALED_MERGE_ZERO, 4065ffd83dbSDimitry Andric GLDFF1S_IMM_MERGE_ZERO, 4075ffd83dbSDimitry Andric 4085ffd83dbSDimitry Andric // Non-temporal gather loads 4095ffd83dbSDimitry Andric GLDNT1_MERGE_ZERO, 4105ffd83dbSDimitry Andric GLDNT1_INDEX_MERGE_ZERO, 4115ffd83dbSDimitry Andric GLDNT1S_MERGE_ZERO, 4125ffd83dbSDimitry Andric 4135ffd83dbSDimitry Andric // Contiguous masked store. 4145ffd83dbSDimitry Andric ST1_PRED, 4155ffd83dbSDimitry Andric 416480093f4SDimitry Andric // Scatter store 4175ffd83dbSDimitry Andric SST1_PRED, 4185ffd83dbSDimitry Andric SST1_SCALED_PRED, 4195ffd83dbSDimitry Andric SST1_UXTW_PRED, 4205ffd83dbSDimitry Andric SST1_SXTW_PRED, 4215ffd83dbSDimitry Andric SST1_UXTW_SCALED_PRED, 4225ffd83dbSDimitry Andric SST1_SXTW_SCALED_PRED, 4235ffd83dbSDimitry Andric SST1_IMM_PRED, 4245ffd83dbSDimitry Andric 4255ffd83dbSDimitry Andric // Non-temporal scatter store 4265ffd83dbSDimitry Andric SSTNT1_PRED, 4275ffd83dbSDimitry Andric SSTNT1_INDEX_PRED, 428480093f4SDimitry Andric 42981ad6265SDimitry Andric // SME 43081ad6265SDimitry Andric RDSVL, 43181ad6265SDimitry Andric REVD_MERGE_PASSTHRU, 43281ad6265SDimitry Andric 433349cc55cSDimitry Andric // Asserts that a function argument (i32) is zero-extended to i8 by 434349cc55cSDimitry Andric // the caller 435349cc55cSDimitry Andric ASSERT_ZEXT_BOOL, 436349cc55cSDimitry Andric 437*bdd1243dSDimitry Andric // 128-bit system register accesses 438*bdd1243dSDimitry Andric // lo64, hi64, chain = MRRS(chain, sysregname) 439*bdd1243dSDimitry Andric MRRS, 440*bdd1243dSDimitry Andric // chain = MSRR(chain, sysregname, lo64, hi64) 441*bdd1243dSDimitry Andric MSRR, 442*bdd1243dSDimitry Andric 44347395794SDimitry Andric // Strict (exception-raising) floating point comparison 44447395794SDimitry Andric STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE, 44547395794SDimitry Andric STRICT_FCMPE, 44647395794SDimitry Andric 4470b57cec5SDimitry Andric // NEON Load/Store with post-increment base updates 4480b57cec5SDimitry Andric LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE, 4490b57cec5SDimitry Andric LD3post, 4500b57cec5SDimitry Andric LD4post, 4510b57cec5SDimitry Andric ST2post, 4520b57cec5SDimitry Andric ST3post, 4530b57cec5SDimitry Andric ST4post, 4540b57cec5SDimitry Andric LD1x2post, 4550b57cec5SDimitry Andric LD1x3post, 4560b57cec5SDimitry Andric LD1x4post, 4570b57cec5SDimitry Andric ST1x2post, 4580b57cec5SDimitry Andric ST1x3post, 4590b57cec5SDimitry Andric ST1x4post, 4600b57cec5SDimitry Andric LD1DUPpost, 4610b57cec5SDimitry Andric LD2DUPpost, 4620b57cec5SDimitry Andric LD3DUPpost, 4630b57cec5SDimitry Andric LD4DUPpost, 4640b57cec5SDimitry Andric LD1LANEpost, 4650b57cec5SDimitry Andric LD2LANEpost, 4660b57cec5SDimitry Andric LD3LANEpost, 4670b57cec5SDimitry Andric LD4LANEpost, 4680b57cec5SDimitry Andric ST2LANEpost, 4690b57cec5SDimitry Andric ST3LANEpost, 4700b57cec5SDimitry Andric ST4LANEpost, 4710b57cec5SDimitry Andric 4720b57cec5SDimitry Andric STG, 4730b57cec5SDimitry Andric STZG, 4740b57cec5SDimitry Andric ST2G, 475480093f4SDimitry Andric STZ2G, 4760b57cec5SDimitry Andric 477480093f4SDimitry Andric LDP, 478*bdd1243dSDimitry Andric LDNP, 4795ffd83dbSDimitry Andric STP, 480e8d8bef9SDimitry Andric STNP, 4811fd87a68SDimitry Andric 4821fd87a68SDimitry Andric // Memory Operations 4831fd87a68SDimitry Andric MOPS_MEMSET, 4841fd87a68SDimitry Andric MOPS_MEMSET_TAGGING, 4851fd87a68SDimitry Andric MOPS_MEMCOPY, 4861fd87a68SDimitry Andric MOPS_MEMMOVE, 4870b57cec5SDimitry Andric }; 4880b57cec5SDimitry Andric 4890b57cec5SDimitry Andric } // end namespace AArch64ISD 4900b57cec5SDimitry Andric 491fe6060f1SDimitry Andric namespace AArch64 { 492fe6060f1SDimitry Andric /// Possible values of current rounding mode, which is specified in bits 493fe6060f1SDimitry Andric /// 23:22 of FPCR. 494fe6060f1SDimitry Andric enum Rounding { 495fe6060f1SDimitry Andric RN = 0, // Round to Nearest 496fe6060f1SDimitry Andric RP = 1, // Round towards Plus infinity 497fe6060f1SDimitry Andric RM = 2, // Round towards Minus infinity 498fe6060f1SDimitry Andric RZ = 3, // Round towards Zero 499fe6060f1SDimitry Andric rmMask = 3 // Bit mask selecting rounding mode 500fe6060f1SDimitry Andric }; 501fe6060f1SDimitry Andric 502fe6060f1SDimitry Andric // Bit position of rounding mode bits in FPCR. 503fe6060f1SDimitry Andric const unsigned RoundingBitsPos = 22; 504fe6060f1SDimitry Andric } // namespace AArch64 505fe6060f1SDimitry Andric 5060b57cec5SDimitry Andric class AArch64Subtarget; 5070b57cec5SDimitry Andric 5080b57cec5SDimitry Andric class AArch64TargetLowering : public TargetLowering { 5090b57cec5SDimitry Andric public: 5100b57cec5SDimitry Andric explicit AArch64TargetLowering(const TargetMachine &TM, 5110b57cec5SDimitry Andric const AArch64Subtarget &STI); 5120b57cec5SDimitry Andric 51381ad6265SDimitry Andric /// Control the following reassociation of operands: (op (op x, c1), y) -> (op 51481ad6265SDimitry Andric /// (op x, y), c1) where N0 is (op x, c1) and N1 is y. 51581ad6265SDimitry Andric bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, 51681ad6265SDimitry Andric SDValue N1) const override; 51781ad6265SDimitry Andric 5180b57cec5SDimitry Andric /// Selects the correct CCAssignFn for a given CallingConvention value. 5190b57cec5SDimitry Andric CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const; 5200b57cec5SDimitry Andric 5210b57cec5SDimitry Andric /// Selects the correct CCAssignFn for a given CallingConvention value. 5220b57cec5SDimitry Andric CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC) const; 5230b57cec5SDimitry Andric 5240b57cec5SDimitry Andric /// Determine which of the bits specified in Mask are known to be either zero 5250b57cec5SDimitry Andric /// or one and return them in the KnownZero/KnownOne bitsets. 5260b57cec5SDimitry Andric void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, 5270b57cec5SDimitry Andric const APInt &DemandedElts, 5280b57cec5SDimitry Andric const SelectionDAG &DAG, 5290b57cec5SDimitry Andric unsigned Depth = 0) const override; 5300b57cec5SDimitry Andric 5318bcb0991SDimitry Andric MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override { 5328bcb0991SDimitry Andric // Returning i64 unconditionally here (i.e. even for ILP32) means that the 5338bcb0991SDimitry Andric // *DAG* representation of pointers will always be 64-bits. They will be 5348bcb0991SDimitry Andric // truncated and extended when transferred to memory, but the 64-bit DAG 5358bcb0991SDimitry Andric // allows us to use AArch64's addressing modes much more easily. 5368bcb0991SDimitry Andric return MVT::getIntegerVT(64); 5378bcb0991SDimitry Andric } 5388bcb0991SDimitry Andric 5395ffd83dbSDimitry Andric bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, 5405ffd83dbSDimitry Andric const APInt &DemandedElts, 5410b57cec5SDimitry Andric TargetLoweringOpt &TLO) const override; 5420b57cec5SDimitry Andric 5430b57cec5SDimitry Andric MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override; 5440b57cec5SDimitry Andric 5450b57cec5SDimitry Andric /// Returns true if the target allows unaligned memory accesses of the 5460b57cec5SDimitry Andric /// specified type. 5470b57cec5SDimitry Andric bool allowsMisalignedMemoryAccesses( 548fe6060f1SDimitry Andric EVT VT, unsigned AddrSpace = 0, Align Alignment = Align(1), 5490b57cec5SDimitry Andric MachineMemOperand::Flags Flags = MachineMemOperand::MONone, 550*bdd1243dSDimitry Andric unsigned *Fast = nullptr) const override; 5518bcb0991SDimitry Andric /// LLT variant. 5525ffd83dbSDimitry Andric bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace, 5535ffd83dbSDimitry Andric Align Alignment, 5545ffd83dbSDimitry Andric MachineMemOperand::Flags Flags, 555*bdd1243dSDimitry Andric unsigned *Fast = nullptr) const override; 5560b57cec5SDimitry Andric 5570b57cec5SDimitry Andric /// Provide custom lowering hooks for some operations. 5580b57cec5SDimitry Andric SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; 5590b57cec5SDimitry Andric 5600b57cec5SDimitry Andric const char *getTargetNodeName(unsigned Opcode) const override; 5610b57cec5SDimitry Andric 5620b57cec5SDimitry Andric SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; 5630b57cec5SDimitry Andric 5640b57cec5SDimitry Andric /// This method returns a target specific FastISel object, or null if the 5650b57cec5SDimitry Andric /// target does not support "fast" ISel. 5660b57cec5SDimitry Andric FastISel *createFastISel(FunctionLoweringInfo &funcInfo, 5670b57cec5SDimitry Andric const TargetLibraryInfo *libInfo) const override; 5680b57cec5SDimitry Andric 5690b57cec5SDimitry Andric bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; 5700b57cec5SDimitry Andric 5710b57cec5SDimitry Andric bool isFPImmLegal(const APFloat &Imm, EVT VT, 5720b57cec5SDimitry Andric bool ForCodeSize) const override; 5730b57cec5SDimitry Andric 5740b57cec5SDimitry Andric /// Return true if the given shuffle mask can be codegen'd directly, or if it 5750b57cec5SDimitry Andric /// should be stack expanded. 5760b57cec5SDimitry Andric bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override; 5770b57cec5SDimitry Andric 578fcaf7f86SDimitry Andric /// Similar to isShuffleMaskLegal. Return true is the given 'select with zero' 579fcaf7f86SDimitry Andric /// shuffle mask can be codegen'd directly. 580fcaf7f86SDimitry Andric bool isVectorClearMaskLegal(ArrayRef<int> M, EVT VT) const override; 581fcaf7f86SDimitry Andric 5820b57cec5SDimitry Andric /// Return the ISD::SETCC ValueType. 5830b57cec5SDimitry Andric EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, 5840b57cec5SDimitry Andric EVT VT) const override; 5850b57cec5SDimitry Andric 5860b57cec5SDimitry Andric SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const; 5870b57cec5SDimitry Andric 5880b57cec5SDimitry Andric MachineBasicBlock *EmitF128CSEL(MachineInstr &MI, 5890b57cec5SDimitry Andric MachineBasicBlock *BB) const; 5900b57cec5SDimitry Andric 5910b57cec5SDimitry Andric MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI, 5920b57cec5SDimitry Andric MachineBasicBlock *BB) const; 5930b57cec5SDimitry Andric 59481ad6265SDimitry Andric MachineBasicBlock *EmitTileLoad(unsigned Opc, unsigned BaseReg, 59581ad6265SDimitry Andric MachineInstr &MI, 59681ad6265SDimitry Andric MachineBasicBlock *BB) const; 59781ad6265SDimitry Andric MachineBasicBlock *EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const; 598*bdd1243dSDimitry Andric MachineBasicBlock *EmitZAInstr(unsigned Opc, unsigned BaseReg, 599*bdd1243dSDimitry Andric MachineInstr &MI, MachineBasicBlock *BB, 600*bdd1243dSDimitry Andric bool HasTile) const; 60181ad6265SDimitry Andric MachineBasicBlock *EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const; 60281ad6265SDimitry Andric 6030b57cec5SDimitry Andric MachineBasicBlock * 6040b57cec5SDimitry Andric EmitInstrWithCustomInserter(MachineInstr &MI, 6050b57cec5SDimitry Andric MachineBasicBlock *MBB) const override; 6060b57cec5SDimitry Andric 6070b57cec5SDimitry Andric bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, 6080b57cec5SDimitry Andric MachineFunction &MF, 6090b57cec5SDimitry Andric unsigned Intrinsic) const override; 6100b57cec5SDimitry Andric 6110b57cec5SDimitry Andric bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, 6120b57cec5SDimitry Andric EVT NewVT) const override; 6130b57cec5SDimitry Andric 6140b57cec5SDimitry Andric bool isTruncateFree(Type *Ty1, Type *Ty2) const override; 6150b57cec5SDimitry Andric bool isTruncateFree(EVT VT1, EVT VT2) const override; 6160b57cec5SDimitry Andric 6170b57cec5SDimitry Andric bool isProfitableToHoist(Instruction *I) const override; 6180b57cec5SDimitry Andric 6190b57cec5SDimitry Andric bool isZExtFree(Type *Ty1, Type *Ty2) const override; 6200b57cec5SDimitry Andric bool isZExtFree(EVT VT1, EVT VT2) const override; 6210b57cec5SDimitry Andric bool isZExtFree(SDValue Val, EVT VT2) const override; 6220b57cec5SDimitry Andric 6230b57cec5SDimitry Andric bool shouldSinkOperands(Instruction *I, 6240b57cec5SDimitry Andric SmallVectorImpl<Use *> &Ops) const override; 6250b57cec5SDimitry Andric 626*bdd1243dSDimitry Andric bool optimizeExtendOrTruncateConversion(Instruction *I, 627*bdd1243dSDimitry Andric Loop *L) const override; 628*bdd1243dSDimitry Andric 6295ffd83dbSDimitry Andric bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override; 6300b57cec5SDimitry Andric 6310b57cec5SDimitry Andric unsigned getMaxSupportedInterleaveFactor() const override { return 4; } 6320b57cec5SDimitry Andric 6330b57cec5SDimitry Andric bool lowerInterleavedLoad(LoadInst *LI, 6340b57cec5SDimitry Andric ArrayRef<ShuffleVectorInst *> Shuffles, 6350b57cec5SDimitry Andric ArrayRef<unsigned> Indices, 6360b57cec5SDimitry Andric unsigned Factor) const override; 6370b57cec5SDimitry Andric bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, 6380b57cec5SDimitry Andric unsigned Factor) const override; 6390b57cec5SDimitry Andric 6400b57cec5SDimitry Andric bool isLegalAddImmediate(int64_t) const override; 6410b57cec5SDimitry Andric bool isLegalICmpImmediate(int64_t) const override; 6420b57cec5SDimitry Andric 64381ad6265SDimitry Andric bool isMulAddWithConstProfitable(SDValue AddNode, 64481ad6265SDimitry Andric SDValue ConstNode) const override; 645349cc55cSDimitry Andric 6460b57cec5SDimitry Andric bool shouldConsiderGEPOffsetSplit() const override; 6470b57cec5SDimitry Andric 6485ffd83dbSDimitry Andric EVT getOptimalMemOpType(const MemOp &Op, 6490b57cec5SDimitry Andric const AttributeList &FuncAttributes) const override; 6500b57cec5SDimitry Andric 6515ffd83dbSDimitry Andric LLT getOptimalMemOpLLT(const MemOp &Op, 6528bcb0991SDimitry Andric const AttributeList &FuncAttributes) const override; 6538bcb0991SDimitry Andric 6540b57cec5SDimitry Andric /// Return true if the addressing mode represented by AM is legal for this 6550b57cec5SDimitry Andric /// target, for a load/store of the specified type. 6560b57cec5SDimitry Andric bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, 6570b57cec5SDimitry Andric unsigned AS, 6580b57cec5SDimitry Andric Instruction *I = nullptr) const override; 6590b57cec5SDimitry Andric 6600b57cec5SDimitry Andric /// Return true if an FMA operation is faster than a pair of fmul and fadd 6610b57cec5SDimitry Andric /// instructions. fmuladd intrinsics will be expanded to FMAs when this method 6620b57cec5SDimitry Andric /// returns true, otherwise fmuladd is expanded to fmul + fadd. 663480093f4SDimitry Andric bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, 664480093f4SDimitry Andric EVT VT) const override; 665480093f4SDimitry Andric bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override; 6660b57cec5SDimitry Andric 667fe6060f1SDimitry Andric bool generateFMAsInMachineCombiner(EVT VT, 668fe6060f1SDimitry Andric CodeGenOpt::Level OptLevel) const override; 669fe6060f1SDimitry Andric 6700b57cec5SDimitry Andric const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override; 6710b57cec5SDimitry Andric 6720b57cec5SDimitry Andric /// Returns false if N is a bit extraction pattern of (X >> C) & Mask. 6730b57cec5SDimitry Andric bool isDesirableToCommuteWithShift(const SDNode *N, 6740b57cec5SDimitry Andric CombineLevel Level) const override; 6750b57cec5SDimitry Andric 676fcaf7f86SDimitry Andric /// Returns false if N is a bit extraction pattern of (X >> C) & Mask. 677fcaf7f86SDimitry Andric bool isDesirableToCommuteXorWithShift(const SDNode *N) const override; 678fcaf7f86SDimitry Andric 67981ad6265SDimitry Andric /// Return true if it is profitable to fold a pair of shifts into a mask. 68081ad6265SDimitry Andric bool shouldFoldConstantShiftPairToMask(const SDNode *N, 68181ad6265SDimitry Andric CombineLevel Level) const override; 68281ad6265SDimitry Andric 6830b57cec5SDimitry Andric /// Returns true if it is beneficial to convert a load of a constant 6840b57cec5SDimitry Andric /// to just the constant itself. 6850b57cec5SDimitry Andric bool shouldConvertConstantLoadToIntImm(const APInt &Imm, 6860b57cec5SDimitry Andric Type *Ty) const override; 6870b57cec5SDimitry Andric 6880b57cec5SDimitry Andric /// Return true if EXTRACT_SUBVECTOR is cheap for this result type 6890b57cec5SDimitry Andric /// with this index. 6900b57cec5SDimitry Andric bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, 6910b57cec5SDimitry Andric unsigned Index) const override; 6920b57cec5SDimitry Andric 6935ffd83dbSDimitry Andric bool shouldFormOverflowOp(unsigned Opcode, EVT VT, 6945ffd83dbSDimitry Andric bool MathUsed) const override { 6955ffd83dbSDimitry Andric // Using overflow ops for overflow checks only should beneficial on 6965ffd83dbSDimitry Andric // AArch64. 6975ffd83dbSDimitry Andric return TargetLowering::shouldFormOverflowOp(Opcode, VT, true); 6985ffd83dbSDimitry Andric } 6995ffd83dbSDimitry Andric 700fe6060f1SDimitry Andric Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, 7010b57cec5SDimitry Andric AtomicOrdering Ord) const override; 702fe6060f1SDimitry Andric Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, 703fe6060f1SDimitry Andric AtomicOrdering Ord) const override; 7040b57cec5SDimitry Andric 705fe6060f1SDimitry Andric void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override; 7060b57cec5SDimitry Andric 707349cc55cSDimitry Andric bool isOpSuitableForLDPSTP(const Instruction *I) const; 708349cc55cSDimitry Andric bool shouldInsertFencesForAtomic(const Instruction *I) const override; 709*bdd1243dSDimitry Andric bool 710*bdd1243dSDimitry Andric shouldInsertTrailingFenceForAtomicStore(const Instruction *I) const override; 711349cc55cSDimitry Andric 7120b57cec5SDimitry Andric TargetLoweringBase::AtomicExpansionKind 7130b57cec5SDimitry Andric shouldExpandAtomicLoadInIR(LoadInst *LI) const override; 71481ad6265SDimitry Andric TargetLoweringBase::AtomicExpansionKind 71581ad6265SDimitry Andric shouldExpandAtomicStoreInIR(StoreInst *SI) const override; 7160b57cec5SDimitry Andric TargetLoweringBase::AtomicExpansionKind 7170b57cec5SDimitry Andric shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override; 7180b57cec5SDimitry Andric 7190b57cec5SDimitry Andric TargetLoweringBase::AtomicExpansionKind 7200b57cec5SDimitry Andric shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override; 7210b57cec5SDimitry Andric 7220b57cec5SDimitry Andric bool useLoadStackGuardNode() const override; 7230b57cec5SDimitry Andric TargetLoweringBase::LegalizeTypeAction 7240b57cec5SDimitry Andric getPreferredVectorAction(MVT VT) const override; 7250b57cec5SDimitry Andric 7260b57cec5SDimitry Andric /// If the target has a standard location for the stack protector cookie, 7270b57cec5SDimitry Andric /// returns the address of that location. Otherwise, returns nullptr. 728fe6060f1SDimitry Andric Value *getIRStackGuard(IRBuilderBase &IRB) const override; 7290b57cec5SDimitry Andric 7300b57cec5SDimitry Andric void insertSSPDeclarations(Module &M) const override; 7310b57cec5SDimitry Andric Value *getSDagStackGuard(const Module &M) const override; 7320b57cec5SDimitry Andric Function *getSSPStackGuardCheck(const Module &M) const override; 7330b57cec5SDimitry Andric 7340b57cec5SDimitry Andric /// If the target has a standard location for the unsafe stack pointer, 7350b57cec5SDimitry Andric /// returns the address of that location. Otherwise, returns nullptr. 736fe6060f1SDimitry Andric Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override; 7370b57cec5SDimitry Andric 7380b57cec5SDimitry Andric /// If a physical register, this returns the register that receives the 7390b57cec5SDimitry Andric /// exception address on entry to an EH pad. 7405ffd83dbSDimitry Andric Register 7410b57cec5SDimitry Andric getExceptionPointerRegister(const Constant *PersonalityFn) const override { 7420b57cec5SDimitry Andric // FIXME: This is a guess. Has this been defined yet? 7430b57cec5SDimitry Andric return AArch64::X0; 7440b57cec5SDimitry Andric } 7450b57cec5SDimitry Andric 7460b57cec5SDimitry Andric /// If a physical register, this returns the register that receives the 7470b57cec5SDimitry Andric /// exception typeid on entry to a landing pad. 7485ffd83dbSDimitry Andric Register 7490b57cec5SDimitry Andric getExceptionSelectorRegister(const Constant *PersonalityFn) const override { 7500b57cec5SDimitry Andric // FIXME: This is a guess. Has this been defined yet? 7510b57cec5SDimitry Andric return AArch64::X1; 7520b57cec5SDimitry Andric } 7530b57cec5SDimitry Andric 7540b57cec5SDimitry Andric bool isIntDivCheap(EVT VT, AttributeList Attr) const override; 7550b57cec5SDimitry Andric 7560b57cec5SDimitry Andric bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, 757349cc55cSDimitry Andric const MachineFunction &MF) const override { 7580b57cec5SDimitry Andric // Do not merge to float value size (128 bytes) if no implicit 7590b57cec5SDimitry Andric // float attribute is set. 7600b57cec5SDimitry Andric 761349cc55cSDimitry Andric bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat); 7620b57cec5SDimitry Andric 7630b57cec5SDimitry Andric if (NoFloat) 7640b57cec5SDimitry Andric return (MemVT.getSizeInBits() <= 64); 7650b57cec5SDimitry Andric return true; 7660b57cec5SDimitry Andric } 7670b57cec5SDimitry Andric 768*bdd1243dSDimitry Andric bool isCheapToSpeculateCttz(Type *) const override { 7690b57cec5SDimitry Andric return true; 7700b57cec5SDimitry Andric } 7710b57cec5SDimitry Andric 772*bdd1243dSDimitry Andric bool isCheapToSpeculateCtlz(Type *) const override { 7730b57cec5SDimitry Andric return true; 7740b57cec5SDimitry Andric } 7750b57cec5SDimitry Andric 7760b57cec5SDimitry Andric bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override; 7770b57cec5SDimitry Andric 7780b57cec5SDimitry Andric bool hasAndNotCompare(SDValue V) const override { 7790b57cec5SDimitry Andric // We can use bics for any scalar. 7800b57cec5SDimitry Andric return V.getValueType().isScalarInteger(); 7810b57cec5SDimitry Andric } 7820b57cec5SDimitry Andric 7830b57cec5SDimitry Andric bool hasAndNot(SDValue Y) const override { 7840b57cec5SDimitry Andric EVT VT = Y.getValueType(); 7850b57cec5SDimitry Andric 7860b57cec5SDimitry Andric if (!VT.isVector()) 7870b57cec5SDimitry Andric return hasAndNotCompare(Y); 7880b57cec5SDimitry Andric 789349cc55cSDimitry Andric TypeSize TS = VT.getSizeInBits(); 790349cc55cSDimitry Andric // TODO: We should be able to use bic/bif too for SVE. 791349cc55cSDimitry Andric return !TS.isScalable() && TS.getFixedValue() >= 64; // vector 'bic' 7920b57cec5SDimitry Andric } 7930b57cec5SDimitry Andric 7948bcb0991SDimitry Andric bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd( 7958bcb0991SDimitry Andric SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, 7968bcb0991SDimitry Andric unsigned OldShiftOpcode, unsigned NewShiftOpcode, 7978bcb0991SDimitry Andric SelectionDAG &DAG) const override; 7988bcb0991SDimitry Andric 799*bdd1243dSDimitry Andric ShiftLegalizationStrategy 800*bdd1243dSDimitry Andric preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, 801*bdd1243dSDimitry Andric unsigned ExpansionFactor) const override; 8020b57cec5SDimitry Andric 8030b57cec5SDimitry Andric bool shouldTransformSignedTruncationCheck(EVT XVT, 8040b57cec5SDimitry Andric unsigned KeptBits) const override { 8050b57cec5SDimitry Andric // For vectors, we don't have a preference.. 8060b57cec5SDimitry Andric if (XVT.isVector()) 8070b57cec5SDimitry Andric return false; 8080b57cec5SDimitry Andric 8090b57cec5SDimitry Andric auto VTIsOk = [](EVT VT) -> bool { 8100b57cec5SDimitry Andric return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || 8110b57cec5SDimitry Andric VT == MVT::i64; 8120b57cec5SDimitry Andric }; 8130b57cec5SDimitry Andric 8140b57cec5SDimitry Andric // We are ok with KeptBitsVT being byte/word/dword, what SXT supports. 8150b57cec5SDimitry Andric // XVT will be larger than KeptBitsVT. 8160b57cec5SDimitry Andric MVT KeptBitsVT = MVT::getIntegerVT(KeptBits); 8170b57cec5SDimitry Andric return VTIsOk(XVT) && VTIsOk(KeptBitsVT); 8180b57cec5SDimitry Andric } 8190b57cec5SDimitry Andric 8200b57cec5SDimitry Andric bool preferIncOfAddToSubOfNot(EVT VT) const override; 8210b57cec5SDimitry Andric 8224824e7fdSDimitry Andric bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override; 8234824e7fdSDimitry Andric 824*bdd1243dSDimitry Andric bool isComplexDeinterleavingSupported() const override; 825*bdd1243dSDimitry Andric bool isComplexDeinterleavingOperationSupported( 826*bdd1243dSDimitry Andric ComplexDeinterleavingOperation Operation, Type *Ty) const override; 827*bdd1243dSDimitry Andric 828*bdd1243dSDimitry Andric Value *createComplexDeinterleavingIR( 829*bdd1243dSDimitry Andric Instruction *I, ComplexDeinterleavingOperation OperationType, 830*bdd1243dSDimitry Andric ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, 831*bdd1243dSDimitry Andric Value *Accumulator = nullptr) const override; 832*bdd1243dSDimitry Andric 8330b57cec5SDimitry Andric bool hasBitPreservingFPLogic(EVT VT) const override { 8340b57cec5SDimitry Andric // FIXME: Is this always true? It should be true for vectors at least. 8350b57cec5SDimitry Andric return VT == MVT::f32 || VT == MVT::f64; 8360b57cec5SDimitry Andric } 8370b57cec5SDimitry Andric 8380b57cec5SDimitry Andric bool supportSplitCSR(MachineFunction *MF) const override { 8390b57cec5SDimitry Andric return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS && 8400b57cec5SDimitry Andric MF->getFunction().hasFnAttribute(Attribute::NoUnwind); 8410b57cec5SDimitry Andric } 8420b57cec5SDimitry Andric void initializeSplitCSR(MachineBasicBlock *Entry) const override; 8430b57cec5SDimitry Andric void insertCopiesSplitCSR( 8440b57cec5SDimitry Andric MachineBasicBlock *Entry, 8450b57cec5SDimitry Andric const SmallVectorImpl<MachineBasicBlock *> &Exits) const override; 8460b57cec5SDimitry Andric 8470b57cec5SDimitry Andric bool supportSwiftError() const override { 8480b57cec5SDimitry Andric return true; 8490b57cec5SDimitry Andric } 8500b57cec5SDimitry Andric 851*bdd1243dSDimitry Andric bool supportKCFIBundles() const override { return true; } 852*bdd1243dSDimitry Andric 8530b57cec5SDimitry Andric /// Enable aggressive FMA fusion on targets that want it. 8540b57cec5SDimitry Andric bool enableAggressiveFMAFusion(EVT VT) const override; 8550b57cec5SDimitry Andric 8560b57cec5SDimitry Andric /// Returns the size of the platform's va_list object. 8570b57cec5SDimitry Andric unsigned getVaListSizeInBits(const DataLayout &DL) const override; 8580b57cec5SDimitry Andric 8590b57cec5SDimitry Andric /// Returns true if \p VecTy is a legal interleaved access type. This 8600b57cec5SDimitry Andric /// function checks the vector element type and the overall width of the 8610b57cec5SDimitry Andric /// vector. 862349cc55cSDimitry Andric bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL, 863349cc55cSDimitry Andric bool &UseScalable) const; 8640b57cec5SDimitry Andric 8650b57cec5SDimitry Andric /// Returns the number of interleaved accesses that will be generated when 8660b57cec5SDimitry Andric /// lowering accesses of the given type. 867349cc55cSDimitry Andric unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL, 868349cc55cSDimitry Andric bool UseScalable) const; 8690b57cec5SDimitry Andric 8705ffd83dbSDimitry Andric MachineMemOperand::Flags getTargetMMOFlags( 8715ffd83dbSDimitry Andric const Instruction &I) const override; 8720b57cec5SDimitry Andric 873fe6060f1SDimitry Andric bool functionArgumentNeedsConsecutiveRegisters( 874fe6060f1SDimitry Andric Type *Ty, CallingConv::ID CallConv, bool isVarArg, 875fe6060f1SDimitry Andric const DataLayout &DL) const override; 876fe6060f1SDimitry Andric 8770b57cec5SDimitry Andric /// Used for exception handling on Win64. 8780b57cec5SDimitry Andric bool needsFixedCatchObjects() const override; 8795ffd83dbSDimitry Andric 8805ffd83dbSDimitry Andric bool fallBackToDAGISel(const Instruction &Inst) const override; 8815ffd83dbSDimitry Andric 8825ffd83dbSDimitry Andric /// SVE code generation for fixed length vectors does not custom lower 8835ffd83dbSDimitry Andric /// BUILD_VECTOR. This makes BUILD_VECTOR legalisation a source of stores to 8845ffd83dbSDimitry Andric /// merge. However, merging them creates a BUILD_VECTOR that is just as 8855ffd83dbSDimitry Andric /// illegal as the original, thus leading to an infinite legalisation loop. 8865ffd83dbSDimitry Andric /// NOTE: Once BUILD_VECTOR is legal or can be custom lowered for all legal 8875ffd83dbSDimitry Andric /// vector types this override can be removed. 888e8d8bef9SDimitry Andric bool mergeStoresAfterLegalization(EVT VT) const override; 8895ffd83dbSDimitry Andric 890fe6060f1SDimitry Andric // If the platform/function should have a redzone, return the size in bytes. 891fe6060f1SDimitry Andric unsigned getRedZoneSize(const Function &F) const { 892fe6060f1SDimitry Andric if (F.hasFnAttribute(Attribute::NoRedZone)) 893fe6060f1SDimitry Andric return 0; 894fe6060f1SDimitry Andric return 128; 895fe6060f1SDimitry Andric } 896fe6060f1SDimitry Andric 89704eeddc0SDimitry Andric bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const; 898fe6060f1SDimitry Andric EVT getPromotedVTForPredicate(EVT VT) const; 899fe6060f1SDimitry Andric 9006e75b2fbSDimitry Andric EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, 9016e75b2fbSDimitry Andric bool AllowUnknown = false) const override; 9026e75b2fbSDimitry Andric 9034824e7fdSDimitry Andric bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override; 9044824e7fdSDimitry Andric 905*bdd1243dSDimitry Andric /// If a change in streaming mode is required on entry to/return from a 906*bdd1243dSDimitry Andric /// function call it emits and returns the corresponding SMSTART or SMSTOP node. 907*bdd1243dSDimitry Andric /// \p Entry tells whether this is before/after the Call, which is necessary 908*bdd1243dSDimitry Andric /// because PSTATE.SM is only queried once. 909*bdd1243dSDimitry Andric SDValue changeStreamingMode(SelectionDAG &DAG, SDLoc DL, bool Enable, 910*bdd1243dSDimitry Andric SDValue Chain, SDValue InFlag, 911*bdd1243dSDimitry Andric SDValue PStateSM, bool Entry) const; 912*bdd1243dSDimitry Andric 913*bdd1243dSDimitry Andric bool isVScaleKnownToBeAPowerOfTwo() const override; 914*bdd1243dSDimitry Andric 915*bdd1243dSDimitry Andric // Normally SVE is only used for byte size vectors that do not fit within a 916*bdd1243dSDimitry Andric // NEON vector. This changes when OverrideNEON is true, allowing SVE to be 917*bdd1243dSDimitry Andric // used for 64bit and 128bit vectors as well. 918*bdd1243dSDimitry Andric bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const; 919*bdd1243dSDimitry Andric 9200b57cec5SDimitry Andric private: 9210b57cec5SDimitry Andric /// Keep a pointer to the AArch64Subtarget around so that we can 9220b57cec5SDimitry Andric /// make the right decision when generating code for different targets. 9230b57cec5SDimitry Andric const AArch64Subtarget *Subtarget; 9240b57cec5SDimitry Andric 9250b57cec5SDimitry Andric bool isExtFreeImpl(const Instruction *Ext) const override; 9260b57cec5SDimitry Andric 927fe6060f1SDimitry Andric void addTypeForNEON(MVT VT); 928*bdd1243dSDimitry Andric void addTypeForStreamingSVE(MVT VT); 9295ffd83dbSDimitry Andric void addTypeForFixedLengthSVE(MVT VT); 9300b57cec5SDimitry Andric void addDRTypeForNEON(MVT VT); 9310b57cec5SDimitry Andric void addQRTypeForNEON(MVT VT); 9320b57cec5SDimitry Andric 933*bdd1243dSDimitry Andric unsigned allocateLazySaveBuffer(SDValue &Chain, const SDLoc &DL, 934*bdd1243dSDimitry Andric SelectionDAG &DAG) const; 935*bdd1243dSDimitry Andric 9360b57cec5SDimitry Andric SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, 9370b57cec5SDimitry Andric bool isVarArg, 9380b57cec5SDimitry Andric const SmallVectorImpl<ISD::InputArg> &Ins, 9390b57cec5SDimitry Andric const SDLoc &DL, SelectionDAG &DAG, 9400b57cec5SDimitry Andric SmallVectorImpl<SDValue> &InVals) const override; 9410b57cec5SDimitry Andric 9420b57cec5SDimitry Andric SDValue LowerCall(CallLoweringInfo & /*CLI*/, 9430b57cec5SDimitry Andric SmallVectorImpl<SDValue> &InVals) const override; 9440b57cec5SDimitry Andric 9450b57cec5SDimitry Andric SDValue LowerCallResult(SDValue Chain, SDValue InFlag, 9460b57cec5SDimitry Andric CallingConv::ID CallConv, bool isVarArg, 947f3fd488fSDimitry Andric const SmallVectorImpl<CCValAssign> &RVLocs, 9480b57cec5SDimitry Andric const SDLoc &DL, SelectionDAG &DAG, 9490b57cec5SDimitry Andric SmallVectorImpl<SDValue> &InVals, bool isThisReturn, 9500b57cec5SDimitry Andric SDValue ThisVal) const; 9510b57cec5SDimitry Andric 952fe6060f1SDimitry Andric SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const; 9530b57cec5SDimitry Andric SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const; 954349cc55cSDimitry Andric SDValue LowerStore128(SDValue Op, SelectionDAG &DAG) const; 955e8d8bef9SDimitry Andric SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const; 956e8d8bef9SDimitry Andric 957e8d8bef9SDimitry Andric SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const; 958e8d8bef9SDimitry Andric SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const; 9590b57cec5SDimitry Andric 960fe6060f1SDimitry Andric SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) const; 961fe6060f1SDimitry Andric 9621fd87a68SDimitry Andric SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const; 9630b57cec5SDimitry Andric SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; 964*bdd1243dSDimitry Andric SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; 9650b57cec5SDimitry Andric 9663a9a9c0cSDimitry Andric bool 9673a9a9c0cSDimitry Andric isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const; 9680b57cec5SDimitry Andric 9690b57cec5SDimitry Andric /// Finds the incoming stack arguments which overlap the given fixed stack 9700b57cec5SDimitry Andric /// object and incorporates their load into the current chain. This prevents 9710b57cec5SDimitry Andric /// an upcoming store from clobbering the stack argument before it's used. 9720b57cec5SDimitry Andric SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, 9730b57cec5SDimitry Andric MachineFrameInfo &MFI, int ClobberedFI) const; 9740b57cec5SDimitry Andric 9750b57cec5SDimitry Andric bool DoesCalleeRestoreStack(CallingConv::ID CallCC, bool TailCallOpt) const; 9760b57cec5SDimitry Andric 9770b57cec5SDimitry Andric void saveVarArgRegisters(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &DL, 9780b57cec5SDimitry Andric SDValue &Chain) const; 9790b57cec5SDimitry Andric 9800b57cec5SDimitry Andric bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, 9810b57cec5SDimitry Andric bool isVarArg, 9820b57cec5SDimitry Andric const SmallVectorImpl<ISD::OutputArg> &Outs, 9830b57cec5SDimitry Andric LLVMContext &Context) const override; 9840b57cec5SDimitry Andric 9850b57cec5SDimitry Andric SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 9860b57cec5SDimitry Andric const SmallVectorImpl<ISD::OutputArg> &Outs, 9870b57cec5SDimitry Andric const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, 9880b57cec5SDimitry Andric SelectionDAG &DAG) const override; 9890b57cec5SDimitry Andric 9900b57cec5SDimitry Andric SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG, 9910b57cec5SDimitry Andric unsigned Flag) const; 9920b57cec5SDimitry Andric SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG, 9930b57cec5SDimitry Andric unsigned Flag) const; 9940b57cec5SDimitry Andric SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG, 9950b57cec5SDimitry Andric unsigned Flag) const; 9960b57cec5SDimitry Andric SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG, 9970b57cec5SDimitry Andric unsigned Flag) const; 9980b57cec5SDimitry Andric template <class NodeTy> 9990b57cec5SDimitry Andric SDValue getGOT(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; 10000b57cec5SDimitry Andric template <class NodeTy> 10010b57cec5SDimitry Andric SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; 10020b57cec5SDimitry Andric template <class NodeTy> 10030b57cec5SDimitry Andric SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; 10040b57cec5SDimitry Andric template <class NodeTy> 10050b57cec5SDimitry Andric SDValue getAddrTiny(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; 10060b57cec5SDimitry Andric SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const; 10070b57cec5SDimitry Andric SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; 10080b57cec5SDimitry Andric SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; 10090b57cec5SDimitry Andric SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; 10100b57cec5SDimitry Andric SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; 1011480093f4SDimitry Andric SDValue LowerELFTLSLocalExec(const GlobalValue *GV, SDValue ThreadBase, 1012480093f4SDimitry Andric const SDLoc &DL, SelectionDAG &DAG) const; 10130b57cec5SDimitry Andric SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL, 10140b57cec5SDimitry Andric SelectionDAG &DAG) const; 10150b57cec5SDimitry Andric SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; 10160b57cec5SDimitry Andric SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; 1017*bdd1243dSDimitry Andric SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const; 10180b57cec5SDimitry Andric SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const; 10190b57cec5SDimitry Andric SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const; 10200b57cec5SDimitry Andric SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const; 10210b57cec5SDimitry Andric SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS, 10220b57cec5SDimitry Andric SDValue TVal, SDValue FVal, const SDLoc &dl, 10230b57cec5SDimitry Andric SelectionDAG &DAG) const; 10240b57cec5SDimitry Andric SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; 10250b57cec5SDimitry Andric SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const; 10260b57cec5SDimitry Andric SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; 10270b57cec5SDimitry Andric SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; 10280b57cec5SDimitry Andric SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const; 10290b57cec5SDimitry Andric SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const; 10300b57cec5SDimitry Andric SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const; 10310b57cec5SDimitry Andric SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const; 10320b57cec5SDimitry Andric SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const; 10330b57cec5SDimitry Andric SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const; 10340b57cec5SDimitry Andric SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const; 10350b57cec5SDimitry Andric SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const; 10360b57cec5SDimitry Andric SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const; 1037*bdd1243dSDimitry Andric SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; 1038fe6060f1SDimitry Andric SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; 10390b57cec5SDimitry Andric SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; 10400b57cec5SDimitry Andric SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; 10410b57cec5SDimitry Andric SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const; 10420b57cec5SDimitry Andric SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; 1043*bdd1243dSDimitry Andric SDValue LowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const; 10440b57cec5SDimitry Andric SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; 10458bcb0991SDimitry Andric SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const; 10465ffd83dbSDimitry Andric SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const; 104781ad6265SDimitry Andric SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG, 104881ad6265SDimitry Andric unsigned NewOp) const; 1049e8d8bef9SDimitry Andric SDValue LowerToScalableOp(SDValue Op, SelectionDAG &DAG) const; 1050fe6060f1SDimitry Andric SDValue LowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const; 10510b57cec5SDimitry Andric SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; 10525ffd83dbSDimitry Andric SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; 1053e8d8bef9SDimitry Andric SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const; 1054e8d8bef9SDimitry Andric SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const; 10550b57cec5SDimitry Andric SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const; 1056fe6060f1SDimitry Andric SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) const; 10570b57cec5SDimitry Andric SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const; 1058fcaf7f86SDimitry Andric SDValue LowerCTPOP_PARITY(SDValue Op, SelectionDAG &DAG) const; 1059e8d8bef9SDimitry Andric SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const; 1060fe6060f1SDimitry Andric SDValue LowerBitreverse(SDValue Op, SelectionDAG &DAG) const; 1061349cc55cSDimitry Andric SDValue LowerMinMax(SDValue Op, SelectionDAG &DAG) const; 10620b57cec5SDimitry Andric SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const; 10630b57cec5SDimitry Andric SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const; 10640b57cec5SDimitry Andric SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const; 10650b57cec5SDimitry Andric SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; 1066349cc55cSDimitry Andric SDValue LowerVectorFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const; 10670b57cec5SDimitry Andric SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; 1068fe6060f1SDimitry Andric SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const; 10690b57cec5SDimitry Andric SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; 1070e8d8bef9SDimitry Andric SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; 10710b57cec5SDimitry Andric SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const; 1072e8d8bef9SDimitry Andric SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) const; 10730b57cec5SDimitry Andric SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; 10740b57cec5SDimitry Andric SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const; 1075fe6060f1SDimitry Andric SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; 10765ffd83dbSDimitry Andric SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const; 10775ffd83dbSDimitry Andric SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const; 10780b57cec5SDimitry Andric SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const; 10790b57cec5SDimitry Andric SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const; 10800b57cec5SDimitry Andric SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const; 10810b57cec5SDimitry Andric SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; 10820b57cec5SDimitry Andric SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain, 10830b57cec5SDimitry Andric SDValue &Size, 10840b57cec5SDimitry Andric SelectionDAG &DAG) const; 10855ffd83dbSDimitry Andric 1086e8d8bef9SDimitry Andric SDValue LowerFixedLengthVectorIntDivideToSVE(SDValue Op, 1087e8d8bef9SDimitry Andric SelectionDAG &DAG) const; 1088e8d8bef9SDimitry Andric SDValue LowerFixedLengthVectorIntExtendToSVE(SDValue Op, 1089e8d8bef9SDimitry Andric SelectionDAG &DAG) const; 10905ffd83dbSDimitry Andric SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const; 1091fe6060f1SDimitry Andric SDValue LowerFixedLengthVectorMLoadToSVE(SDValue Op, SelectionDAG &DAG) const; 1092e8d8bef9SDimitry Andric SDValue LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp, SelectionDAG &DAG) const; 1093e8d8bef9SDimitry Andric SDValue LowerPredReductionToSVE(SDValue ScalarOp, SelectionDAG &DAG) const; 1094e8d8bef9SDimitry Andric SDValue LowerReductionToSVE(unsigned Opcode, SDValue ScalarOp, 1095e8d8bef9SDimitry Andric SelectionDAG &DAG) const; 1096e8d8bef9SDimitry Andric SDValue LowerFixedLengthVectorSelectToSVE(SDValue Op, SelectionDAG &DAG) const; 1097e8d8bef9SDimitry Andric SDValue LowerFixedLengthVectorSetccToSVE(SDValue Op, SelectionDAG &DAG) const; 10985ffd83dbSDimitry Andric SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const; 1099fe6060f1SDimitry Andric SDValue LowerFixedLengthVectorMStoreToSVE(SDValue Op, 1100fe6060f1SDimitry Andric SelectionDAG &DAG) const; 11015ffd83dbSDimitry Andric SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op, 11025ffd83dbSDimitry Andric SelectionDAG &DAG) const; 1103fe6060f1SDimitry Andric SDValue LowerFixedLengthExtractVectorElt(SDValue Op, SelectionDAG &DAG) const; 1104fe6060f1SDimitry Andric SDValue LowerFixedLengthInsertVectorElt(SDValue Op, SelectionDAG &DAG) const; 1105fe6060f1SDimitry Andric SDValue LowerFixedLengthBitcastToSVE(SDValue Op, SelectionDAG &DAG) const; 1106fe6060f1SDimitry Andric SDValue LowerFixedLengthConcatVectorsToSVE(SDValue Op, 1107fe6060f1SDimitry Andric SelectionDAG &DAG) const; 1108fe6060f1SDimitry Andric SDValue LowerFixedLengthFPExtendToSVE(SDValue Op, SelectionDAG &DAG) const; 1109fe6060f1SDimitry Andric SDValue LowerFixedLengthFPRoundToSVE(SDValue Op, SelectionDAG &DAG) const; 1110fe6060f1SDimitry Andric SDValue LowerFixedLengthIntToFPToSVE(SDValue Op, SelectionDAG &DAG) const; 1111fe6060f1SDimitry Andric SDValue LowerFixedLengthFPToIntToSVE(SDValue Op, SelectionDAG &DAG) const; 1112fe6060f1SDimitry Andric SDValue LowerFixedLengthVECTOR_SHUFFLEToSVE(SDValue Op, 1113fe6060f1SDimitry Andric SelectionDAG &DAG) const; 11140b57cec5SDimitry Andric 11150b57cec5SDimitry Andric SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, 11160b57cec5SDimitry Andric SmallVectorImpl<SDNode *> &Created) const override; 111781ad6265SDimitry Andric SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, 111881ad6265SDimitry Andric SmallVectorImpl<SDNode *> &Created) const override; 11190b57cec5SDimitry Andric SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, 11200b57cec5SDimitry Andric int &ExtraSteps, bool &UseOneConst, 11210b57cec5SDimitry Andric bool Reciprocal) const override; 11220b57cec5SDimitry Andric SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, 11230b57cec5SDimitry Andric int &ExtraSteps) const override; 1124e8d8bef9SDimitry Andric SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, 1125e8d8bef9SDimitry Andric const DenormalMode &Mode) const override; 1126e8d8bef9SDimitry Andric SDValue getSqrtResultForDenormInput(SDValue Operand, 1127e8d8bef9SDimitry Andric SelectionDAG &DAG) const override; 11280b57cec5SDimitry Andric unsigned combineRepeatedFPDivisors() const override; 11290b57cec5SDimitry Andric 11300b57cec5SDimitry Andric ConstraintType getConstraintType(StringRef Constraint) const override; 1131480093f4SDimitry Andric Register getRegisterByName(const char* RegName, LLT VT, 11328bcb0991SDimitry Andric const MachineFunction &MF) const override; 11330b57cec5SDimitry Andric 11340b57cec5SDimitry Andric /// Examine constraint string and operand type and determine a weight value. 11350b57cec5SDimitry Andric /// The operand object must already have been set up with the operand type. 11360b57cec5SDimitry Andric ConstraintWeight 11370b57cec5SDimitry Andric getSingleConstraintMatchWeight(AsmOperandInfo &info, 11380b57cec5SDimitry Andric const char *constraint) const override; 11390b57cec5SDimitry Andric 11400b57cec5SDimitry Andric std::pair<unsigned, const TargetRegisterClass *> 11410b57cec5SDimitry Andric getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 11420b57cec5SDimitry Andric StringRef Constraint, MVT VT) const override; 11430b57cec5SDimitry Andric 11440b57cec5SDimitry Andric const char *LowerXConstraint(EVT ConstraintVT) const override; 11450b57cec5SDimitry Andric 11460b57cec5SDimitry Andric void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 11470b57cec5SDimitry Andric std::vector<SDValue> &Ops, 11480b57cec5SDimitry Andric SelectionDAG &DAG) const override; 11490b57cec5SDimitry Andric 11500b57cec5SDimitry Andric unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override { 11510b57cec5SDimitry Andric if (ConstraintCode == "Q") 11520b57cec5SDimitry Andric return InlineAsm::Constraint_Q; 11530b57cec5SDimitry Andric // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are 11540b57cec5SDimitry Andric // followed by llvm_unreachable so we'll leave them unimplemented in 11550b57cec5SDimitry Andric // the backend for now. 11560b57cec5SDimitry Andric return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); 11570b57cec5SDimitry Andric } 11580b57cec5SDimitry Andric 1159fe6060f1SDimitry Andric bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const override; 116081ad6265SDimitry Andric bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const override; 1161480093f4SDimitry Andric bool isVectorLoadExtDesirable(SDValue ExtVal) const override; 11620b57cec5SDimitry Andric bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override; 11630b57cec5SDimitry Andric bool mayBeEmittedAsTailCall(const CallInst *CI) const override; 1164*bdd1243dSDimitry Andric bool getIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, 1165*bdd1243dSDimitry Andric SDValue &Offset, ISD::MemIndexedMode &AM, 1166*bdd1243dSDimitry Andric bool &IsInc, SelectionDAG &DAG) const; 11670b57cec5SDimitry Andric bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, 11680b57cec5SDimitry Andric ISD::MemIndexedMode &AM, 11690b57cec5SDimitry Andric SelectionDAG &DAG) const override; 11700b57cec5SDimitry Andric bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, 11710b57cec5SDimitry Andric SDValue &Offset, ISD::MemIndexedMode &AM, 11720b57cec5SDimitry Andric SelectionDAG &DAG) const override; 11730b57cec5SDimitry Andric 11740b57cec5SDimitry Andric void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results, 11750b57cec5SDimitry Andric SelectionDAG &DAG) const override; 1176fe6060f1SDimitry Andric void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results, 1177fe6060f1SDimitry Andric SelectionDAG &DAG) const; 11785ffd83dbSDimitry Andric void ReplaceExtractSubVectorResults(SDNode *N, 11795ffd83dbSDimitry Andric SmallVectorImpl<SDValue> &Results, 11805ffd83dbSDimitry Andric SelectionDAG &DAG) const; 11810b57cec5SDimitry Andric 11820b57cec5SDimitry Andric bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override; 11830b57cec5SDimitry Andric 11840b57cec5SDimitry Andric void finalizeLowering(MachineFunction &MF) const override; 11855ffd83dbSDimitry Andric 11865ffd83dbSDimitry Andric bool shouldLocalize(const MachineInstr &MI, 11875ffd83dbSDimitry Andric const TargetTransformInfo *TTI) const override; 11885ffd83dbSDimitry Andric 1189fe6060f1SDimitry Andric bool SimplifyDemandedBitsForTargetNode(SDValue Op, 1190fe6060f1SDimitry Andric const APInt &OriginalDemandedBits, 1191fe6060f1SDimitry Andric const APInt &OriginalDemandedElts, 1192fe6060f1SDimitry Andric KnownBits &Known, 1193fe6060f1SDimitry Andric TargetLoweringOpt &TLO, 1194fe6060f1SDimitry Andric unsigned Depth) const override; 1195fe6060f1SDimitry Andric 119681ad6265SDimitry Andric bool isTargetCanonicalConstantNode(SDValue Op) const override; 119781ad6265SDimitry Andric 1198e8d8bef9SDimitry Andric // With the exception of data-predicate transitions, no instructions are 1199e8d8bef9SDimitry Andric // required to cast between legal scalable vector types. However: 1200e8d8bef9SDimitry Andric // 1. Packed and unpacked types have different bit lengths, meaning BITCAST 1201e8d8bef9SDimitry Andric // is not universally useable. 1202e8d8bef9SDimitry Andric // 2. Most unpacked integer types are not legal and thus integer extends 1203e8d8bef9SDimitry Andric // cannot be used to convert between unpacked and packed types. 1204e8d8bef9SDimitry Andric // These can make "bitcasting" a multiphase process. REINTERPRET_CAST is used 1205e8d8bef9SDimitry Andric // to transition between unpacked and packed types of the same element type, 1206e8d8bef9SDimitry Andric // with BITCAST used otherwise. 1207753f127fSDimitry Andric // This function does not handle predicate bitcasts. 1208e8d8bef9SDimitry Andric SDValue getSVESafeBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) const; 1209fe6060f1SDimitry Andric 1210*bdd1243dSDimitry Andric // Returns the runtime value for PSTATE.SM. When the function is streaming- 1211*bdd1243dSDimitry Andric // compatible, this generates a call to __arm_sme_state. 1212*bdd1243dSDimitry Andric SDValue getPStateSM(SelectionDAG &DAG, SDValue Chain, SMEAttrs Attrs, 1213*bdd1243dSDimitry Andric SDLoc DL, EVT VT) const; 1214*bdd1243dSDimitry Andric 121504eeddc0SDimitry Andric bool isConstantUnsignedBitfieldExtractLegal(unsigned Opc, LLT Ty1, 1216fe6060f1SDimitry Andric LLT Ty2) const override; 12170b57cec5SDimitry Andric }; 12180b57cec5SDimitry Andric 12190b57cec5SDimitry Andric namespace AArch64 { 12200b57cec5SDimitry Andric FastISel *createFastISel(FunctionLoweringInfo &funcInfo, 12210b57cec5SDimitry Andric const TargetLibraryInfo *libInfo); 12220b57cec5SDimitry Andric } // end namespace AArch64 12230b57cec5SDimitry Andric 12240b57cec5SDimitry Andric } // end namespace llvm 12250b57cec5SDimitry Andric 12260b57cec5SDimitry Andric #endif 1227