1 //=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file implements the WebAssemblyTargetLowering class. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #include "WebAssemblyISelLowering.h" 15 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" 16 #include "Utils/WebAssemblyTypeUtilities.h" 17 #include "Utils/WebAssemblyUtilities.h" 18 #include "WebAssemblyMachineFunctionInfo.h" 19 #include "WebAssemblySubtarget.h" 20 #include "WebAssemblyTargetMachine.h" 21 #include "llvm/CodeGen/CallingConvLower.h" 22 #include "llvm/CodeGen/MachineFrameInfo.h" 23 #include "llvm/CodeGen/MachineFunctionPass.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/MachineJumpTableInfo.h" 26 #include "llvm/CodeGen/MachineModuleInfo.h" 27 #include "llvm/CodeGen/MachineRegisterInfo.h" 28 #include "llvm/CodeGen/SelectionDAG.h" 29 #include "llvm/CodeGen/SelectionDAGNodes.h" 30 #include "llvm/IR/DiagnosticInfo.h" 31 #include "llvm/IR/DiagnosticPrinter.h" 32 #include "llvm/IR/Function.h" 33 #include "llvm/IR/Intrinsics.h" 34 #include "llvm/IR/IntrinsicsWebAssembly.h" 35 #include "llvm/Support/Debug.h" 36 #include "llvm/Support/ErrorHandling.h" 37 #include "llvm/Support/KnownBits.h" 38 #include "llvm/Support/MathExtras.h" 39 #include "llvm/Support/raw_ostream.h" 40 #include "llvm/Target/TargetOptions.h" 41 using namespace llvm; 42 43 #define DEBUG_TYPE "wasm-lower" 44 45 WebAssemblyTargetLowering::WebAssemblyTargetLowering( 46 const TargetMachine &TM, const WebAssemblySubtarget &STI) 47 : TargetLowering(TM), Subtarget(&STI) { 48 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32; 49 50 // Booleans always contain 0 or 1. 51 setBooleanContents(ZeroOrOneBooleanContent); 52 // Except in SIMD vectors 53 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 54 // We don't know the microarchitecture here, so just reduce register pressure. 55 setSchedulingPreference(Sched::RegPressure); 56 // Tell ISel that we have a stack pointer. 57 setStackPointerRegisterToSaveRestore( 58 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32); 59 // Set up the register classes. 60 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass); 61 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass); 62 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass); 63 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass); 64 if (Subtarget->hasSIMD128()) { 65 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass); 66 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass); 67 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass); 68 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass); 69 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass); 70 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass); 71 } 72 if (Subtarget->hasReferenceTypes()) { 73 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass); 74 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass); 75 } 76 // Compute derived properties from the register classes. 77 computeRegisterProperties(Subtarget->getRegisterInfo()); 78 79 // Transform loads and stores to pointers in address space 1 to loads and 80 // stores to WebAssembly global variables, outside linear memory. 81 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) { 82 setOperationAction(ISD::LOAD, T, Custom); 83 setOperationAction(ISD::STORE, T, Custom); 84 } 85 if (Subtarget->hasSIMD128()) { 86 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, 87 MVT::v2f64}) { 88 setOperationAction(ISD::LOAD, T, Custom); 89 setOperationAction(ISD::STORE, T, Custom); 90 } 91 } 92 if (Subtarget->hasReferenceTypes()) { 93 // We need custom load and store lowering for both externref, funcref and 94 // Other. The MVT::Other here represents tables of reference types. 95 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) { 96 setOperationAction(ISD::LOAD, T, Custom); 97 setOperationAction(ISD::STORE, T, Custom); 98 } 99 } 100 101 setOperationAction(ISD::GlobalAddress, MVTPtr, Custom); 102 setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom); 103 setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom); 104 setOperationAction(ISD::JumpTable, MVTPtr, Custom); 105 setOperationAction(ISD::BlockAddress, MVTPtr, Custom); 106 setOperationAction(ISD::BRIND, MVT::Other, Custom); 107 108 // Take the default expansion for va_arg, va_copy, and va_end. There is no 109 // default action for va_start, so we do that custom. 110 setOperationAction(ISD::VASTART, MVT::Other, Custom); 111 setOperationAction(ISD::VAARG, MVT::Other, Expand); 112 setOperationAction(ISD::VACOPY, MVT::Other, Expand); 113 setOperationAction(ISD::VAEND, MVT::Other, Expand); 114 115 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { 116 // Don't expand the floating-point types to constant pools. 117 setOperationAction(ISD::ConstantFP, T, Legal); 118 // Expand floating-point comparisons. 119 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE, 120 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE}) 121 setCondCodeAction(CC, T, Expand); 122 // Expand floating-point library function operators. 123 for (auto Op : 124 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA}) 125 setOperationAction(Op, T, Expand); 126 // Note supported floating-point library function operators that otherwise 127 // default to expand. 128 for (auto Op : 129 {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT}) 130 setOperationAction(Op, T, Legal); 131 // Support minimum and maximum, which otherwise default to expand. 132 setOperationAction(ISD::FMINIMUM, T, Legal); 133 setOperationAction(ISD::FMAXIMUM, T, Legal); 134 // WebAssembly currently has no builtin f16 support. 135 setOperationAction(ISD::FP16_TO_FP, T, Expand); 136 setOperationAction(ISD::FP_TO_FP16, T, Expand); 137 setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand); 138 setTruncStoreAction(T, MVT::f16, Expand); 139 } 140 141 // Expand unavailable integer operations. 142 for (auto Op : 143 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU, 144 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS, 145 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) { 146 for (auto T : {MVT::i32, MVT::i64}) 147 setOperationAction(Op, T, Expand); 148 if (Subtarget->hasSIMD128()) 149 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) 150 setOperationAction(Op, T, Expand); 151 } 152 153 if (Subtarget->hasNontrappingFPToInt()) 154 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) 155 for (auto T : {MVT::i32, MVT::i64}) 156 setOperationAction(Op, T, Custom); 157 158 // SIMD-specific configuration 159 if (Subtarget->hasSIMD128()) { 160 // Hoist bitcasts out of shuffles 161 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 162 163 // Combine extends of extract_subvectors into widening ops 164 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}); 165 166 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into 167 // conversions ops 168 setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND, 169 ISD::EXTRACT_SUBVECTOR}); 170 171 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa 172 // into conversion ops 173 setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, 174 ISD::FP_ROUND, ISD::CONCAT_VECTORS}); 175 176 setTargetDAGCombine(ISD::TRUNCATE); 177 178 // Support saturating add for i8x16 and i16x8 179 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT}) 180 for (auto T : {MVT::v16i8, MVT::v8i16}) 181 setOperationAction(Op, T, Legal); 182 183 // Support integer abs 184 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) 185 setOperationAction(ISD::ABS, T, Legal); 186 187 // Custom lower BUILD_VECTORs to minimize number of replace_lanes 188 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, 189 MVT::v2f64}) 190 setOperationAction(ISD::BUILD_VECTOR, T, Custom); 191 192 // We have custom shuffle lowering to expose the shuffle mask 193 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, 194 MVT::v2f64}) 195 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom); 196 197 // Support splatting 198 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, 199 MVT::v2f64}) 200 setOperationAction(ISD::SPLAT_VECTOR, T, Legal); 201 202 // Custom lowering since wasm shifts must have a scalar shift amount 203 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) 204 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) 205 setOperationAction(Op, T, Custom); 206 207 // Custom lower lane accesses to expand out variable indices 208 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) 209 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, 210 MVT::v2f64}) 211 setOperationAction(Op, T, Custom); 212 213 // There is no i8x16.mul instruction 214 setOperationAction(ISD::MUL, MVT::v16i8, Expand); 215 216 // There is no vector conditional select instruction 217 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, 218 MVT::v2f64}) 219 setOperationAction(ISD::SELECT_CC, T, Expand); 220 221 // Expand integer operations supported for scalars but not SIMD 222 for (auto Op : 223 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR}) 224 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) 225 setOperationAction(Op, T, Expand); 226 227 // But we do have integer min and max operations 228 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) 229 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) 230 setOperationAction(Op, T, Legal); 231 232 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz. 233 setOperationAction(ISD::CTPOP, MVT::v16i8, Legal); 234 setOperationAction(ISD::CTLZ, MVT::v16i8, Expand); 235 setOperationAction(ISD::CTTZ, MVT::v16i8, Expand); 236 237 // Custom lower bit counting operations for other types to scalarize them. 238 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}) 239 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64}) 240 setOperationAction(Op, T, Custom); 241 242 // Expand float operations supported for scalars but not SIMD 243 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, 244 ISD::FEXP, ISD::FEXP2, ISD::FRINT}) 245 for (auto T : {MVT::v4f32, MVT::v2f64}) 246 setOperationAction(Op, T, Expand); 247 248 // Unsigned comparison operations are unavailable for i64x2 vectors. 249 for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE}) 250 setCondCodeAction(CC, MVT::v2i64, Custom); 251 252 // 64x2 conversions are not in the spec 253 for (auto Op : 254 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}) 255 for (auto T : {MVT::v2i64, MVT::v2f64}) 256 setOperationAction(Op, T, Expand); 257 258 // But saturating fp_to_int converstions are 259 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) 260 setOperationAction(Op, MVT::v4i32, Custom); 261 } 262 263 // As a special case, these operators use the type to mean the type to 264 // sign-extend from. 265 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 266 if (!Subtarget->hasSignExt()) { 267 // Sign extends are legal only when extending a vector extract 268 auto Action = Subtarget->hasSIMD128() ? Custom : Expand; 269 for (auto T : {MVT::i8, MVT::i16, MVT::i32}) 270 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action); 271 } 272 for (auto T : MVT::integer_fixedlen_vector_valuetypes()) 273 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand); 274 275 // Dynamic stack allocation: use the default expansion. 276 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 277 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 278 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand); 279 280 setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 281 setOperationAction(ISD::FrameIndex, MVT::i64, Custom); 282 setOperationAction(ISD::CopyToReg, MVT::Other, Custom); 283 284 // Expand these forms; we pattern-match the forms that we can handle in isel. 285 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) 286 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC}) 287 setOperationAction(Op, T, Expand); 288 289 // We have custom switch handling. 290 setOperationAction(ISD::BR_JT, MVT::Other, Custom); 291 292 // WebAssembly doesn't have: 293 // - Floating-point extending loads. 294 // - Floating-point truncating stores. 295 // - i1 extending loads. 296 // - truncating SIMD stores and most extending loads 297 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); 298 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 299 for (auto T : MVT::integer_valuetypes()) 300 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}) 301 setLoadExtAction(Ext, T, MVT::i1, Promote); 302 if (Subtarget->hasSIMD128()) { 303 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, 304 MVT::v2f64}) { 305 for (auto MemT : MVT::fixedlen_vector_valuetypes()) { 306 if (MVT(T) != MemT) { 307 setTruncStoreAction(T, MemT, Expand); 308 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}) 309 setLoadExtAction(Ext, T, MemT, Expand); 310 } 311 } 312 } 313 // But some vector extending loads are legal 314 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) { 315 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal); 316 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal); 317 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal); 318 } 319 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal); 320 } 321 322 // Don't do anything clever with build_pairs 323 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 324 325 // Trap lowers to wasm unreachable 326 setOperationAction(ISD::TRAP, MVT::Other, Legal); 327 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); 328 329 // Exception handling intrinsics 330 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 331 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 332 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 333 334 setMaxAtomicSizeInBitsSupported(64); 335 336 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is 337 // consistent with the f64 and f128 names. 338 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); 339 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); 340 341 // Define the emscripten name for return address helper. 342 // TODO: when implementing other Wasm backends, make this generic or only do 343 // this on emscripten depending on what they end up doing. 344 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address"); 345 346 // Always convert switches to br_tables unless there is only one case, which 347 // is equivalent to a simple branch. This reduces code size for wasm, and we 348 // defer possible jump table optimizations to the VM. 349 setMinimumJumpTableEntries(2); 350 } 351 352 MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL, 353 uint32_t AS) const { 354 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF) 355 return MVT::externref; 356 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF) 357 return MVT::funcref; 358 return TargetLowering::getPointerTy(DL, AS); 359 } 360 361 MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL, 362 uint32_t AS) const { 363 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF) 364 return MVT::externref; 365 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF) 366 return MVT::funcref; 367 return TargetLowering::getPointerMemTy(DL, AS); 368 } 369 370 TargetLowering::AtomicExpansionKind 371 WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { 372 // We have wasm instructions for these 373 switch (AI->getOperation()) { 374 case AtomicRMWInst::Add: 375 case AtomicRMWInst::Sub: 376 case AtomicRMWInst::And: 377 case AtomicRMWInst::Or: 378 case AtomicRMWInst::Xor: 379 case AtomicRMWInst::Xchg: 380 return AtomicExpansionKind::None; 381 default: 382 break; 383 } 384 return AtomicExpansionKind::CmpXChg; 385 } 386 387 bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const { 388 // Implementation copied from X86TargetLowering. 389 unsigned Opc = VecOp.getOpcode(); 390 391 // Assume target opcodes can't be scalarized. 392 // TODO - do we have any exceptions? 393 if (Opc >= ISD::BUILTIN_OP_END) 394 return false; 395 396 // If the vector op is not supported, try to convert to scalar. 397 EVT VecVT = VecOp.getValueType(); 398 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT)) 399 return true; 400 401 // If the vector op is supported, but the scalar op is not, the transform may 402 // not be worthwhile. 403 EVT ScalarVT = VecVT.getScalarType(); 404 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT); 405 } 406 407 FastISel *WebAssemblyTargetLowering::createFastISel( 408 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const { 409 return WebAssembly::createFastISel(FuncInfo, LibInfo); 410 } 411 412 MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/, 413 EVT VT) const { 414 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1); 415 if (BitWidth > 1 && BitWidth < 8) 416 BitWidth = 8; 417 418 if (BitWidth > 64) { 419 // The shift will be lowered to a libcall, and compiler-rt libcalls expect 420 // the count to be an i32. 421 BitWidth = 32; 422 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && 423 "32-bit shift counts ought to be enough for anyone"); 424 } 425 426 MVT Result = MVT::getIntegerVT(BitWidth); 427 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE && 428 "Unable to represent scalar shift amount type"); 429 return Result; 430 } 431 432 // Lower an fp-to-int conversion operator from the LLVM opcode, which has an 433 // undefined result on invalid/overflow, to the WebAssembly opcode, which 434 // traps on invalid/overflow. 435 static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL, 436 MachineBasicBlock *BB, 437 const TargetInstrInfo &TII, 438 bool IsUnsigned, bool Int64, 439 bool Float64, unsigned LoweredOpcode) { 440 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 441 442 Register OutReg = MI.getOperand(0).getReg(); 443 Register InReg = MI.getOperand(1).getReg(); 444 445 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32; 446 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32; 447 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32; 448 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32; 449 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32; 450 unsigned Eqz = WebAssembly::EQZ_I32; 451 unsigned And = WebAssembly::AND_I32; 452 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN; 453 int64_t Substitute = IsUnsigned ? 0 : Limit; 454 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit; 455 auto &Context = BB->getParent()->getFunction().getContext(); 456 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context); 457 458 const BasicBlock *LLVMBB = BB->getBasicBlock(); 459 MachineFunction *F = BB->getParent(); 460 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB); 461 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB); 462 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB); 463 464 MachineFunction::iterator It = ++BB->getIterator(); 465 F->insert(It, FalseMBB); 466 F->insert(It, TrueMBB); 467 F->insert(It, DoneMBB); 468 469 // Transfer the remainder of BB and its successor edges to DoneMBB. 470 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end()); 471 DoneMBB->transferSuccessorsAndUpdatePHIs(BB); 472 473 BB->addSuccessor(TrueMBB); 474 BB->addSuccessor(FalseMBB); 475 TrueMBB->addSuccessor(DoneMBB); 476 FalseMBB->addSuccessor(DoneMBB); 477 478 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg; 479 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); 480 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); 481 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); 482 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); 483 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg)); 484 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg)); 485 486 MI.eraseFromParent(); 487 // For signed numbers, we can do a single comparison to determine whether 488 // fabs(x) is within range. 489 if (IsUnsigned) { 490 Tmp0 = InReg; 491 } else { 492 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg); 493 } 494 BuildMI(BB, DL, TII.get(FConst), Tmp1) 495 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal))); 496 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1); 497 498 // For unsigned numbers, we have to do a separate comparison with zero. 499 if (IsUnsigned) { 500 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); 501 Register SecondCmpReg = 502 MRI.createVirtualRegister(&WebAssembly::I32RegClass); 503 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); 504 BuildMI(BB, DL, TII.get(FConst), Tmp1) 505 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0))); 506 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1); 507 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg); 508 CmpReg = AndReg; 509 } 510 511 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg); 512 513 // Create the CFG diamond to select between doing the conversion or using 514 // the substitute value. 515 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg); 516 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg); 517 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB); 518 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute); 519 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg) 520 .addReg(FalseReg) 521 .addMBB(FalseMBB) 522 .addReg(TrueReg) 523 .addMBB(TrueMBB); 524 525 return DoneMBB; 526 } 527 528 static MachineBasicBlock * 529 LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, 530 const WebAssemblySubtarget *Subtarget, 531 const TargetInstrInfo &TII) { 532 MachineInstr &CallParams = *CallResults.getPrevNode(); 533 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS); 534 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS || 535 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS); 536 537 bool IsIndirect = CallParams.getOperand(0).isReg(); 538 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS; 539 540 bool IsFuncrefCall = false; 541 if (IsIndirect) { 542 Register Reg = CallParams.getOperand(0).getReg(); 543 const MachineFunction *MF = BB->getParent(); 544 const MachineRegisterInfo &MRI = MF->getRegInfo(); 545 const TargetRegisterClass *TRC = MRI.getRegClass(Reg); 546 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass); 547 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes()); 548 } 549 550 unsigned CallOp; 551 if (IsIndirect && IsRetCall) { 552 CallOp = WebAssembly::RET_CALL_INDIRECT; 553 } else if (IsIndirect) { 554 CallOp = WebAssembly::CALL_INDIRECT; 555 } else if (IsRetCall) { 556 CallOp = WebAssembly::RET_CALL; 557 } else { 558 CallOp = WebAssembly::CALL; 559 } 560 561 MachineFunction &MF = *BB->getParent(); 562 const MCInstrDesc &MCID = TII.get(CallOp); 563 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL)); 564 565 // See if we must truncate the function pointer. 566 // CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers 567 // as 64-bit for uniformity with other pointer types. 568 // See also: WebAssemblyFastISel::selectCall 569 if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) { 570 Register Reg32 = 571 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass); 572 auto &FnPtr = CallParams.getOperand(0); 573 BuildMI(*BB, CallResults.getIterator(), DL, 574 TII.get(WebAssembly::I32_WRAP_I64), Reg32) 575 .addReg(FnPtr.getReg()); 576 FnPtr.setReg(Reg32); 577 } 578 579 // Move the function pointer to the end of the arguments for indirect calls 580 if (IsIndirect) { 581 auto FnPtr = CallParams.getOperand(0); 582 CallParams.removeOperand(0); 583 584 // For funcrefs, call_indirect is done through __funcref_call_table and the 585 // funcref is always installed in slot 0 of the table, therefore instead of 586 // having the function pointer added at the end of the params list, a zero 587 // (the index in 588 // __funcref_call_table is added). 589 if (IsFuncrefCall) { 590 Register RegZero = 591 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass); 592 MachineInstrBuilder MIBC0 = 593 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0); 594 595 BB->insert(CallResults.getIterator(), MIBC0); 596 MachineInstrBuilder(MF, CallParams).addReg(RegZero); 597 } else 598 CallParams.addOperand(FnPtr); 599 } 600 601 for (auto Def : CallResults.defs()) 602 MIB.add(Def); 603 604 if (IsIndirect) { 605 // Placeholder for the type index. 606 MIB.addImm(0); 607 // The table into which this call_indirect indexes. 608 MCSymbolWasm *Table = IsFuncrefCall 609 ? WebAssembly::getOrCreateFuncrefCallTableSymbol( 610 MF.getContext(), Subtarget) 611 : WebAssembly::getOrCreateFunctionTableSymbol( 612 MF.getContext(), Subtarget); 613 if (Subtarget->hasReferenceTypes()) { 614 MIB.addSym(Table); 615 } else { 616 // For the MVP there is at most one table whose number is 0, but we can't 617 // write a table symbol or issue relocations. Instead we just ensure the 618 // table is live and write a zero. 619 Table->setNoStrip(); 620 MIB.addImm(0); 621 } 622 } 623 624 for (auto Use : CallParams.uses()) 625 MIB.add(Use); 626 627 BB->insert(CallResults.getIterator(), MIB); 628 CallParams.eraseFromParent(); 629 CallResults.eraseFromParent(); 630 631 // If this is a funcref call, to avoid hidden GC roots, we need to clear the 632 // table slot with ref.null upon call_indirect return. 633 // 634 // This generates the following code, which comes right after a call_indirect 635 // of a funcref: 636 // 637 // i32.const 0 638 // ref.null func 639 // table.set __funcref_call_table 640 if (IsIndirect && IsFuncrefCall) { 641 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol( 642 MF.getContext(), Subtarget); 643 Register RegZero = 644 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass); 645 MachineInstr *Const0 = 646 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0); 647 BB->insertAfter(MIB.getInstr()->getIterator(), Const0); 648 649 Register RegFuncref = 650 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass); 651 MachineInstr *RefNull = 652 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref); 653 BB->insertAfter(Const0->getIterator(), RefNull); 654 655 MachineInstr *TableSet = 656 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF)) 657 .addSym(Table) 658 .addReg(RegZero) 659 .addReg(RegFuncref); 660 BB->insertAfter(RefNull->getIterator(), TableSet); 661 } 662 663 return BB; 664 } 665 666 MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter( 667 MachineInstr &MI, MachineBasicBlock *BB) const { 668 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); 669 DebugLoc DL = MI.getDebugLoc(); 670 671 switch (MI.getOpcode()) { 672 default: 673 llvm_unreachable("Unexpected instr type to insert"); 674 case WebAssembly::FP_TO_SINT_I32_F32: 675 return LowerFPToInt(MI, DL, BB, TII, false, false, false, 676 WebAssembly::I32_TRUNC_S_F32); 677 case WebAssembly::FP_TO_UINT_I32_F32: 678 return LowerFPToInt(MI, DL, BB, TII, true, false, false, 679 WebAssembly::I32_TRUNC_U_F32); 680 case WebAssembly::FP_TO_SINT_I64_F32: 681 return LowerFPToInt(MI, DL, BB, TII, false, true, false, 682 WebAssembly::I64_TRUNC_S_F32); 683 case WebAssembly::FP_TO_UINT_I64_F32: 684 return LowerFPToInt(MI, DL, BB, TII, true, true, false, 685 WebAssembly::I64_TRUNC_U_F32); 686 case WebAssembly::FP_TO_SINT_I32_F64: 687 return LowerFPToInt(MI, DL, BB, TII, false, false, true, 688 WebAssembly::I32_TRUNC_S_F64); 689 case WebAssembly::FP_TO_UINT_I32_F64: 690 return LowerFPToInt(MI, DL, BB, TII, true, false, true, 691 WebAssembly::I32_TRUNC_U_F64); 692 case WebAssembly::FP_TO_SINT_I64_F64: 693 return LowerFPToInt(MI, DL, BB, TII, false, true, true, 694 WebAssembly::I64_TRUNC_S_F64); 695 case WebAssembly::FP_TO_UINT_I64_F64: 696 return LowerFPToInt(MI, DL, BB, TII, true, true, true, 697 WebAssembly::I64_TRUNC_U_F64); 698 case WebAssembly::CALL_RESULTS: 699 case WebAssembly::RET_CALL_RESULTS: 700 return LowerCallResults(MI, DL, BB, Subtarget, TII); 701 } 702 } 703 704 const char * 705 WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const { 706 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) { 707 case WebAssemblyISD::FIRST_NUMBER: 708 case WebAssemblyISD::FIRST_MEM_OPCODE: 709 break; 710 #define HANDLE_NODETYPE(NODE) \ 711 case WebAssemblyISD::NODE: \ 712 return "WebAssemblyISD::" #NODE; 713 #define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE) 714 #include "WebAssemblyISD.def" 715 #undef HANDLE_MEM_NODETYPE 716 #undef HANDLE_NODETYPE 717 } 718 return nullptr; 719 } 720 721 std::pair<unsigned, const TargetRegisterClass *> 722 WebAssemblyTargetLowering::getRegForInlineAsmConstraint( 723 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { 724 // First, see if this is a constraint that directly corresponds to a 725 // WebAssembly register class. 726 if (Constraint.size() == 1) { 727 switch (Constraint[0]) { 728 case 'r': 729 assert(VT != MVT::iPTR && "Pointer MVT not expected here"); 730 if (Subtarget->hasSIMD128() && VT.isVector()) { 731 if (VT.getSizeInBits() == 128) 732 return std::make_pair(0U, &WebAssembly::V128RegClass); 733 } 734 if (VT.isInteger() && !VT.isVector()) { 735 if (VT.getSizeInBits() <= 32) 736 return std::make_pair(0U, &WebAssembly::I32RegClass); 737 if (VT.getSizeInBits() <= 64) 738 return std::make_pair(0U, &WebAssembly::I64RegClass); 739 } 740 if (VT.isFloatingPoint() && !VT.isVector()) { 741 switch (VT.getSizeInBits()) { 742 case 32: 743 return std::make_pair(0U, &WebAssembly::F32RegClass); 744 case 64: 745 return std::make_pair(0U, &WebAssembly::F64RegClass); 746 default: 747 break; 748 } 749 } 750 break; 751 default: 752 break; 753 } 754 } 755 756 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 757 } 758 759 bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const { 760 // Assume ctz is a relatively cheap operation. 761 return true; 762 } 763 764 bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const { 765 // Assume clz is a relatively cheap operation. 766 return true; 767 } 768 769 bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL, 770 const AddrMode &AM, 771 Type *Ty, unsigned AS, 772 Instruction *I) const { 773 // WebAssembly offsets are added as unsigned without wrapping. The 774 // isLegalAddressingMode gives us no way to determine if wrapping could be 775 // happening, so we approximate this by accepting only non-negative offsets. 776 if (AM.BaseOffs < 0) 777 return false; 778 779 // WebAssembly has no scale register operands. 780 if (AM.Scale != 0) 781 return false; 782 783 // Everything else is legal. 784 return true; 785 } 786 787 bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses( 788 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/, 789 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const { 790 // WebAssembly supports unaligned accesses, though it should be declared 791 // with the p2align attribute on loads and stores which do so, and there 792 // may be a performance impact. We tell LLVM they're "fast" because 793 // for the kinds of things that LLVM uses this for (merging adjacent stores 794 // of constants, etc.), WebAssembly implementations will either want the 795 // unaligned access or they'll split anyway. 796 if (Fast) 797 *Fast = 1; 798 return true; 799 } 800 801 bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT, 802 AttributeList Attr) const { 803 // The current thinking is that wasm engines will perform this optimization, 804 // so we can save on code size. 805 return true; 806 } 807 808 bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const { 809 EVT ExtT = ExtVal.getValueType(); 810 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0); 811 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) || 812 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) || 813 (ExtT == MVT::v2i64 && MemT == MVT::v2i32); 814 } 815 816 bool WebAssemblyTargetLowering::isOffsetFoldingLegal( 817 const GlobalAddressSDNode *GA) const { 818 // Wasm doesn't support function addresses with offsets 819 const GlobalValue *GV = GA->getGlobal(); 820 return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA); 821 } 822 823 EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL, 824 LLVMContext &C, 825 EVT VT) const { 826 if (VT.isVector()) 827 return VT.changeVectorElementTypeToInteger(); 828 829 // So far, all branch instructions in Wasm take an I32 condition. 830 // The default TargetLowering::getSetCCResultType returns the pointer size, 831 // which would be useful to reduce instruction counts when testing 832 // against 64-bit pointers/values if at some point Wasm supports that. 833 return EVT::getIntegerVT(C, 32); 834 } 835 836 bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 837 const CallInst &I, 838 MachineFunction &MF, 839 unsigned Intrinsic) const { 840 switch (Intrinsic) { 841 case Intrinsic::wasm_memory_atomic_notify: 842 Info.opc = ISD::INTRINSIC_W_CHAIN; 843 Info.memVT = MVT::i32; 844 Info.ptrVal = I.getArgOperand(0); 845 Info.offset = 0; 846 Info.align = Align(4); 847 // atomic.notify instruction does not really load the memory specified with 848 // this argument, but MachineMemOperand should either be load or store, so 849 // we set this to a load. 850 // FIXME Volatile isn't really correct, but currently all LLVM atomic 851 // instructions are treated as volatiles in the backend, so we should be 852 // consistent. The same applies for wasm_atomic_wait intrinsics too. 853 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; 854 return true; 855 case Intrinsic::wasm_memory_atomic_wait32: 856 Info.opc = ISD::INTRINSIC_W_CHAIN; 857 Info.memVT = MVT::i32; 858 Info.ptrVal = I.getArgOperand(0); 859 Info.offset = 0; 860 Info.align = Align(4); 861 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; 862 return true; 863 case Intrinsic::wasm_memory_atomic_wait64: 864 Info.opc = ISD::INTRINSIC_W_CHAIN; 865 Info.memVT = MVT::i64; 866 Info.ptrVal = I.getArgOperand(0); 867 Info.offset = 0; 868 Info.align = Align(8); 869 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; 870 return true; 871 default: 872 return false; 873 } 874 } 875 876 void WebAssemblyTargetLowering::computeKnownBitsForTargetNode( 877 const SDValue Op, KnownBits &Known, const APInt &DemandedElts, 878 const SelectionDAG &DAG, unsigned Depth) const { 879 switch (Op.getOpcode()) { 880 default: 881 break; 882 case ISD::INTRINSIC_WO_CHAIN: { 883 unsigned IntNo = Op.getConstantOperandVal(0); 884 switch (IntNo) { 885 default: 886 break; 887 case Intrinsic::wasm_bitmask: { 888 unsigned BitWidth = Known.getBitWidth(); 889 EVT VT = Op.getOperand(1).getSimpleValueType(); 890 unsigned PossibleBits = VT.getVectorNumElements(); 891 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits); 892 Known.Zero |= ZeroMask; 893 break; 894 } 895 } 896 } 897 } 898 } 899 900 TargetLoweringBase::LegalizeTypeAction 901 WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const { 902 if (VT.isFixedLengthVector()) { 903 MVT EltVT = VT.getVectorElementType(); 904 // We have legal vector types with these lane types, so widening the 905 // vector would let us use some of the lanes directly without having to 906 // extend or truncate values. 907 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 || 908 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64) 909 return TypeWidenVector; 910 } 911 912 return TargetLoweringBase::getPreferredVectorAction(VT); 913 } 914 915 bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts( 916 SDValue Op, const TargetLoweringOpt &TLO) const { 917 // ISel process runs DAGCombiner after legalization; this step is called 918 // SelectionDAG optimization phase. This post-legalization combining process 919 // runs DAGCombiner on each node, and if there was a change to be made, 920 // re-runs legalization again on it and its user nodes to make sure 921 // everythiing is in a legalized state. 922 // 923 // The legalization calls lowering routines, and we do our custom lowering for 924 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements 925 // into zeros. But there is a set of routines in DAGCombiner that turns unused 926 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts 927 // turns unused vector elements into undefs. But this routine does not work 928 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This 929 // combination can result in a infinite loop, in which undefs are converted to 930 // zeros in legalization and back to undefs in combining. 931 // 932 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from 933 // running for build_vectors. 934 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys) 935 return false; 936 return true; 937 } 938 939 //===----------------------------------------------------------------------===// 940 // WebAssembly Lowering private implementation. 941 //===----------------------------------------------------------------------===// 942 943 //===----------------------------------------------------------------------===// 944 // Lowering Code 945 //===----------------------------------------------------------------------===// 946 947 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) { 948 MachineFunction &MF = DAG.getMachineFunction(); 949 DAG.getContext()->diagnose( 950 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc())); 951 } 952 953 // Test whether the given calling convention is supported. 954 static bool callingConvSupported(CallingConv::ID CallConv) { 955 // We currently support the language-independent target-independent 956 // conventions. We don't yet have a way to annotate calls with properties like 957 // "cold", and we don't have any call-clobbered registers, so these are mostly 958 // all handled the same. 959 return CallConv == CallingConv::C || CallConv == CallingConv::Fast || 960 CallConv == CallingConv::Cold || 961 CallConv == CallingConv::PreserveMost || 962 CallConv == CallingConv::PreserveAll || 963 CallConv == CallingConv::CXX_FAST_TLS || 964 CallConv == CallingConv::WASM_EmscriptenInvoke || 965 CallConv == CallingConv::Swift; 966 } 967 968 SDValue 969 WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI, 970 SmallVectorImpl<SDValue> &InVals) const { 971 SelectionDAG &DAG = CLI.DAG; 972 SDLoc DL = CLI.DL; 973 SDValue Chain = CLI.Chain; 974 SDValue Callee = CLI.Callee; 975 MachineFunction &MF = DAG.getMachineFunction(); 976 auto Layout = MF.getDataLayout(); 977 978 CallingConv::ID CallConv = CLI.CallConv; 979 if (!callingConvSupported(CallConv)) 980 fail(DL, DAG, 981 "WebAssembly doesn't support language-specific or target-specific " 982 "calling conventions yet"); 983 if (CLI.IsPatchPoint) 984 fail(DL, DAG, "WebAssembly doesn't support patch point yet"); 985 986 if (CLI.IsTailCall) { 987 auto NoTail = [&](const char *Msg) { 988 if (CLI.CB && CLI.CB->isMustTailCall()) 989 fail(DL, DAG, Msg); 990 CLI.IsTailCall = false; 991 }; 992 993 if (!Subtarget->hasTailCall()) 994 NoTail("WebAssembly 'tail-call' feature not enabled"); 995 996 // Varargs calls cannot be tail calls because the buffer is on the stack 997 if (CLI.IsVarArg) 998 NoTail("WebAssembly does not support varargs tail calls"); 999 1000 // Do not tail call unless caller and callee return types match 1001 const Function &F = MF.getFunction(); 1002 const TargetMachine &TM = getTargetMachine(); 1003 Type *RetTy = F.getReturnType(); 1004 SmallVector<MVT, 4> CallerRetTys; 1005 SmallVector<MVT, 4> CalleeRetTys; 1006 computeLegalValueVTs(F, TM, RetTy, CallerRetTys); 1007 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys); 1008 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() && 1009 std::equal(CallerRetTys.begin(), CallerRetTys.end(), 1010 CalleeRetTys.begin()); 1011 if (!TypesMatch) 1012 NoTail("WebAssembly tail call requires caller and callee return types to " 1013 "match"); 1014 1015 // If pointers to local stack values are passed, we cannot tail call 1016 if (CLI.CB) { 1017 for (auto &Arg : CLI.CB->args()) { 1018 Value *Val = Arg.get(); 1019 // Trace the value back through pointer operations 1020 while (true) { 1021 Value *Src = Val->stripPointerCastsAndAliases(); 1022 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src)) 1023 Src = GEP->getPointerOperand(); 1024 if (Val == Src) 1025 break; 1026 Val = Src; 1027 } 1028 if (isa<AllocaInst>(Val)) { 1029 NoTail( 1030 "WebAssembly does not support tail calling with stack arguments"); 1031 break; 1032 } 1033 } 1034 } 1035 } 1036 1037 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 1038 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 1039 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 1040 1041 // The generic code may have added an sret argument. If we're lowering an 1042 // invoke function, the ABI requires that the function pointer be the first 1043 // argument, so we may have to swap the arguments. 1044 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 && 1045 Outs[0].Flags.isSRet()) { 1046 std::swap(Outs[0], Outs[1]); 1047 std::swap(OutVals[0], OutVals[1]); 1048 } 1049 1050 bool HasSwiftSelfArg = false; 1051 bool HasSwiftErrorArg = false; 1052 unsigned NumFixedArgs = 0; 1053 for (unsigned I = 0; I < Outs.size(); ++I) { 1054 const ISD::OutputArg &Out = Outs[I]; 1055 SDValue &OutVal = OutVals[I]; 1056 HasSwiftSelfArg |= Out.Flags.isSwiftSelf(); 1057 HasSwiftErrorArg |= Out.Flags.isSwiftError(); 1058 if (Out.Flags.isNest()) 1059 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments"); 1060 if (Out.Flags.isInAlloca()) 1061 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments"); 1062 if (Out.Flags.isInConsecutiveRegs()) 1063 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments"); 1064 if (Out.Flags.isInConsecutiveRegsLast()) 1065 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments"); 1066 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) { 1067 auto &MFI = MF.getFrameInfo(); 1068 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(), 1069 Out.Flags.getNonZeroByValAlign(), 1070 /*isSS=*/false); 1071 SDValue SizeNode = 1072 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32); 1073 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout)); 1074 Chain = DAG.getMemcpy( 1075 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(), 1076 /*isVolatile*/ false, /*AlwaysInline=*/false, 1077 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo()); 1078 OutVal = FINode; 1079 } 1080 // Count the number of fixed args *after* legalization. 1081 NumFixedArgs += Out.IsFixed; 1082 } 1083 1084 bool IsVarArg = CLI.IsVarArg; 1085 auto PtrVT = getPointerTy(Layout); 1086 1087 // For swiftcc, emit additional swiftself and swifterror arguments 1088 // if there aren't. These additional arguments are also added for callee 1089 // signature They are necessary to match callee and caller signature for 1090 // indirect call. 1091 if (CallConv == CallingConv::Swift) { 1092 if (!HasSwiftSelfArg) { 1093 NumFixedArgs++; 1094 ISD::OutputArg Arg; 1095 Arg.Flags.setSwiftSelf(); 1096 CLI.Outs.push_back(Arg); 1097 SDValue ArgVal = DAG.getUNDEF(PtrVT); 1098 CLI.OutVals.push_back(ArgVal); 1099 } 1100 if (!HasSwiftErrorArg) { 1101 NumFixedArgs++; 1102 ISD::OutputArg Arg; 1103 Arg.Flags.setSwiftError(); 1104 CLI.Outs.push_back(Arg); 1105 SDValue ArgVal = DAG.getUNDEF(PtrVT); 1106 CLI.OutVals.push_back(ArgVal); 1107 } 1108 } 1109 1110 // Analyze operands of the call, assigning locations to each operand. 1111 SmallVector<CCValAssign, 16> ArgLocs; 1112 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 1113 1114 if (IsVarArg) { 1115 // Outgoing non-fixed arguments are placed in a buffer. First 1116 // compute their offsets and the total amount of buffer space needed. 1117 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) { 1118 const ISD::OutputArg &Out = Outs[I]; 1119 SDValue &Arg = OutVals[I]; 1120 EVT VT = Arg.getValueType(); 1121 assert(VT != MVT::iPTR && "Legalized args should be concrete"); 1122 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); 1123 Align Alignment = 1124 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty)); 1125 unsigned Offset = 1126 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment); 1127 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(), 1128 Offset, VT.getSimpleVT(), 1129 CCValAssign::Full)); 1130 } 1131 } 1132 1133 unsigned NumBytes = CCInfo.getAlignedCallFrameSize(); 1134 1135 SDValue FINode; 1136 if (IsVarArg && NumBytes) { 1137 // For non-fixed arguments, next emit stores to store the argument values 1138 // to the stack buffer at the offsets computed above. 1139 int FI = MF.getFrameInfo().CreateStackObject(NumBytes, 1140 Layout.getStackAlignment(), 1141 /*isSS=*/false); 1142 unsigned ValNo = 0; 1143 SmallVector<SDValue, 8> Chains; 1144 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) { 1145 assert(ArgLocs[ValNo].getValNo() == ValNo && 1146 "ArgLocs should remain in order and only hold varargs args"); 1147 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset(); 1148 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout)); 1149 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode, 1150 DAG.getConstant(Offset, DL, PtrVT)); 1151 Chains.push_back( 1152 DAG.getStore(Chain, DL, Arg, Add, 1153 MachinePointerInfo::getFixedStack(MF, FI, Offset))); 1154 } 1155 if (!Chains.empty()) 1156 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 1157 } else if (IsVarArg) { 1158 FINode = DAG.getIntPtrConstant(0, DL); 1159 } 1160 1161 if (Callee->getOpcode() == ISD::GlobalAddress) { 1162 // If the callee is a GlobalAddress node (quite common, every direct call 1163 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress 1164 // doesn't at MO_GOT which is not needed for direct calls. 1165 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee); 1166 Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, 1167 getPointerTy(DAG.getDataLayout()), 1168 GA->getOffset()); 1169 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL, 1170 getPointerTy(DAG.getDataLayout()), Callee); 1171 } 1172 1173 // Compute the operands for the CALLn node. 1174 SmallVector<SDValue, 16> Ops; 1175 Ops.push_back(Chain); 1176 Ops.push_back(Callee); 1177 1178 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs 1179 // isn't reliable. 1180 Ops.append(OutVals.begin(), 1181 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end()); 1182 // Add a pointer to the vararg buffer. 1183 if (IsVarArg) 1184 Ops.push_back(FINode); 1185 1186 SmallVector<EVT, 8> InTys; 1187 for (const auto &In : Ins) { 1188 assert(!In.Flags.isByVal() && "byval is not valid for return values"); 1189 assert(!In.Flags.isNest() && "nest is not valid for return values"); 1190 if (In.Flags.isInAlloca()) 1191 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values"); 1192 if (In.Flags.isInConsecutiveRegs()) 1193 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values"); 1194 if (In.Flags.isInConsecutiveRegsLast()) 1195 fail(DL, DAG, 1196 "WebAssembly hasn't implemented cons regs last return values"); 1197 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in 1198 // registers. 1199 InTys.push_back(In.VT); 1200 } 1201 1202 // Lastly, if this is a call to a funcref we need to add an instruction 1203 // table.set to the chain and transform the call. 1204 if (CLI.CB && 1205 WebAssembly::isFuncrefType(CLI.CB->getCalledOperand()->getType())) { 1206 // In the absence of function references proposal where a funcref call is 1207 // lowered to call_ref, using reference types we generate a table.set to set 1208 // the funcref to a special table used solely for this purpose, followed by 1209 // a call_indirect. Here we just generate the table set, and return the 1210 // SDValue of the table.set so that LowerCall can finalize the lowering by 1211 // generating the call_indirect. 1212 SDValue Chain = Ops[0]; 1213 1214 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol( 1215 MF.getContext(), Subtarget); 1216 SDValue Sym = DAG.getMCSymbol(Table, PtrVT); 1217 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32); 1218 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee}; 1219 SDValue TableSet = DAG.getMemIntrinsicNode( 1220 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps, 1221 MVT::funcref, 1222 // Machine Mem Operand args 1223 MachinePointerInfo( 1224 WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF), 1225 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()), 1226 MachineMemOperand::MOStore); 1227 1228 Ops[0] = TableSet; // The new chain is the TableSet itself 1229 } 1230 1231 if (CLI.IsTailCall) { 1232 // ret_calls do not return values to the current frame 1233 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 1234 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops); 1235 } 1236 1237 InTys.push_back(MVT::Other); 1238 SDVTList InTyList = DAG.getVTList(InTys); 1239 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops); 1240 1241 for (size_t I = 0; I < Ins.size(); ++I) 1242 InVals.push_back(Res.getValue(I)); 1243 1244 // Return the chain 1245 return Res.getValue(Ins.size()); 1246 } 1247 1248 bool WebAssemblyTargetLowering::CanLowerReturn( 1249 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/, 1250 const SmallVectorImpl<ISD::OutputArg> &Outs, 1251 LLVMContext & /*Context*/) const { 1252 // WebAssembly can only handle returning tuples with multivalue enabled 1253 return Subtarget->hasMultivalue() || Outs.size() <= 1; 1254 } 1255 1256 SDValue WebAssemblyTargetLowering::LowerReturn( 1257 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/, 1258 const SmallVectorImpl<ISD::OutputArg> &Outs, 1259 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, 1260 SelectionDAG &DAG) const { 1261 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) && 1262 "MVP WebAssembly can only return up to one value"); 1263 if (!callingConvSupported(CallConv)) 1264 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions"); 1265 1266 SmallVector<SDValue, 4> RetOps(1, Chain); 1267 RetOps.append(OutVals.begin(), OutVals.end()); 1268 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps); 1269 1270 // Record the number and types of the return values. 1271 for (const ISD::OutputArg &Out : Outs) { 1272 assert(!Out.Flags.isByVal() && "byval is not valid for return values"); 1273 assert(!Out.Flags.isNest() && "nest is not valid for return values"); 1274 assert(Out.IsFixed && "non-fixed return value is not valid"); 1275 if (Out.Flags.isInAlloca()) 1276 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results"); 1277 if (Out.Flags.isInConsecutiveRegs()) 1278 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results"); 1279 if (Out.Flags.isInConsecutiveRegsLast()) 1280 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results"); 1281 } 1282 1283 return Chain; 1284 } 1285 1286 SDValue WebAssemblyTargetLowering::LowerFormalArguments( 1287 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 1288 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 1289 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 1290 if (!callingConvSupported(CallConv)) 1291 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions"); 1292 1293 MachineFunction &MF = DAG.getMachineFunction(); 1294 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>(); 1295 1296 // Set up the incoming ARGUMENTS value, which serves to represent the liveness 1297 // of the incoming values before they're represented by virtual registers. 1298 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS); 1299 1300 bool HasSwiftErrorArg = false; 1301 bool HasSwiftSelfArg = false; 1302 for (const ISD::InputArg &In : Ins) { 1303 HasSwiftSelfArg |= In.Flags.isSwiftSelf(); 1304 HasSwiftErrorArg |= In.Flags.isSwiftError(); 1305 if (In.Flags.isInAlloca()) 1306 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments"); 1307 if (In.Flags.isNest()) 1308 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments"); 1309 if (In.Flags.isInConsecutiveRegs()) 1310 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments"); 1311 if (In.Flags.isInConsecutiveRegsLast()) 1312 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments"); 1313 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in 1314 // registers. 1315 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT, 1316 DAG.getTargetConstant(InVals.size(), 1317 DL, MVT::i32)) 1318 : DAG.getUNDEF(In.VT)); 1319 1320 // Record the number and types of arguments. 1321 MFI->addParam(In.VT); 1322 } 1323 1324 // For swiftcc, emit additional swiftself and swifterror arguments 1325 // if there aren't. These additional arguments are also added for callee 1326 // signature They are necessary to match callee and caller signature for 1327 // indirect call. 1328 auto PtrVT = getPointerTy(MF.getDataLayout()); 1329 if (CallConv == CallingConv::Swift) { 1330 if (!HasSwiftSelfArg) { 1331 MFI->addParam(PtrVT); 1332 } 1333 if (!HasSwiftErrorArg) { 1334 MFI->addParam(PtrVT); 1335 } 1336 } 1337 // Varargs are copied into a buffer allocated by the caller, and a pointer to 1338 // the buffer is passed as an argument. 1339 if (IsVarArg) { 1340 MVT PtrVT = getPointerTy(MF.getDataLayout()); 1341 Register VarargVreg = 1342 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT)); 1343 MFI->setVarargBufferVreg(VarargVreg); 1344 Chain = DAG.getCopyToReg( 1345 Chain, DL, VarargVreg, 1346 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT, 1347 DAG.getTargetConstant(Ins.size(), DL, MVT::i32))); 1348 MFI->addParam(PtrVT); 1349 } 1350 1351 // Record the number and types of arguments and results. 1352 SmallVector<MVT, 4> Params; 1353 SmallVector<MVT, 4> Results; 1354 computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(), 1355 MF.getFunction(), DAG.getTarget(), Params, Results); 1356 for (MVT VT : Results) 1357 MFI->addResult(VT); 1358 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify 1359 // the param logic here with ComputeSignatureVTs 1360 assert(MFI->getParams().size() == Params.size() && 1361 std::equal(MFI->getParams().begin(), MFI->getParams().end(), 1362 Params.begin())); 1363 1364 return Chain; 1365 } 1366 1367 void WebAssemblyTargetLowering::ReplaceNodeResults( 1368 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 1369 switch (N->getOpcode()) { 1370 case ISD::SIGN_EXTEND_INREG: 1371 // Do not add any results, signifying that N should not be custom lowered 1372 // after all. This happens because simd128 turns on custom lowering for 1373 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an 1374 // illegal type. 1375 break; 1376 default: 1377 llvm_unreachable( 1378 "ReplaceNodeResults not implemented for this op for WebAssembly!"); 1379 } 1380 } 1381 1382 //===----------------------------------------------------------------------===// 1383 // Custom lowering hooks. 1384 //===----------------------------------------------------------------------===// 1385 1386 SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op, 1387 SelectionDAG &DAG) const { 1388 SDLoc DL(Op); 1389 switch (Op.getOpcode()) { 1390 default: 1391 llvm_unreachable("unimplemented operation lowering"); 1392 return SDValue(); 1393 case ISD::FrameIndex: 1394 return LowerFrameIndex(Op, DAG); 1395 case ISD::GlobalAddress: 1396 return LowerGlobalAddress(Op, DAG); 1397 case ISD::GlobalTLSAddress: 1398 return LowerGlobalTLSAddress(Op, DAG); 1399 case ISD::ExternalSymbol: 1400 return LowerExternalSymbol(Op, DAG); 1401 case ISD::JumpTable: 1402 return LowerJumpTable(Op, DAG); 1403 case ISD::BR_JT: 1404 return LowerBR_JT(Op, DAG); 1405 case ISD::VASTART: 1406 return LowerVASTART(Op, DAG); 1407 case ISD::BlockAddress: 1408 case ISD::BRIND: 1409 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos"); 1410 return SDValue(); 1411 case ISD::RETURNADDR: 1412 return LowerRETURNADDR(Op, DAG); 1413 case ISD::FRAMEADDR: 1414 return LowerFRAMEADDR(Op, DAG); 1415 case ISD::CopyToReg: 1416 return LowerCopyToReg(Op, DAG); 1417 case ISD::EXTRACT_VECTOR_ELT: 1418 case ISD::INSERT_VECTOR_ELT: 1419 return LowerAccessVectorElement(Op, DAG); 1420 case ISD::INTRINSIC_VOID: 1421 case ISD::INTRINSIC_WO_CHAIN: 1422 case ISD::INTRINSIC_W_CHAIN: 1423 return LowerIntrinsic(Op, DAG); 1424 case ISD::SIGN_EXTEND_INREG: 1425 return LowerSIGN_EXTEND_INREG(Op, DAG); 1426 case ISD::BUILD_VECTOR: 1427 return LowerBUILD_VECTOR(Op, DAG); 1428 case ISD::VECTOR_SHUFFLE: 1429 return LowerVECTOR_SHUFFLE(Op, DAG); 1430 case ISD::SETCC: 1431 return LowerSETCC(Op, DAG); 1432 case ISD::SHL: 1433 case ISD::SRA: 1434 case ISD::SRL: 1435 return LowerShift(Op, DAG); 1436 case ISD::FP_TO_SINT_SAT: 1437 case ISD::FP_TO_UINT_SAT: 1438 return LowerFP_TO_INT_SAT(Op, DAG); 1439 case ISD::LOAD: 1440 return LowerLoad(Op, DAG); 1441 case ISD::STORE: 1442 return LowerStore(Op, DAG); 1443 case ISD::CTPOP: 1444 case ISD::CTLZ: 1445 case ISD::CTTZ: 1446 return DAG.UnrollVectorOp(Op.getNode()); 1447 } 1448 } 1449 1450 static bool IsWebAssemblyGlobal(SDValue Op) { 1451 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) 1452 return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace()); 1453 1454 return false; 1455 } 1456 1457 static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op, 1458 SelectionDAG &DAG) { 1459 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op); 1460 if (!FI) 1461 return std::nullopt; 1462 1463 auto &MF = DAG.getMachineFunction(); 1464 return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex()); 1465 } 1466 1467 SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op, 1468 SelectionDAG &DAG) const { 1469 SDLoc DL(Op); 1470 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); 1471 const SDValue &Value = SN->getValue(); 1472 const SDValue &Base = SN->getBasePtr(); 1473 const SDValue &Offset = SN->getOffset(); 1474 1475 if (IsWebAssemblyGlobal(Base)) { 1476 if (!Offset->isUndef()) 1477 report_fatal_error("unexpected offset when storing to webassembly global", 1478 false); 1479 1480 SDVTList Tys = DAG.getVTList(MVT::Other); 1481 SDValue Ops[] = {SN->getChain(), Value, Base}; 1482 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops, 1483 SN->getMemoryVT(), SN->getMemOperand()); 1484 } 1485 1486 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) { 1487 if (!Offset->isUndef()) 1488 report_fatal_error("unexpected offset when storing to webassembly local", 1489 false); 1490 1491 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32); 1492 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain. 1493 SDValue Ops[] = {SN->getChain(), Idx, Value}; 1494 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops); 1495 } 1496 1497 if (WebAssembly::isWasmVarAddressSpace(SN->getAddressSpace())) 1498 report_fatal_error( 1499 "Encountered an unlowerable store to the wasm_var address space", 1500 false); 1501 1502 return Op; 1503 } 1504 1505 SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op, 1506 SelectionDAG &DAG) const { 1507 SDLoc DL(Op); 1508 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); 1509 const SDValue &Base = LN->getBasePtr(); 1510 const SDValue &Offset = LN->getOffset(); 1511 1512 if (IsWebAssemblyGlobal(Base)) { 1513 if (!Offset->isUndef()) 1514 report_fatal_error( 1515 "unexpected offset when loading from webassembly global", false); 1516 1517 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other); 1518 SDValue Ops[] = {LN->getChain(), Base}; 1519 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops, 1520 LN->getMemoryVT(), LN->getMemOperand()); 1521 } 1522 1523 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) { 1524 if (!Offset->isUndef()) 1525 report_fatal_error( 1526 "unexpected offset when loading from webassembly local", false); 1527 1528 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32); 1529 EVT LocalVT = LN->getValueType(0); 1530 SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT, 1531 {LN->getChain(), Idx}); 1532 SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL); 1533 assert(Result->getNumValues() == 2 && "Loads must carry a chain!"); 1534 return Result; 1535 } 1536 1537 if (WebAssembly::isWasmVarAddressSpace(LN->getAddressSpace())) 1538 report_fatal_error( 1539 "Encountered an unlowerable load from the wasm_var address space", 1540 false); 1541 1542 return Op; 1543 } 1544 1545 SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op, 1546 SelectionDAG &DAG) const { 1547 SDValue Src = Op.getOperand(2); 1548 if (isa<FrameIndexSDNode>(Src.getNode())) { 1549 // CopyToReg nodes don't support FrameIndex operands. Other targets select 1550 // the FI to some LEA-like instruction, but since we don't have that, we 1551 // need to insert some kind of instruction that can take an FI operand and 1552 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy 1553 // local.copy between Op and its FI operand. 1554 SDValue Chain = Op.getOperand(0); 1555 SDLoc DL(Op); 1556 Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg(); 1557 EVT VT = Src.getValueType(); 1558 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32 1559 : WebAssembly::COPY_I64, 1560 DL, VT, Src), 1561 0); 1562 return Op.getNode()->getNumValues() == 1 1563 ? DAG.getCopyToReg(Chain, DL, Reg, Copy) 1564 : DAG.getCopyToReg(Chain, DL, Reg, Copy, 1565 Op.getNumOperands() == 4 ? Op.getOperand(3) 1566 : SDValue()); 1567 } 1568 return SDValue(); 1569 } 1570 1571 SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op, 1572 SelectionDAG &DAG) const { 1573 int FI = cast<FrameIndexSDNode>(Op)->getIndex(); 1574 return DAG.getTargetFrameIndex(FI, Op.getValueType()); 1575 } 1576 1577 SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op, 1578 SelectionDAG &DAG) const { 1579 SDLoc DL(Op); 1580 1581 if (!Subtarget->getTargetTriple().isOSEmscripten()) { 1582 fail(DL, DAG, 1583 "Non-Emscripten WebAssembly hasn't implemented " 1584 "__builtin_return_address"); 1585 return SDValue(); 1586 } 1587 1588 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 1589 return SDValue(); 1590 1591 unsigned Depth = Op.getConstantOperandVal(0); 1592 MakeLibCallOptions CallOptions; 1593 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(), 1594 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL) 1595 .first; 1596 } 1597 1598 SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op, 1599 SelectionDAG &DAG) const { 1600 // Non-zero depths are not supported by WebAssembly currently. Use the 1601 // legalizer's default expansion, which is to return 0 (what this function is 1602 // documented to do). 1603 if (Op.getConstantOperandVal(0) > 0) 1604 return SDValue(); 1605 1606 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true); 1607 EVT VT = Op.getValueType(); 1608 Register FP = 1609 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction()); 1610 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT); 1611 } 1612 1613 SDValue 1614 WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op, 1615 SelectionDAG &DAG) const { 1616 SDLoc DL(Op); 1617 const auto *GA = cast<GlobalAddressSDNode>(Op); 1618 1619 MachineFunction &MF = DAG.getMachineFunction(); 1620 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory()) 1621 report_fatal_error("cannot use thread-local storage without bulk memory", 1622 false); 1623 1624 const GlobalValue *GV = GA->getGlobal(); 1625 1626 // Currently only Emscripten supports dynamic linking with threads. Therefore, 1627 // on other targets, if we have thread-local storage, only the local-exec 1628 // model is possible. 1629 auto model = Subtarget->getTargetTriple().isOSEmscripten() 1630 ? GV->getThreadLocalMode() 1631 : GlobalValue::LocalExecTLSModel; 1632 1633 // Unsupported TLS modes 1634 assert(model != GlobalValue::NotThreadLocal); 1635 assert(model != GlobalValue::InitialExecTLSModel); 1636 1637 if (model == GlobalValue::LocalExecTLSModel || 1638 model == GlobalValue::LocalDynamicTLSModel || 1639 (model == GlobalValue::GeneralDynamicTLSModel && 1640 getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) { 1641 // For DSO-local TLS variables we use offset from __tls_base 1642 1643 MVT PtrVT = getPointerTy(DAG.getDataLayout()); 1644 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64 1645 : WebAssembly::GLOBAL_GET_I32; 1646 const char *BaseName = MF.createExternalSymbolName("__tls_base"); 1647 1648 SDValue BaseAddr( 1649 DAG.getMachineNode(GlobalGet, DL, PtrVT, 1650 DAG.getTargetExternalSymbol(BaseName, PtrVT)), 1651 0); 1652 1653 SDValue TLSOffset = DAG.getTargetGlobalAddress( 1654 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL); 1655 SDValue SymOffset = 1656 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset); 1657 1658 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset); 1659 } 1660 1661 assert(model == GlobalValue::GeneralDynamicTLSModel); 1662 1663 EVT VT = Op.getValueType(); 1664 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, 1665 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, 1666 GA->getOffset(), 1667 WebAssemblyII::MO_GOT_TLS)); 1668 } 1669 1670 SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op, 1671 SelectionDAG &DAG) const { 1672 SDLoc DL(Op); 1673 const auto *GA = cast<GlobalAddressSDNode>(Op); 1674 EVT VT = Op.getValueType(); 1675 assert(GA->getTargetFlags() == 0 && 1676 "Unexpected target flags on generic GlobalAddressSDNode"); 1677 if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace())) 1678 fail(DL, DAG, "Invalid address space for WebAssembly target"); 1679 1680 unsigned OperandFlags = 0; 1681 if (isPositionIndependent()) { 1682 const GlobalValue *GV = GA->getGlobal(); 1683 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) { 1684 MachineFunction &MF = DAG.getMachineFunction(); 1685 MVT PtrVT = getPointerTy(MF.getDataLayout()); 1686 const char *BaseName; 1687 if (GV->getValueType()->isFunctionTy()) { 1688 BaseName = MF.createExternalSymbolName("__table_base"); 1689 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL; 1690 } else { 1691 BaseName = MF.createExternalSymbolName("__memory_base"); 1692 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL; 1693 } 1694 SDValue BaseAddr = 1695 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, 1696 DAG.getTargetExternalSymbol(BaseName, PtrVT)); 1697 1698 SDValue SymAddr = DAG.getNode( 1699 WebAssemblyISD::WrapperREL, DL, VT, 1700 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(), 1701 OperandFlags)); 1702 1703 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr); 1704 } 1705 OperandFlags = WebAssemblyII::MO_GOT; 1706 } 1707 1708 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, 1709 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, 1710 GA->getOffset(), OperandFlags)); 1711 } 1712 1713 SDValue 1714 WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op, 1715 SelectionDAG &DAG) const { 1716 SDLoc DL(Op); 1717 const auto *ES = cast<ExternalSymbolSDNode>(Op); 1718 EVT VT = Op.getValueType(); 1719 assert(ES->getTargetFlags() == 0 && 1720 "Unexpected target flags on generic ExternalSymbolSDNode"); 1721 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, 1722 DAG.getTargetExternalSymbol(ES->getSymbol(), VT)); 1723 } 1724 1725 SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op, 1726 SelectionDAG &DAG) const { 1727 // There's no need for a Wrapper node because we always incorporate a jump 1728 // table operand into a BR_TABLE instruction, rather than ever 1729 // materializing it in a register. 1730 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 1731 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(), 1732 JT->getTargetFlags()); 1733 } 1734 1735 SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op, 1736 SelectionDAG &DAG) const { 1737 SDLoc DL(Op); 1738 SDValue Chain = Op.getOperand(0); 1739 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1)); 1740 SDValue Index = Op.getOperand(2); 1741 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags"); 1742 1743 SmallVector<SDValue, 8> Ops; 1744 Ops.push_back(Chain); 1745 Ops.push_back(Index); 1746 1747 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo(); 1748 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs; 1749 1750 // Add an operand for each case. 1751 for (auto *MBB : MBBs) 1752 Ops.push_back(DAG.getBasicBlock(MBB)); 1753 1754 // Add the first MBB as a dummy default target for now. This will be replaced 1755 // with the proper default target (and the preceding range check eliminated) 1756 // if possible by WebAssemblyFixBrTableDefaults. 1757 Ops.push_back(DAG.getBasicBlock(*MBBs.begin())); 1758 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops); 1759 } 1760 1761 SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op, 1762 SelectionDAG &DAG) const { 1763 SDLoc DL(Op); 1764 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout()); 1765 1766 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>(); 1767 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1768 1769 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL, 1770 MFI->getVarargBufferVreg(), PtrVT); 1771 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1), 1772 MachinePointerInfo(SV)); 1773 } 1774 1775 SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op, 1776 SelectionDAG &DAG) const { 1777 MachineFunction &MF = DAG.getMachineFunction(); 1778 unsigned IntNo; 1779 switch (Op.getOpcode()) { 1780 case ISD::INTRINSIC_VOID: 1781 case ISD::INTRINSIC_W_CHAIN: 1782 IntNo = Op.getConstantOperandVal(1); 1783 break; 1784 case ISD::INTRINSIC_WO_CHAIN: 1785 IntNo = Op.getConstantOperandVal(0); 1786 break; 1787 default: 1788 llvm_unreachable("Invalid intrinsic"); 1789 } 1790 SDLoc DL(Op); 1791 1792 switch (IntNo) { 1793 default: 1794 return SDValue(); // Don't custom lower most intrinsics. 1795 1796 case Intrinsic::wasm_lsda: { 1797 auto PtrVT = getPointerTy(MF.getDataLayout()); 1798 const char *SymName = MF.createExternalSymbolName( 1799 "GCC_except_table" + std::to_string(MF.getFunctionNumber())); 1800 if (isPositionIndependent()) { 1801 SDValue Node = DAG.getTargetExternalSymbol( 1802 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL); 1803 const char *BaseName = MF.createExternalSymbolName("__memory_base"); 1804 SDValue BaseAddr = 1805 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, 1806 DAG.getTargetExternalSymbol(BaseName, PtrVT)); 1807 SDValue SymAddr = 1808 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node); 1809 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr); 1810 } 1811 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT); 1812 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node); 1813 } 1814 1815 case Intrinsic::wasm_shuffle: { 1816 // Drop in-chain and replace undefs, but otherwise pass through unchanged 1817 SDValue Ops[18]; 1818 size_t OpIdx = 0; 1819 Ops[OpIdx++] = Op.getOperand(1); 1820 Ops[OpIdx++] = Op.getOperand(2); 1821 while (OpIdx < 18) { 1822 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1); 1823 if (MaskIdx.isUndef() || 1824 cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) { 1825 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32); 1826 } else { 1827 Ops[OpIdx++] = MaskIdx; 1828 } 1829 } 1830 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops); 1831 } 1832 } 1833 } 1834 1835 SDValue 1836 WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 1837 SelectionDAG &DAG) const { 1838 SDLoc DL(Op); 1839 // If sign extension operations are disabled, allow sext_inreg only if operand 1840 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign 1841 // extension operations, but allowing sext_inreg in this context lets us have 1842 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg 1843 // everywhere would be simpler in this file, but would necessitate large and 1844 // brittle patterns to undo the expansion and select extract_lane_s 1845 // instructions. 1846 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128()); 1847 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1848 return SDValue(); 1849 1850 const SDValue &Extract = Op.getOperand(0); 1851 MVT VecT = Extract.getOperand(0).getSimpleValueType(); 1852 if (VecT.getVectorElementType().getSizeInBits() > 32) 1853 return SDValue(); 1854 MVT ExtractedLaneT = 1855 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT(); 1856 MVT ExtractedVecT = 1857 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits()); 1858 if (ExtractedVecT == VecT) 1859 return Op; 1860 1861 // Bitcast vector to appropriate type to ensure ISel pattern coverage 1862 const SDNode *Index = Extract.getOperand(1).getNode(); 1863 if (!isa<ConstantSDNode>(Index)) 1864 return SDValue(); 1865 unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue(); 1866 unsigned Scale = 1867 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements(); 1868 assert(Scale > 1); 1869 SDValue NewIndex = 1870 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0)); 1871 SDValue NewExtract = DAG.getNode( 1872 ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(), 1873 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex); 1874 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract, 1875 Op.getOperand(1)); 1876 } 1877 1878 static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) { 1879 SDLoc DL(Op); 1880 if (Op.getValueType() != MVT::v2f64) 1881 return SDValue(); 1882 1883 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec, 1884 unsigned &Index) -> bool { 1885 switch (Op.getOpcode()) { 1886 case ISD::SINT_TO_FP: 1887 Opcode = WebAssemblyISD::CONVERT_LOW_S; 1888 break; 1889 case ISD::UINT_TO_FP: 1890 Opcode = WebAssemblyISD::CONVERT_LOW_U; 1891 break; 1892 case ISD::FP_EXTEND: 1893 Opcode = WebAssemblyISD::PROMOTE_LOW; 1894 break; 1895 default: 1896 return false; 1897 } 1898 1899 auto ExtractVector = Op.getOperand(0); 1900 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1901 return false; 1902 1903 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode())) 1904 return false; 1905 1906 SrcVec = ExtractVector.getOperand(0); 1907 Index = ExtractVector.getConstantOperandVal(1); 1908 return true; 1909 }; 1910 1911 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex; 1912 SDValue LHSSrcVec, RHSSrcVec; 1913 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) || 1914 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex)) 1915 return SDValue(); 1916 1917 if (LHSOpcode != RHSOpcode) 1918 return SDValue(); 1919 1920 MVT ExpectedSrcVT; 1921 switch (LHSOpcode) { 1922 case WebAssemblyISD::CONVERT_LOW_S: 1923 case WebAssemblyISD::CONVERT_LOW_U: 1924 ExpectedSrcVT = MVT::v4i32; 1925 break; 1926 case WebAssemblyISD::PROMOTE_LOW: 1927 ExpectedSrcVT = MVT::v4f32; 1928 break; 1929 } 1930 if (LHSSrcVec.getValueType() != ExpectedSrcVT) 1931 return SDValue(); 1932 1933 auto Src = LHSSrcVec; 1934 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) { 1935 // Shuffle the source vector so that the converted lanes are the low lanes. 1936 Src = DAG.getVectorShuffle( 1937 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec, 1938 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1}); 1939 } 1940 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src); 1941 } 1942 1943 SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op, 1944 SelectionDAG &DAG) const { 1945 if (auto ConvertLow = LowerConvertLow(Op, DAG)) 1946 return ConvertLow; 1947 1948 SDLoc DL(Op); 1949 const EVT VecT = Op.getValueType(); 1950 const EVT LaneT = Op.getOperand(0).getValueType(); 1951 const size_t Lanes = Op.getNumOperands(); 1952 bool CanSwizzle = VecT == MVT::v16i8; 1953 1954 // BUILD_VECTORs are lowered to the instruction that initializes the highest 1955 // possible number of lanes at once followed by a sequence of replace_lane 1956 // instructions to individually initialize any remaining lanes. 1957 1958 // TODO: Tune this. For example, lanewise swizzling is very expensive, so 1959 // swizzled lanes should be given greater weight. 1960 1961 // TODO: Investigate looping rather than always extracting/replacing specific 1962 // lanes to fill gaps. 1963 1964 auto IsConstant = [](const SDValue &V) { 1965 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP; 1966 }; 1967 1968 // Returns the source vector and index vector pair if they exist. Checks for: 1969 // (extract_vector_elt 1970 // $src, 1971 // (sign_extend_inreg (extract_vector_elt $indices, $i)) 1972 // ) 1973 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) { 1974 auto Bail = std::make_pair(SDValue(), SDValue()); 1975 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1976 return Bail; 1977 const SDValue &SwizzleSrc = Lane->getOperand(0); 1978 const SDValue &IndexExt = Lane->getOperand(1); 1979 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG) 1980 return Bail; 1981 const SDValue &Index = IndexExt->getOperand(0); 1982 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1983 return Bail; 1984 const SDValue &SwizzleIndices = Index->getOperand(0); 1985 if (SwizzleSrc.getValueType() != MVT::v16i8 || 1986 SwizzleIndices.getValueType() != MVT::v16i8 || 1987 Index->getOperand(1)->getOpcode() != ISD::Constant || 1988 Index->getConstantOperandVal(1) != I) 1989 return Bail; 1990 return std::make_pair(SwizzleSrc, SwizzleIndices); 1991 }; 1992 1993 // If the lane is extracted from another vector at a constant index, return 1994 // that vector. The source vector must not have more lanes than the dest 1995 // because the shufflevector indices are in terms of the destination lanes and 1996 // would not be able to address the smaller individual source lanes. 1997 auto GetShuffleSrc = [&](const SDValue &Lane) { 1998 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1999 return SDValue(); 2000 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode())) 2001 return SDValue(); 2002 if (Lane->getOperand(0).getValueType().getVectorNumElements() > 2003 VecT.getVectorNumElements()) 2004 return SDValue(); 2005 return Lane->getOperand(0); 2006 }; 2007 2008 using ValueEntry = std::pair<SDValue, size_t>; 2009 SmallVector<ValueEntry, 16> SplatValueCounts; 2010 2011 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>; 2012 SmallVector<SwizzleEntry, 16> SwizzleCounts; 2013 2014 using ShuffleEntry = std::pair<SDValue, size_t>; 2015 SmallVector<ShuffleEntry, 16> ShuffleCounts; 2016 2017 auto AddCount = [](auto &Counts, const auto &Val) { 2018 auto CountIt = 2019 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; }); 2020 if (CountIt == Counts.end()) { 2021 Counts.emplace_back(Val, 1); 2022 } else { 2023 CountIt->second++; 2024 } 2025 }; 2026 2027 auto GetMostCommon = [](auto &Counts) { 2028 auto CommonIt = 2029 std::max_element(Counts.begin(), Counts.end(), llvm::less_second()); 2030 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector"); 2031 return *CommonIt; 2032 }; 2033 2034 size_t NumConstantLanes = 0; 2035 2036 // Count eligible lanes for each type of vector creation op 2037 for (size_t I = 0; I < Lanes; ++I) { 2038 const SDValue &Lane = Op->getOperand(I); 2039 if (Lane.isUndef()) 2040 continue; 2041 2042 AddCount(SplatValueCounts, Lane); 2043 2044 if (IsConstant(Lane)) 2045 NumConstantLanes++; 2046 if (auto ShuffleSrc = GetShuffleSrc(Lane)) 2047 AddCount(ShuffleCounts, ShuffleSrc); 2048 if (CanSwizzle) { 2049 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane); 2050 if (SwizzleSrcs.first) 2051 AddCount(SwizzleCounts, SwizzleSrcs); 2052 } 2053 } 2054 2055 SDValue SplatValue; 2056 size_t NumSplatLanes; 2057 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts); 2058 2059 SDValue SwizzleSrc; 2060 SDValue SwizzleIndices; 2061 size_t NumSwizzleLanes = 0; 2062 if (SwizzleCounts.size()) 2063 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices), 2064 NumSwizzleLanes) = GetMostCommon(SwizzleCounts); 2065 2066 // Shuffles can draw from up to two vectors, so find the two most common 2067 // sources. 2068 SDValue ShuffleSrc1, ShuffleSrc2; 2069 size_t NumShuffleLanes = 0; 2070 if (ShuffleCounts.size()) { 2071 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts); 2072 llvm::erase_if(ShuffleCounts, 2073 [&](const auto &Pair) { return Pair.first == ShuffleSrc1; }); 2074 } 2075 if (ShuffleCounts.size()) { 2076 size_t AdditionalShuffleLanes; 2077 std::tie(ShuffleSrc2, AdditionalShuffleLanes) = 2078 GetMostCommon(ShuffleCounts); 2079 NumShuffleLanes += AdditionalShuffleLanes; 2080 } 2081 2082 // Predicate returning true if the lane is properly initialized by the 2083 // original instruction 2084 std::function<bool(size_t, const SDValue &)> IsLaneConstructed; 2085 SDValue Result; 2086 // Prefer swizzles over shuffles over vector consts over splats 2087 if (NumSwizzleLanes >= NumShuffleLanes && 2088 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) { 2089 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc, 2090 SwizzleIndices); 2091 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices); 2092 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) { 2093 return Swizzled == GetSwizzleSrcs(I, Lane); 2094 }; 2095 } else if (NumShuffleLanes >= NumConstantLanes && 2096 NumShuffleLanes >= NumSplatLanes) { 2097 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8; 2098 size_t DestLaneCount = VecT.getVectorNumElements(); 2099 size_t Scale1 = 1; 2100 size_t Scale2 = 1; 2101 SDValue Src1 = ShuffleSrc1; 2102 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT); 2103 if (Src1.getValueType() != VecT) { 2104 size_t LaneSize = 2105 Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8; 2106 assert(LaneSize > DestLaneSize); 2107 Scale1 = LaneSize / DestLaneSize; 2108 Src1 = DAG.getBitcast(VecT, Src1); 2109 } 2110 if (Src2.getValueType() != VecT) { 2111 size_t LaneSize = 2112 Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8; 2113 assert(LaneSize > DestLaneSize); 2114 Scale2 = LaneSize / DestLaneSize; 2115 Src2 = DAG.getBitcast(VecT, Src2); 2116 } 2117 2118 int Mask[16]; 2119 assert(DestLaneCount <= 16); 2120 for (size_t I = 0; I < DestLaneCount; ++I) { 2121 const SDValue &Lane = Op->getOperand(I); 2122 SDValue Src = GetShuffleSrc(Lane); 2123 if (Src == ShuffleSrc1) { 2124 Mask[I] = Lane->getConstantOperandVal(1) * Scale1; 2125 } else if (Src && Src == ShuffleSrc2) { 2126 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2; 2127 } else { 2128 Mask[I] = -1; 2129 } 2130 } 2131 ArrayRef<int> MaskRef(Mask, DestLaneCount); 2132 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef); 2133 IsLaneConstructed = [&](size_t, const SDValue &Lane) { 2134 auto Src = GetShuffleSrc(Lane); 2135 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2); 2136 }; 2137 } else if (NumConstantLanes >= NumSplatLanes) { 2138 SmallVector<SDValue, 16> ConstLanes; 2139 for (const SDValue &Lane : Op->op_values()) { 2140 if (IsConstant(Lane)) { 2141 // Values may need to be fixed so that they will sign extend to be 2142 // within the expected range during ISel. Check whether the value is in 2143 // bounds based on the lane bit width and if it is out of bounds, lop 2144 // off the extra bits and subtract 2^n to reflect giving the high bit 2145 // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it 2146 // cannot possibly be out of range. 2147 auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode()); 2148 int64_t Val = Const ? Const->getSExtValue() : 0; 2149 uint64_t LaneBits = 128 / Lanes; 2150 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && 2151 "Unexpected out of bounds negative value"); 2152 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) { 2153 auto NewVal = ((uint64_t)Val % (1ll << LaneBits)) - (1ll << LaneBits); 2154 ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT)); 2155 } else { 2156 ConstLanes.push_back(Lane); 2157 } 2158 } else if (LaneT.isFloatingPoint()) { 2159 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT)); 2160 } else { 2161 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT)); 2162 } 2163 } 2164 Result = DAG.getBuildVector(VecT, DL, ConstLanes); 2165 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) { 2166 return IsConstant(Lane); 2167 }; 2168 } else { 2169 // Use a splat (which might be selected as a load splat) 2170 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue); 2171 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) { 2172 return Lane == SplatValue; 2173 }; 2174 } 2175 2176 assert(Result); 2177 assert(IsLaneConstructed); 2178 2179 // Add replace_lane instructions for any unhandled values 2180 for (size_t I = 0; I < Lanes; ++I) { 2181 const SDValue &Lane = Op->getOperand(I); 2182 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane)) 2183 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane, 2184 DAG.getConstant(I, DL, MVT::i32)); 2185 } 2186 2187 return Result; 2188 } 2189 2190 SDValue 2191 WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 2192 SelectionDAG &DAG) const { 2193 SDLoc DL(Op); 2194 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask(); 2195 MVT VecType = Op.getOperand(0).getSimpleValueType(); 2196 assert(VecType.is128BitVector() && "Unexpected shuffle vector type"); 2197 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8; 2198 2199 // Space for two vector args and sixteen mask indices 2200 SDValue Ops[18]; 2201 size_t OpIdx = 0; 2202 Ops[OpIdx++] = Op.getOperand(0); 2203 Ops[OpIdx++] = Op.getOperand(1); 2204 2205 // Expand mask indices to byte indices and materialize them as operands 2206 for (int M : Mask) { 2207 for (size_t J = 0; J < LaneBytes; ++J) { 2208 // Lower undefs (represented by -1 in mask) to {0..J}, which use a 2209 // whole lane of vector input, to allow further reduction at VM. E.g. 2210 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle. 2211 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J; 2212 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32); 2213 } 2214 } 2215 2216 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops); 2217 } 2218 2219 SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op, 2220 SelectionDAG &DAG) const { 2221 SDLoc DL(Op); 2222 // The legalizer does not know how to expand the unsupported comparison modes 2223 // of i64x2 vectors, so we manually unroll them here. 2224 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64); 2225 SmallVector<SDValue, 2> LHS, RHS; 2226 DAG.ExtractVectorElements(Op->getOperand(0), LHS); 2227 DAG.ExtractVectorElements(Op->getOperand(1), RHS); 2228 const SDValue &CC = Op->getOperand(2); 2229 auto MakeLane = [&](unsigned I) { 2230 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I], 2231 DAG.getConstant(uint64_t(-1), DL, MVT::i64), 2232 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC); 2233 }; 2234 return DAG.getBuildVector(Op->getValueType(0), DL, 2235 {MakeLane(0), MakeLane(1)}); 2236 } 2237 2238 SDValue 2239 WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op, 2240 SelectionDAG &DAG) const { 2241 // Allow constant lane indices, expand variable lane indices 2242 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode(); 2243 if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef()) { 2244 // Ensure the index type is i32 to match the tablegen patterns 2245 uint64_t Idx = cast<ConstantSDNode>(IdxNode)->getZExtValue(); 2246 SmallVector<SDValue, 3> Ops(Op.getNode()->ops()); 2247 Ops[Op.getNumOperands() - 1] = 2248 DAG.getConstant(Idx, SDLoc(IdxNode), MVT::i32); 2249 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Ops); 2250 } 2251 // Perform default expansion 2252 return SDValue(); 2253 } 2254 2255 static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) { 2256 EVT LaneT = Op.getSimpleValueType().getVectorElementType(); 2257 // 32-bit and 64-bit unrolled shifts will have proper semantics 2258 if (LaneT.bitsGE(MVT::i32)) 2259 return DAG.UnrollVectorOp(Op.getNode()); 2260 // Otherwise mask the shift value to get proper semantics from 32-bit shift 2261 SDLoc DL(Op); 2262 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements(); 2263 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32); 2264 unsigned ShiftOpcode = Op.getOpcode(); 2265 SmallVector<SDValue, 16> ShiftedElements; 2266 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32); 2267 SmallVector<SDValue, 16> ShiftElements; 2268 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32); 2269 SmallVector<SDValue, 16> UnrolledOps; 2270 for (size_t i = 0; i < NumLanes; ++i) { 2271 SDValue MaskedShiftValue = 2272 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask); 2273 SDValue ShiftedValue = ShiftedElements[i]; 2274 if (ShiftOpcode == ISD::SRA) 2275 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, 2276 ShiftedValue, DAG.getValueType(LaneT)); 2277 UnrolledOps.push_back( 2278 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue)); 2279 } 2280 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps); 2281 } 2282 2283 SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op, 2284 SelectionDAG &DAG) const { 2285 SDLoc DL(Op); 2286 2287 // Only manually lower vector shifts 2288 assert(Op.getSimpleValueType().isVector()); 2289 2290 auto ShiftVal = DAG.getSplatValue(Op.getOperand(1)); 2291 if (!ShiftVal) 2292 return unrollVectorShift(Op, DAG); 2293 2294 // Use anyext because none of the high bits can affect the shift 2295 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32); 2296 2297 unsigned Opcode; 2298 switch (Op.getOpcode()) { 2299 case ISD::SHL: 2300 Opcode = WebAssemblyISD::VEC_SHL; 2301 break; 2302 case ISD::SRA: 2303 Opcode = WebAssemblyISD::VEC_SHR_S; 2304 break; 2305 case ISD::SRL: 2306 Opcode = WebAssemblyISD::VEC_SHR_U; 2307 break; 2308 default: 2309 llvm_unreachable("unexpected opcode"); 2310 } 2311 2312 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal); 2313 } 2314 2315 SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op, 2316 SelectionDAG &DAG) const { 2317 SDLoc DL(Op); 2318 EVT ResT = Op.getValueType(); 2319 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2320 2321 if ((ResT == MVT::i32 || ResT == MVT::i64) && 2322 (SatVT == MVT::i32 || SatVT == MVT::i64)) 2323 return Op; 2324 2325 if (ResT == MVT::v4i32 && SatVT == MVT::i32) 2326 return Op; 2327 2328 return SDValue(); 2329 } 2330 2331 //===----------------------------------------------------------------------===// 2332 // Custom DAG combine hooks 2333 //===----------------------------------------------------------------------===// 2334 static SDValue 2335 performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { 2336 auto &DAG = DCI.DAG; 2337 auto Shuffle = cast<ShuffleVectorSDNode>(N); 2338 2339 // Hoist vector bitcasts that don't change the number of lanes out of unary 2340 // shuffles, where they are less likely to get in the way of other combines. 2341 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) -> 2342 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask)))) 2343 SDValue Bitcast = N->getOperand(0); 2344 if (Bitcast.getOpcode() != ISD::BITCAST) 2345 return SDValue(); 2346 if (!N->getOperand(1).isUndef()) 2347 return SDValue(); 2348 SDValue CastOp = Bitcast.getOperand(0); 2349 MVT SrcType = CastOp.getSimpleValueType(); 2350 MVT DstType = Bitcast.getSimpleValueType(); 2351 if (!SrcType.is128BitVector() || 2352 SrcType.getVectorNumElements() != DstType.getVectorNumElements()) 2353 return SDValue(); 2354 SDValue NewShuffle = DAG.getVectorShuffle( 2355 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask()); 2356 return DAG.getBitcast(DstType, NewShuffle); 2357 } 2358 2359 /// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get 2360 /// split up into scalar instructions during legalization, and the vector 2361 /// extending instructions are selected in performVectorExtendCombine below. 2362 static SDValue 2363 performVectorExtendToFPCombine(SDNode *N, 2364 TargetLowering::DAGCombinerInfo &DCI) { 2365 auto &DAG = DCI.DAG; 2366 assert(N->getOpcode() == ISD::UINT_TO_FP || 2367 N->getOpcode() == ISD::SINT_TO_FP); 2368 2369 EVT InVT = N->getOperand(0)->getValueType(0); 2370 EVT ResVT = N->getValueType(0); 2371 MVT ExtVT; 2372 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8)) 2373 ExtVT = MVT::v4i32; 2374 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8)) 2375 ExtVT = MVT::v2i32; 2376 else 2377 return SDValue(); 2378 2379 unsigned Op = 2380 N->getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; 2381 SDValue Conv = DAG.getNode(Op, SDLoc(N), ExtVT, N->getOperand(0)); 2382 return DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, Conv); 2383 } 2384 2385 static SDValue 2386 performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { 2387 auto &DAG = DCI.DAG; 2388 assert(N->getOpcode() == ISD::SIGN_EXTEND || 2389 N->getOpcode() == ISD::ZERO_EXTEND); 2390 2391 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if 2392 // possible before the extract_subvector can be expanded. 2393 auto Extract = N->getOperand(0); 2394 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR) 2395 return SDValue(); 2396 auto Source = Extract.getOperand(0); 2397 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1)); 2398 if (IndexNode == nullptr) 2399 return SDValue(); 2400 auto Index = IndexNode->getZExtValue(); 2401 2402 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the 2403 // extracted subvector is the low or high half of its source. 2404 EVT ResVT = N->getValueType(0); 2405 if (ResVT == MVT::v8i16) { 2406 if (Extract.getValueType() != MVT::v8i8 || 2407 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8)) 2408 return SDValue(); 2409 } else if (ResVT == MVT::v4i32) { 2410 if (Extract.getValueType() != MVT::v4i16 || 2411 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4)) 2412 return SDValue(); 2413 } else if (ResVT == MVT::v2i64) { 2414 if (Extract.getValueType() != MVT::v2i32 || 2415 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2)) 2416 return SDValue(); 2417 } else { 2418 return SDValue(); 2419 } 2420 2421 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND; 2422 bool IsLow = Index == 0; 2423 2424 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S 2425 : WebAssemblyISD::EXTEND_HIGH_S) 2426 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U 2427 : WebAssemblyISD::EXTEND_HIGH_U); 2428 2429 return DAG.getNode(Op, SDLoc(N), ResVT, Source); 2430 } 2431 2432 static SDValue 2433 performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { 2434 auto &DAG = DCI.DAG; 2435 2436 auto GetWasmConversionOp = [](unsigned Op) { 2437 switch (Op) { 2438 case ISD::FP_TO_SINT_SAT: 2439 return WebAssemblyISD::TRUNC_SAT_ZERO_S; 2440 case ISD::FP_TO_UINT_SAT: 2441 return WebAssemblyISD::TRUNC_SAT_ZERO_U; 2442 case ISD::FP_ROUND: 2443 return WebAssemblyISD::DEMOTE_ZERO; 2444 } 2445 llvm_unreachable("unexpected op"); 2446 }; 2447 2448 auto IsZeroSplat = [](SDValue SplatVal) { 2449 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode()); 2450 APInt SplatValue, SplatUndef; 2451 unsigned SplatBitSize; 2452 bool HasAnyUndefs; 2453 return Splat && 2454 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, 2455 HasAnyUndefs) && 2456 SplatValue == 0; 2457 }; 2458 2459 if (N->getOpcode() == ISD::CONCAT_VECTORS) { 2460 // Combine this: 2461 // 2462 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0))) 2463 // 2464 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x). 2465 // 2466 // Or this: 2467 // 2468 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0))) 2469 // 2470 // into (f32x4.demote_zero_f64x2 $x). 2471 EVT ResVT; 2472 EVT ExpectedConversionType; 2473 auto Conversion = N->getOperand(0); 2474 auto ConversionOp = Conversion.getOpcode(); 2475 switch (ConversionOp) { 2476 case ISD::FP_TO_SINT_SAT: 2477 case ISD::FP_TO_UINT_SAT: 2478 ResVT = MVT::v4i32; 2479 ExpectedConversionType = MVT::v2i32; 2480 break; 2481 case ISD::FP_ROUND: 2482 ResVT = MVT::v4f32; 2483 ExpectedConversionType = MVT::v2f32; 2484 break; 2485 default: 2486 return SDValue(); 2487 } 2488 2489 if (N->getValueType(0) != ResVT) 2490 return SDValue(); 2491 2492 if (Conversion.getValueType() != ExpectedConversionType) 2493 return SDValue(); 2494 2495 auto Source = Conversion.getOperand(0); 2496 if (Source.getValueType() != MVT::v2f64) 2497 return SDValue(); 2498 2499 if (!IsZeroSplat(N->getOperand(1)) || 2500 N->getOperand(1).getValueType() != ExpectedConversionType) 2501 return SDValue(); 2502 2503 unsigned Op = GetWasmConversionOp(ConversionOp); 2504 return DAG.getNode(Op, SDLoc(N), ResVT, Source); 2505 } 2506 2507 // Combine this: 2508 // 2509 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32) 2510 // 2511 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x). 2512 // 2513 // Or this: 2514 // 2515 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0))))) 2516 // 2517 // into (f32x4.demote_zero_f64x2 $x). 2518 EVT ResVT; 2519 auto ConversionOp = N->getOpcode(); 2520 switch (ConversionOp) { 2521 case ISD::FP_TO_SINT_SAT: 2522 case ISD::FP_TO_UINT_SAT: 2523 ResVT = MVT::v4i32; 2524 break; 2525 case ISD::FP_ROUND: 2526 ResVT = MVT::v4f32; 2527 break; 2528 default: 2529 llvm_unreachable("unexpected op"); 2530 } 2531 2532 if (N->getValueType(0) != ResVT) 2533 return SDValue(); 2534 2535 auto Concat = N->getOperand(0); 2536 if (Concat.getValueType() != MVT::v4f64) 2537 return SDValue(); 2538 2539 auto Source = Concat.getOperand(0); 2540 if (Source.getValueType() != MVT::v2f64) 2541 return SDValue(); 2542 2543 if (!IsZeroSplat(Concat.getOperand(1)) || 2544 Concat.getOperand(1).getValueType() != MVT::v2f64) 2545 return SDValue(); 2546 2547 unsigned Op = GetWasmConversionOp(ConversionOp); 2548 return DAG.getNode(Op, SDLoc(N), ResVT, Source); 2549 } 2550 2551 // Helper to extract VectorWidth bits from Vec, starting from IdxVal. 2552 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, 2553 const SDLoc &DL, unsigned VectorWidth) { 2554 EVT VT = Vec.getValueType(); 2555 EVT ElVT = VT.getVectorElementType(); 2556 unsigned Factor = VT.getSizeInBits() / VectorWidth; 2557 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, 2558 VT.getVectorNumElements() / Factor); 2559 2560 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR 2561 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits(); 2562 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2"); 2563 2564 // This is the index of the first element of the VectorWidth-bit chunk 2565 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits. 2566 IdxVal &= ~(ElemsPerChunk - 1); 2567 2568 // If the input is a buildvector just emit a smaller one. 2569 if (Vec.getOpcode() == ISD::BUILD_VECTOR) 2570 return DAG.getBuildVector(ResultVT, DL, 2571 Vec->ops().slice(IdxVal, ElemsPerChunk)); 2572 2573 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL); 2574 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx); 2575 } 2576 2577 // Helper to recursively truncate vector elements in half with NARROW_U. DstVT 2578 // is the expected destination value type after recursion. In is the initial 2579 // input. Note that the input should have enough leading zero bits to prevent 2580 // NARROW_U from saturating results. 2581 static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, 2582 SelectionDAG &DAG) { 2583 EVT SrcVT = In.getValueType(); 2584 2585 // No truncation required, we might get here due to recursive calls. 2586 if (SrcVT == DstVT) 2587 return In; 2588 2589 unsigned SrcSizeInBits = SrcVT.getSizeInBits(); 2590 unsigned NumElems = SrcVT.getVectorNumElements(); 2591 if (!isPowerOf2_32(NumElems)) 2592 return SDValue(); 2593 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation"); 2594 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation"); 2595 2596 LLVMContext &Ctx = *DAG.getContext(); 2597 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2); 2598 2599 // Narrow to the largest type possible: 2600 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u. 2601 EVT InVT = MVT::i16, OutVT = MVT::i8; 2602 if (SrcVT.getScalarSizeInBits() > 16) { 2603 InVT = MVT::i32; 2604 OutVT = MVT::i16; 2605 } 2606 unsigned SubSizeInBits = SrcSizeInBits / 2; 2607 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits()); 2608 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits()); 2609 2610 // Split lower/upper subvectors. 2611 SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits); 2612 SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits); 2613 2614 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors. 2615 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) { 2616 Lo = DAG.getBitcast(InVT, Lo); 2617 Hi = DAG.getBitcast(InVT, Hi); 2618 SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi); 2619 return DAG.getBitcast(DstVT, Res); 2620 } 2621 2622 // Recursively narrow lower/upper subvectors, concat result and narrow again. 2623 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2); 2624 Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG); 2625 Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG); 2626 2627 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems); 2628 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi); 2629 return truncateVectorWithNARROW(DstVT, Res, DL, DAG); 2630 } 2631 2632 static SDValue performTruncateCombine(SDNode *N, 2633 TargetLowering::DAGCombinerInfo &DCI) { 2634 auto &DAG = DCI.DAG; 2635 2636 SDValue In = N->getOperand(0); 2637 EVT InVT = In.getValueType(); 2638 if (!InVT.isSimple()) 2639 return SDValue(); 2640 2641 EVT OutVT = N->getValueType(0); 2642 if (!OutVT.isVector()) 2643 return SDValue(); 2644 2645 EVT OutSVT = OutVT.getVectorElementType(); 2646 EVT InSVT = InVT.getVectorElementType(); 2647 // Currently only cover truncate to v16i8 or v8i16. 2648 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) && 2649 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector())) 2650 return SDValue(); 2651 2652 SDLoc DL(N); 2653 APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(), 2654 OutVT.getScalarSizeInBits()); 2655 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT)); 2656 return truncateVectorWithNARROW(OutVT, In, DL, DAG); 2657 } 2658 2659 SDValue 2660 WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N, 2661 DAGCombinerInfo &DCI) const { 2662 switch (N->getOpcode()) { 2663 default: 2664 return SDValue(); 2665 case ISD::VECTOR_SHUFFLE: 2666 return performVECTOR_SHUFFLECombine(N, DCI); 2667 case ISD::SIGN_EXTEND: 2668 case ISD::ZERO_EXTEND: 2669 return performVectorExtendCombine(N, DCI); 2670 case ISD::UINT_TO_FP: 2671 case ISD::SINT_TO_FP: 2672 return performVectorExtendToFPCombine(N, DCI); 2673 case ISD::FP_TO_SINT_SAT: 2674 case ISD::FP_TO_UINT_SAT: 2675 case ISD::FP_ROUND: 2676 case ISD::CONCAT_VECTORS: 2677 return performVectorTruncZeroCombine(N, DCI); 2678 case ISD::TRUNCATE: 2679 return performTruncateCombine(N, DCI); 2680 } 2681 } 2682