1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the interfaces that BPF uses to lower LLVM code into a 10 // selection DAG. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "BPFISelLowering.h" 15 #include "BPF.h" 16 #include "BPFSubtarget.h" 17 #include "BPFTargetMachine.h" 18 #include "llvm/CodeGen/CallingConvLower.h" 19 #include "llvm/CodeGen/MachineFrameInfo.h" 20 #include "llvm/CodeGen/MachineFunction.h" 21 #include "llvm/CodeGen/MachineInstrBuilder.h" 22 #include "llvm/CodeGen/MachineRegisterInfo.h" 23 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 24 #include "llvm/CodeGen/ValueTypes.h" 25 #include "llvm/IR/DiagnosticInfo.h" 26 #include "llvm/IR/DiagnosticPrinter.h" 27 #include "llvm/Support/Debug.h" 28 #include "llvm/Support/ErrorHandling.h" 29 #include "llvm/Support/raw_ostream.h" 30 using namespace llvm; 31 32 #define DEBUG_TYPE "bpf-lower" 33 34 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", 35 cl::Hidden, cl::init(false), 36 cl::desc("Expand memcpy into load/store pairs in order")); 37 38 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) { 39 MachineFunction &MF = DAG.getMachineFunction(); 40 DAG.getContext()->diagnose( 41 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc())); 42 } 43 44 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg, 45 SDValue Val) { 46 MachineFunction &MF = DAG.getMachineFunction(); 47 std::string Str; 48 raw_string_ostream OS(Str); 49 OS << Msg; 50 Val->print(OS); 51 OS.flush(); 52 DAG.getContext()->diagnose( 53 DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc())); 54 } 55 56 BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, 57 const BPFSubtarget &STI) 58 : TargetLowering(TM) { 59 60 // Set up the register classes. 61 addRegisterClass(MVT::i64, &BPF::GPRRegClass); 62 if (STI.getHasAlu32()) 63 addRegisterClass(MVT::i32, &BPF::GPR32RegClass); 64 65 // Compute derived properties from the register classes 66 computeRegisterProperties(STI.getRegisterInfo()); 67 68 setStackPointerRegisterToSaveRestore(BPF::R11); 69 70 setOperationAction(ISD::BR_CC, MVT::i64, Custom); 71 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 72 setOperationAction(ISD::BRIND, MVT::Other, Expand); 73 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 74 75 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 76 77 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom); 78 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 79 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 80 81 // Set unsupported atomic operations as Custom so 82 // we can emit better error messages than fatal error 83 // from selectiondag. 84 for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) { 85 if (VT == MVT::i32) { 86 if (STI.getHasAlu32()) 87 continue; 88 } else { 89 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom); 90 } 91 92 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom); 93 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom); 94 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom); 95 setOperationAction(ISD::ATOMIC_SWAP, VT, Custom); 96 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom); 97 } 98 99 for (auto VT : { MVT::i32, MVT::i64 }) { 100 if (VT == MVT::i32 && !STI.getHasAlu32()) 101 continue; 102 103 setOperationAction(ISD::SDIVREM, VT, Expand); 104 setOperationAction(ISD::UDIVREM, VT, Expand); 105 setOperationAction(ISD::SREM, VT, Expand); 106 setOperationAction(ISD::MULHU, VT, Expand); 107 setOperationAction(ISD::MULHS, VT, Expand); 108 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 109 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 110 setOperationAction(ISD::ROTR, VT, Expand); 111 setOperationAction(ISD::ROTL, VT, Expand); 112 setOperationAction(ISD::SHL_PARTS, VT, Expand); 113 setOperationAction(ISD::SRL_PARTS, VT, Expand); 114 setOperationAction(ISD::SRA_PARTS, VT, Expand); 115 setOperationAction(ISD::CTPOP, VT, Expand); 116 117 setOperationAction(ISD::SETCC, VT, Expand); 118 setOperationAction(ISD::SELECT, VT, Expand); 119 setOperationAction(ISD::SELECT_CC, VT, Custom); 120 } 121 122 if (STI.getHasAlu32()) { 123 setOperationAction(ISD::BSWAP, MVT::i32, Promote); 124 setOperationAction(ISD::BR_CC, MVT::i32, 125 STI.getHasJmp32() ? Custom : Promote); 126 } 127 128 setOperationAction(ISD::CTTZ, MVT::i64, Custom); 129 setOperationAction(ISD::CTLZ, MVT::i64, Custom); 130 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Custom); 131 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); 132 133 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 134 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand); 135 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand); 136 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Expand); 137 138 // Extended load operations for i1 types must be promoted 139 for (MVT VT : MVT::integer_valuetypes()) { 140 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 141 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 142 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 143 144 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); 145 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Expand); 146 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); 147 } 148 149 setBooleanContents(ZeroOrOneBooleanContent); 150 151 // Function alignments 152 setMinFunctionAlignment(Align(8)); 153 setPrefFunctionAlignment(Align(8)); 154 155 if (BPFExpandMemcpyInOrder) { 156 // LLVM generic code will try to expand memcpy into load/store pairs at this 157 // stage which is before quite a few IR optimization passes, therefore the 158 // loads and stores could potentially be moved apart from each other which 159 // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT 160 // compilers. 161 // 162 // When -bpf-expand-memcpy-in-order specified, we want to defer the expand 163 // of memcpy to later stage in IR optimization pipeline so those load/store 164 // pairs won't be touched and could be kept in order. Hence, we set 165 // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores 166 // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy. 167 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 0; 168 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 0; 169 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 0; 170 MaxLoadsPerMemcmp = 0; 171 } else { 172 // inline memcpy() for kernel to see explicit copy 173 unsigned CommonMaxStores = 174 STI.getSelectionDAGInfo()->getCommonMaxStoresPerMemFunc(); 175 176 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores; 177 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores; 178 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = CommonMaxStores; 179 MaxLoadsPerMemcmp = MaxLoadsPerMemcmpOptSize = CommonMaxStores; 180 } 181 182 // CPU/Feature control 183 HasAlu32 = STI.getHasAlu32(); 184 HasJmp32 = STI.getHasJmp32(); 185 HasJmpExt = STI.getHasJmpExt(); 186 } 187 188 bool BPFTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 189 return false; 190 } 191 192 bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 193 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 194 return false; 195 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 196 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 197 return NumBits1 > NumBits2; 198 } 199 200 bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 201 if (!VT1.isInteger() || !VT2.isInteger()) 202 return false; 203 unsigned NumBits1 = VT1.getSizeInBits(); 204 unsigned NumBits2 = VT2.getSizeInBits(); 205 return NumBits1 > NumBits2; 206 } 207 208 bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const { 209 if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 210 return false; 211 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 212 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 213 return NumBits1 == 32 && NumBits2 == 64; 214 } 215 216 bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const { 217 if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger()) 218 return false; 219 unsigned NumBits1 = VT1.getSizeInBits(); 220 unsigned NumBits2 = VT2.getSizeInBits(); 221 return NumBits1 == 32 && NumBits2 == 64; 222 } 223 224 BPFTargetLowering::ConstraintType 225 BPFTargetLowering::getConstraintType(StringRef Constraint) const { 226 if (Constraint.size() == 1) { 227 switch (Constraint[0]) { 228 default: 229 break; 230 case 'w': 231 return C_RegisterClass; 232 } 233 } 234 235 return TargetLowering::getConstraintType(Constraint); 236 } 237 238 std::pair<unsigned, const TargetRegisterClass *> 239 BPFTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 240 StringRef Constraint, 241 MVT VT) const { 242 if (Constraint.size() == 1) 243 // GCC Constraint Letters 244 switch (Constraint[0]) { 245 case 'r': // GENERAL_REGS 246 return std::make_pair(0U, &BPF::GPRRegClass); 247 case 'w': 248 if (HasAlu32) 249 return std::make_pair(0U, &BPF::GPR32RegClass); 250 break; 251 default: 252 break; 253 } 254 255 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 256 } 257 258 void BPFTargetLowering::ReplaceNodeResults( 259 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 260 const char *err_msg; 261 uint32_t Opcode = N->getOpcode(); 262 switch (Opcode) { 263 default: 264 report_fatal_error("Unhandled custom legalization"); 265 case ISD::ATOMIC_LOAD_ADD: 266 case ISD::ATOMIC_LOAD_AND: 267 case ISD::ATOMIC_LOAD_OR: 268 case ISD::ATOMIC_LOAD_XOR: 269 case ISD::ATOMIC_SWAP: 270 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: 271 if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD) 272 err_msg = "Unsupported atomic operations, please use 32/64 bit version"; 273 else 274 err_msg = "Unsupported atomic operations, please use 64 bit version"; 275 break; 276 } 277 278 SDLoc DL(N); 279 fail(DL, DAG, err_msg); 280 } 281 282 SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 283 switch (Op.getOpcode()) { 284 case ISD::BR_CC: 285 return LowerBR_CC(Op, DAG); 286 case ISD::GlobalAddress: 287 return LowerGlobalAddress(Op, DAG); 288 case ISD::SELECT_CC: 289 return LowerSELECT_CC(Op, DAG); 290 case ISD::DYNAMIC_STACKALLOC: 291 report_fatal_error("Unsupported dynamic stack allocation"); 292 default: 293 llvm_unreachable("unimplemented operand"); 294 } 295 } 296 297 // Calling Convention Implementation 298 #include "BPFGenCallingConv.inc" 299 300 SDValue BPFTargetLowering::LowerFormalArguments( 301 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 302 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 303 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 304 switch (CallConv) { 305 default: 306 report_fatal_error("Unsupported calling convention"); 307 case CallingConv::C: 308 case CallingConv::Fast: 309 break; 310 } 311 312 MachineFunction &MF = DAG.getMachineFunction(); 313 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 314 315 // Assign locations to all of the incoming arguments. 316 SmallVector<CCValAssign, 16> ArgLocs; 317 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 318 CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64); 319 320 for (auto &VA : ArgLocs) { 321 if (VA.isRegLoc()) { 322 // Arguments passed in registers 323 EVT RegVT = VA.getLocVT(); 324 MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy; 325 switch (SimpleTy) { 326 default: { 327 errs() << "LowerFormalArguments Unhandled argument type: " 328 << RegVT.getEVTString() << '\n'; 329 llvm_unreachable(nullptr); 330 } 331 case MVT::i32: 332 case MVT::i64: 333 Register VReg = RegInfo.createVirtualRegister( 334 SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass); 335 RegInfo.addLiveIn(VA.getLocReg(), VReg); 336 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT); 337 338 // If this is an value that has been promoted to wider types, insert an 339 // assert[sz]ext to capture this, then truncate to the right size. 340 if (VA.getLocInfo() == CCValAssign::SExt) 341 ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue, 342 DAG.getValueType(VA.getValVT())); 343 else if (VA.getLocInfo() == CCValAssign::ZExt) 344 ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue, 345 DAG.getValueType(VA.getValVT())); 346 347 if (VA.getLocInfo() != CCValAssign::Full) 348 ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue); 349 350 InVals.push_back(ArgValue); 351 352 break; 353 } 354 } else { 355 fail(DL, DAG, "defined with too many args"); 356 InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT())); 357 } 358 } 359 360 if (IsVarArg || MF.getFunction().hasStructRetAttr()) { 361 fail(DL, DAG, "functions with VarArgs or StructRet are not supported"); 362 } 363 364 return Chain; 365 } 366 367 const unsigned BPFTargetLowering::MaxArgs = 5; 368 369 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 370 SmallVectorImpl<SDValue> &InVals) const { 371 SelectionDAG &DAG = CLI.DAG; 372 auto &Outs = CLI.Outs; 373 auto &OutVals = CLI.OutVals; 374 auto &Ins = CLI.Ins; 375 SDValue Chain = CLI.Chain; 376 SDValue Callee = CLI.Callee; 377 bool &IsTailCall = CLI.IsTailCall; 378 CallingConv::ID CallConv = CLI.CallConv; 379 bool IsVarArg = CLI.IsVarArg; 380 MachineFunction &MF = DAG.getMachineFunction(); 381 382 // BPF target does not support tail call optimization. 383 IsTailCall = false; 384 385 switch (CallConv) { 386 default: 387 report_fatal_error("Unsupported calling convention"); 388 case CallingConv::Fast: 389 case CallingConv::C: 390 break; 391 } 392 393 // Analyze operands of the call, assigning locations to each operand. 394 SmallVector<CCValAssign, 16> ArgLocs; 395 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 396 397 CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64); 398 399 unsigned NumBytes = CCInfo.getNextStackOffset(); 400 401 if (Outs.size() > MaxArgs) 402 fail(CLI.DL, DAG, "too many args to ", Callee); 403 404 for (auto &Arg : Outs) { 405 ISD::ArgFlagsTy Flags = Arg.Flags; 406 if (!Flags.isByVal()) 407 continue; 408 409 fail(CLI.DL, DAG, "pass by value not supported ", Callee); 410 } 411 412 auto PtrVT = getPointerTy(MF.getDataLayout()); 413 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL); 414 415 SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass; 416 417 // Walk arg assignments 418 for (unsigned i = 0, 419 e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs); 420 i != e; ++i) { 421 CCValAssign &VA = ArgLocs[i]; 422 SDValue Arg = OutVals[i]; 423 424 // Promote the value if needed. 425 switch (VA.getLocInfo()) { 426 default: 427 llvm_unreachable("Unknown loc info"); 428 case CCValAssign::Full: 429 break; 430 case CCValAssign::SExt: 431 Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg); 432 break; 433 case CCValAssign::ZExt: 434 Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg); 435 break; 436 case CCValAssign::AExt: 437 Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg); 438 break; 439 } 440 441 // Push arguments into RegsToPass vector 442 if (VA.isRegLoc()) 443 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 444 else 445 llvm_unreachable("call arg pass bug"); 446 } 447 448 SDValue InFlag; 449 450 // Build a sequence of copy-to-reg nodes chained together with token chain and 451 // flag operands which copy the outgoing args into registers. The InFlag in 452 // necessary since all emitted instructions must be stuck together. 453 for (auto &Reg : RegsToPass) { 454 Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag); 455 InFlag = Chain.getValue(1); 456 } 457 458 // If the callee is a GlobalAddress node (quite common, every direct call is) 459 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it. 460 // Likewise ExternalSymbol -> TargetExternalSymbol. 461 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 462 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT, 463 G->getOffset(), 0); 464 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 465 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0); 466 fail(CLI.DL, DAG, Twine("A call to built-in function '" 467 + StringRef(E->getSymbol()) 468 + "' is not supported.")); 469 } 470 471 // Returns a chain & a flag for retval copy to use. 472 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 473 SmallVector<SDValue, 8> Ops; 474 Ops.push_back(Chain); 475 Ops.push_back(Callee); 476 477 // Add argument registers to the end of the list so that they are 478 // known live into the call. 479 for (auto &Reg : RegsToPass) 480 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType())); 481 482 if (InFlag.getNode()) 483 Ops.push_back(InFlag); 484 485 Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops); 486 InFlag = Chain.getValue(1); 487 488 // Create the CALLSEQ_END node. 489 Chain = DAG.getCALLSEQ_END( 490 Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true), 491 DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL); 492 InFlag = Chain.getValue(1); 493 494 // Handle result values, copying them out of physregs into vregs that we 495 // return. 496 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG, 497 InVals); 498 } 499 500 SDValue 501 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 502 bool IsVarArg, 503 const SmallVectorImpl<ISD::OutputArg> &Outs, 504 const SmallVectorImpl<SDValue> &OutVals, 505 const SDLoc &DL, SelectionDAG &DAG) const { 506 unsigned Opc = BPFISD::RET_FLAG; 507 508 // CCValAssign - represent the assignment of the return value to a location 509 SmallVector<CCValAssign, 16> RVLocs; 510 MachineFunction &MF = DAG.getMachineFunction(); 511 512 // CCState - Info about the registers and stack slot. 513 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext()); 514 515 if (MF.getFunction().getReturnType()->isAggregateType()) { 516 fail(DL, DAG, "only integer returns supported"); 517 return DAG.getNode(Opc, DL, MVT::Other, Chain); 518 } 519 520 // Analize return values. 521 CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64); 522 523 SDValue Flag; 524 SmallVector<SDValue, 4> RetOps(1, Chain); 525 526 // Copy the result values into the output registers. 527 for (unsigned i = 0; i != RVLocs.size(); ++i) { 528 CCValAssign &VA = RVLocs[i]; 529 assert(VA.isRegLoc() && "Can only return in registers!"); 530 531 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag); 532 533 // Guarantee that all emitted copies are stuck together, 534 // avoiding something bad. 535 Flag = Chain.getValue(1); 536 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 537 } 538 539 RetOps[0] = Chain; // Update chain. 540 541 // Add the flag if we have it. 542 if (Flag.getNode()) 543 RetOps.push_back(Flag); 544 545 return DAG.getNode(Opc, DL, MVT::Other, RetOps); 546 } 547 548 SDValue BPFTargetLowering::LowerCallResult( 549 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg, 550 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 551 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 552 553 MachineFunction &MF = DAG.getMachineFunction(); 554 // Assign locations to each value returned by this call. 555 SmallVector<CCValAssign, 16> RVLocs; 556 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext()); 557 558 if (Ins.size() >= 2) { 559 fail(DL, DAG, "only small returns supported"); 560 for (unsigned i = 0, e = Ins.size(); i != e; ++i) 561 InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT)); 562 return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1); 563 } 564 565 CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64); 566 567 // Copy all of the result registers out of their specified physreg. 568 for (auto &Val : RVLocs) { 569 Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(), 570 Val.getValVT(), InFlag).getValue(1); 571 InFlag = Chain.getValue(2); 572 InVals.push_back(Chain.getValue(0)); 573 } 574 575 return Chain; 576 } 577 578 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) { 579 switch (CC) { 580 default: 581 break; 582 case ISD::SETULT: 583 case ISD::SETULE: 584 case ISD::SETLT: 585 case ISD::SETLE: 586 CC = ISD::getSetCCSwappedOperands(CC); 587 std::swap(LHS, RHS); 588 break; 589 } 590 } 591 592 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 593 SDValue Chain = Op.getOperand(0); 594 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 595 SDValue LHS = Op.getOperand(2); 596 SDValue RHS = Op.getOperand(3); 597 SDValue Dest = Op.getOperand(4); 598 SDLoc DL(Op); 599 600 if (!getHasJmpExt()) 601 NegateCC(LHS, RHS, CC); 602 603 return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS, 604 DAG.getConstant(CC, DL, LHS.getValueType()), Dest); 605 } 606 607 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 608 SDValue LHS = Op.getOperand(0); 609 SDValue RHS = Op.getOperand(1); 610 SDValue TrueV = Op.getOperand(2); 611 SDValue FalseV = Op.getOperand(3); 612 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 613 SDLoc DL(Op); 614 615 if (!getHasJmpExt()) 616 NegateCC(LHS, RHS, CC); 617 618 SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType()); 619 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 620 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV}; 621 622 return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops); 623 } 624 625 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { 626 switch ((BPFISD::NodeType)Opcode) { 627 case BPFISD::FIRST_NUMBER: 628 break; 629 case BPFISD::RET_FLAG: 630 return "BPFISD::RET_FLAG"; 631 case BPFISD::CALL: 632 return "BPFISD::CALL"; 633 case BPFISD::SELECT_CC: 634 return "BPFISD::SELECT_CC"; 635 case BPFISD::BR_CC: 636 return "BPFISD::BR_CC"; 637 case BPFISD::Wrapper: 638 return "BPFISD::Wrapper"; 639 case BPFISD::MEMCPY: 640 return "BPFISD::MEMCPY"; 641 } 642 return nullptr; 643 } 644 645 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op, 646 SelectionDAG &DAG) const { 647 auto N = cast<GlobalAddressSDNode>(Op); 648 assert(N->getOffset() == 0 && "Invalid offset for global address"); 649 650 SDLoc DL(Op); 651 const GlobalValue *GV = N->getGlobal(); 652 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64); 653 654 return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA); 655 } 656 657 unsigned 658 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB, 659 unsigned Reg, bool isSigned) const { 660 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo(); 661 const TargetRegisterClass *RC = getRegClassFor(MVT::i64); 662 int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri; 663 MachineFunction *F = BB->getParent(); 664 DebugLoc DL = MI.getDebugLoc(); 665 666 MachineRegisterInfo &RegInfo = F->getRegInfo(); 667 668 if (!isSigned) { 669 Register PromotedReg0 = RegInfo.createVirtualRegister(RC); 670 BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg); 671 return PromotedReg0; 672 } 673 Register PromotedReg0 = RegInfo.createVirtualRegister(RC); 674 Register PromotedReg1 = RegInfo.createVirtualRegister(RC); 675 Register PromotedReg2 = RegInfo.createVirtualRegister(RC); 676 BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg); 677 BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1) 678 .addReg(PromotedReg0).addImm(32); 679 BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2) 680 .addReg(PromotedReg1).addImm(32); 681 682 return PromotedReg2; 683 } 684 685 MachineBasicBlock * 686 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI, 687 MachineBasicBlock *BB) 688 const { 689 MachineFunction *MF = MI.getParent()->getParent(); 690 MachineRegisterInfo &MRI = MF->getRegInfo(); 691 MachineInstrBuilder MIB(*MF, MI); 692 unsigned ScratchReg; 693 694 // This function does custom insertion during lowering BPFISD::MEMCPY which 695 // only has two register operands from memcpy semantics, the copy source 696 // address and the copy destination address. 697 // 698 // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need 699 // a third scratch register to serve as the destination register of load and 700 // source register of store. 701 // 702 // The scratch register here is with the Define | Dead | EarlyClobber flags. 703 // The EarlyClobber flag has the semantic property that the operand it is 704 // attached to is clobbered before the rest of the inputs are read. Hence it 705 // must be unique among the operands to the instruction. The Define flag is 706 // needed to coerce the machine verifier that an Undef value isn't a problem 707 // as we anyway is loading memory into it. The Dead flag is needed as the 708 // value in scratch isn't supposed to be used by any other instruction. 709 ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass); 710 MIB.addReg(ScratchReg, 711 RegState::Define | RegState::Dead | RegState::EarlyClobber); 712 713 return BB; 714 } 715 716 MachineBasicBlock * 717 BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 718 MachineBasicBlock *BB) const { 719 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo(); 720 DebugLoc DL = MI.getDebugLoc(); 721 unsigned Opc = MI.getOpcode(); 722 bool isSelectRROp = (Opc == BPF::Select || 723 Opc == BPF::Select_64_32 || 724 Opc == BPF::Select_32 || 725 Opc == BPF::Select_32_64); 726 727 bool isMemcpyOp = Opc == BPF::MEMCPY; 728 729 #ifndef NDEBUG 730 bool isSelectRIOp = (Opc == BPF::Select_Ri || 731 Opc == BPF::Select_Ri_64_32 || 732 Opc == BPF::Select_Ri_32 || 733 Opc == BPF::Select_Ri_32_64); 734 735 736 assert((isSelectRROp || isSelectRIOp || isMemcpyOp) && 737 "Unexpected instr type to insert"); 738 #endif 739 740 if (isMemcpyOp) 741 return EmitInstrWithCustomInserterMemcpy(MI, BB); 742 743 bool is32BitCmp = (Opc == BPF::Select_32 || 744 Opc == BPF::Select_32_64 || 745 Opc == BPF::Select_Ri_32 || 746 Opc == BPF::Select_Ri_32_64); 747 748 // To "insert" a SELECT instruction, we actually have to insert the diamond 749 // control-flow pattern. The incoming instruction knows the destination vreg 750 // to set, the condition code register to branch on, the true/false values to 751 // select between, and a branch opcode to use. 752 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 753 MachineFunction::iterator I = ++BB->getIterator(); 754 755 // ThisMBB: 756 // ... 757 // TrueVal = ... 758 // jmp_XX r1, r2 goto Copy1MBB 759 // fallthrough --> Copy0MBB 760 MachineBasicBlock *ThisMBB = BB; 761 MachineFunction *F = BB->getParent(); 762 MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 763 MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB); 764 765 F->insert(I, Copy0MBB); 766 F->insert(I, Copy1MBB); 767 // Update machine-CFG edges by transferring all successors of the current 768 // block to the new block which will contain the Phi node for the select. 769 Copy1MBB->splice(Copy1MBB->begin(), BB, 770 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 771 Copy1MBB->transferSuccessorsAndUpdatePHIs(BB); 772 // Next, add the true and fallthrough blocks as its successors. 773 BB->addSuccessor(Copy0MBB); 774 BB->addSuccessor(Copy1MBB); 775 776 // Insert Branch if Flag 777 int CC = MI.getOperand(3).getImm(); 778 int NewCC; 779 switch (CC) { 780 #define SET_NEWCC(X, Y) \ 781 case ISD::X: \ 782 if (is32BitCmp && HasJmp32) \ 783 NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \ 784 else \ 785 NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \ 786 break 787 SET_NEWCC(SETGT, JSGT); 788 SET_NEWCC(SETUGT, JUGT); 789 SET_NEWCC(SETGE, JSGE); 790 SET_NEWCC(SETUGE, JUGE); 791 SET_NEWCC(SETEQ, JEQ); 792 SET_NEWCC(SETNE, JNE); 793 SET_NEWCC(SETLT, JSLT); 794 SET_NEWCC(SETULT, JULT); 795 SET_NEWCC(SETLE, JSLE); 796 SET_NEWCC(SETULE, JULE); 797 default: 798 report_fatal_error("unimplemented select CondCode " + Twine(CC)); 799 } 800 801 Register LHS = MI.getOperand(1).getReg(); 802 bool isSignedCmp = (CC == ISD::SETGT || 803 CC == ISD::SETGE || 804 CC == ISD::SETLT || 805 CC == ISD::SETLE); 806 807 // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need 808 // to be promoted, however if the 32-bit comparison operands are destination 809 // registers then they are implicitly zero-extended already, there is no 810 // need of explicit zero-extend sequence for them. 811 // 812 // We simply do extension for all situations in this method, but we will 813 // try to remove those unnecessary in BPFMIPeephole pass. 814 if (is32BitCmp && !HasJmp32) 815 LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp); 816 817 if (isSelectRROp) { 818 Register RHS = MI.getOperand(2).getReg(); 819 820 if (is32BitCmp && !HasJmp32) 821 RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp); 822 823 BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB); 824 } else { 825 int64_t imm32 = MI.getOperand(2).getImm(); 826 // Check before we build J*_ri instruction. 827 assert (isInt<32>(imm32)); 828 BuildMI(BB, DL, TII.get(NewCC)) 829 .addReg(LHS).addImm(imm32).addMBB(Copy1MBB); 830 } 831 832 // Copy0MBB: 833 // %FalseValue = ... 834 // # fallthrough to Copy1MBB 835 BB = Copy0MBB; 836 837 // Update machine-CFG edges 838 BB->addSuccessor(Copy1MBB); 839 840 // Copy1MBB: 841 // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ] 842 // ... 843 BB = Copy1MBB; 844 BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg()) 845 .addReg(MI.getOperand(5).getReg()) 846 .addMBB(Copy0MBB) 847 .addReg(MI.getOperand(4).getReg()) 848 .addMBB(ThisMBB); 849 850 MI.eraseFromParent(); // The pseudo instruction is gone now. 851 return BB; 852 } 853 854 EVT BPFTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &, 855 EVT VT) const { 856 return getHasAlu32() ? MVT::i32 : MVT::i64; 857 } 858 859 MVT BPFTargetLowering::getScalarShiftAmountTy(const DataLayout &DL, 860 EVT VT) const { 861 return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64; 862 } 863 864 bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL, 865 const AddrMode &AM, Type *Ty, 866 unsigned AS, 867 Instruction *I) const { 868 // No global is ever allowed as a base. 869 if (AM.BaseGV) 870 return false; 871 872 switch (AM.Scale) { 873 case 0: // "r+i" or just "i", depending on HasBaseReg. 874 break; 875 case 1: 876 if (!AM.HasBaseReg) // allow "r+i". 877 break; 878 return false; // disallow "r+r" or "r+r+i". 879 default: 880 return false; 881 } 882 883 return true; 884 } 885