1//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file describes the BPF instructions in TableGen format. 10// 11//===----------------------------------------------------------------------===// 12 13include "BPFInstrFormats.td" 14 15// Instruction Operands and Patterns 16 17// These are target-independent nodes, but have target-specific formats. 18def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>, 19 SDTCisVT<1, iPTR>]>; 20def SDT_BPFCallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>; 21def SDT_BPFCall : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; 22def SDT_BPFSetFlag : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>; 23def SDT_BPFSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>, 24 SDTCisSameAs<0, 4>, 25 SDTCisSameAs<4, 5>]>; 26def SDT_BPFBrCC : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>, 27 SDTCisVT<3, OtherVT>]>; 28def SDT_BPFWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, 29 SDTCisPtrTy<0>]>; 30def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>, 31 SDTCisVT<1, i64>, 32 SDTCisVT<2, i64>, 33 SDTCisVT<3, i64>]>; 34 35def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall, 36 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, 37 SDNPVariadic]>; 38def BPFretglue : SDNode<"BPFISD::RET_GLUE", SDTNone, 39 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>; 40def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart, 41 [SDNPHasChain, SDNPOutGlue]>; 42def BPFcallseq_end : SDNode<"ISD::CALLSEQ_END", SDT_BPFCallSeqEnd, 43 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; 44def BPFbrcc : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC, 45 [SDNPHasChain, SDNPOutGlue, SDNPInGlue]>; 46 47def BPFselectcc : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC>; 48def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; 49def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, 50 [SDNPHasChain, SDNPInGlue, SDNPOutGlue, 51 SDNPMayStore, SDNPMayLoad]>; 52def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; 53def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; 54def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; 55def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">; 56def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; 57def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">; 58def BPFHasBswap : Predicate<"Subtarget->hasBswap()">; 59def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">; 60def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">; 61def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">; 62def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">; 63def BPFHasLoadAcqStoreRel : Predicate<"Subtarget->hasLoadAcqStoreRel()">; 64 65class ImmediateAsmOperand<string name> : AsmOperandClass { 66 let Name = name; 67 let RenderMethod = "addImmOperands"; 68 let DiagnosticType = !strconcat("Invalid", name); 69} 70 71def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">; 72 73def brtarget : Operand<OtherVT> { 74 let PrintMethod = "printBrTargetOperand"; 75 let ParserMatchClass = ImmediateAsmOperand<"BrTarget">; 76} 77def calltarget : Operand<i64>; 78 79def u64imm : Operand<i64> { 80 let PrintMethod = "printImm64Operand"; 81} 82 83def s16imm : Operand<i16> { 84 let ParserMatchClass = SImm16AsmOperand; 85} 86 87def gpr_or_imm : Operand<i64>; 88 89def i64immSExt32 : PatLeaf<(i64 imm), 90 [{return isInt<32>(N->getSExtValue()); }]>; 91def i32immSExt32 : PatLeaf<(i32 imm), 92 [{return isInt<32>(N->getSExtValue()); }]>; 93def i64immZExt32 : PatLeaf<(i64 imm), 94 [{return isUInt<32>(N->getZExtValue()); }]>; 95 96def imm_to_i64 : SDNodeXForm<timm, [{ 97 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64); 98}]>; 99 100// Addressing modes. 101def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>; 102def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>; 103 104// Address operands 105def MEMri : Operand<i64> { 106 let PrintMethod = "printMemOperand"; 107 let EncoderMethod = "getMemoryOpValue"; 108 let DecoderMethod = "decodeMemoryOpValue"; 109 let MIOperandInfo = (ops GPR, s16imm); 110} 111 112// Conditional code predicates - used for pattern matching for jump instructions 113def BPF_CC_EQ : PatLeaf<(i64 imm), 114 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 115def BPF_CC_NE : PatLeaf<(i64 imm), 116 [{return (N->getZExtValue() == ISD::SETNE);}]>; 117def BPF_CC_GE : PatLeaf<(i64 imm), 118 [{return (N->getZExtValue() == ISD::SETGE);}]>; 119def BPF_CC_GT : PatLeaf<(i64 imm), 120 [{return (N->getZExtValue() == ISD::SETGT);}]>; 121def BPF_CC_GTU : PatLeaf<(i64 imm), 122 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 123def BPF_CC_GEU : PatLeaf<(i64 imm), 124 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 125def BPF_CC_LE : PatLeaf<(i64 imm), 126 [{return (N->getZExtValue() == ISD::SETLE);}]>; 127def BPF_CC_LT : PatLeaf<(i64 imm), 128 [{return (N->getZExtValue() == ISD::SETLT);}]>; 129def BPF_CC_LTU : PatLeaf<(i64 imm), 130 [{return (N->getZExtValue() == ISD::SETULT);}]>; 131def BPF_CC_LEU : PatLeaf<(i64 imm), 132 [{return (N->getZExtValue() == ISD::SETULE);}]>; 133def BPF_CC_EQ_32 : PatLeaf<(i32 imm), 134 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 135def BPF_CC_NE_32 : PatLeaf<(i32 imm), 136 [{return (N->getZExtValue() == ISD::SETNE);}]>; 137def BPF_CC_GE_32 : PatLeaf<(i32 imm), 138 [{return (N->getZExtValue() == ISD::SETGE);}]>; 139def BPF_CC_GT_32 : PatLeaf<(i32 imm), 140 [{return (N->getZExtValue() == ISD::SETGT);}]>; 141def BPF_CC_GTU_32 : PatLeaf<(i32 imm), 142 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 143def BPF_CC_GEU_32 : PatLeaf<(i32 imm), 144 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 145def BPF_CC_LE_32 : PatLeaf<(i32 imm), 146 [{return (N->getZExtValue() == ISD::SETLE);}]>; 147def BPF_CC_LT_32 : PatLeaf<(i32 imm), 148 [{return (N->getZExtValue() == ISD::SETLT);}]>; 149def BPF_CC_LTU_32 : PatLeaf<(i32 imm), 150 [{return (N->getZExtValue() == ISD::SETULT);}]>; 151def BPF_CC_LEU_32 : PatLeaf<(i32 imm), 152 [{return (N->getZExtValue() == ISD::SETULE);}]>; 153def NoCond : PatLeaf<(vt)> {} 154 155// For arithmetic and jump instructions the 8-bit 'code' 156// field is divided into three parts: 157// 158// +----------------+--------+--------------------+ 159// | 4 bits | 1 bit | 3 bits | 160// | operation code | source | instruction class | 161// +----------------+--------+--------------------+ 162// (MSB) (LSB) 163class TYPE_ALU_JMP<bits<4> op, bits<1> srctype, 164 dag outs, dag ins, string asmstr, list<dag> pattern> 165 : InstBPF<outs, ins, asmstr, pattern> { 166 167 let Inst{63-60} = op; 168 let Inst{59} = srctype; 169} 170 171//For load and store instructions the 8-bit 'code' field is divided as: 172// 173// +--------+--------+-------------------+ 174// | 3 bits | 2 bits | 3 bits | 175// | mode | size | instruction class | 176// +--------+--------+-------------------+ 177// (MSB) (LSB) 178class TYPE_LD_ST<bits<3> mode, bits<2> size, 179 dag outs, dag ins, string asmstr, list<dag> pattern> 180 : InstBPF<outs, ins, asmstr, pattern> { 181 182 let Inst{63-61} = mode; 183 let Inst{60-59} = size; 184} 185 186// jump instructions 187class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 188 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 189 (outs), 190 (ins GPR:$dst, GPR:$src, brtarget:$BrDst), 191 "if $dst "#OpcodeStr#" $src goto $BrDst", 192 [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> { 193 bits<4> dst; 194 bits<4> src; 195 bits<16> BrDst; 196 197 let Inst{55-52} = src; 198 let Inst{51-48} = dst; 199 let Inst{47-32} = BrDst; 200 let BPFClass = BPF_JMP; 201} 202 203class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 204 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 205 (outs), 206 (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst), 207 "if $dst "#OpcodeStr#" $imm goto $BrDst", 208 [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> { 209 bits<4> dst; 210 bits<16> BrDst; 211 bits<32> imm; 212 213 let Inst{51-48} = dst; 214 let Inst{47-32} = BrDst; 215 let Inst{31-0} = imm; 216 let BPFClass = BPF_JMP; 217} 218 219class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 220 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 221 (outs), 222 (ins brtarget:$BrDst), 223 !strconcat(OpcodeStr, " $BrDst"), 224 Pattern> { 225 bits<16> BrDst; 226 227 let Inst{47-32} = BrDst; 228 let BPFClass = BPF_JMP; 229} 230 231class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 232 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 233 (outs), 234 (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst), 235 "if $dst "#OpcodeStr#" $src goto $BrDst", 236 [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> { 237 bits<4> dst; 238 bits<4> src; 239 bits<16> BrDst; 240 241 let Inst{55-52} = src; 242 let Inst{51-48} = dst; 243 let Inst{47-32} = BrDst; 244 let BPFClass = BPF_JMP32; 245} 246 247class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 248 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 249 (outs), 250 (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst), 251 "if $dst "#OpcodeStr#" $imm goto $BrDst", 252 [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> { 253 bits<4> dst; 254 bits<16> BrDst; 255 bits<32> imm; 256 257 let Inst{51-48} = dst; 258 let Inst{47-32} = BrDst; 259 let Inst{31-0} = imm; 260 let BPFClass = BPF_JMP32; 261} 262 263multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> { 264 def _rr : JMP_RR<Opc, OpcodeStr, Cond>; 265 def _ri : JMP_RI<Opc, OpcodeStr, Cond>; 266 def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>; 267 def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>; 268} 269 270let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in { 271// cmp+goto instructions 272defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>; 273defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>; 274defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>; 275defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>; 276defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>; 277defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>; 278defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>; 279defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>; 280defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>; 281defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>; 282defm JSET : J<BPF_JSET, "&", NoCond, NoCond>; 283def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>; 284} 285 286// ALU instructions 287class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off, 288 dag outs, dag ins, string asmstr, list<dag> pattern> 289 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> { 290 bits<4> dst; 291 bits<32> imm; 292 293 let Inst{51-48} = dst; 294 let Inst{47-32} = off; 295 let Inst{31-0} = imm; 296 let BPFClass = Class; 297} 298 299class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off, 300 dag outs, dag ins, string asmstr, list<dag> pattern> 301 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> { 302 bits<4> dst; 303 bits<4> src; 304 305 let Inst{55-52} = src; 306 let Inst{51-48} = dst; 307 let Inst{47-32} = off; 308 let BPFClass = Class; 309} 310 311multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { 312 def _rr : ALU_RR<BPF_ALU64, Opc, off, 313 (outs GPR:$dst), 314 (ins GPR:$src2, GPR:$src), 315 "$dst "#OpcodeStr#" $src", 316 [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>; 317 def _ri : ALU_RI<BPF_ALU64, Opc, off, 318 (outs GPR:$dst), 319 (ins GPR:$src2, i64imm:$imm), 320 "$dst "#OpcodeStr#" $imm", 321 [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>; 322 def _rr_32 : ALU_RR<BPF_ALU, Opc, off, 323 (outs GPR32:$dst), 324 (ins GPR32:$src2, GPR32:$src), 325 "$dst "#OpcodeStr#" $src", 326 [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>; 327 def _ri_32 : ALU_RI<BPF_ALU, Opc, off, 328 (outs GPR32:$dst), 329 (ins GPR32:$src2, i32imm:$imm), 330 "$dst "#OpcodeStr#" $imm", 331 [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>; 332} 333 334let Constraints = "$dst = $src2" in { 335let isAsCheapAsAMove = 1 in { 336 defm ADD : ALU<BPF_ADD, 0, "+=", add>; 337 defm SUB : ALU<BPF_SUB, 0, "-=", sub>; 338 defm OR : ALU<BPF_OR, 0, "|=", or>; 339 defm AND : ALU<BPF_AND, 0, "&=", and>; 340 defm SLL : ALU<BPF_LSH, 0, "<<=", shl>; 341 defm SRL : ALU<BPF_RSH, 0, ">>=", srl>; 342 defm XOR : ALU<BPF_XOR, 0, "^=", xor>; 343 defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>; 344} 345 defm MUL : ALU<BPF_MUL, 0, "*=", mul>; 346 defm DIV : ALU<BPF_DIV, 0, "/=", udiv>; 347 defm MOD : ALU<BPF_MOD, 0, "%=", urem>; 348 349 let Predicates = [BPFHasSdivSmod] in { 350 defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>; 351 defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>; 352 } 353} 354 355class NEG_RR<BPFOpClass Class, BPFArithOp Opc, 356 dag outs, dag ins, string asmstr, list<dag> pattern> 357 : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> { 358 bits<4> dst; 359 360 let Inst{51-48} = dst; 361 let BPFClass = Class; 362} 363 364let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in { 365 def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src), 366 "$dst = -$src", 367 [(set GPR:$dst, (ineg i64:$src))]>; 368 def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src), 369 "$dst = -$src", 370 [(set GPR32:$dst, (ineg i32:$src))]>; 371} 372 373class LD_IMM64<bits<4> Pseudo, string OpcodeStr> 374 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 375 (outs GPR:$dst), 376 (ins u64imm:$imm), 377 "$dst "#OpcodeStr#" ${imm} ll", 378 [(set GPR:$dst, (i64 imm:$imm))]> { 379 380 bits<4> dst; 381 bits<64> imm; 382 383 let Inst{51-48} = dst; 384 let Inst{55-52} = Pseudo; 385 let Inst{47-32} = 0; 386 let Inst{31-0} = imm{31-0}; 387 let BPFClass = BPF_LD; 388} 389 390let isReMaterializable = 1, isAsCheapAsAMove = 1 in { 391def LD_imm64 : LD_IMM64<0, "=">; 392def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0, 393 (outs GPR:$dst), 394 (ins GPR:$src), 395 "$dst = $src", 396 []>; 397def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0, 398 (outs GPR:$dst), 399 (ins i64imm:$imm), 400 "$dst = $imm", 401 [(set GPR:$dst, (i64 i64immSExt32:$imm))]>; 402def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0, 403 (outs GPR32:$dst), 404 (ins GPR32:$src), 405 "$dst = $src", 406 []>; 407def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0, 408 (outs GPR32:$dst), 409 (ins i32imm:$imm), 410 "$dst = $imm", 411 [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>; 412 413let Predicates = [BPFHasMovsx] in { 414 def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8, 415 (outs GPR:$dst), (ins GPR:$src), 416 "$dst = (s8)$src", 417 [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>; 418 def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16, 419 (outs GPR:$dst), (ins GPR:$src), 420 "$dst = (s16)$src", 421 [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>; 422 def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32, 423 (outs GPR:$dst), (ins GPR:$src), 424 "$dst = (s32)$src", 425 [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>; 426 def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8, 427 (outs GPR32:$dst), (ins GPR32:$src), 428 "$dst = (s8)$src", 429 [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>; 430 def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16, 431 (outs GPR32:$dst), (ins GPR32:$src), 432 "$dst = (s16)$src", 433 [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>; 434} 435} 436 437def ADDR_SPACE_CAST 438 : ALU_RR<BPF_ALU64, BPF_MOV, 64, 439 (outs GPR:$dst), 440 (ins GPR:$src, i64imm:$dst_as, i64imm:$src_as), 441 "$dst = addr_space_cast($src, $dst_as, $src_as)", 442 []> { 443 bits<64> dst_as; 444 bits<64> src_as; 445 446 let Inst{47-32} = 1; 447 let Inst{31-16} = dst_as{15-0}; 448 let Inst{15-0} = src_as{15-0}; 449} 450 451def SrcAddrSpace : SDNodeXForm<addrspacecast, [{ 452 return CurDAG->getTargetConstant( 453 cast<AddrSpaceCastSDNode>(N)->getSrcAddressSpace(), 454 SDLoc(N), MVT::i64); 455}]>; 456 457def DstAddrSpace : SDNodeXForm<addrspacecast, [{ 458 return CurDAG->getTargetConstant( 459 cast<AddrSpaceCastSDNode>(N)->getDestAddressSpace(), 460 SDLoc(N), MVT::i64); 461}]>; 462 463def : Pat<(addrspacecast:$this GPR:$src), 464 (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>; 465 466def FI_ri 467 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 468 (outs GPR:$dst), 469 (ins MEMri:$addr), 470 "lea\t$dst, $addr", 471 [(set i64:$dst, FIri:$addr)]> { 472 // This is a tentative instruction, and will be replaced 473 // with MOV_rr and ADD_ri in PEI phase 474 let Inst{51-48} = 0; 475 let Inst{55-52} = 2; 476 let Inst{47-32} = 0; 477 let Inst{31-0} = 0; 478 let BPFClass = BPF_LD; 479 bit isPseudo = true; 480} 481 482def LD_pseudo 483 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 484 (outs GPR:$dst), 485 (ins i64imm:$pseudo, u64imm:$imm), 486 "ld_pseudo\t$dst, $pseudo, $imm", 487 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> { 488 489 bits<4> dst; 490 bits<64> imm; 491 bits<4> pseudo; 492 493 let Inst{51-48} = dst; 494 let Inst{55-52} = pseudo; 495 let Inst{47-32} = 0; 496 let Inst{31-0} = imm{31-0}; 497 let BPFClass = BPF_LD; 498} 499 500// STORE instructions 501class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 502 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 503 (outs), 504 (ins GPR:$src, MEMri:$addr), 505 "*("#OpcodeStr#" *)($addr) = $src", 506 Pattern> { 507 bits<4> src; 508 bits<20> addr; 509 510 let Inst{51-48} = addr{19-16}; // base reg 511 let Inst{55-52} = src; 512 let Inst{47-32} = addr{15-0}; // offset 513 let BPFClass = BPF_STX; 514} 515 516class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 517 : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>; 518 519let Predicates = [BPFNoALU32] in { 520 def STW : STOREi64<BPF_W, "u32", truncstorei32>; 521 def STH : STOREi64<BPF_H, "u16", truncstorei16>; 522 def STB : STOREi64<BPF_B, "u8", truncstorei8>; 523} 524def STD : STOREi64<BPF_DW, "u64", store>; 525 526class STORE_imm<BPFWidthModifer SizeOp, 527 string OpcodeStr, dag Pattern> 528 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 529 (outs), 530 (ins i64imm:$imm, MEMri:$addr), 531 "*("#OpcodeStr#" *)($addr) = $imm", 532 [Pattern]> { 533 bits<20> addr; 534 bits<32> imm; 535 536 let Inst{51-48} = addr{19-16}; // base reg 537 let Inst{47-32} = addr{15-0}; // offset 538 let Inst{31-0} = imm; 539 let BPFClass = BPF_ST; 540} 541 542let Predicates = [BPFHasStoreImm] in { 543 // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for 544 // value stored to memory: 545 // - it is fine to generate such write when immediate is -1 546 // - it is incorrect to generate such write when immediate is 547 // +0xffff_ffff. 548 // 549 // In the latter case two instructions would be generated instead of 550 // one BPF_ST: 551 // rA = 0xffffffff ll ; LD_imm64 552 // *(u64 *)(rB + 0) = rA ; STX 553 // 554 // For BPF_{B,H,W} the size of value stored matches size of the immediate. 555 def STD_imm : STORE_imm<BPF_DW, "u64", (store (i64 i64immSExt32:$imm), ADDRri:$addr)>; 556 def STW_imm : STORE_imm<BPF_W, "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 557 def STH_imm : STORE_imm<BPF_H, "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 558 def STB_imm : STORE_imm<BPF_B, "u8", (truncstorei8 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 559} 560 561let Predicates = [BPFHasALU32, BPFHasStoreImm] in { 562 def : Pat<(store (i32 imm:$src), ADDRri:$dst), 563 (STW_imm (imm_to_i64 $src), ADDRri:$dst)>; 564 def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst), 565 (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 566 def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst), 567 (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 568} 569 570class STORE_RELEASE<BPFWidthModifer SizeOp, string OpcodeStr, RegisterClass RegTp> 571 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 572 (outs), 573 (ins RegTp:$src, MEMri:$addr), 574 "store_release(("#OpcodeStr#" *)($addr), $src)", 575 []> { 576 bits<4> src; 577 bits<20> addr; 578 579 let Inst{51-48} = addr{19-16}; // base reg 580 let Inst{55-52} = src; 581 let Inst{47-32} = addr{15-0}; // offset 582 let Inst{8-4} = BPF_STORE_REL.Value; 583 let BPFClass = BPF_STX; 584} 585 586class STORE_RELEASEi64<BPFWidthModifer Opc, string OpcodeStr> 587 : STORE_RELEASE<Opc, OpcodeStr, GPR>; 588 589class relaxed_store<PatFrag base> 590 : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> { 591 let IsAtomic = 1; 592 let IsAtomicOrderingReleaseOrStronger = 0; 593} 594 595class releasing_store<PatFrag base> 596 : PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> { 597 let IsAtomic = 1; 598 let IsAtomicOrderingRelease = 1; 599} 600 601let Predicates = [BPFHasLoadAcqStoreRel] in { 602 def STDREL : STORE_RELEASEi64<BPF_DW, "u64">; 603 604 foreach P = [[relaxed_store<atomic_store_64>, STD], 605 [releasing_store<atomic_store_64>, STDREL], 606 ] in { 607 def : Pat<(P[0] GPR:$val, ADDRri:$addr), (P[1] GPR:$val, ADDRri:$addr)>; 608 } 609} 610 611// LOAD instructions 612class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 613 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 614 (outs GPR:$dst), 615 (ins MEMri:$addr), 616 "$dst = *("#OpcodeStr#" *)($addr)", 617 Pattern> { 618 bits<4> dst; 619 bits<20> addr; 620 621 let Inst{51-48} = dst; 622 let Inst{55-52} = addr{19-16}; 623 let Inst{47-32} = addr{15-0}; 624 let BPFClass = BPF_LDX; 625} 626 627class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 628 : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>; 629 630let isCodeGenOnly = 1 in { 631 class CORE_LD<RegisterClass RegClass, string Sz> 632 : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 633 (outs RegClass:$dst), 634 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 635 "$dst = core_ld"#Sz#"($opcode, $src, $offset)", 636 []>; 637 def CORE_LD64 : CORE_LD<GPR, "64">; 638 def CORE_LD32 : CORE_LD<GPR32, "32">; 639 def CORE_ST : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 640 (outs), 641 (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset), 642 "core_st($src, $opcode, $ptr, $offset)", 643 []>; 644 let Constraints = "$dst = $src" in { 645 def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0, 646 (outs GPR:$dst), 647 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 648 "$dst = core_shift($opcode, $src, $offset)", 649 []>; 650 } 651} 652 653let Predicates = [BPFNoALU32] in { 654 def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>; 655 def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>; 656 def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>; 657} 658 659let Predicates = [BPFHasLdsx] in { 660 def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>; 661 def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>; 662 def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8", sextloadi8>; 663} 664 665def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>; 666 667class LOAD_ACQUIRE<BPFWidthModifer SizeOp, string OpcodeStr, RegisterClass RegTp> 668 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 669 (outs RegTp:$dst), 670 (ins MEMri:$addr), 671 "$dst = load_acquire(("#OpcodeStr#" *)($addr))", 672 []> { 673 bits<4> dst; 674 bits<20> addr; 675 676 let Inst{51-48} = dst; 677 let Inst{55-52} = addr{19-16}; // base reg 678 let Inst{47-32} = addr{15-0}; // offset 679 let Inst{8-4} = BPF_LOAD_ACQ.Value; 680 let BPFClass = BPF_STX; 681} 682 683class LOAD_ACQUIREi64<BPFWidthModifer SizeOp, string OpcodeStr> 684 : LOAD_ACQUIRE<SizeOp, OpcodeStr, GPR>; 685 686class relaxed_load<PatFrags base> 687 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 688 let IsAtomic = 1; 689 let IsAtomicOrderingAcquireOrStronger = 0; 690} 691 692class acquiring_load<PatFrags base> 693 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 694 let IsAtomic = 1; 695 let IsAtomicOrderingAcquire = 1; 696} 697 698let Predicates = [BPFHasLoadAcqStoreRel] in { 699 def LDDACQ : LOAD_ACQUIREi64<BPF_DW, "u64">; 700 701 foreach P = [[relaxed_load<atomic_load_nonext_64>, LDD], 702 [acquiring_load<atomic_load_nonext_64>, LDDACQ], 703 ] in { 704 def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>; 705 } 706} 707 708class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 709 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 710 (outs), 711 (ins brtarget:$BrDst), 712 !strconcat(OpcodeStr, " $BrDst"), 713 Pattern> { 714 bits<16> BrDst; 715 716 let Inst{47-32} = BrDst; 717 let BPFClass = BPF_JMP; 718} 719 720class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 721 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 722 (outs), 723 (ins brtarget:$BrDst), 724 !strconcat(OpcodeStr, " $BrDst"), 725 Pattern> { 726 bits<32> BrDst; 727 728 let Inst{31-0} = BrDst; 729 let BPFClass = BPF_JMP32; 730} 731 732class CALL<string OpcodeStr> 733 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value, 734 (outs), 735 (ins calltarget:$BrDst), 736 !strconcat(OpcodeStr, " $BrDst"), 737 []> { 738 bits<32> BrDst; 739 740 let Inst{31-0} = BrDst; 741 let BPFClass = BPF_JMP; 742} 743 744class CALLX<string OpcodeStr> 745 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value, 746 (outs), 747 (ins GPR:$BrDst), 748 !strconcat(OpcodeStr, " $BrDst"), 749 []> { 750 bits<4> BrDst; 751 752 let Inst{51-48} = BrDst; 753 let BPFClass = BPF_JMP; 754} 755 756// Jump always 757let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in { 758 def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>; 759 def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>; 760} 761 762// Jump and link 763let isCall=1, hasDelaySlot=0, Uses = [R11] in { 764 def JAL : CALL<"call">; 765 def JALX : CALLX<"callx">; 766} 767 768class NOP_I<string OpcodeStr> 769 : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value, 770 (outs), 771 (ins i32imm:$imm), 772 !strconcat(OpcodeStr, "\t$imm"), 773 []> { 774 // mov r0, r0 == nop 775 let Inst{55-52} = 0; 776 let Inst{51-48} = 0; 777 let BPFClass = BPF_ALU64; 778} 779 780let hasSideEffects = 0, isCodeGenOnly = 1 in 781 def NOP : NOP_I<"nop">; 782 783class RET<string OpcodeStr> 784 : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value, 785 (outs), 786 (ins), 787 !strconcat(OpcodeStr, ""), 788 [(BPFretglue)]> { 789 let Inst{31-0} = 0; 790 let BPFClass = BPF_JMP; 791} 792 793let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1, 794 isNotDuplicable = 1 in { 795 def RET : RET<"exit">; 796} 797 798// ADJCALLSTACKDOWN/UP pseudo insns 799let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in { 800def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 801 "#ADJCALLSTACKDOWN $amt1 $amt2", 802 [(BPFcallseq_start timm:$amt1, timm:$amt2)]>; 803def ADJCALLSTACKUP : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 804 "#ADJCALLSTACKUP $amt1 $amt2", 805 [(BPFcallseq_end timm:$amt1, timm:$amt2)]>; 806} 807 808let usesCustomInserter = 1, isCodeGenOnly = 1 in { 809 def Select : Pseudo<(outs GPR:$dst), 810 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 811 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 812 [(set i64:$dst, 813 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>; 814 def Select_Ri : Pseudo<(outs GPR:$dst), 815 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 816 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 817 [(set i64:$dst, 818 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>; 819 def Select_64_32 : Pseudo<(outs GPR32:$dst), 820 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 821 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 822 [(set i32:$dst, 823 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>; 824 def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst), 825 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 826 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 827 [(set i32:$dst, 828 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>; 829 def Select_32 : Pseudo<(outs GPR32:$dst), 830 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 831 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 832 [(set i32:$dst, 833 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>; 834 def Select_Ri_32 : Pseudo<(outs GPR32:$dst), 835 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 836 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 837 [(set i32:$dst, 838 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>; 839 def Select_32_64 : Pseudo<(outs GPR:$dst), 840 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 841 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 842 [(set i64:$dst, 843 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>; 844 def Select_Ri_32_64 : Pseudo<(outs GPR:$dst), 845 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 846 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 847 [(set i64:$dst, 848 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>; 849} 850 851// load 64-bit global addr into register 852def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>; 853def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>; 854 855// 0xffffFFFF doesn't fit into simm32, optimize common case 856def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)), 857 (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>; 858 859// Calls 860def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>; 861def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>; 862def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>; 863def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>; 864 865// Loads 866let Predicates = [BPFNoALU32] in { 867 def : Pat<(i64 (extloadi8 ADDRri:$src)), (i64 (LDB ADDRri:$src))>; 868 def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>; 869 def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>; 870} 871 872// Atomic add, and, or, xor 873class ATOMIC_NOFETCH<BPFWidthModifer SizeOp, string OpType, RegisterClass RegTp, 874 BPFArithOp Opc, string Opstr> 875 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 876 (outs RegTp:$dst), 877 (ins MEMri:$addr, RegTp:$val), 878 "lock *(" #OpType# " *)($addr) " #Opstr# "= $val", 879 []> { 880 bits<4> dst; 881 bits<20> addr; 882 883 let Inst{51-48} = addr{19-16}; // base reg 884 let Inst{55-52} = dst; 885 let Inst{47-32} = addr{15-0}; // offset 886 let Inst{7-4} = Opc.Value; 887 let BPFClass = BPF_STX; 888} 889 890let Constraints = "$dst = $val" in { 891 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 892 def XADDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_ADD, "+">; 893 def XANDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_AND, "&">; 894 def XORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_OR, "|">; 895 def XXORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_XOR, "^">; 896 } 897 def XADDW : ATOMIC_NOFETCH<BPF_W, "u32", GPR, BPF_ADD, "+">; 898 def XADDD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_ADD, "+">; 899 def XANDD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_AND, "&">; 900 def XORD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_OR, "|">; 901 def XXORD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_XOR, "^">; 902} 903 904let Predicates = [BPFNoALU32] in { 905 def : Pat<(atomic_load_add_i32 ADDRri:$addr, GPR:$val), 906 (XADDW ADDRri:$addr, GPR:$val)>; 907 def : Pat<(atomic_load_add_i64 ADDRri:$addr, GPR:$val), 908 (XADDD ADDRri:$addr, GPR:$val)>; 909} 910 911// Atomic Fetch-and-<add, and, or, xor> operations 912class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr> 913 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 914 (outs GPR:$dst), 915 (ins MEMri:$addr, GPR:$val), 916 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 917 []> { 918 bits<4> dst; 919 bits<20> addr; 920 921 let Inst{51-48} = addr{19-16}; // base reg 922 let Inst{55-52} = dst; 923 let Inst{47-32} = addr{15-0}; // offset 924 let Inst{7-4} = Opc.Value; 925 let Inst{3-0} = BPF_FETCH.Value; 926 let BPFClass = BPF_STX; 927} 928 929class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr> 930 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 931 (outs GPR32:$dst), 932 (ins MEMri:$addr, GPR32:$val), 933 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 934 []> { 935 bits<4> dst; 936 bits<20> addr; 937 938 let Inst{51-48} = addr{19-16}; // base reg 939 let Inst{55-52} = dst; 940 let Inst{47-32} = addr{15-0}; // offset 941 let Inst{7-4} = Opc.Value; 942 let Inst{3-0} = BPF_FETCH.Value; 943 let BPFClass = BPF_STX; 944} 945 946let Constraints = "$dst = $val" in { 947 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 948 def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add">; 949 def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and">; 950 def XFORW32 : XFALU32<BPF_W, BPF_OR, "u32", "or">; 951 def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor">; 952 } 953 954 let Predicates = [BPFHasALU32] in { 955 def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add">; 956 } 957 def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and">; 958 def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or">; 959 def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor">; 960} 961 962let Predicates = [BPFHasALU32] in { 963 foreach P = [// add 964 [atomic_load_add_i32_monotonic, XADDW32], 965 [atomic_load_add_i32_acquire, XFADDW32], 966 [atomic_load_add_i32_release, XFADDW32], 967 [atomic_load_add_i32_acq_rel, XFADDW32], 968 [atomic_load_add_i32_seq_cst, XFADDW32], 969 // and 970 [atomic_load_and_i32_monotonic, XANDW32], 971 [atomic_load_and_i32_acquire, XFANDW32], 972 [atomic_load_and_i32_release, XFANDW32], 973 [atomic_load_and_i32_acq_rel, XFANDW32], 974 [atomic_load_and_i32_seq_cst, XFANDW32], 975 // or 976 [atomic_load_or_i32_monotonic, XORW32], 977 [atomic_load_or_i32_acquire, XFORW32], 978 [atomic_load_or_i32_release, XFORW32], 979 [atomic_load_or_i32_acq_rel, XFORW32], 980 [atomic_load_or_i32_seq_cst, XFORW32], 981 // xor 982 [atomic_load_xor_i32_monotonic, XXORW32], 983 [atomic_load_xor_i32_acquire, XFXORW32], 984 [atomic_load_xor_i32_release, XFXORW32], 985 [atomic_load_xor_i32_acq_rel, XFXORW32], 986 [atomic_load_xor_i32_seq_cst, XFXORW32], 987 ] in { 988 def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1] ADDRri:$addr, GPR32:$val)>; 989 } 990 991 // atomic_load_sub can be represented as a neg followed 992 // by an atomic_load_add. 993 foreach P = [[atomic_load_sub_i32_monotonic, XADDW32], 994 [atomic_load_sub_i32_acquire, XFADDW32], 995 [atomic_load_sub_i32_release, XFADDW32], 996 [atomic_load_sub_i32_acq_rel, XFADDW32], 997 [atomic_load_sub_i32_seq_cst, XFADDW32], 998 ] in { 999 def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1] ADDRri:$addr, (NEG_32 GPR32:$val))>; 1000 } 1001 1002 foreach P = [// add 1003 [atomic_load_add_i64_monotonic, XADDD], 1004 [atomic_load_add_i64_acquire, XFADDD], 1005 [atomic_load_add_i64_release, XFADDD], 1006 [atomic_load_add_i64_acq_rel, XFADDD], 1007 [atomic_load_add_i64_seq_cst, XFADDD], 1008 ] in { 1009 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, GPR:$val)>; 1010 } 1011} 1012 1013foreach P = [[atomic_load_sub_i64_monotonic, XADDD], 1014 [atomic_load_sub_i64_acquire, XFADDD], 1015 [atomic_load_sub_i64_release, XFADDD], 1016 [atomic_load_sub_i64_acq_rel, XFADDD], 1017 [atomic_load_sub_i64_seq_cst, XFADDD], 1018 ] in { 1019 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, (NEG_64 GPR:$val))>; 1020} 1021 1022// Borrow the idea from X86InstrFragments.td 1023class binop_no_use<SDPatternOperator operator> 1024 : PatFrag<(ops node:$A, node:$B), 1025 (operator node:$A, node:$B), 1026 [{ return SDValue(N, 0).use_empty(); }]>; 1027 1028class binop_has_use<SDPatternOperator operator> 1029 : PatFrag<(ops node:$A, node:$B), 1030 (operator node:$A, node:$B), 1031 [{ return !SDValue(N, 0).use_empty(); }]>; 1032 1033foreach op = [add, and, or, xor] in { 1034def atomic_load_ # op # _i64_monotonic_nu: 1035 binop_no_use <!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>; 1036def atomic_load_ # op # _i64_monotonic_hu: 1037 binop_has_use<!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>; 1038} 1039 1040foreach P = [// and 1041 [atomic_load_and_i64_monotonic_nu, XANDD], 1042 [atomic_load_and_i64_monotonic_hu, XFANDD], 1043 [atomic_load_and_i64_acquire, XFANDD], 1044 [atomic_load_and_i64_release, XFANDD], 1045 [atomic_load_and_i64_acq_rel, XFANDD], 1046 [atomic_load_and_i64_seq_cst, XFANDD], 1047 // or 1048 [atomic_load_or_i64_monotonic_nu, XORD], 1049 [atomic_load_or_i64_monotonic_hu, XFORD], 1050 [atomic_load_or_i64_acquire, XFORD], 1051 [atomic_load_or_i64_release, XFORD], 1052 [atomic_load_or_i64_acq_rel, XFORD], 1053 [atomic_load_or_i64_seq_cst, XFORD], 1054 // xor 1055 [atomic_load_xor_i64_monotonic_nu, XXORD], 1056 [atomic_load_xor_i64_monotonic_hu, XFXORD], 1057 [atomic_load_xor_i64_acquire, XFXORD], 1058 [atomic_load_xor_i64_release, XFXORD], 1059 [atomic_load_xor_i64_acq_rel, XFXORD], 1060 [atomic_load_xor_i64_seq_cst, XFXORD], 1061 ] in { 1062 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, GPR:$val)>; 1063} 1064 1065// Atomic Exchange 1066class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1067 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1068 (outs GPR:$dst), 1069 (ins MEMri:$addr, GPR:$val), 1070 "$dst = xchg_"#OpcodeStr#"($addr, $val)", 1071 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> { 1072 bits<4> dst; 1073 bits<20> addr; 1074 1075 let Inst{51-48} = addr{19-16}; // base reg 1076 let Inst{55-52} = dst; 1077 let Inst{47-32} = addr{15-0}; // offset 1078 let Inst{7-4} = BPF_XCHG.Value; 1079 let Inst{3-0} = BPF_FETCH.Value; 1080 let BPFClass = BPF_STX; 1081} 1082 1083class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1084 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1085 (outs GPR32:$dst), 1086 (ins MEMri:$addr, GPR32:$val), 1087 "$dst = xchg32_"#OpcodeStr#"($addr, $val)", 1088 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> { 1089 bits<4> dst; 1090 bits<20> addr; 1091 1092 let Inst{51-48} = addr{19-16}; // base reg 1093 let Inst{55-52} = dst; 1094 let Inst{47-32} = addr{15-0}; // offset 1095 let Inst{7-4} = BPF_XCHG.Value; 1096 let Inst{3-0} = BPF_FETCH.Value; 1097 let BPFClass = BPF_STX; 1098} 1099 1100let Constraints = "$dst = $val" in { 1101 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1102 def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_i32>; 1103 } 1104 1105 def XCHGD : XCHG<BPF_DW, "64", atomic_swap_i64>; 1106} 1107 1108// Compare-And-Exchange 1109class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1110 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1111 (outs), 1112 (ins MEMri:$addr, GPR:$new), 1113 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)", 1114 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> { 1115 bits<4> new; 1116 bits<20> addr; 1117 1118 let Inst{51-48} = addr{19-16}; // base reg 1119 let Inst{55-52} = new; 1120 let Inst{47-32} = addr{15-0}; // offset 1121 let Inst{7-4} = BPF_CMPXCHG.Value; 1122 let Inst{3-0} = BPF_FETCH.Value; 1123 let BPFClass = BPF_STX; 1124} 1125 1126class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1127 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1128 (outs), 1129 (ins MEMri:$addr, GPR32:$new), 1130 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)", 1131 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> { 1132 bits<4> new; 1133 bits<20> addr; 1134 1135 let Inst{51-48} = addr{19-16}; // base reg 1136 let Inst{55-52} = new; 1137 let Inst{47-32} = addr{15-0}; // offset 1138 let Inst{7-4} = BPF_CMPXCHG.Value; 1139 let Inst{3-0} = BPF_FETCH.Value; 1140 let BPFClass = BPF_STX; 1141} 1142 1143let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0], 1144 DecoderNamespace = "BPFALU32" in { 1145 def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_i32>; 1146} 1147 1148let Defs = [R0], Uses = [R0] in { 1149 def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_i64>; 1150} 1151 1152// bswap16, bswap32, bswap64 1153class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern> 1154 : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value, 1155 (outs GPR:$dst), 1156 (ins GPR:$src), 1157 "$dst = "#OpcodeStr#" $src", 1158 Pattern> { 1159 bits<4> dst; 1160 1161 let Inst{51-48} = dst; 1162 let Inst{31-0} = SizeOp; 1163 let BPFClass = Class; 1164} 1165 1166 1167let Constraints = "$dst = $src" in { 1168 let Predicates = [BPFHasBswap] in { 1169 def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1170 def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1171 def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 1172 } 1173 1174 let Predicates = [BPFNoBswap] in { 1175 let Predicates = [BPFIsLittleEndian] in { 1176 def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1177 def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1178 def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>; 1179 } 1180 let Predicates = [BPFIsBigEndian] in { 1181 def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1182 def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1183 def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 1184 } 1185 } 1186} 1187 1188let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1, 1189 hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in { 1190class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1191 : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value, 1192 (outs), 1193 (ins GPR:$skb, i64imm:$imm), 1194 "r0 = *("#OpcodeStr#" *)skb[$imm]", 1195 [(set R0, (OpNode GPR:$skb, i64immSExt32:$imm))]> { 1196 bits<32> imm; 1197 1198 let Inst{31-0} = imm; 1199 let BPFClass = BPF_LD; 1200} 1201 1202class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1203 : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value, 1204 (outs), 1205 (ins GPR:$skb, GPR:$val), 1206 "r0 = *("#OpcodeStr#" *)skb[$val]", 1207 [(set R0, (OpNode GPR:$skb, GPR:$val))]> { 1208 bits<4> val; 1209 1210 let Inst{55-52} = val; 1211 let BPFClass = BPF_LD; 1212} 1213} 1214 1215def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>; 1216def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>; 1217def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>; 1218 1219def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>; 1220def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>; 1221def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>; 1222 1223let isCodeGenOnly = 1 in { 1224 def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0, 1225 (outs GPR:$dst), (ins GPR32:$src), 1226 "$dst = $src", []>; 1227} 1228 1229let Predicates = [BPFNoMovsx] in { 1230 def : Pat<(i64 (sext GPR32:$src)), 1231 (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>; 1232} 1233 1234let Predicates = [BPFHasMovsx] in { 1235 def : Pat<(i64 (sext GPR32:$src)), 1236 (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>; 1237} 1238 1239def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>; 1240 1241// For i64 -> i32 truncation, use the 32-bit subregister directly. 1242def : Pat<(i32 (trunc GPR:$src)), 1243 (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>; 1244 1245// For i32 -> i64 anyext, we don't care about the high bits. 1246def : Pat<(i64 (anyext GPR32:$src)), 1247 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>; 1248 1249class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 1250 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 1251 (outs), 1252 (ins GPR32:$src, MEMri:$addr), 1253 "*("#OpcodeStr#" *)($addr) = $src", 1254 Pattern> { 1255 bits<4> src; 1256 bits<20> addr; 1257 1258 let Inst{51-48} = addr{19-16}; // base reg 1259 let Inst{55-52} = src; 1260 let Inst{47-32} = addr{15-0}; // offset 1261 let BPFClass = BPF_STX; 1262} 1263 1264class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 1265 : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>; 1266 1267class STORE_RELEASEi32<BPFWidthModifer Opc, string OpcodeStr> 1268 : STORE_RELEASE<Opc, OpcodeStr, GPR32>; 1269 1270let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1271 def STW32 : STOREi32<BPF_W, "u32", store>; 1272 def STH32 : STOREi32<BPF_H, "u16", truncstorei16>; 1273 def STB32 : STOREi32<BPF_B, "u8", truncstorei8>; 1274 1275 let Predicates = [BPFHasLoadAcqStoreRel] in { 1276 def STWREL32 : STORE_RELEASEi32<BPF_W, "u32">; 1277 def STHREL32 : STORE_RELEASEi32<BPF_H, "u16">; 1278 def STBREL32 : STORE_RELEASEi32<BPF_B, "u8">; 1279 } 1280} 1281 1282class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 1283 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 1284 (outs GPR32:$dst), 1285 (ins MEMri:$addr), 1286 "$dst = *("#OpcodeStr#" *)($addr)", 1287 Pattern> { 1288 bits<4> dst; 1289 bits<20> addr; 1290 1291 let Inst{51-48} = dst; 1292 let Inst{55-52} = addr{19-16}; 1293 let Inst{47-32} = addr{15-0}; 1294 let BPFClass = BPF_LDX; 1295} 1296 1297class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 1298 : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>; 1299 1300class LOAD_ACQUIREi32<BPFWidthModifer SizeOp, string OpcodeStr> 1301 : LOAD_ACQUIRE<SizeOp, OpcodeStr, GPR32>; 1302 1303let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1304 def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>; 1305 def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>; 1306 def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>; 1307 1308 let Predicates = [BPFHasLoadAcqStoreRel] in { 1309 def LDWACQ32 : LOAD_ACQUIREi32<BPF_W, "u32">; 1310 def LDHACQ32 : LOAD_ACQUIREi32<BPF_H, "u16">; 1311 def LDBACQ32 : LOAD_ACQUIREi32<BPF_B, "u8">; 1312 } 1313} 1314 1315let Predicates = [BPFHasALU32] in { 1316 def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst), 1317 (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1318 def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst), 1319 (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1320 def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst), 1321 (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1322 def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>; 1323 def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>; 1324 1325 let Predicates = [BPFHasLdsx] in { 1326 def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>; 1327 def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>; 1328 } 1329 1330 def : Pat<(i64 (zextloadi8 ADDRri:$src)), 1331 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1332 def : Pat<(i64 (zextloadi16 ADDRri:$src)), 1333 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1334 def : Pat<(i64 (zextloadi32 ADDRri:$src)), 1335 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1336 def : Pat<(i64 (extloadi8 ADDRri:$src)), 1337 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1338 def : Pat<(i64 (extloadi16 ADDRri:$src)), 1339 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1340 def : Pat<(i64 (extloadi32 ADDRri:$src)), 1341 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1342 1343 let Predicates = [BPFHasLoadAcqStoreRel] in { 1344 foreach P = [[relaxed_load<atomic_load_nonext_32>, LDW32], 1345 [relaxed_load<atomic_load_azext_16>, LDH32], 1346 [relaxed_load<atomic_load_azext_8>, LDB32], 1347 [acquiring_load<atomic_load_nonext_32>, LDWACQ32], 1348 [acquiring_load<atomic_load_azext_16>, LDHACQ32], 1349 [acquiring_load<atomic_load_azext_8>, LDBACQ32], 1350 ] in { 1351 def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>; 1352 } 1353 } 1354 1355 let Predicates = [BPFHasLoadAcqStoreRel] in { 1356 foreach P = [[relaxed_store<atomic_store_32>, STW32], 1357 [relaxed_store<atomic_store_16>, STH32], 1358 [relaxed_store<atomic_store_8>, STB32], 1359 [releasing_store<atomic_store_32>, STWREL32], 1360 [releasing_store<atomic_store_16>, STHREL32], 1361 [releasing_store<atomic_store_8>, STBREL32], 1362 ] in { 1363 def : Pat<(P[0] GPR32:$val, ADDRri:$addr), (P[1] GPR32:$val, ADDRri:$addr)>; 1364 } 1365 } 1366} 1367 1368let usesCustomInserter = 1, isCodeGenOnly = 1 in { 1369 def MEMCPY : Pseudo< 1370 (outs), 1371 (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops), 1372 "#memcpy dst: $dst, src: $src, len: $len, align: $align", 1373 [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>; 1374} 1375