1//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file describes the BPF instructions in TableGen format. 10// 11//===----------------------------------------------------------------------===// 12 13include "BPFInstrFormats.td" 14 15// Instruction Operands and Patterns 16 17// These are target-independent nodes, but have target-specific formats. 18def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>, 19 SDTCisVT<1, iPTR>]>; 20def SDT_BPFCallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>; 21def SDT_BPFCall : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; 22def SDT_BPFSetFlag : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>; 23def SDT_BPFSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>, 24 SDTCisSameAs<0, 4>, 25 SDTCisSameAs<4, 5>]>; 26def SDT_BPFBrCC : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>, 27 SDTCisVT<3, OtherVT>]>; 28def SDT_BPFWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, 29 SDTCisPtrTy<0>]>; 30def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>, 31 SDTCisVT<1, i64>, 32 SDTCisVT<2, i64>, 33 SDTCisVT<3, i64>]>; 34 35def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall, 36 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, 37 SDNPVariadic]>; 38def BPFretglue : SDNode<"BPFISD::RET_GLUE", SDTNone, 39 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>; 40def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart, 41 [SDNPHasChain, SDNPOutGlue]>; 42def BPFcallseq_end : SDNode<"ISD::CALLSEQ_END", SDT_BPFCallSeqEnd, 43 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; 44def BPFbrcc : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC, 45 [SDNPHasChain, SDNPOutGlue, SDNPInGlue]>; 46 47def BPFselectcc : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC, [SDNPInGlue]>; 48def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; 49def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, 50 [SDNPHasChain, SDNPInGlue, SDNPOutGlue, 51 SDNPMayStore, SDNPMayLoad]>; 52def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; 53def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; 54def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; 55def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">; 56def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; 57def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">; 58def BPFHasBswap : Predicate<"Subtarget->hasBswap()">; 59def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">; 60def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">; 61def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">; 62def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">; 63 64class ImmediateAsmOperand<string name> : AsmOperandClass { 65 let Name = name; 66 let RenderMethod = "addImmOperands"; 67 let DiagnosticType = !strconcat("Invalid", name); 68} 69 70def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">; 71 72def brtarget : Operand<OtherVT> { 73 let PrintMethod = "printBrTargetOperand"; 74 let ParserMatchClass = ImmediateAsmOperand<"BrTarget">; 75} 76def calltarget : Operand<i64>; 77 78def u64imm : Operand<i64> { 79 let PrintMethod = "printImm64Operand"; 80} 81 82def s16imm : Operand<i16> { 83 let ParserMatchClass = SImm16AsmOperand; 84} 85 86def gpr_or_imm : Operand<i64>; 87 88def i64immSExt32 : PatLeaf<(i64 imm), 89 [{return isInt<32>(N->getSExtValue()); }]>; 90def i32immSExt32 : PatLeaf<(i32 imm), 91 [{return isInt<32>(N->getSExtValue()); }]>; 92def i64immZExt32 : PatLeaf<(i64 imm), 93 [{return isUInt<32>(N->getZExtValue()); }]>; 94 95def imm_to_i64 : SDNodeXForm<timm, [{ 96 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64); 97}]>; 98 99// Addressing modes. 100def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>; 101def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>; 102 103// Address operands 104def MEMri : Operand<i64> { 105 let PrintMethod = "printMemOperand"; 106 let EncoderMethod = "getMemoryOpValue"; 107 let DecoderMethod = "decodeMemoryOpValue"; 108 let MIOperandInfo = (ops GPR, s16imm); 109} 110 111// Conditional code predicates - used for pattern matching for jump instructions 112def BPF_CC_EQ : PatLeaf<(i64 imm), 113 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 114def BPF_CC_NE : PatLeaf<(i64 imm), 115 [{return (N->getZExtValue() == ISD::SETNE);}]>; 116def BPF_CC_GE : PatLeaf<(i64 imm), 117 [{return (N->getZExtValue() == ISD::SETGE);}]>; 118def BPF_CC_GT : PatLeaf<(i64 imm), 119 [{return (N->getZExtValue() == ISD::SETGT);}]>; 120def BPF_CC_GTU : PatLeaf<(i64 imm), 121 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 122def BPF_CC_GEU : PatLeaf<(i64 imm), 123 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 124def BPF_CC_LE : PatLeaf<(i64 imm), 125 [{return (N->getZExtValue() == ISD::SETLE);}]>; 126def BPF_CC_LT : PatLeaf<(i64 imm), 127 [{return (N->getZExtValue() == ISD::SETLT);}]>; 128def BPF_CC_LTU : PatLeaf<(i64 imm), 129 [{return (N->getZExtValue() == ISD::SETULT);}]>; 130def BPF_CC_LEU : PatLeaf<(i64 imm), 131 [{return (N->getZExtValue() == ISD::SETULE);}]>; 132def BPF_CC_EQ_32 : PatLeaf<(i32 imm), 133 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 134def BPF_CC_NE_32 : PatLeaf<(i32 imm), 135 [{return (N->getZExtValue() == ISD::SETNE);}]>; 136def BPF_CC_GE_32 : PatLeaf<(i32 imm), 137 [{return (N->getZExtValue() == ISD::SETGE);}]>; 138def BPF_CC_GT_32 : PatLeaf<(i32 imm), 139 [{return (N->getZExtValue() == ISD::SETGT);}]>; 140def BPF_CC_GTU_32 : PatLeaf<(i32 imm), 141 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 142def BPF_CC_GEU_32 : PatLeaf<(i32 imm), 143 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 144def BPF_CC_LE_32 : PatLeaf<(i32 imm), 145 [{return (N->getZExtValue() == ISD::SETLE);}]>; 146def BPF_CC_LT_32 : PatLeaf<(i32 imm), 147 [{return (N->getZExtValue() == ISD::SETLT);}]>; 148def BPF_CC_LTU_32 : PatLeaf<(i32 imm), 149 [{return (N->getZExtValue() == ISD::SETULT);}]>; 150def BPF_CC_LEU_32 : PatLeaf<(i32 imm), 151 [{return (N->getZExtValue() == ISD::SETULE);}]>; 152def NoCond : PatLeaf<(vt)> {} 153 154// For arithmetic and jump instructions the 8-bit 'code' 155// field is divided into three parts: 156// 157// +----------------+--------+--------------------+ 158// | 4 bits | 1 bit | 3 bits | 159// | operation code | source | instruction class | 160// +----------------+--------+--------------------+ 161// (MSB) (LSB) 162class TYPE_ALU_JMP<bits<4> op, bits<1> srctype, 163 dag outs, dag ins, string asmstr, list<dag> pattern> 164 : InstBPF<outs, ins, asmstr, pattern> { 165 166 let Inst{63-60} = op; 167 let Inst{59} = srctype; 168} 169 170//For load and store instructions the 8-bit 'code' field is divided as: 171// 172// +--------+--------+-------------------+ 173// | 3 bits | 2 bits | 3 bits | 174// | mode | size | instruction class | 175// +--------+--------+-------------------+ 176// (MSB) (LSB) 177class TYPE_LD_ST<bits<3> mode, bits<2> size, 178 dag outs, dag ins, string asmstr, list<dag> pattern> 179 : InstBPF<outs, ins, asmstr, pattern> { 180 181 let Inst{63-61} = mode; 182 let Inst{60-59} = size; 183} 184 185// jump instructions 186class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 187 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 188 (outs), 189 (ins GPR:$dst, GPR:$src, brtarget:$BrDst), 190 "if $dst "#OpcodeStr#" $src goto $BrDst", 191 [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> { 192 bits<4> dst; 193 bits<4> src; 194 bits<16> BrDst; 195 196 let Inst{55-52} = src; 197 let Inst{51-48} = dst; 198 let Inst{47-32} = BrDst; 199 let BPFClass = BPF_JMP; 200} 201 202class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 203 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 204 (outs), 205 (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst), 206 "if $dst "#OpcodeStr#" $imm goto $BrDst", 207 [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> { 208 bits<4> dst; 209 bits<16> BrDst; 210 bits<32> imm; 211 212 let Inst{51-48} = dst; 213 let Inst{47-32} = BrDst; 214 let Inst{31-0} = imm; 215 let BPFClass = BPF_JMP; 216} 217 218class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 219 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 220 (outs), 221 (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst), 222 "if $dst "#OpcodeStr#" $src goto $BrDst", 223 [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> { 224 bits<4> dst; 225 bits<4> src; 226 bits<16> BrDst; 227 228 let Inst{55-52} = src; 229 let Inst{51-48} = dst; 230 let Inst{47-32} = BrDst; 231 let BPFClass = BPF_JMP32; 232} 233 234class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 235 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 236 (outs), 237 (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst), 238 "if $dst "#OpcodeStr#" $imm goto $BrDst", 239 [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> { 240 bits<4> dst; 241 bits<16> BrDst; 242 bits<32> imm; 243 244 let Inst{51-48} = dst; 245 let Inst{47-32} = BrDst; 246 let Inst{31-0} = imm; 247 let BPFClass = BPF_JMP32; 248} 249 250multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> { 251 def _rr : JMP_RR<Opc, OpcodeStr, Cond>; 252 def _ri : JMP_RI<Opc, OpcodeStr, Cond>; 253 def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>; 254 def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>; 255} 256 257let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in { 258// cmp+goto instructions 259defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>; 260defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>; 261defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>; 262defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>; 263defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>; 264defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>; 265defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>; 266defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>; 267defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>; 268defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>; 269defm JSET : J<BPF_JSET, "&", NoCond, NoCond>; 270} 271 272// ALU instructions 273class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off, 274 dag outs, dag ins, string asmstr, list<dag> pattern> 275 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> { 276 bits<4> dst; 277 bits<32> imm; 278 279 let Inst{51-48} = dst; 280 let Inst{47-32} = off; 281 let Inst{31-0} = imm; 282 let BPFClass = Class; 283} 284 285class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off, 286 dag outs, dag ins, string asmstr, list<dag> pattern> 287 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> { 288 bits<4> dst; 289 bits<4> src; 290 291 let Inst{55-52} = src; 292 let Inst{51-48} = dst; 293 let Inst{47-32} = off; 294 let BPFClass = Class; 295} 296 297multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { 298 def _rr : ALU_RR<BPF_ALU64, Opc, off, 299 (outs GPR:$dst), 300 (ins GPR:$src2, GPR:$src), 301 "$dst "#OpcodeStr#" $src", 302 [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>; 303 def _ri : ALU_RI<BPF_ALU64, Opc, off, 304 (outs GPR:$dst), 305 (ins GPR:$src2, i64imm:$imm), 306 "$dst "#OpcodeStr#" $imm", 307 [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>; 308 def _rr_32 : ALU_RR<BPF_ALU, Opc, off, 309 (outs GPR32:$dst), 310 (ins GPR32:$src2, GPR32:$src), 311 "$dst "#OpcodeStr#" $src", 312 [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>; 313 def _ri_32 : ALU_RI<BPF_ALU, Opc, off, 314 (outs GPR32:$dst), 315 (ins GPR32:$src2, i32imm:$imm), 316 "$dst "#OpcodeStr#" $imm", 317 [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>; 318} 319 320let Constraints = "$dst = $src2" in { 321let isAsCheapAsAMove = 1 in { 322 defm ADD : ALU<BPF_ADD, 0, "+=", add>; 323 defm SUB : ALU<BPF_SUB, 0, "-=", sub>; 324 defm OR : ALU<BPF_OR, 0, "|=", or>; 325 defm AND : ALU<BPF_AND, 0, "&=", and>; 326 defm SLL : ALU<BPF_LSH, 0, "<<=", shl>; 327 defm SRL : ALU<BPF_RSH, 0, ">>=", srl>; 328 defm XOR : ALU<BPF_XOR, 0, "^=", xor>; 329 defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>; 330} 331 defm MUL : ALU<BPF_MUL, 0, "*=", mul>; 332 defm DIV : ALU<BPF_DIV, 0, "/=", udiv>; 333 defm MOD : ALU<BPF_MOD, 0, "%=", urem>; 334 335 let Predicates = [BPFHasSdivSmod] in { 336 defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>; 337 defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>; 338 } 339} 340 341class NEG_RR<BPFOpClass Class, BPFArithOp Opc, 342 dag outs, dag ins, string asmstr, list<dag> pattern> 343 : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> { 344 bits<4> dst; 345 346 let Inst{51-48} = dst; 347 let BPFClass = Class; 348} 349 350let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in { 351 def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src), 352 "$dst = -$src", 353 [(set GPR:$dst, (ineg i64:$src))]>; 354 def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src), 355 "$dst = -$src", 356 [(set GPR32:$dst, (ineg i32:$src))]>; 357} 358 359class LD_IMM64<bits<4> Pseudo, string OpcodeStr> 360 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 361 (outs GPR:$dst), 362 (ins u64imm:$imm), 363 "$dst "#OpcodeStr#" ${imm} ll", 364 [(set GPR:$dst, (i64 imm:$imm))]> { 365 366 bits<4> dst; 367 bits<64> imm; 368 369 let Inst{51-48} = dst; 370 let Inst{55-52} = Pseudo; 371 let Inst{47-32} = 0; 372 let Inst{31-0} = imm{31-0}; 373 let BPFClass = BPF_LD; 374} 375 376let isReMaterializable = 1, isAsCheapAsAMove = 1 in { 377def LD_imm64 : LD_IMM64<0, "=">; 378def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0, 379 (outs GPR:$dst), 380 (ins GPR:$src), 381 "$dst = $src", 382 []>; 383def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0, 384 (outs GPR:$dst), 385 (ins i64imm:$imm), 386 "$dst = $imm", 387 [(set GPR:$dst, (i64 i64immSExt32:$imm))]>; 388def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0, 389 (outs GPR32:$dst), 390 (ins GPR32:$src), 391 "$dst = $src", 392 []>; 393def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0, 394 (outs GPR32:$dst), 395 (ins i32imm:$imm), 396 "$dst = $imm", 397 [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>; 398 399let Predicates = [BPFHasMovsx] in { 400 def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8, 401 (outs GPR:$dst), (ins GPR:$src), 402 "$dst = (s8)$src", 403 [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>; 404 def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16, 405 (outs GPR:$dst), (ins GPR:$src), 406 "$dst = (s16)$src", 407 [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>; 408 def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32, 409 (outs GPR:$dst), (ins GPR:$src), 410 "$dst = (s32)$src", 411 [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>; 412 def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8, 413 (outs GPR32:$dst), (ins GPR32:$src), 414 "$dst = (s8)$src", 415 [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>; 416 def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16, 417 (outs GPR32:$dst), (ins GPR32:$src), 418 "$dst = (s16)$src", 419 [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>; 420} 421} 422 423def FI_ri 424 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 425 (outs GPR:$dst), 426 (ins MEMri:$addr), 427 "lea\t$dst, $addr", 428 [(set i64:$dst, FIri:$addr)]> { 429 // This is a tentative instruction, and will be replaced 430 // with MOV_rr and ADD_ri in PEI phase 431 let Inst{51-48} = 0; 432 let Inst{55-52} = 2; 433 let Inst{47-32} = 0; 434 let Inst{31-0} = 0; 435 let BPFClass = BPF_LD; 436 bit isPseudo = true; 437} 438 439def LD_pseudo 440 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 441 (outs GPR:$dst), 442 (ins i64imm:$pseudo, u64imm:$imm), 443 "ld_pseudo\t$dst, $pseudo, $imm", 444 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> { 445 446 bits<4> dst; 447 bits<64> imm; 448 bits<4> pseudo; 449 450 let Inst{51-48} = dst; 451 let Inst{55-52} = pseudo; 452 let Inst{47-32} = 0; 453 let Inst{31-0} = imm{31-0}; 454 let BPFClass = BPF_LD; 455} 456 457// STORE instructions 458class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 459 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 460 (outs), 461 (ins GPR:$src, MEMri:$addr), 462 "*("#OpcodeStr#" *)($addr) = $src", 463 Pattern> { 464 bits<4> src; 465 bits<20> addr; 466 467 let Inst{51-48} = addr{19-16}; // base reg 468 let Inst{55-52} = src; 469 let Inst{47-32} = addr{15-0}; // offset 470 let BPFClass = BPF_STX; 471} 472 473class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 474 : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>; 475 476let Predicates = [BPFNoALU32] in { 477 def STW : STOREi64<BPF_W, "u32", truncstorei32>; 478 def STH : STOREi64<BPF_H, "u16", truncstorei16>; 479 def STB : STOREi64<BPF_B, "u8", truncstorei8>; 480} 481def STD : STOREi64<BPF_DW, "u64", store>; 482 483class STORE_imm<BPFWidthModifer SizeOp, 484 string OpcodeStr, dag Pattern> 485 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 486 (outs), 487 (ins i64imm:$imm, MEMri:$addr), 488 "*("#OpcodeStr#" *)($addr) = $imm", 489 [Pattern]> { 490 bits<20> addr; 491 bits<32> imm; 492 493 let Inst{51-48} = addr{19-16}; // base reg 494 let Inst{47-32} = addr{15-0}; // offset 495 let Inst{31-0} = imm; 496 let BPFClass = BPF_ST; 497} 498 499let Predicates = [BPFHasStoreImm] in { 500 // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for 501 // value stored to memory: 502 // - it is fine to generate such write when immediate is -1 503 // - it is incorrect to generate such write when immediate is 504 // +0xffff_ffff. 505 // 506 // In the latter case two instructions would be generated instead of 507 // one BPF_ST: 508 // rA = 0xffffffff ll ; LD_imm64 509 // *(u64 *)(rB + 0) = rA ; STX 510 // 511 // For BPF_{B,H,W} the size of value stored matches size of the immediate. 512 def STD_imm : STORE_imm<BPF_DW, "u64", (store (i64 i64immSExt32:$imm), ADDRri:$addr)>; 513 def STW_imm : STORE_imm<BPF_W, "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 514 def STH_imm : STORE_imm<BPF_H, "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 515 def STB_imm : STORE_imm<BPF_B, "u8", (truncstorei8 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 516} 517 518let Predicates = [BPFHasALU32, BPFHasStoreImm] in { 519 def : Pat<(store (i32 imm:$src), ADDRri:$dst), 520 (STW_imm (imm_to_i64 $src), ADDRri:$dst)>; 521 def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst), 522 (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 523 def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst), 524 (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 525} 526 527// LOAD instructions 528class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 529 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 530 (outs GPR:$dst), 531 (ins MEMri:$addr), 532 "$dst = *("#OpcodeStr#" *)($addr)", 533 Pattern> { 534 bits<4> dst; 535 bits<20> addr; 536 537 let Inst{51-48} = dst; 538 let Inst{55-52} = addr{19-16}; 539 let Inst{47-32} = addr{15-0}; 540 let BPFClass = BPF_LDX; 541} 542 543class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 544 : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>; 545 546let isCodeGenOnly = 1 in { 547 class CORE_LD<RegisterClass RegClass, string Sz> 548 : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 549 (outs RegClass:$dst), 550 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 551 "$dst = core_ld"#Sz#"($opcode, $src, $offset)", 552 []>; 553 def CORE_LD64 : CORE_LD<GPR, "64">; 554 def CORE_LD32 : CORE_LD<GPR32, "32">; 555 def CORE_ST : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 556 (outs), 557 (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset), 558 "core_st($src, $opcode, $ptr, $offset)", 559 []>; 560 let Constraints = "$dst = $src" in { 561 def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0, 562 (outs GPR:$dst), 563 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 564 "$dst = core_shift($opcode, $src, $offset)", 565 []>; 566 } 567} 568 569let Predicates = [BPFNoALU32] in { 570 def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>; 571 def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>; 572 def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>; 573} 574 575let Predicates = [BPFHasLdsx] in { 576 def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>; 577 def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>; 578 def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8", sextloadi8>; 579} 580 581def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>; 582 583class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 584 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 585 (outs), 586 (ins brtarget:$BrDst), 587 !strconcat(OpcodeStr, " $BrDst"), 588 Pattern> { 589 bits<16> BrDst; 590 591 let Inst{47-32} = BrDst; 592 let BPFClass = BPF_JMP; 593} 594 595class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 596 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 597 (outs), 598 (ins brtarget:$BrDst), 599 !strconcat(OpcodeStr, " $BrDst"), 600 Pattern> { 601 bits<32> BrDst; 602 603 let Inst{31-0} = BrDst; 604 let BPFClass = BPF_JMP32; 605} 606 607class CALL<string OpcodeStr> 608 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value, 609 (outs), 610 (ins calltarget:$BrDst), 611 !strconcat(OpcodeStr, " $BrDst"), 612 []> { 613 bits<32> BrDst; 614 615 let Inst{31-0} = BrDst; 616 let BPFClass = BPF_JMP; 617} 618 619class CALLX<string OpcodeStr> 620 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value, 621 (outs), 622 (ins GPR:$BrDst), 623 !strconcat(OpcodeStr, " $BrDst"), 624 []> { 625 bits<32> BrDst; 626 627 let Inst{31-0} = BrDst; 628 let BPFClass = BPF_JMP; 629} 630 631// Jump always 632let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in { 633 def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>; 634 def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>; 635} 636 637// Jump and link 638let isCall=1, hasDelaySlot=0, Uses = [R11], 639 // Potentially clobbered registers 640 Defs = [R0, R1, R2, R3, R4, R5] in { 641 def JAL : CALL<"call">; 642 def JALX : CALLX<"callx">; 643} 644 645class NOP_I<string OpcodeStr> 646 : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value, 647 (outs), 648 (ins i32imm:$imm), 649 !strconcat(OpcodeStr, "\t$imm"), 650 []> { 651 // mov r0, r0 == nop 652 let Inst{55-52} = 0; 653 let Inst{51-48} = 0; 654 let BPFClass = BPF_ALU64; 655} 656 657let hasSideEffects = 0, isCodeGenOnly = 1 in 658 def NOP : NOP_I<"nop">; 659 660class RET<string OpcodeStr> 661 : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value, 662 (outs), 663 (ins), 664 !strconcat(OpcodeStr, ""), 665 [(BPFretglue)]> { 666 let Inst{31-0} = 0; 667 let BPFClass = BPF_JMP; 668} 669 670let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1, 671 isNotDuplicable = 1 in { 672 def RET : RET<"exit">; 673} 674 675// ADJCALLSTACKDOWN/UP pseudo insns 676let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in { 677def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 678 "#ADJCALLSTACKDOWN $amt1 $amt2", 679 [(BPFcallseq_start timm:$amt1, timm:$amt2)]>; 680def ADJCALLSTACKUP : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 681 "#ADJCALLSTACKUP $amt1 $amt2", 682 [(BPFcallseq_end timm:$amt1, timm:$amt2)]>; 683} 684 685let usesCustomInserter = 1, isCodeGenOnly = 1 in { 686 def Select : Pseudo<(outs GPR:$dst), 687 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 688 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 689 [(set i64:$dst, 690 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>; 691 def Select_Ri : Pseudo<(outs GPR:$dst), 692 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 693 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 694 [(set i64:$dst, 695 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>; 696 def Select_64_32 : Pseudo<(outs GPR32:$dst), 697 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 698 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 699 [(set i32:$dst, 700 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>; 701 def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst), 702 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 703 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 704 [(set i32:$dst, 705 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>; 706 def Select_32 : Pseudo<(outs GPR32:$dst), 707 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 708 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 709 [(set i32:$dst, 710 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>; 711 def Select_Ri_32 : Pseudo<(outs GPR32:$dst), 712 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 713 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 714 [(set i32:$dst, 715 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>; 716 def Select_32_64 : Pseudo<(outs GPR:$dst), 717 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 718 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 719 [(set i64:$dst, 720 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>; 721 def Select_Ri_32_64 : Pseudo<(outs GPR:$dst), 722 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 723 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 724 [(set i64:$dst, 725 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>; 726} 727 728// load 64-bit global addr into register 729def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>; 730 731// 0xffffFFFF doesn't fit into simm32, optimize common case 732def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)), 733 (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>; 734 735// Calls 736def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>; 737def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>; 738def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>; 739def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>; 740 741// Loads 742let Predicates = [BPFNoALU32] in { 743 def : Pat<(i64 (extloadi8 ADDRri:$src)), (i64 (LDB ADDRri:$src))>; 744 def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>; 745 def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>; 746} 747 748// Atomic XADD for BPFNoALU32 749class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 750 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 751 (outs GPR:$dst), 752 (ins MEMri:$addr, GPR:$val), 753 "lock *("#OpcodeStr#" *)($addr) += $val", 754 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> { 755 bits<4> dst; 756 bits<20> addr; 757 758 let Inst{51-48} = addr{19-16}; // base reg 759 let Inst{55-52} = dst; 760 let Inst{47-32} = addr{15-0}; // offset 761 let Inst{7-4} = BPF_ADD.Value; 762 let BPFClass = BPF_STX; 763} 764 765let Constraints = "$dst = $val" in { 766 let Predicates = [BPFNoALU32] in { 767 def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>; 768 } 769} 770 771// Atomic add, and, or, xor 772class ATOMIC_NOFETCH<BPFArithOp Opc, string Opstr> 773 : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_DW.Value, 774 (outs GPR:$dst), 775 (ins MEMri:$addr, GPR:$val), 776 "lock *(u64 *)($addr) " #Opstr# "= $val", 777 []> { 778 bits<4> dst; 779 bits<20> addr; 780 781 let Inst{51-48} = addr{19-16}; // base reg 782 let Inst{55-52} = dst; 783 let Inst{47-32} = addr{15-0}; // offset 784 let Inst{7-4} = Opc.Value; 785 let BPFClass = BPF_STX; 786} 787 788class ATOMIC32_NOFETCH<BPFArithOp Opc, string Opstr> 789 : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_W.Value, 790 (outs GPR32:$dst), 791 (ins MEMri:$addr, GPR32:$val), 792 "lock *(u32 *)($addr) " #Opstr# "= $val", 793 []> { 794 bits<4> dst; 795 bits<20> addr; 796 797 let Inst{51-48} = addr{19-16}; // base reg 798 let Inst{55-52} = dst; 799 let Inst{47-32} = addr{15-0}; // offset 800 let Inst{7-4} = Opc.Value; 801 let BPFClass = BPF_STX; 802} 803 804let Constraints = "$dst = $val" in { 805 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 806 def XADDW32 : ATOMIC32_NOFETCH<BPF_ADD, "+">; 807 def XANDW32 : ATOMIC32_NOFETCH<BPF_AND, "&">; 808 def XORW32 : ATOMIC32_NOFETCH<BPF_OR, "|">; 809 def XXORW32 : ATOMIC32_NOFETCH<BPF_XOR, "^">; 810 } 811 812 def XADDD : ATOMIC_NOFETCH<BPF_ADD, "+">; 813 def XANDD : ATOMIC_NOFETCH<BPF_AND, "&">; 814 def XORD : ATOMIC_NOFETCH<BPF_OR, "|">; 815 def XXORD : ATOMIC_NOFETCH<BPF_XOR, "^">; 816} 817 818// Atomic Fetch-and-<add, and, or, xor> operations 819class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, 820 string OpcStr, PatFrag OpNode> 821 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 822 (outs GPR:$dst), 823 (ins MEMri:$addr, GPR:$val), 824 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 825 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> { 826 bits<4> dst; 827 bits<20> addr; 828 829 let Inst{51-48} = addr{19-16}; // base reg 830 let Inst{55-52} = dst; 831 let Inst{47-32} = addr{15-0}; // offset 832 let Inst{7-4} = Opc.Value; 833 let Inst{3-0} = BPF_FETCH.Value; 834 let BPFClass = BPF_STX; 835} 836 837class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, 838 string OpcStr, PatFrag OpNode> 839 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 840 (outs GPR32:$dst), 841 (ins MEMri:$addr, GPR32:$val), 842 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 843 [(set GPR32:$dst, (OpNode ADDRri:$addr, GPR32:$val))]> { 844 bits<4> dst; 845 bits<20> addr; 846 847 let Inst{51-48} = addr{19-16}; // base reg 848 let Inst{55-52} = dst; 849 let Inst{47-32} = addr{15-0}; // offset 850 let Inst{7-4} = Opc.Value; 851 let Inst{3-0} = BPF_FETCH.Value; 852 let BPFClass = BPF_STX; 853} 854 855let Constraints = "$dst = $val" in { 856 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 857 def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add", atomic_load_add_32>; 858 def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and", atomic_load_and_32>; 859 def XFORW32 : XFALU32<BPF_W, BPF_OR, "u32", "or", atomic_load_or_32>; 860 def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor", atomic_load_xor_32>; 861 } 862 863 def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_64>; 864 def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and", atomic_load_and_64>; 865 def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or", atomic_load_or_64>; 866 def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor", atomic_load_xor_64>; 867} 868 869// atomic_load_sub can be represented as a neg followed 870// by an atomic_load_add. 871def : Pat<(atomic_load_sub_32 ADDRri:$addr, GPR32:$val), 872 (XFADDW32 ADDRri:$addr, (NEG_32 GPR32:$val))>; 873def : Pat<(atomic_load_sub_64 ADDRri:$addr, GPR:$val), 874 (XFADDD ADDRri:$addr, (NEG_64 GPR:$val))>; 875 876// Atomic Exchange 877class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 878 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 879 (outs GPR:$dst), 880 (ins MEMri:$addr, GPR:$val), 881 "$dst = xchg_"#OpcodeStr#"($addr, $val)", 882 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> { 883 bits<4> dst; 884 bits<20> addr; 885 886 let Inst{51-48} = addr{19-16}; // base reg 887 let Inst{55-52} = dst; 888 let Inst{47-32} = addr{15-0}; // offset 889 let Inst{7-4} = BPF_XCHG.Value; 890 let Inst{3-0} = BPF_FETCH.Value; 891 let BPFClass = BPF_STX; 892} 893 894class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 895 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 896 (outs GPR32:$dst), 897 (ins MEMri:$addr, GPR32:$val), 898 "$dst = xchg32_"#OpcodeStr#"($addr, $val)", 899 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> { 900 bits<4> dst; 901 bits<20> addr; 902 903 let Inst{51-48} = addr{19-16}; // base reg 904 let Inst{55-52} = dst; 905 let Inst{47-32} = addr{15-0}; // offset 906 let Inst{7-4} = BPF_XCHG.Value; 907 let Inst{3-0} = BPF_FETCH.Value; 908 let BPFClass = BPF_STX; 909} 910 911let Constraints = "$dst = $val" in { 912 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 913 def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_32>; 914 } 915 916 def XCHGD : XCHG<BPF_DW, "64", atomic_swap_64>; 917} 918 919// Compare-And-Exchange 920class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 921 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 922 (outs), 923 (ins MEMri:$addr, GPR:$new), 924 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)", 925 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> { 926 bits<4> new; 927 bits<20> addr; 928 929 let Inst{51-48} = addr{19-16}; // base reg 930 let Inst{55-52} = new; 931 let Inst{47-32} = addr{15-0}; // offset 932 let Inst{7-4} = BPF_CMPXCHG.Value; 933 let Inst{3-0} = BPF_FETCH.Value; 934 let BPFClass = BPF_STX; 935} 936 937class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 938 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 939 (outs), 940 (ins MEMri:$addr, GPR32:$new), 941 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)", 942 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> { 943 bits<4> new; 944 bits<20> addr; 945 946 let Inst{51-48} = addr{19-16}; // base reg 947 let Inst{55-52} = new; 948 let Inst{47-32} = addr{15-0}; // offset 949 let Inst{7-4} = BPF_CMPXCHG.Value; 950 let Inst{3-0} = BPF_FETCH.Value; 951 let BPFClass = BPF_STX; 952} 953 954let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0], 955 DecoderNamespace = "BPFALU32" in { 956 def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_32>; 957} 958 959let Defs = [R0], Uses = [R0] in { 960 def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_64>; 961} 962 963// bswap16, bswap32, bswap64 964class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern> 965 : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value, 966 (outs GPR:$dst), 967 (ins GPR:$src), 968 "$dst = "#OpcodeStr#" $src", 969 Pattern> { 970 bits<4> dst; 971 972 let Inst{51-48} = dst; 973 let Inst{31-0} = SizeOp; 974 let BPFClass = Class; 975} 976 977 978let Constraints = "$dst = $src" in { 979 let Predicates = [BPFHasBswap] in { 980 def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 981 def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 982 def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 983 } 984 985 let Predicates = [BPFNoBswap] in { 986 let Predicates = [BPFIsLittleEndian] in { 987 def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 988 def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 989 def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>; 990 } 991 let Predicates = [BPFIsBigEndian] in { 992 def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 993 def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 994 def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 995 } 996 } 997} 998 999let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1, 1000 hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in { 1001class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1002 : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value, 1003 (outs), 1004 (ins GPR:$skb, i64imm:$imm), 1005 "r0 = *("#OpcodeStr#" *)skb[$imm]", 1006 [(set R0, (OpNode GPR:$skb, i64immSExt32:$imm))]> { 1007 bits<32> imm; 1008 1009 let Inst{31-0} = imm; 1010 let BPFClass = BPF_LD; 1011} 1012 1013class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1014 : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value, 1015 (outs), 1016 (ins GPR:$skb, GPR:$val), 1017 "r0 = *("#OpcodeStr#" *)skb[$val]", 1018 [(set R0, (OpNode GPR:$skb, GPR:$val))]> { 1019 bits<4> val; 1020 1021 let Inst{55-52} = val; 1022 let BPFClass = BPF_LD; 1023} 1024} 1025 1026def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>; 1027def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>; 1028def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>; 1029 1030def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>; 1031def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>; 1032def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>; 1033 1034let isCodeGenOnly = 1 in { 1035 def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0, 1036 (outs GPR:$dst), (ins GPR32:$src), 1037 "$dst = $src", []>; 1038} 1039 1040let Predicates = [BPFNoMovsx] in { 1041 def : Pat<(i64 (sext GPR32:$src)), 1042 (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>; 1043} 1044 1045let Predicates = [BPFHasMovsx] in { 1046 def : Pat<(i64 (sext GPR32:$src)), 1047 (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>; 1048} 1049 1050def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>; 1051 1052// For i64 -> i32 truncation, use the 32-bit subregister directly. 1053def : Pat<(i32 (trunc GPR:$src)), 1054 (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>; 1055 1056// For i32 -> i64 anyext, we don't care about the high bits. 1057def : Pat<(i64 (anyext GPR32:$src)), 1058 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>; 1059 1060class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 1061 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 1062 (outs), 1063 (ins GPR32:$src, MEMri:$addr), 1064 "*("#OpcodeStr#" *)($addr) = $src", 1065 Pattern> { 1066 bits<4> src; 1067 bits<20> addr; 1068 1069 let Inst{51-48} = addr{19-16}; // base reg 1070 let Inst{55-52} = src; 1071 let Inst{47-32} = addr{15-0}; // offset 1072 let BPFClass = BPF_STX; 1073} 1074 1075class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 1076 : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>; 1077 1078let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1079 def STW32 : STOREi32<BPF_W, "u32", store>; 1080 def STH32 : STOREi32<BPF_H, "u16", truncstorei16>; 1081 def STB32 : STOREi32<BPF_B, "u8", truncstorei8>; 1082} 1083 1084class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 1085 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 1086 (outs GPR32:$dst), 1087 (ins MEMri:$addr), 1088 "$dst = *("#OpcodeStr#" *)($addr)", 1089 Pattern> { 1090 bits<4> dst; 1091 bits<20> addr; 1092 1093 let Inst{51-48} = dst; 1094 let Inst{55-52} = addr{19-16}; 1095 let Inst{47-32} = addr{15-0}; 1096 let BPFClass = BPF_LDX; 1097} 1098 1099class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 1100 : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>; 1101 1102let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1103 def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>; 1104 def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>; 1105 def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>; 1106} 1107 1108let Predicates = [BPFHasALU32] in { 1109 def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst), 1110 (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1111 def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst), 1112 (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1113 def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst), 1114 (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1115 def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>; 1116 def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>; 1117 1118 let Predicates = [BPFHasLdsx] in { 1119 def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>; 1120 def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>; 1121 } 1122 1123 def : Pat<(i64 (zextloadi8 ADDRri:$src)), 1124 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1125 def : Pat<(i64 (zextloadi16 ADDRri:$src)), 1126 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1127 def : Pat<(i64 (zextloadi32 ADDRri:$src)), 1128 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1129 def : Pat<(i64 (extloadi8 ADDRri:$src)), 1130 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1131 def : Pat<(i64 (extloadi16 ADDRri:$src)), 1132 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1133 def : Pat<(i64 (extloadi32 ADDRri:$src)), 1134 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1135} 1136 1137let usesCustomInserter = 1, isCodeGenOnly = 1 in { 1138 def MEMCPY : Pseudo< 1139 (outs), 1140 (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops), 1141 "#memcpy dst: $dst, src: $src, len: $len, align: $align", 1142 [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>; 1143} 1144