1//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file describes the BPF instructions in TableGen format. 10// 11//===----------------------------------------------------------------------===// 12 13include "BPFInstrFormats.td" 14 15// Instruction Operands and Patterns 16 17// These are target-independent nodes, but have target-specific formats. 18def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>, 19 SDTCisVT<1, iPTR>]>; 20def SDT_BPFCallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>; 21def SDT_BPFCall : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; 22def SDT_BPFSetFlag : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>; 23def SDT_BPFSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>, 24 SDTCisSameAs<0, 4>, 25 SDTCisSameAs<4, 5>]>; 26def SDT_BPFBrCC : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>, 27 SDTCisVT<3, OtherVT>]>; 28def SDT_BPFWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, 29 SDTCisPtrTy<0>]>; 30def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>, 31 SDTCisVT<1, i64>, 32 SDTCisVT<2, i64>, 33 SDTCisVT<3, i64>]>; 34 35def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall, 36 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, 37 SDNPVariadic]>; 38def BPFretglue : SDNode<"BPFISD::RET_GLUE", SDTNone, 39 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>; 40def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart, 41 [SDNPHasChain, SDNPOutGlue]>; 42def BPFcallseq_end : SDNode<"ISD::CALLSEQ_END", SDT_BPFCallSeqEnd, 43 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; 44def BPFbrcc : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC, 45 [SDNPHasChain, SDNPOutGlue, SDNPInGlue]>; 46 47def BPFselectcc : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC, [SDNPInGlue]>; 48def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; 49def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, 50 [SDNPHasChain, SDNPInGlue, SDNPOutGlue, 51 SDNPMayStore, SDNPMayLoad]>; 52def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; 53def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; 54def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; 55def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">; 56def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; 57def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">; 58def BPFHasBswap : Predicate<"Subtarget->hasBswap()">; 59def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">; 60def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">; 61def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">; 62def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">; 63 64class ImmediateAsmOperand<string name> : AsmOperandClass { 65 let Name = name; 66 let RenderMethod = "addImmOperands"; 67 let DiagnosticType = !strconcat("Invalid", name); 68} 69 70def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">; 71 72def brtarget : Operand<OtherVT> { 73 let PrintMethod = "printBrTargetOperand"; 74 let ParserMatchClass = ImmediateAsmOperand<"BrTarget">; 75} 76def calltarget : Operand<i64>; 77 78def u64imm : Operand<i64> { 79 let PrintMethod = "printImm64Operand"; 80} 81 82def s16imm : Operand<i16> { 83 let ParserMatchClass = SImm16AsmOperand; 84} 85 86def gpr_or_imm : Operand<i64>; 87 88def i64immSExt32 : PatLeaf<(i64 imm), 89 [{return isInt<32>(N->getSExtValue()); }]>; 90def i32immSExt32 : PatLeaf<(i32 imm), 91 [{return isInt<32>(N->getSExtValue()); }]>; 92def i64immZExt32 : PatLeaf<(i64 imm), 93 [{return isUInt<32>(N->getZExtValue()); }]>; 94 95def imm_to_i64 : SDNodeXForm<timm, [{ 96 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64); 97}]>; 98 99// Addressing modes. 100def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>; 101def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>; 102 103// Address operands 104def MEMri : Operand<i64> { 105 let PrintMethod = "printMemOperand"; 106 let EncoderMethod = "getMemoryOpValue"; 107 let DecoderMethod = "decodeMemoryOpValue"; 108 let MIOperandInfo = (ops GPR, s16imm); 109} 110 111// Conditional code predicates - used for pattern matching for jump instructions 112def BPF_CC_EQ : PatLeaf<(i64 imm), 113 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 114def BPF_CC_NE : PatLeaf<(i64 imm), 115 [{return (N->getZExtValue() == ISD::SETNE);}]>; 116def BPF_CC_GE : PatLeaf<(i64 imm), 117 [{return (N->getZExtValue() == ISD::SETGE);}]>; 118def BPF_CC_GT : PatLeaf<(i64 imm), 119 [{return (N->getZExtValue() == ISD::SETGT);}]>; 120def BPF_CC_GTU : PatLeaf<(i64 imm), 121 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 122def BPF_CC_GEU : PatLeaf<(i64 imm), 123 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 124def BPF_CC_LE : PatLeaf<(i64 imm), 125 [{return (N->getZExtValue() == ISD::SETLE);}]>; 126def BPF_CC_LT : PatLeaf<(i64 imm), 127 [{return (N->getZExtValue() == ISD::SETLT);}]>; 128def BPF_CC_LTU : PatLeaf<(i64 imm), 129 [{return (N->getZExtValue() == ISD::SETULT);}]>; 130def BPF_CC_LEU : PatLeaf<(i64 imm), 131 [{return (N->getZExtValue() == ISD::SETULE);}]>; 132def BPF_CC_EQ_32 : PatLeaf<(i32 imm), 133 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 134def BPF_CC_NE_32 : PatLeaf<(i32 imm), 135 [{return (N->getZExtValue() == ISD::SETNE);}]>; 136def BPF_CC_GE_32 : PatLeaf<(i32 imm), 137 [{return (N->getZExtValue() == ISD::SETGE);}]>; 138def BPF_CC_GT_32 : PatLeaf<(i32 imm), 139 [{return (N->getZExtValue() == ISD::SETGT);}]>; 140def BPF_CC_GTU_32 : PatLeaf<(i32 imm), 141 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 142def BPF_CC_GEU_32 : PatLeaf<(i32 imm), 143 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 144def BPF_CC_LE_32 : PatLeaf<(i32 imm), 145 [{return (N->getZExtValue() == ISD::SETLE);}]>; 146def BPF_CC_LT_32 : PatLeaf<(i32 imm), 147 [{return (N->getZExtValue() == ISD::SETLT);}]>; 148def BPF_CC_LTU_32 : PatLeaf<(i32 imm), 149 [{return (N->getZExtValue() == ISD::SETULT);}]>; 150def BPF_CC_LEU_32 : PatLeaf<(i32 imm), 151 [{return (N->getZExtValue() == ISD::SETULE);}]>; 152def NoCond : PatLeaf<(vt)> {} 153 154// For arithmetic and jump instructions the 8-bit 'code' 155// field is divided into three parts: 156// 157// +----------------+--------+--------------------+ 158// | 4 bits | 1 bit | 3 bits | 159// | operation code | source | instruction class | 160// +----------------+--------+--------------------+ 161// (MSB) (LSB) 162class TYPE_ALU_JMP<bits<4> op, bits<1> srctype, 163 dag outs, dag ins, string asmstr, list<dag> pattern> 164 : InstBPF<outs, ins, asmstr, pattern> { 165 166 let Inst{63-60} = op; 167 let Inst{59} = srctype; 168} 169 170//For load and store instructions the 8-bit 'code' field is divided as: 171// 172// +--------+--------+-------------------+ 173// | 3 bits | 2 bits | 3 bits | 174// | mode | size | instruction class | 175// +--------+--------+-------------------+ 176// (MSB) (LSB) 177class TYPE_LD_ST<bits<3> mode, bits<2> size, 178 dag outs, dag ins, string asmstr, list<dag> pattern> 179 : InstBPF<outs, ins, asmstr, pattern> { 180 181 let Inst{63-61} = mode; 182 let Inst{60-59} = size; 183} 184 185// jump instructions 186class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 187 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 188 (outs), 189 (ins GPR:$dst, GPR:$src, brtarget:$BrDst), 190 "if $dst "#OpcodeStr#" $src goto $BrDst", 191 [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> { 192 bits<4> dst; 193 bits<4> src; 194 bits<16> BrDst; 195 196 let Inst{55-52} = src; 197 let Inst{51-48} = dst; 198 let Inst{47-32} = BrDst; 199 let BPFClass = BPF_JMP; 200} 201 202class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 203 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 204 (outs), 205 (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst), 206 "if $dst "#OpcodeStr#" $imm goto $BrDst", 207 [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> { 208 bits<4> dst; 209 bits<16> BrDst; 210 bits<32> imm; 211 212 let Inst{51-48} = dst; 213 let Inst{47-32} = BrDst; 214 let Inst{31-0} = imm; 215 let BPFClass = BPF_JMP; 216} 217 218class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 219 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 220 (outs), 221 (ins brtarget:$BrDst), 222 !strconcat(OpcodeStr, " $BrDst"), 223 Pattern> { 224 bits<16> BrDst; 225 226 let Inst{47-32} = BrDst; 227 let BPFClass = BPF_JMP; 228} 229 230class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 231 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 232 (outs), 233 (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst), 234 "if $dst "#OpcodeStr#" $src goto $BrDst", 235 [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> { 236 bits<4> dst; 237 bits<4> src; 238 bits<16> BrDst; 239 240 let Inst{55-52} = src; 241 let Inst{51-48} = dst; 242 let Inst{47-32} = BrDst; 243 let BPFClass = BPF_JMP32; 244} 245 246class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 247 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 248 (outs), 249 (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst), 250 "if $dst "#OpcodeStr#" $imm goto $BrDst", 251 [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> { 252 bits<4> dst; 253 bits<16> BrDst; 254 bits<32> imm; 255 256 let Inst{51-48} = dst; 257 let Inst{47-32} = BrDst; 258 let Inst{31-0} = imm; 259 let BPFClass = BPF_JMP32; 260} 261 262multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> { 263 def _rr : JMP_RR<Opc, OpcodeStr, Cond>; 264 def _ri : JMP_RI<Opc, OpcodeStr, Cond>; 265 def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>; 266 def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>; 267} 268 269let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in { 270// cmp+goto instructions 271defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>; 272defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>; 273defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>; 274defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>; 275defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>; 276defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>; 277defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>; 278defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>; 279defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>; 280defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>; 281defm JSET : J<BPF_JSET, "&", NoCond, NoCond>; 282def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>; 283} 284 285// ALU instructions 286class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off, 287 dag outs, dag ins, string asmstr, list<dag> pattern> 288 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> { 289 bits<4> dst; 290 bits<32> imm; 291 292 let Inst{51-48} = dst; 293 let Inst{47-32} = off; 294 let Inst{31-0} = imm; 295 let BPFClass = Class; 296} 297 298class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off, 299 dag outs, dag ins, string asmstr, list<dag> pattern> 300 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> { 301 bits<4> dst; 302 bits<4> src; 303 304 let Inst{55-52} = src; 305 let Inst{51-48} = dst; 306 let Inst{47-32} = off; 307 let BPFClass = Class; 308} 309 310multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { 311 def _rr : ALU_RR<BPF_ALU64, Opc, off, 312 (outs GPR:$dst), 313 (ins GPR:$src2, GPR:$src), 314 "$dst "#OpcodeStr#" $src", 315 [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>; 316 def _ri : ALU_RI<BPF_ALU64, Opc, off, 317 (outs GPR:$dst), 318 (ins GPR:$src2, i64imm:$imm), 319 "$dst "#OpcodeStr#" $imm", 320 [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>; 321 def _rr_32 : ALU_RR<BPF_ALU, Opc, off, 322 (outs GPR32:$dst), 323 (ins GPR32:$src2, GPR32:$src), 324 "$dst "#OpcodeStr#" $src", 325 [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>; 326 def _ri_32 : ALU_RI<BPF_ALU, Opc, off, 327 (outs GPR32:$dst), 328 (ins GPR32:$src2, i32imm:$imm), 329 "$dst "#OpcodeStr#" $imm", 330 [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>; 331} 332 333let Constraints = "$dst = $src2" in { 334let isAsCheapAsAMove = 1 in { 335 defm ADD : ALU<BPF_ADD, 0, "+=", add>; 336 defm SUB : ALU<BPF_SUB, 0, "-=", sub>; 337 defm OR : ALU<BPF_OR, 0, "|=", or>; 338 defm AND : ALU<BPF_AND, 0, "&=", and>; 339 defm SLL : ALU<BPF_LSH, 0, "<<=", shl>; 340 defm SRL : ALU<BPF_RSH, 0, ">>=", srl>; 341 defm XOR : ALU<BPF_XOR, 0, "^=", xor>; 342 defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>; 343} 344 defm MUL : ALU<BPF_MUL, 0, "*=", mul>; 345 defm DIV : ALU<BPF_DIV, 0, "/=", udiv>; 346 defm MOD : ALU<BPF_MOD, 0, "%=", urem>; 347 348 let Predicates = [BPFHasSdivSmod] in { 349 defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>; 350 defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>; 351 } 352} 353 354class NEG_RR<BPFOpClass Class, BPFArithOp Opc, 355 dag outs, dag ins, string asmstr, list<dag> pattern> 356 : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> { 357 bits<4> dst; 358 359 let Inst{51-48} = dst; 360 let BPFClass = Class; 361} 362 363let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in { 364 def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src), 365 "$dst = -$src", 366 [(set GPR:$dst, (ineg i64:$src))]>; 367 def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src), 368 "$dst = -$src", 369 [(set GPR32:$dst, (ineg i32:$src))]>; 370} 371 372class LD_IMM64<bits<4> Pseudo, string OpcodeStr> 373 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 374 (outs GPR:$dst), 375 (ins u64imm:$imm), 376 "$dst "#OpcodeStr#" ${imm} ll", 377 [(set GPR:$dst, (i64 imm:$imm))]> { 378 379 bits<4> dst; 380 bits<64> imm; 381 382 let Inst{51-48} = dst; 383 let Inst{55-52} = Pseudo; 384 let Inst{47-32} = 0; 385 let Inst{31-0} = imm{31-0}; 386 let BPFClass = BPF_LD; 387} 388 389let isReMaterializable = 1, isAsCheapAsAMove = 1 in { 390def LD_imm64 : LD_IMM64<0, "=">; 391def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0, 392 (outs GPR:$dst), 393 (ins GPR:$src), 394 "$dst = $src", 395 []>; 396def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0, 397 (outs GPR:$dst), 398 (ins i64imm:$imm), 399 "$dst = $imm", 400 [(set GPR:$dst, (i64 i64immSExt32:$imm))]>; 401def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0, 402 (outs GPR32:$dst), 403 (ins GPR32:$src), 404 "$dst = $src", 405 []>; 406def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0, 407 (outs GPR32:$dst), 408 (ins i32imm:$imm), 409 "$dst = $imm", 410 [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>; 411 412let Predicates = [BPFHasMovsx] in { 413 def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8, 414 (outs GPR:$dst), (ins GPR:$src), 415 "$dst = (s8)$src", 416 [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>; 417 def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16, 418 (outs GPR:$dst), (ins GPR:$src), 419 "$dst = (s16)$src", 420 [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>; 421 def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32, 422 (outs GPR:$dst), (ins GPR:$src), 423 "$dst = (s32)$src", 424 [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>; 425 def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8, 426 (outs GPR32:$dst), (ins GPR32:$src), 427 "$dst = (s8)$src", 428 [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>; 429 def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16, 430 (outs GPR32:$dst), (ins GPR32:$src), 431 "$dst = (s16)$src", 432 [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>; 433} 434} 435 436def ADDR_SPACE_CAST 437 : ALU_RR<BPF_ALU64, BPF_MOV, 64, 438 (outs GPR:$dst), 439 (ins GPR:$src, i64imm:$dst_as, i64imm:$src_as), 440 "$dst = addr_space_cast($src, $dst_as, $src_as)", 441 []> { 442 bits<64> dst_as; 443 bits<64> src_as; 444 445 let Inst{47-32} = 1; 446 let Inst{31-16} = dst_as{15-0}; 447 let Inst{15-0} = src_as{15-0}; 448} 449 450def SrcAddrSpace : SDNodeXForm<addrspacecast, [{ 451 return CurDAG->getTargetConstant( 452 cast<AddrSpaceCastSDNode>(N)->getSrcAddressSpace(), 453 SDLoc(N), MVT::i64); 454}]>; 455 456def DstAddrSpace : SDNodeXForm<addrspacecast, [{ 457 return CurDAG->getTargetConstant( 458 cast<AddrSpaceCastSDNode>(N)->getDestAddressSpace(), 459 SDLoc(N), MVT::i64); 460}]>; 461 462def : Pat<(addrspacecast:$this GPR:$src), 463 (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>; 464 465def FI_ri 466 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 467 (outs GPR:$dst), 468 (ins MEMri:$addr), 469 "lea\t$dst, $addr", 470 [(set i64:$dst, FIri:$addr)]> { 471 // This is a tentative instruction, and will be replaced 472 // with MOV_rr and ADD_ri in PEI phase 473 let Inst{51-48} = 0; 474 let Inst{55-52} = 2; 475 let Inst{47-32} = 0; 476 let Inst{31-0} = 0; 477 let BPFClass = BPF_LD; 478 bit isPseudo = true; 479} 480 481def LD_pseudo 482 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 483 (outs GPR:$dst), 484 (ins i64imm:$pseudo, u64imm:$imm), 485 "ld_pseudo\t$dst, $pseudo, $imm", 486 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> { 487 488 bits<4> dst; 489 bits<64> imm; 490 bits<4> pseudo; 491 492 let Inst{51-48} = dst; 493 let Inst{55-52} = pseudo; 494 let Inst{47-32} = 0; 495 let Inst{31-0} = imm{31-0}; 496 let BPFClass = BPF_LD; 497} 498 499// STORE instructions 500class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 501 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 502 (outs), 503 (ins GPR:$src, MEMri:$addr), 504 "*("#OpcodeStr#" *)($addr) = $src", 505 Pattern> { 506 bits<4> src; 507 bits<20> addr; 508 509 let Inst{51-48} = addr{19-16}; // base reg 510 let Inst{55-52} = src; 511 let Inst{47-32} = addr{15-0}; // offset 512 let BPFClass = BPF_STX; 513} 514 515class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 516 : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>; 517 518let Predicates = [BPFNoALU32] in { 519 def STW : STOREi64<BPF_W, "u32", truncstorei32>; 520 def STH : STOREi64<BPF_H, "u16", truncstorei16>; 521 def STB : STOREi64<BPF_B, "u8", truncstorei8>; 522} 523def STD : STOREi64<BPF_DW, "u64", store>; 524 525class STORE_imm<BPFWidthModifer SizeOp, 526 string OpcodeStr, dag Pattern> 527 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 528 (outs), 529 (ins i64imm:$imm, MEMri:$addr), 530 "*("#OpcodeStr#" *)($addr) = $imm", 531 [Pattern]> { 532 bits<20> addr; 533 bits<32> imm; 534 535 let Inst{51-48} = addr{19-16}; // base reg 536 let Inst{47-32} = addr{15-0}; // offset 537 let Inst{31-0} = imm; 538 let BPFClass = BPF_ST; 539} 540 541let Predicates = [BPFHasStoreImm] in { 542 // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for 543 // value stored to memory: 544 // - it is fine to generate such write when immediate is -1 545 // - it is incorrect to generate such write when immediate is 546 // +0xffff_ffff. 547 // 548 // In the latter case two instructions would be generated instead of 549 // one BPF_ST: 550 // rA = 0xffffffff ll ; LD_imm64 551 // *(u64 *)(rB + 0) = rA ; STX 552 // 553 // For BPF_{B,H,W} the size of value stored matches size of the immediate. 554 def STD_imm : STORE_imm<BPF_DW, "u64", (store (i64 i64immSExt32:$imm), ADDRri:$addr)>; 555 def STW_imm : STORE_imm<BPF_W, "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 556 def STH_imm : STORE_imm<BPF_H, "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 557 def STB_imm : STORE_imm<BPF_B, "u8", (truncstorei8 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 558} 559 560let Predicates = [BPFHasALU32, BPFHasStoreImm] in { 561 def : Pat<(store (i32 imm:$src), ADDRri:$dst), 562 (STW_imm (imm_to_i64 $src), ADDRri:$dst)>; 563 def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst), 564 (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 565 def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst), 566 (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 567} 568 569// LOAD instructions 570class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 571 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 572 (outs GPR:$dst), 573 (ins MEMri:$addr), 574 "$dst = *("#OpcodeStr#" *)($addr)", 575 Pattern> { 576 bits<4> dst; 577 bits<20> addr; 578 579 let Inst{51-48} = dst; 580 let Inst{55-52} = addr{19-16}; 581 let Inst{47-32} = addr{15-0}; 582 let BPFClass = BPF_LDX; 583} 584 585class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 586 : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>; 587 588let isCodeGenOnly = 1 in { 589 class CORE_LD<RegisterClass RegClass, string Sz> 590 : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 591 (outs RegClass:$dst), 592 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 593 "$dst = core_ld"#Sz#"($opcode, $src, $offset)", 594 []>; 595 def CORE_LD64 : CORE_LD<GPR, "64">; 596 def CORE_LD32 : CORE_LD<GPR32, "32">; 597 def CORE_ST : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 598 (outs), 599 (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset), 600 "core_st($src, $opcode, $ptr, $offset)", 601 []>; 602 let Constraints = "$dst = $src" in { 603 def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0, 604 (outs GPR:$dst), 605 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 606 "$dst = core_shift($opcode, $src, $offset)", 607 []>; 608 } 609} 610 611let Predicates = [BPFNoALU32] in { 612 def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>; 613 def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>; 614 def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>; 615} 616 617let Predicates = [BPFHasLdsx] in { 618 def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>; 619 def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>; 620 def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8", sextloadi8>; 621} 622 623def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>; 624 625class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 626 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 627 (outs), 628 (ins brtarget:$BrDst), 629 !strconcat(OpcodeStr, " $BrDst"), 630 Pattern> { 631 bits<16> BrDst; 632 633 let Inst{47-32} = BrDst; 634 let BPFClass = BPF_JMP; 635} 636 637class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 638 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 639 (outs), 640 (ins brtarget:$BrDst), 641 !strconcat(OpcodeStr, " $BrDst"), 642 Pattern> { 643 bits<32> BrDst; 644 645 let Inst{31-0} = BrDst; 646 let BPFClass = BPF_JMP32; 647} 648 649class CALL<string OpcodeStr> 650 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value, 651 (outs), 652 (ins calltarget:$BrDst), 653 !strconcat(OpcodeStr, " $BrDst"), 654 []> { 655 bits<32> BrDst; 656 657 let Inst{31-0} = BrDst; 658 let BPFClass = BPF_JMP; 659} 660 661class CALLX<string OpcodeStr> 662 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value, 663 (outs), 664 (ins GPR:$BrDst), 665 !strconcat(OpcodeStr, " $BrDst"), 666 []> { 667 bits<4> BrDst; 668 669 let Inst{51-48} = BrDst; 670 let BPFClass = BPF_JMP; 671} 672 673// Jump always 674let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in { 675 def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>; 676 def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>; 677} 678 679// Jump and link 680let isCall=1, hasDelaySlot=0, Uses = [R11], 681 // Potentially clobbered registers 682 Defs = [R0, R1, R2, R3, R4, R5] in { 683 def JAL : CALL<"call">; 684 def JALX : CALLX<"callx">; 685} 686 687class NOP_I<string OpcodeStr> 688 : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value, 689 (outs), 690 (ins i32imm:$imm), 691 !strconcat(OpcodeStr, "\t$imm"), 692 []> { 693 // mov r0, r0 == nop 694 let Inst{55-52} = 0; 695 let Inst{51-48} = 0; 696 let BPFClass = BPF_ALU64; 697} 698 699let hasSideEffects = 0, isCodeGenOnly = 1 in 700 def NOP : NOP_I<"nop">; 701 702class RET<string OpcodeStr> 703 : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value, 704 (outs), 705 (ins), 706 !strconcat(OpcodeStr, ""), 707 [(BPFretglue)]> { 708 let Inst{31-0} = 0; 709 let BPFClass = BPF_JMP; 710} 711 712let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1, 713 isNotDuplicable = 1 in { 714 def RET : RET<"exit">; 715} 716 717// ADJCALLSTACKDOWN/UP pseudo insns 718let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in { 719def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 720 "#ADJCALLSTACKDOWN $amt1 $amt2", 721 [(BPFcallseq_start timm:$amt1, timm:$amt2)]>; 722def ADJCALLSTACKUP : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 723 "#ADJCALLSTACKUP $amt1 $amt2", 724 [(BPFcallseq_end timm:$amt1, timm:$amt2)]>; 725} 726 727let usesCustomInserter = 1, isCodeGenOnly = 1 in { 728 def Select : Pseudo<(outs GPR:$dst), 729 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 730 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 731 [(set i64:$dst, 732 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>; 733 def Select_Ri : Pseudo<(outs GPR:$dst), 734 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 735 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 736 [(set i64:$dst, 737 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>; 738 def Select_64_32 : Pseudo<(outs GPR32:$dst), 739 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 740 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 741 [(set i32:$dst, 742 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>; 743 def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst), 744 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 745 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 746 [(set i32:$dst, 747 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>; 748 def Select_32 : Pseudo<(outs GPR32:$dst), 749 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 750 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 751 [(set i32:$dst, 752 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>; 753 def Select_Ri_32 : Pseudo<(outs GPR32:$dst), 754 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 755 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 756 [(set i32:$dst, 757 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>; 758 def Select_32_64 : Pseudo<(outs GPR:$dst), 759 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 760 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 761 [(set i64:$dst, 762 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>; 763 def Select_Ri_32_64 : Pseudo<(outs GPR:$dst), 764 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 765 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 766 [(set i64:$dst, 767 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>; 768} 769 770// load 64-bit global addr into register 771def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>; 772def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>; 773 774// 0xffffFFFF doesn't fit into simm32, optimize common case 775def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)), 776 (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>; 777 778// Calls 779def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>; 780def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>; 781def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>; 782def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>; 783 784// Loads 785let Predicates = [BPFNoALU32] in { 786 def : Pat<(i64 (extloadi8 ADDRri:$src)), (i64 (LDB ADDRri:$src))>; 787 def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>; 788 def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>; 789} 790 791// Atomic XADD for BPFNoALU32 792class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 793 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 794 (outs GPR:$dst), 795 (ins MEMri:$addr, GPR:$val), 796 "lock *("#OpcodeStr#" *)($addr) += $val", 797 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> { 798 bits<4> dst; 799 bits<20> addr; 800 801 let Inst{51-48} = addr{19-16}; // base reg 802 let Inst{55-52} = dst; 803 let Inst{47-32} = addr{15-0}; // offset 804 let Inst{7-4} = BPF_ADD.Value; 805 let BPFClass = BPF_STX; 806} 807 808let Constraints = "$dst = $val" in { 809 let Predicates = [BPFNoALU32] in { 810 def XADDW : XADD<BPF_W, "u32", atomic_load_add_i32>; 811 } 812} 813 814// Atomic add, and, or, xor 815class ATOMIC_NOFETCH<BPFArithOp Opc, string Opstr> 816 : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_DW.Value, 817 (outs GPR:$dst), 818 (ins MEMri:$addr, GPR:$val), 819 "lock *(u64 *)($addr) " #Opstr# "= $val", 820 []> { 821 bits<4> dst; 822 bits<20> addr; 823 824 let Inst{51-48} = addr{19-16}; // base reg 825 let Inst{55-52} = dst; 826 let Inst{47-32} = addr{15-0}; // offset 827 let Inst{7-4} = Opc.Value; 828 let BPFClass = BPF_STX; 829} 830 831class ATOMIC32_NOFETCH<BPFArithOp Opc, string Opstr> 832 : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_W.Value, 833 (outs GPR32:$dst), 834 (ins MEMri:$addr, GPR32:$val), 835 "lock *(u32 *)($addr) " #Opstr# "= $val", 836 []> { 837 bits<4> dst; 838 bits<20> addr; 839 840 let Inst{51-48} = addr{19-16}; // base reg 841 let Inst{55-52} = dst; 842 let Inst{47-32} = addr{15-0}; // offset 843 let Inst{7-4} = Opc.Value; 844 let BPFClass = BPF_STX; 845} 846 847let Constraints = "$dst = $val" in { 848 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 849 def XADDW32 : ATOMIC32_NOFETCH<BPF_ADD, "+">; 850 def XANDW32 : ATOMIC32_NOFETCH<BPF_AND, "&">; 851 def XORW32 : ATOMIC32_NOFETCH<BPF_OR, "|">; 852 def XXORW32 : ATOMIC32_NOFETCH<BPF_XOR, "^">; 853 } 854 855 def XADDD : ATOMIC_NOFETCH<BPF_ADD, "+">; 856 def XANDD : ATOMIC_NOFETCH<BPF_AND, "&">; 857 def XORD : ATOMIC_NOFETCH<BPF_OR, "|">; 858 def XXORD : ATOMIC_NOFETCH<BPF_XOR, "^">; 859} 860 861// Atomic Fetch-and-<add, and, or, xor> operations 862class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, 863 string OpcStr, PatFrag OpNode> 864 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 865 (outs GPR:$dst), 866 (ins MEMri:$addr, GPR:$val), 867 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 868 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> { 869 bits<4> dst; 870 bits<20> addr; 871 872 let Inst{51-48} = addr{19-16}; // base reg 873 let Inst{55-52} = dst; 874 let Inst{47-32} = addr{15-0}; // offset 875 let Inst{7-4} = Opc.Value; 876 let Inst{3-0} = BPF_FETCH.Value; 877 let BPFClass = BPF_STX; 878} 879 880class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, 881 string OpcStr, PatFrag OpNode> 882 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 883 (outs GPR32:$dst), 884 (ins MEMri:$addr, GPR32:$val), 885 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 886 [(set GPR32:$dst, (OpNode ADDRri:$addr, GPR32:$val))]> { 887 bits<4> dst; 888 bits<20> addr; 889 890 let Inst{51-48} = addr{19-16}; // base reg 891 let Inst{55-52} = dst; 892 let Inst{47-32} = addr{15-0}; // offset 893 let Inst{7-4} = Opc.Value; 894 let Inst{3-0} = BPF_FETCH.Value; 895 let BPFClass = BPF_STX; 896} 897 898let Constraints = "$dst = $val" in { 899 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 900 def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add", atomic_load_add_i32>; 901 def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and", atomic_load_and_i32>; 902 def XFORW32 : XFALU32<BPF_W, BPF_OR, "u32", "or", atomic_load_or_i32>; 903 def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor", atomic_load_xor_i32>; 904 } 905 906 def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_i64>; 907 def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and", atomic_load_and_i64>; 908 def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or", atomic_load_or_i64>; 909 def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor", atomic_load_xor_i64>; 910} 911 912// atomic_load_sub can be represented as a neg followed 913// by an atomic_load_add. 914def : Pat<(atomic_load_sub_i32 ADDRri:$addr, GPR32:$val), 915 (XFADDW32 ADDRri:$addr, (NEG_32 GPR32:$val))>; 916def : Pat<(atomic_load_sub_i64 ADDRri:$addr, GPR:$val), 917 (XFADDD ADDRri:$addr, (NEG_64 GPR:$val))>; 918 919// Atomic Exchange 920class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 921 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 922 (outs GPR:$dst), 923 (ins MEMri:$addr, GPR:$val), 924 "$dst = xchg_"#OpcodeStr#"($addr, $val)", 925 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> { 926 bits<4> dst; 927 bits<20> addr; 928 929 let Inst{51-48} = addr{19-16}; // base reg 930 let Inst{55-52} = dst; 931 let Inst{47-32} = addr{15-0}; // offset 932 let Inst{7-4} = BPF_XCHG.Value; 933 let Inst{3-0} = BPF_FETCH.Value; 934 let BPFClass = BPF_STX; 935} 936 937class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 938 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 939 (outs GPR32:$dst), 940 (ins MEMri:$addr, GPR32:$val), 941 "$dst = xchg32_"#OpcodeStr#"($addr, $val)", 942 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> { 943 bits<4> dst; 944 bits<20> addr; 945 946 let Inst{51-48} = addr{19-16}; // base reg 947 let Inst{55-52} = dst; 948 let Inst{47-32} = addr{15-0}; // offset 949 let Inst{7-4} = BPF_XCHG.Value; 950 let Inst{3-0} = BPF_FETCH.Value; 951 let BPFClass = BPF_STX; 952} 953 954let Constraints = "$dst = $val" in { 955 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 956 def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_i32>; 957 } 958 959 def XCHGD : XCHG<BPF_DW, "64", atomic_swap_i64>; 960} 961 962// Compare-And-Exchange 963class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 964 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 965 (outs), 966 (ins MEMri:$addr, GPR:$new), 967 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)", 968 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> { 969 bits<4> new; 970 bits<20> addr; 971 972 let Inst{51-48} = addr{19-16}; // base reg 973 let Inst{55-52} = new; 974 let Inst{47-32} = addr{15-0}; // offset 975 let Inst{7-4} = BPF_CMPXCHG.Value; 976 let Inst{3-0} = BPF_FETCH.Value; 977 let BPFClass = BPF_STX; 978} 979 980class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 981 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 982 (outs), 983 (ins MEMri:$addr, GPR32:$new), 984 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)", 985 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> { 986 bits<4> new; 987 bits<20> addr; 988 989 let Inst{51-48} = addr{19-16}; // base reg 990 let Inst{55-52} = new; 991 let Inst{47-32} = addr{15-0}; // offset 992 let Inst{7-4} = BPF_CMPXCHG.Value; 993 let Inst{3-0} = BPF_FETCH.Value; 994 let BPFClass = BPF_STX; 995} 996 997let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0], 998 DecoderNamespace = "BPFALU32" in { 999 def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_i32>; 1000} 1001 1002let Defs = [R0], Uses = [R0] in { 1003 def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_i64>; 1004} 1005 1006// bswap16, bswap32, bswap64 1007class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern> 1008 : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value, 1009 (outs GPR:$dst), 1010 (ins GPR:$src), 1011 "$dst = "#OpcodeStr#" $src", 1012 Pattern> { 1013 bits<4> dst; 1014 1015 let Inst{51-48} = dst; 1016 let Inst{31-0} = SizeOp; 1017 let BPFClass = Class; 1018} 1019 1020 1021let Constraints = "$dst = $src" in { 1022 let Predicates = [BPFHasBswap] in { 1023 def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1024 def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1025 def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 1026 } 1027 1028 let Predicates = [BPFNoBswap] in { 1029 let Predicates = [BPFIsLittleEndian] in { 1030 def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1031 def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1032 def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>; 1033 } 1034 let Predicates = [BPFIsBigEndian] in { 1035 def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1036 def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1037 def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 1038 } 1039 } 1040} 1041 1042let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1, 1043 hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in { 1044class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1045 : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value, 1046 (outs), 1047 (ins GPR:$skb, i64imm:$imm), 1048 "r0 = *("#OpcodeStr#" *)skb[$imm]", 1049 [(set R0, (OpNode GPR:$skb, i64immSExt32:$imm))]> { 1050 bits<32> imm; 1051 1052 let Inst{31-0} = imm; 1053 let BPFClass = BPF_LD; 1054} 1055 1056class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1057 : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value, 1058 (outs), 1059 (ins GPR:$skb, GPR:$val), 1060 "r0 = *("#OpcodeStr#" *)skb[$val]", 1061 [(set R0, (OpNode GPR:$skb, GPR:$val))]> { 1062 bits<4> val; 1063 1064 let Inst{55-52} = val; 1065 let BPFClass = BPF_LD; 1066} 1067} 1068 1069def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>; 1070def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>; 1071def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>; 1072 1073def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>; 1074def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>; 1075def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>; 1076 1077let isCodeGenOnly = 1 in { 1078 def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0, 1079 (outs GPR:$dst), (ins GPR32:$src), 1080 "$dst = $src", []>; 1081} 1082 1083let Predicates = [BPFNoMovsx] in { 1084 def : Pat<(i64 (sext GPR32:$src)), 1085 (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>; 1086} 1087 1088let Predicates = [BPFHasMovsx] in { 1089 def : Pat<(i64 (sext GPR32:$src)), 1090 (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>; 1091} 1092 1093def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>; 1094 1095// For i64 -> i32 truncation, use the 32-bit subregister directly. 1096def : Pat<(i32 (trunc GPR:$src)), 1097 (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>; 1098 1099// For i32 -> i64 anyext, we don't care about the high bits. 1100def : Pat<(i64 (anyext GPR32:$src)), 1101 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>; 1102 1103class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 1104 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 1105 (outs), 1106 (ins GPR32:$src, MEMri:$addr), 1107 "*("#OpcodeStr#" *)($addr) = $src", 1108 Pattern> { 1109 bits<4> src; 1110 bits<20> addr; 1111 1112 let Inst{51-48} = addr{19-16}; // base reg 1113 let Inst{55-52} = src; 1114 let Inst{47-32} = addr{15-0}; // offset 1115 let BPFClass = BPF_STX; 1116} 1117 1118class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 1119 : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>; 1120 1121let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1122 def STW32 : STOREi32<BPF_W, "u32", store>; 1123 def STH32 : STOREi32<BPF_H, "u16", truncstorei16>; 1124 def STB32 : STOREi32<BPF_B, "u8", truncstorei8>; 1125} 1126 1127class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 1128 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 1129 (outs GPR32:$dst), 1130 (ins MEMri:$addr), 1131 "$dst = *("#OpcodeStr#" *)($addr)", 1132 Pattern> { 1133 bits<4> dst; 1134 bits<20> addr; 1135 1136 let Inst{51-48} = dst; 1137 let Inst{55-52} = addr{19-16}; 1138 let Inst{47-32} = addr{15-0}; 1139 let BPFClass = BPF_LDX; 1140} 1141 1142class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 1143 : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>; 1144 1145let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1146 def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>; 1147 def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>; 1148 def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>; 1149} 1150 1151let Predicates = [BPFHasALU32] in { 1152 def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst), 1153 (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1154 def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst), 1155 (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1156 def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst), 1157 (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1158 def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>; 1159 def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>; 1160 1161 let Predicates = [BPFHasLdsx] in { 1162 def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>; 1163 def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>; 1164 } 1165 1166 def : Pat<(i64 (zextloadi8 ADDRri:$src)), 1167 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1168 def : Pat<(i64 (zextloadi16 ADDRri:$src)), 1169 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1170 def : Pat<(i64 (zextloadi32 ADDRri:$src)), 1171 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1172 def : Pat<(i64 (extloadi8 ADDRri:$src)), 1173 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1174 def : Pat<(i64 (extloadi16 ADDRri:$src)), 1175 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1176 def : Pat<(i64 (extloadi32 ADDRri:$src)), 1177 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1178} 1179 1180let usesCustomInserter = 1, isCodeGenOnly = 1 in { 1181 def MEMCPY : Pseudo< 1182 (outs), 1183 (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops), 1184 "#memcpy dst: $dst, src: $src, len: $len, align: $align", 1185 [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>; 1186} 1187