xref: /freebsd/contrib/llvm-project/llvm/lib/Target/BPF/BPFInstrInfo.td (revision 95eb4b873b6a8b527c5bd78d7191975dfca38998)
1//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file describes the BPF instructions in TableGen format.
10//
11//===----------------------------------------------------------------------===//
12
13include "BPFInstrFormats.td"
14
15// Instruction Operands and Patterns
16
17// These are target-independent nodes, but have target-specific formats.
18def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>,
19                                          SDTCisVT<1, iPTR>]>;
20def SDT_BPFCallSeqEnd   : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>;
21def SDT_BPFCall         : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>;
22def SDT_BPFSetFlag      : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>;
23def SDT_BPFSelectCC     : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>,
24                                               SDTCisSameAs<0, 4>,
25                                               SDTCisSameAs<4, 5>]>;
26def SDT_BPFBrCC         : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>,
27                                               SDTCisVT<3, OtherVT>]>;
28def SDT_BPFWrapper      : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
29                                               SDTCisPtrTy<0>]>;
30def SDT_BPFMEMCPY       : SDTypeProfile<0, 4, [SDTCisVT<0, i64>,
31                                               SDTCisVT<1, i64>,
32                                               SDTCisVT<2, i64>,
33                                               SDTCisVT<3, i64>]>;
34
35def BPFcall         : SDNode<"BPFISD::CALL", SDT_BPFCall,
36                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
37                              SDNPVariadic]>;
38def BPFretglue      : SDNode<"BPFISD::RET_GLUE", SDTNone,
39                             [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
40def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart,
41                             [SDNPHasChain, SDNPOutGlue]>;
42def BPFcallseq_end  : SDNode<"ISD::CALLSEQ_END",   SDT_BPFCallSeqEnd,
43                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
44def BPFbrcc         : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC,
45                             [SDNPHasChain, SDNPOutGlue, SDNPInGlue]>;
46
47def BPFselectcc     : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC, [SDNPInGlue]>;
48def BPFWrapper      : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>;
49def BPFmemcpy       : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY,
50                             [SDNPHasChain, SDNPInGlue, SDNPOutGlue,
51                              SDNPMayStore, SDNPMayLoad]>;
52def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">;
53def BPFIsBigEndian    : Predicate<"!Subtarget->isLittleEndian()">;
54def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">;
55def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">;
56def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">;
57def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">;
58def BPFHasBswap : Predicate<"Subtarget->hasBswap()">;
59def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">;
60def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">;
61def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">;
62def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">;
63
64class ImmediateAsmOperand<string name> : AsmOperandClass {
65  let Name = name;
66  let RenderMethod = "addImmOperands";
67  let DiagnosticType = !strconcat("Invalid", name);
68}
69
70def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">;
71
72def brtarget : Operand<OtherVT> {
73  let PrintMethod = "printBrTargetOperand";
74  let ParserMatchClass = ImmediateAsmOperand<"BrTarget">;
75}
76def calltarget : Operand<i64>;
77
78def u64imm   : Operand<i64> {
79  let PrintMethod = "printImm64Operand";
80}
81
82def s16imm : Operand<i16> {
83  let ParserMatchClass = SImm16AsmOperand;
84}
85
86def gpr_or_imm : Operand<i64>;
87
88def i64immSExt32 : PatLeaf<(i64 imm),
89                [{return isInt<32>(N->getSExtValue()); }]>;
90def i32immSExt32 : PatLeaf<(i32 imm),
91                [{return isInt<32>(N->getSExtValue()); }]>;
92def i64immZExt32 : PatLeaf<(i64 imm),
93                [{return isUInt<32>(N->getZExtValue()); }]>;
94
95def imm_to_i64 : SDNodeXForm<timm, [{
96  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64);
97}]>;
98
99// Addressing modes.
100def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>;
101def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>;
102
103// Address operands
104def MEMri : Operand<i64> {
105  let PrintMethod = "printMemOperand";
106  let EncoderMethod = "getMemoryOpValue";
107  let DecoderMethod = "decodeMemoryOpValue";
108  let MIOperandInfo = (ops GPR, s16imm);
109}
110
111// Conditional code predicates - used for pattern matching for jump instructions
112def BPF_CC_EQ  : PatLeaf<(i64 imm),
113                         [{return (N->getZExtValue() == ISD::SETEQ);}]>;
114def BPF_CC_NE  : PatLeaf<(i64 imm),
115                         [{return (N->getZExtValue() == ISD::SETNE);}]>;
116def BPF_CC_GE  : PatLeaf<(i64 imm),
117                         [{return (N->getZExtValue() == ISD::SETGE);}]>;
118def BPF_CC_GT  : PatLeaf<(i64 imm),
119                         [{return (N->getZExtValue() == ISD::SETGT);}]>;
120def BPF_CC_GTU : PatLeaf<(i64 imm),
121                         [{return (N->getZExtValue() == ISD::SETUGT);}]>;
122def BPF_CC_GEU : PatLeaf<(i64 imm),
123                         [{return (N->getZExtValue() == ISD::SETUGE);}]>;
124def BPF_CC_LE  : PatLeaf<(i64 imm),
125                         [{return (N->getZExtValue() == ISD::SETLE);}]>;
126def BPF_CC_LT  : PatLeaf<(i64 imm),
127                         [{return (N->getZExtValue() == ISD::SETLT);}]>;
128def BPF_CC_LTU : PatLeaf<(i64 imm),
129                         [{return (N->getZExtValue() == ISD::SETULT);}]>;
130def BPF_CC_LEU : PatLeaf<(i64 imm),
131                         [{return (N->getZExtValue() == ISD::SETULE);}]>;
132def BPF_CC_EQ_32  : PatLeaf<(i32 imm),
133                         [{return (N->getZExtValue() == ISD::SETEQ);}]>;
134def BPF_CC_NE_32  : PatLeaf<(i32 imm),
135                         [{return (N->getZExtValue() == ISD::SETNE);}]>;
136def BPF_CC_GE_32  : PatLeaf<(i32 imm),
137                         [{return (N->getZExtValue() == ISD::SETGE);}]>;
138def BPF_CC_GT_32  : PatLeaf<(i32 imm),
139                         [{return (N->getZExtValue() == ISD::SETGT);}]>;
140def BPF_CC_GTU_32 : PatLeaf<(i32 imm),
141                         [{return (N->getZExtValue() == ISD::SETUGT);}]>;
142def BPF_CC_GEU_32 : PatLeaf<(i32 imm),
143                         [{return (N->getZExtValue() == ISD::SETUGE);}]>;
144def BPF_CC_LE_32  : PatLeaf<(i32 imm),
145                         [{return (N->getZExtValue() == ISD::SETLE);}]>;
146def BPF_CC_LT_32  : PatLeaf<(i32 imm),
147                         [{return (N->getZExtValue() == ISD::SETLT);}]>;
148def BPF_CC_LTU_32 : PatLeaf<(i32 imm),
149                         [{return (N->getZExtValue() == ISD::SETULT);}]>;
150def BPF_CC_LEU_32 : PatLeaf<(i32 imm),
151                         [{return (N->getZExtValue() == ISD::SETULE);}]>;
152def NoCond : PatLeaf<(vt)> {}
153
154// For arithmetic and jump instructions the 8-bit 'code'
155// field is divided into three parts:
156//
157//  +----------------+--------+--------------------+
158//  |   4 bits       |  1 bit |   3 bits           |
159//  | operation code | source | instruction class  |
160//  +----------------+--------+--------------------+
161//  (MSB)                                      (LSB)
162class TYPE_ALU_JMP<bits<4> op, bits<1> srctype,
163                   dag outs, dag ins, string asmstr, list<dag> pattern>
164  : InstBPF<outs, ins, asmstr, pattern> {
165
166  let Inst{63-60} = op;
167  let Inst{59} = srctype;
168}
169
170//For load and store instructions the 8-bit 'code' field is divided as:
171//
172//  +--------+--------+-------------------+
173//  | 3 bits | 2 bits |   3 bits          |
174//  |  mode  |  size  | instruction class |
175//  +--------+--------+-------------------+
176//  (MSB)                             (LSB)
177class TYPE_LD_ST<bits<3> mode, bits<2> size,
178                 dag outs, dag ins, string asmstr, list<dag> pattern>
179  : InstBPF<outs, ins, asmstr, pattern> {
180
181  let Inst{63-61} = mode;
182  let Inst{60-59} = size;
183}
184
185// jump instructions
186class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
187    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
188                   (outs),
189                   (ins GPR:$dst, GPR:$src, brtarget:$BrDst),
190                   "if $dst "#OpcodeStr#" $src goto $BrDst",
191                   [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> {
192  bits<4> dst;
193  bits<4> src;
194  bits<16> BrDst;
195
196  let Inst{55-52} = src;
197  let Inst{51-48} = dst;
198  let Inst{47-32} = BrDst;
199  let BPFClass = BPF_JMP;
200}
201
202class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
203    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
204                   (outs),
205                   (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst),
206                   "if $dst "#OpcodeStr#" $imm goto $BrDst",
207                   [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> {
208  bits<4> dst;
209  bits<16> BrDst;
210  bits<32> imm;
211
212  let Inst{51-48} = dst;
213  let Inst{47-32} = BrDst;
214  let Inst{31-0} = imm;
215  let BPFClass = BPF_JMP;
216}
217
218class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
219    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
220                   (outs),
221                   (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst),
222                   "if $dst "#OpcodeStr#" $src goto $BrDst",
223                   [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> {
224  bits<4> dst;
225  bits<4> src;
226  bits<16> BrDst;
227
228  let Inst{55-52} = src;
229  let Inst{51-48} = dst;
230  let Inst{47-32} = BrDst;
231  let BPFClass = BPF_JMP32;
232}
233
234class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
235    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
236                   (outs),
237                   (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst),
238                   "if $dst "#OpcodeStr#" $imm goto $BrDst",
239                   [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> {
240  bits<4> dst;
241  bits<16> BrDst;
242  bits<32> imm;
243
244  let Inst{51-48} = dst;
245  let Inst{47-32} = BrDst;
246  let Inst{31-0} = imm;
247  let BPFClass = BPF_JMP32;
248}
249
250multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> {
251  def _rr : JMP_RR<Opc, OpcodeStr, Cond>;
252  def _ri : JMP_RI<Opc, OpcodeStr, Cond>;
253  def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>;
254  def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>;
255}
256
257let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in {
258// cmp+goto instructions
259defm JEQ  : J<BPF_JEQ, "==",  BPF_CC_EQ, BPF_CC_EQ_32>;
260defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>;
261defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>;
262defm JNE  : J<BPF_JNE, "!=",  BPF_CC_NE, BPF_CC_NE_32>;
263defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>;
264defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>;
265defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>;
266defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>;
267defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
268defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
269defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;
270}
271
272// ALU instructions
273class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off,
274             dag outs, dag ins, string asmstr, list<dag> pattern>
275    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> {
276  bits<4> dst;
277  bits<32> imm;
278
279  let Inst{51-48} = dst;
280  let Inst{47-32} = off;
281  let Inst{31-0} = imm;
282  let BPFClass = Class;
283}
284
285class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off,
286             dag outs, dag ins, string asmstr, list<dag> pattern>
287    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> {
288  bits<4> dst;
289  bits<4> src;
290
291  let Inst{55-52} = src;
292  let Inst{51-48} = dst;
293  let Inst{47-32} = off;
294  let BPFClass = Class;
295}
296
297multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> {
298  def _rr : ALU_RR<BPF_ALU64, Opc, off,
299                   (outs GPR:$dst),
300                   (ins GPR:$src2, GPR:$src),
301                   "$dst "#OpcodeStr#" $src",
302                   [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>;
303  def _ri : ALU_RI<BPF_ALU64, Opc, off,
304                   (outs GPR:$dst),
305                   (ins GPR:$src2, i64imm:$imm),
306                   "$dst "#OpcodeStr#" $imm",
307                   [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>;
308  def _rr_32 : ALU_RR<BPF_ALU, Opc, off,
309                   (outs GPR32:$dst),
310                   (ins GPR32:$src2, GPR32:$src),
311                   "$dst "#OpcodeStr#" $src",
312                   [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>;
313  def _ri_32 : ALU_RI<BPF_ALU, Opc, off,
314                   (outs GPR32:$dst),
315                   (ins GPR32:$src2, i32imm:$imm),
316                   "$dst "#OpcodeStr#" $imm",
317                   [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>;
318}
319
320let Constraints = "$dst = $src2" in {
321let isAsCheapAsAMove = 1 in {
322  defm ADD : ALU<BPF_ADD, 0, "+=", add>;
323  defm SUB : ALU<BPF_SUB, 0, "-=", sub>;
324  defm OR  : ALU<BPF_OR, 0, "|=", or>;
325  defm AND : ALU<BPF_AND, 0, "&=", and>;
326  defm SLL : ALU<BPF_LSH, 0, "<<=", shl>;
327  defm SRL : ALU<BPF_RSH, 0, ">>=", srl>;
328  defm XOR : ALU<BPF_XOR, 0, "^=", xor>;
329  defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>;
330}
331  defm MUL : ALU<BPF_MUL, 0, "*=", mul>;
332  defm DIV : ALU<BPF_DIV, 0, "/=", udiv>;
333  defm MOD : ALU<BPF_MOD, 0, "%=", urem>;
334
335  let Predicates = [BPFHasSdivSmod] in {
336    defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>;
337    defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>;
338  }
339}
340
341class NEG_RR<BPFOpClass Class, BPFArithOp Opc,
342             dag outs, dag ins, string asmstr, list<dag> pattern>
343    : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> {
344  bits<4> dst;
345
346  let Inst{51-48} = dst;
347  let BPFClass = Class;
348}
349
350let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in {
351  def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src),
352                     "$dst = -$src",
353                     [(set GPR:$dst, (ineg i64:$src))]>;
354  def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src),
355                     "$dst = -$src",
356                     [(set GPR32:$dst, (ineg i32:$src))]>;
357}
358
359class LD_IMM64<bits<4> Pseudo, string OpcodeStr>
360    : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
361                 (outs GPR:$dst),
362                 (ins u64imm:$imm),
363                 "$dst "#OpcodeStr#" ${imm} ll",
364                 [(set GPR:$dst, (i64 imm:$imm))]> {
365
366  bits<4> dst;
367  bits<64> imm;
368
369  let Inst{51-48} = dst;
370  let Inst{55-52} = Pseudo;
371  let Inst{47-32} = 0;
372  let Inst{31-0} = imm{31-0};
373  let BPFClass = BPF_LD;
374}
375
376let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
377def LD_imm64 : LD_IMM64<0, "=">;
378def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0,
379                    (outs GPR:$dst),
380                    (ins GPR:$src),
381                    "$dst = $src",
382                    []>;
383def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0,
384                    (outs GPR:$dst),
385                    (ins i64imm:$imm),
386                    "$dst = $imm",
387                    [(set GPR:$dst, (i64 i64immSExt32:$imm))]>;
388def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0,
389                    (outs GPR32:$dst),
390                    (ins GPR32:$src),
391                    "$dst = $src",
392                    []>;
393def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0,
394                    (outs GPR32:$dst),
395                    (ins i32imm:$imm),
396                    "$dst = $imm",
397                    [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>;
398
399let Predicates = [BPFHasMovsx] in {
400  def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8,
401                      (outs GPR:$dst), (ins GPR:$src),
402                      "$dst = (s8)$src",
403                      [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>;
404  def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16,
405                      (outs GPR:$dst), (ins GPR:$src),
406                      "$dst = (s16)$src",
407                      [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>;
408  def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32,
409                      (outs GPR:$dst), (ins GPR:$src),
410                      "$dst = (s32)$src",
411                      [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>;
412  def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8,
413                      (outs GPR32:$dst), (ins GPR32:$src),
414                      "$dst = (s8)$src",
415                      [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>;
416  def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16,
417                      (outs GPR32:$dst), (ins GPR32:$src),
418                      "$dst = (s16)$src",
419                      [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>;
420}
421}
422
423def FI_ri
424    : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
425                 (outs GPR:$dst),
426                 (ins MEMri:$addr),
427                 "lea\t$dst, $addr",
428                 [(set i64:$dst, FIri:$addr)]> {
429  // This is a tentative instruction, and will be replaced
430  // with MOV_rr and ADD_ri in PEI phase
431  let Inst{51-48} = 0;
432  let Inst{55-52} = 2;
433  let Inst{47-32} = 0;
434  let Inst{31-0} = 0;
435  let BPFClass = BPF_LD;
436  bit isPseudo = true;
437}
438
439def LD_pseudo
440    : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
441                 (outs GPR:$dst),
442                 (ins i64imm:$pseudo, u64imm:$imm),
443                 "ld_pseudo\t$dst, $pseudo, $imm",
444                 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> {
445
446  bits<4> dst;
447  bits<64> imm;
448  bits<4> pseudo;
449
450  let Inst{51-48} = dst;
451  let Inst{55-52} = pseudo;
452  let Inst{47-32} = 0;
453  let Inst{31-0} = imm{31-0};
454  let BPFClass = BPF_LD;
455}
456
457// STORE instructions
458class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern>
459    : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
460                 (outs),
461                 (ins GPR:$src, MEMri:$addr),
462                 "*("#OpcodeStr#" *)($addr) = $src",
463                 Pattern> {
464  bits<4> src;
465  bits<20> addr;
466
467  let Inst{51-48} = addr{19-16}; // base reg
468  let Inst{55-52} = src;
469  let Inst{47-32} = addr{15-0}; // offset
470  let BPFClass = BPF_STX;
471}
472
473class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>
474    : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>;
475
476let Predicates = [BPFNoALU32] in {
477  def STW : STOREi64<BPF_W, "u32", truncstorei32>;
478  def STH : STOREi64<BPF_H, "u16", truncstorei16>;
479  def STB : STOREi64<BPF_B, "u8", truncstorei8>;
480}
481def STD : STOREi64<BPF_DW, "u64", store>;
482
483class STORE_imm<BPFWidthModifer SizeOp,
484                string OpcodeStr, dag Pattern>
485    : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
486                 (outs),
487                 (ins i64imm:$imm, MEMri:$addr),
488                 "*("#OpcodeStr#" *)($addr) = $imm",
489                 [Pattern]> {
490  bits<20> addr;
491  bits<32> imm;
492
493  let Inst{51-48} = addr{19-16}; // base reg
494  let Inst{47-32} = addr{15-0};  // offset
495  let Inst{31-0} = imm;
496  let BPFClass = BPF_ST;
497}
498
499let Predicates = [BPFHasStoreImm] in {
500  // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for
501  // value stored to memory:
502  // - it is fine to generate such write when immediate is -1
503  // - it is incorrect to generate such write when immediate is
504  //   +0xffff_ffff.
505  //
506  // In the latter case two instructions would be generated instead of
507  // one BPF_ST:
508  //   rA = 0xffffffff ll     ; LD_imm64
509  //   *(u64 *)(rB + 0) = rA  ; STX
510  //
511  // For BPF_{B,H,W} the size of value stored matches size of the immediate.
512  def STD_imm : STORE_imm<BPF_DW, "u64", (store         (i64 i64immSExt32:$imm), ADDRri:$addr)>;
513  def STW_imm : STORE_imm<BPF_W,  "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>;
514  def STH_imm : STORE_imm<BPF_H,  "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>;
515  def STB_imm : STORE_imm<BPF_B,  "u8",  (truncstorei8  (i64 i64immZExt32:$imm), ADDRri:$addr)>;
516}
517
518let Predicates = [BPFHasALU32, BPFHasStoreImm] in {
519  def : Pat<(store (i32 imm:$src), ADDRri:$dst),
520            (STW_imm (imm_to_i64 $src), ADDRri:$dst)>;
521  def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst),
522            (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>;
523  def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst),
524            (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>;
525}
526
527// LOAD instructions
528class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern>
529    : TYPE_LD_ST<ModOp.Value, SizeOp.Value,
530                 (outs GPR:$dst),
531                 (ins MEMri:$addr),
532                 "$dst = *("#OpcodeStr#" *)($addr)",
533                 Pattern> {
534  bits<4> dst;
535  bits<20> addr;
536
537  let Inst{51-48} = dst;
538  let Inst{55-52} = addr{19-16};
539  let Inst{47-32} = addr{15-0};
540  let BPFClass = BPF_LDX;
541}
542
543class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode>
544    : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>;
545
546let isCodeGenOnly = 1 in {
547  class CORE_LD<RegisterClass RegClass, string Sz>
548                : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,
549                             (outs RegClass:$dst),
550                             (ins u64imm:$opcode, GPR:$src, u64imm:$offset),
551                             "$dst = core_ld"#Sz#"($opcode, $src, $offset)",
552                             []>;
553  def CORE_LD64 : CORE_LD<GPR, "64">;
554  def CORE_LD32 : CORE_LD<GPR32, "32">;
555  def CORE_ST   : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,
556                             (outs),
557                             (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset),
558                             "core_st($src, $opcode, $ptr, $offset)",
559                             []>;
560  let Constraints = "$dst = $src" in {
561    def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0,
562                             (outs GPR:$dst),
563                             (ins u64imm:$opcode, GPR:$src, u64imm:$offset),
564                             "$dst = core_shift($opcode, $src, $offset)",
565                             []>;
566  }
567}
568
569let Predicates = [BPFNoALU32] in {
570  def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>;
571  def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>;
572  def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>;
573}
574
575let Predicates = [BPFHasLdsx] in {
576  def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>;
577  def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>;
578  def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8",  sextloadi8>;
579}
580
581def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>;
582
583class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
584    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
585                   (outs),
586                   (ins brtarget:$BrDst),
587                   !strconcat(OpcodeStr, " $BrDst"),
588                   Pattern> {
589  bits<16> BrDst;
590
591  let Inst{47-32} = BrDst;
592  let BPFClass = BPF_JMP;
593}
594
595class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
596    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
597                   (outs),
598                   (ins brtarget:$BrDst),
599                   !strconcat(OpcodeStr, " $BrDst"),
600                   Pattern> {
601  bits<32> BrDst;
602
603  let Inst{31-0} = BrDst;
604  let BPFClass = BPF_JMP32;
605}
606
607class CALL<string OpcodeStr>
608    : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value,
609                   (outs),
610                   (ins calltarget:$BrDst),
611                   !strconcat(OpcodeStr, " $BrDst"),
612                   []> {
613  bits<32> BrDst;
614
615  let Inst{31-0} = BrDst;
616  let BPFClass = BPF_JMP;
617}
618
619class CALLX<string OpcodeStr>
620    : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value,
621                   (outs),
622                   (ins GPR:$BrDst),
623                   !strconcat(OpcodeStr, " $BrDst"),
624                   []> {
625  bits<32> BrDst;
626
627  let Inst{31-0} = BrDst;
628  let BPFClass = BPF_JMP;
629}
630
631// Jump always
632let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in {
633  def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>;
634  def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>;
635}
636
637// Jump and link
638let isCall=1, hasDelaySlot=0, Uses = [R11],
639    // Potentially clobbered registers
640    Defs = [R0, R1, R2, R3, R4, R5] in {
641  def JAL  : CALL<"call">;
642  def JALX  : CALLX<"callx">;
643}
644
645class NOP_I<string OpcodeStr>
646    : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value,
647                   (outs),
648                   (ins i32imm:$imm),
649                   !strconcat(OpcodeStr, "\t$imm"),
650                   []> {
651  // mov r0, r0 == nop
652  let Inst{55-52} = 0;
653  let Inst{51-48} = 0;
654  let BPFClass = BPF_ALU64;
655}
656
657let hasSideEffects = 0, isCodeGenOnly = 1 in
658  def NOP : NOP_I<"nop">;
659
660class RET<string OpcodeStr>
661    : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value,
662                   (outs),
663                   (ins),
664                   !strconcat(OpcodeStr, ""),
665                   [(BPFretglue)]> {
666  let Inst{31-0} = 0;
667  let BPFClass = BPF_JMP;
668}
669
670let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1,
671    isNotDuplicable = 1 in {
672  def RET : RET<"exit">;
673}
674
675// ADJCALLSTACKDOWN/UP pseudo insns
676let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in {
677def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2),
678                              "#ADJCALLSTACKDOWN $amt1 $amt2",
679                              [(BPFcallseq_start timm:$amt1, timm:$amt2)]>;
680def ADJCALLSTACKUP   : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2),
681                              "#ADJCALLSTACKUP $amt1 $amt2",
682                              [(BPFcallseq_end timm:$amt1, timm:$amt2)]>;
683}
684
685let usesCustomInserter = 1, isCodeGenOnly = 1 in {
686  def Select : Pseudo<(outs GPR:$dst),
687                      (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2),
688                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
689                      [(set i64:$dst,
690                       (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>;
691  def Select_Ri : Pseudo<(outs GPR:$dst),
692                      (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2),
693                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
694                      [(set i64:$dst,
695                       (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>;
696  def Select_64_32 : Pseudo<(outs GPR32:$dst),
697                      (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2),
698                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
699                      [(set i32:$dst,
700                       (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>;
701  def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst),
702                      (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2),
703                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
704                      [(set i32:$dst,
705                       (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>;
706  def Select_32 : Pseudo<(outs GPR32:$dst),
707                      (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2),
708                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
709                      [(set i32:$dst,
710                       (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>;
711  def Select_Ri_32 : Pseudo<(outs GPR32:$dst),
712                      (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2),
713                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
714                      [(set i32:$dst,
715                       (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>;
716  def Select_32_64 : Pseudo<(outs GPR:$dst),
717                      (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2),
718                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
719                      [(set i64:$dst,
720                       (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>;
721  def Select_Ri_32_64 : Pseudo<(outs GPR:$dst),
722                      (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2),
723                      "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2",
724                      [(set i64:$dst,
725                       (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>;
726}
727
728// load 64-bit global addr into register
729def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>;
730
731// 0xffffFFFF doesn't fit into simm32, optimize common case
732def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)),
733          (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>;
734
735// Calls
736def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>;
737def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>;
738def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>;
739def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>;
740
741// Loads
742let Predicates = [BPFNoALU32] in {
743  def : Pat<(i64 (extloadi8  ADDRri:$src)), (i64 (LDB ADDRri:$src))>;
744  def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>;
745  def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>;
746}
747
748// Atomic XADD for BPFNoALU32
749class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
750    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
751                 (outs GPR:$dst),
752                 (ins MEMri:$addr, GPR:$val),
753                 "lock *("#OpcodeStr#" *)($addr) += $val",
754                 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> {
755  bits<4> dst;
756  bits<20> addr;
757
758  let Inst{51-48} = addr{19-16}; // base reg
759  let Inst{55-52} = dst;
760  let Inst{47-32} = addr{15-0}; // offset
761  let Inst{7-4} = BPF_ADD.Value;
762  let BPFClass = BPF_STX;
763}
764
765let Constraints = "$dst = $val" in {
766  let Predicates = [BPFNoALU32] in {
767    def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>;
768  }
769}
770
771// Atomic add, and, or, xor
772class ATOMIC_NOFETCH<BPFArithOp Opc, string Opstr>
773    : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_DW.Value,
774                 (outs GPR:$dst),
775                 (ins MEMri:$addr, GPR:$val),
776                 "lock *(u64 *)($addr) " #Opstr# "= $val",
777                 []> {
778  bits<4> dst;
779  bits<20> addr;
780
781  let Inst{51-48} = addr{19-16}; // base reg
782  let Inst{55-52} = dst;
783  let Inst{47-32} = addr{15-0}; // offset
784  let Inst{7-4} = Opc.Value;
785  let BPFClass = BPF_STX;
786}
787
788class ATOMIC32_NOFETCH<BPFArithOp Opc, string Opstr>
789    : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_W.Value,
790                 (outs GPR32:$dst),
791                 (ins MEMri:$addr, GPR32:$val),
792                 "lock *(u32 *)($addr) " #Opstr# "= $val",
793                 []> {
794  bits<4> dst;
795  bits<20> addr;
796
797  let Inst{51-48} = addr{19-16}; // base reg
798  let Inst{55-52} = dst;
799  let Inst{47-32} = addr{15-0}; // offset
800  let Inst{7-4} = Opc.Value;
801  let BPFClass = BPF_STX;
802}
803
804let Constraints = "$dst = $val" in {
805  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
806    def XADDW32 : ATOMIC32_NOFETCH<BPF_ADD, "+">;
807    def XANDW32 : ATOMIC32_NOFETCH<BPF_AND, "&">;
808    def XORW32  : ATOMIC32_NOFETCH<BPF_OR, "|">;
809    def XXORW32 : ATOMIC32_NOFETCH<BPF_XOR, "^">;
810  }
811
812  def XADDD  : ATOMIC_NOFETCH<BPF_ADD, "+">;
813  def XANDD  : ATOMIC_NOFETCH<BPF_AND, "&">;
814  def XORD   : ATOMIC_NOFETCH<BPF_OR, "|">;
815  def XXORD  : ATOMIC_NOFETCH<BPF_XOR, "^">;
816}
817
818// Atomic Fetch-and-<add, and, or, xor> operations
819class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr,
820              string OpcStr, PatFrag OpNode>
821    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
822                 (outs GPR:$dst),
823                 (ins MEMri:$addr, GPR:$val),
824                 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
825                 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> {
826  bits<4> dst;
827  bits<20> addr;
828
829  let Inst{51-48} = addr{19-16}; // base reg
830  let Inst{55-52} = dst;
831  let Inst{47-32} = addr{15-0}; // offset
832  let Inst{7-4} = Opc.Value;
833  let Inst{3-0} = BPF_FETCH.Value;
834  let BPFClass = BPF_STX;
835}
836
837class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr,
838              string OpcStr, PatFrag OpNode>
839    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
840                 (outs GPR32:$dst),
841                 (ins MEMri:$addr, GPR32:$val),
842                 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
843                 [(set GPR32:$dst, (OpNode ADDRri:$addr, GPR32:$val))]> {
844  bits<4> dst;
845  bits<20> addr;
846
847  let Inst{51-48} = addr{19-16}; // base reg
848  let Inst{55-52} = dst;
849  let Inst{47-32} = addr{15-0}; // offset
850  let Inst{7-4} = Opc.Value;
851  let Inst{3-0} = BPF_FETCH.Value;
852  let BPFClass = BPF_STX;
853}
854
855let Constraints = "$dst = $val" in {
856  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
857    def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add", atomic_load_add_32>;
858    def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and", atomic_load_and_32>;
859    def XFORW32  : XFALU32<BPF_W, BPF_OR,  "u32", "or",  atomic_load_or_32>;
860    def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor", atomic_load_xor_32>;
861  }
862
863  def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_64>;
864  def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and", atomic_load_and_64>;
865  def XFORD  : XFALU64<BPF_DW, BPF_OR,  "u64", "or",  atomic_load_or_64>;
866  def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor", atomic_load_xor_64>;
867}
868
869// atomic_load_sub can be represented as a neg followed
870// by an atomic_load_add.
871def : Pat<(atomic_load_sub_32 ADDRri:$addr, GPR32:$val),
872          (XFADDW32 ADDRri:$addr, (NEG_32 GPR32:$val))>;
873def : Pat<(atomic_load_sub_64 ADDRri:$addr, GPR:$val),
874          (XFADDD ADDRri:$addr, (NEG_64 GPR:$val))>;
875
876// Atomic Exchange
877class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
878    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
879                 (outs GPR:$dst),
880                 (ins MEMri:$addr, GPR:$val),
881                 "$dst = xchg_"#OpcodeStr#"($addr, $val)",
882                 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> {
883  bits<4> dst;
884  bits<20> addr;
885
886  let Inst{51-48} = addr{19-16}; // base reg
887  let Inst{55-52} = dst;
888  let Inst{47-32} = addr{15-0}; // offset
889  let Inst{7-4} = BPF_XCHG.Value;
890  let Inst{3-0} = BPF_FETCH.Value;
891  let BPFClass = BPF_STX;
892}
893
894class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
895    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
896                 (outs GPR32:$dst),
897                 (ins MEMri:$addr, GPR32:$val),
898                 "$dst = xchg32_"#OpcodeStr#"($addr, $val)",
899                 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> {
900  bits<4> dst;
901  bits<20> addr;
902
903  let Inst{51-48} = addr{19-16}; // base reg
904  let Inst{55-52} = dst;
905  let Inst{47-32} = addr{15-0}; // offset
906  let Inst{7-4} = BPF_XCHG.Value;
907  let Inst{3-0} = BPF_FETCH.Value;
908  let BPFClass = BPF_STX;
909}
910
911let Constraints = "$dst = $val" in {
912  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
913    def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_32>;
914  }
915
916  def XCHGD : XCHG<BPF_DW, "64", atomic_swap_64>;
917}
918
919// Compare-And-Exchange
920class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
921    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
922                 (outs),
923                 (ins MEMri:$addr, GPR:$new),
924                 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)",
925                 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> {
926  bits<4> new;
927  bits<20> addr;
928
929  let Inst{51-48} = addr{19-16}; // base reg
930  let Inst{55-52} = new;
931  let Inst{47-32} = addr{15-0}; // offset
932  let Inst{7-4} = BPF_CMPXCHG.Value;
933  let Inst{3-0} = BPF_FETCH.Value;
934  let BPFClass = BPF_STX;
935}
936
937class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
938    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
939                 (outs),
940                 (ins MEMri:$addr, GPR32:$new),
941                 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)",
942                 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> {
943  bits<4> new;
944  bits<20> addr;
945
946  let Inst{51-48} = addr{19-16}; // base reg
947  let Inst{55-52} = new;
948  let Inst{47-32} = addr{15-0}; // offset
949  let Inst{7-4} = BPF_CMPXCHG.Value;
950  let Inst{3-0} = BPF_FETCH.Value;
951  let BPFClass = BPF_STX;
952}
953
954let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0],
955    DecoderNamespace = "BPFALU32" in {
956  def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_32>;
957}
958
959let Defs = [R0], Uses = [R0] in {
960  def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_64>;
961}
962
963// bswap16, bswap32, bswap64
964class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern>
965    : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value,
966                   (outs GPR:$dst),
967                   (ins GPR:$src),
968                   "$dst = "#OpcodeStr#" $src",
969                   Pattern> {
970  bits<4> dst;
971
972  let Inst{51-48} = dst;
973  let Inst{31-0} = SizeOp;
974  let BPFClass = Class;
975}
976
977
978let Constraints = "$dst = $src" in {
979  let Predicates = [BPFHasBswap] in {
980    def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;
981    def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;
982    def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>;
983  }
984
985  let Predicates = [BPFNoBswap] in {
986    let Predicates = [BPFIsLittleEndian] in {
987        def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;
988        def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;
989        def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>;
990    }
991    let Predicates = [BPFIsBigEndian] in {
992        def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>;
993        def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>;
994        def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>;
995    }
996  }
997}
998
999let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1,
1000    hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in {
1001class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode>
1002    : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value,
1003                 (outs),
1004                 (ins GPR:$skb, i64imm:$imm),
1005                 "r0 = *("#OpcodeStr#" *)skb[$imm]",
1006                 [(set R0, (OpNode GPR:$skb, i64immSExt32:$imm))]> {
1007  bits<32> imm;
1008
1009  let Inst{31-0} = imm;
1010  let BPFClass = BPF_LD;
1011}
1012
1013class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode>
1014    : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value,
1015                 (outs),
1016                 (ins GPR:$skb, GPR:$val),
1017                 "r0 = *("#OpcodeStr#" *)skb[$val]",
1018                 [(set R0, (OpNode GPR:$skb, GPR:$val))]> {
1019  bits<4> val;
1020
1021  let Inst{55-52} = val;
1022  let BPFClass = BPF_LD;
1023}
1024}
1025
1026def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>;
1027def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>;
1028def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>;
1029
1030def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>;
1031def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>;
1032def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>;
1033
1034let isCodeGenOnly = 1 in {
1035  def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0,
1036                         (outs GPR:$dst), (ins GPR32:$src),
1037                         "$dst = $src", []>;
1038}
1039
1040let Predicates = [BPFNoMovsx] in {
1041  def : Pat<(i64 (sext GPR32:$src)),
1042            (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>;
1043}
1044
1045let Predicates = [BPFHasMovsx] in {
1046  def : Pat<(i64 (sext GPR32:$src)),
1047            (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>;
1048}
1049
1050def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>;
1051
1052// For i64 -> i32 truncation, use the 32-bit subregister directly.
1053def : Pat<(i32 (trunc GPR:$src)),
1054          (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>;
1055
1056// For i32 -> i64 anyext, we don't care about the high bits.
1057def : Pat<(i64 (anyext GPR32:$src)),
1058          (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>;
1059
1060class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern>
1061    : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
1062                 (outs),
1063                 (ins GPR32:$src, MEMri:$addr),
1064                 "*("#OpcodeStr#" *)($addr) = $src",
1065                 Pattern> {
1066  bits<4> src;
1067  bits<20> addr;
1068
1069  let Inst{51-48} = addr{19-16}; // base reg
1070  let Inst{55-52} = src;
1071  let Inst{47-32} = addr{15-0}; // offset
1072  let BPFClass = BPF_STX;
1073}
1074
1075class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>
1076    : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>;
1077
1078let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
1079  def STW32 : STOREi32<BPF_W, "u32", store>;
1080  def STH32 : STOREi32<BPF_H, "u16", truncstorei16>;
1081  def STB32 : STOREi32<BPF_B, "u8", truncstorei8>;
1082}
1083
1084class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern>
1085    : TYPE_LD_ST<ModOp.Value, SizeOp.Value,
1086                (outs GPR32:$dst),
1087                (ins MEMri:$addr),
1088                "$dst = *("#OpcodeStr#" *)($addr)",
1089                Pattern> {
1090  bits<4> dst;
1091  bits<20> addr;
1092
1093  let Inst{51-48} = dst;
1094  let Inst{55-52} = addr{19-16};
1095  let Inst{47-32} = addr{15-0};
1096  let BPFClass = BPF_LDX;
1097}
1098
1099class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode>
1100    : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>;
1101
1102let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
1103  def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>;
1104  def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>;
1105  def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>;
1106}
1107
1108let Predicates = [BPFHasALU32] in {
1109  def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst),
1110            (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;
1111  def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst),
1112            (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;
1113  def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst),
1114            (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>;
1115  def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>;
1116  def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>;
1117
1118  let Predicates = [BPFHasLdsx] in {
1119    def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>;
1120    def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>;
1121  }
1122
1123  def : Pat<(i64 (zextloadi8  ADDRri:$src)),
1124            (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>;
1125  def : Pat<(i64 (zextloadi16 ADDRri:$src)),
1126            (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>;
1127  def : Pat<(i64 (zextloadi32 ADDRri:$src)),
1128            (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>;
1129  def : Pat<(i64 (extloadi8  ADDRri:$src)),
1130            (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>;
1131  def : Pat<(i64 (extloadi16 ADDRri:$src)),
1132            (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>;
1133  def : Pat<(i64 (extloadi32 ADDRri:$src)),
1134            (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>;
1135}
1136
1137let usesCustomInserter = 1, isCodeGenOnly = 1 in {
1138    def MEMCPY : Pseudo<
1139      (outs),
1140      (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops),
1141      "#memcpy dst: $dst, src: $src, len: $len, align: $align",
1142      [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>;
1143}
1144