xref: /freebsd/contrib/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td (revision 8ddb146abcdf061be9f2c0db7e391697dafad85c) 
1// WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// WebAssembly SIMD operand code-gen constructs.
11///
12//===----------------------------------------------------------------------===//
13
14// Instructions using the SIMD opcode prefix and requiring one of the SIMD
15// feature predicates.
16multiclass ABSTRACT_SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
17                           list<dag> pattern_r, string asmstr_r,
18                           string asmstr_s, bits<32> simdop,
19                           Predicate simd_level> {
20  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
21              !if(!ge(simdop, 0x100),
22                  !or(0xfd0000, !and(0xffff, simdop)),
23                  !or(0xfd00, !and(0xff, simdop)))>,
24            Requires<[simd_level]>;
25}
26
27multiclass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
28                  list<dag> pattern_r, string asmstr_r = "",
29                  string asmstr_s = "", bits<32> simdop = -1> {
30  defm "" : ABSTRACT_SIMD_I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r,
31                            asmstr_s, simdop, HasSIMD128>;
32}
33
34multiclass RELAXED_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
35                     list<dag> pattern_r, string asmstr_r = "",
36                     string asmstr_s = "", bits<32> simdop = -1> {
37  defm "" : ABSTRACT_SIMD_I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r,
38                            asmstr_s, simdop, HasRelaxedSIMD>;
39}
40
41
42defm "" : ARGUMENT<V128, v16i8>;
43defm "" : ARGUMENT<V128, v8i16>;
44defm "" : ARGUMENT<V128, v4i32>;
45defm "" : ARGUMENT<V128, v2i64>;
46defm "" : ARGUMENT<V128, v4f32>;
47defm "" : ARGUMENT<V128, v2f64>;
48
49// Constrained immediate argument types
50foreach SIZE = [8, 16] in
51def ImmI#SIZE : ImmLeaf<i32,
52  "return -(1 << ("#SIZE#" - 1)) <= Imm && Imm < (1 << ("#SIZE#" - 1));"
53>;
54foreach SIZE = [2, 4, 8, 16, 32] in
55def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
56
57// Create vector with identical lanes: splat
58def splat2 : PatFrag<(ops node:$x), (build_vector $x, $x)>;
59def splat4 : PatFrag<(ops node:$x), (build_vector $x, $x, $x, $x)>;
60def splat8 : PatFrag<(ops node:$x), (build_vector $x, $x, $x, $x,
61                                                  $x, $x, $x, $x)>;
62def splat16 : PatFrag<(ops node:$x),
63                      (build_vector $x, $x, $x, $x, $x, $x, $x, $x,
64                                    $x, $x, $x, $x, $x, $x, $x, $x)>;
65
66class Vec {
67  ValueType vt;
68  ValueType int_vt;
69  ValueType lane_vt;
70  WebAssemblyRegClass lane_rc;
71  int lane_bits;
72  ImmLeaf lane_idx;
73  PatFrag splat;
74  string prefix;
75  Vec split;
76}
77
78def I8x16 : Vec {
79  let vt = v16i8;
80  let int_vt = vt;
81  let lane_vt = i32;
82  let lane_rc = I32;
83  let lane_bits = 8;
84  let lane_idx = LaneIdx16;
85  let splat = splat16;
86  let prefix = "i8x16";
87}
88
89def I16x8 : Vec {
90  let vt = v8i16;
91  let int_vt = vt;
92  let lane_vt = i32;
93  let lane_rc = I32;
94  let lane_bits = 16;
95  let lane_idx = LaneIdx8;
96  let splat = splat8;
97  let prefix = "i16x8";
98  let split = I8x16;
99}
100
101def I32x4 : Vec {
102  let vt = v4i32;
103  let int_vt = vt;
104  let lane_vt = i32;
105  let lane_rc = I32;
106  let lane_bits = 32;
107  let lane_idx = LaneIdx4;
108  let splat = splat4;
109  let prefix = "i32x4";
110  let split = I16x8;
111}
112
113def I64x2 : Vec {
114  let vt = v2i64;
115  let int_vt = vt;
116  let lane_vt = i64;
117  let lane_rc = I64;
118  let lane_bits = 64;
119  let lane_idx = LaneIdx2;
120  let splat = splat2;
121  let prefix = "i64x2";
122  let split = I32x4;
123}
124
125def F32x4 : Vec {
126  let vt = v4f32;
127  let int_vt = v4i32;
128  let lane_vt = f32;
129  let lane_rc = F32;
130  let lane_bits = 32;
131  let lane_idx = LaneIdx4;
132  let splat = splat4;
133  let prefix = "f32x4";
134}
135
136def F64x2 : Vec {
137  let vt = v2f64;
138  let int_vt = v2i64;
139  let lane_vt = f64;
140  let lane_rc = F64;
141  let lane_bits = 64;
142  let lane_idx = LaneIdx2;
143  let splat = splat2;
144  let prefix = "f64x2";
145}
146
147defvar AllVecs = [I8x16, I16x8, I32x4, I64x2, F32x4, F64x2];
148defvar IntVecs = [I8x16, I16x8, I32x4, I64x2];
149
150//===----------------------------------------------------------------------===//
151// Load and store
152//===----------------------------------------------------------------------===//
153
154// Load: v128.load
155let mayLoad = 1, UseNamedOperandTable = 1 in {
156defm LOAD_V128_A32 :
157  SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
158         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
159         "v128.load\t$dst, ${off}(${addr})$p2align",
160         "v128.load\t$off$p2align", 0>;
161defm LOAD_V128_A64 :
162  SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
163         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
164         "v128.load\t$dst, ${off}(${addr})$p2align",
165         "v128.load\t$off$p2align", 0>;
166}
167
168// Def load patterns from WebAssemblyInstrMemory.td for vector types
169foreach vec = AllVecs in {
170defm : LoadPatNoOffset<vec.vt, load, "LOAD_V128">;
171defm : LoadPatImmOff<vec.vt, load, regPlusImm, "LOAD_V128">;
172defm : LoadPatImmOff<vec.vt, load, or_is_add, "LOAD_V128">;
173defm : LoadPatOffsetOnly<vec.vt, load, "LOAD_V128">;
174defm : LoadPatGlobalAddrOffOnly<vec.vt, load, "LOAD_V128">;
175}
176
177// v128.loadX_splat
178multiclass SIMDLoadSplat<int size, bits<32> simdop> {
179  let mayLoad = 1, UseNamedOperandTable = 1 in {
180  defm LOAD#size#_SPLAT_A32 :
181    SIMD_I<(outs V128:$dst),
182           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
183           (outs),
184           (ins P2Align:$p2align, offset32_op:$off), [],
185           "v128.load"#size#"_splat\t$dst, ${off}(${addr})$p2align",
186           "v128.load"#size#"_splat\t$off$p2align", simdop>;
187  defm LOAD#size#_SPLAT_A64 :
188    SIMD_I<(outs V128:$dst),
189           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
190           (outs),
191           (ins P2Align:$p2align, offset64_op:$off), [],
192           "v128.load"#size#"_splat\t$dst, ${off}(${addr})$p2align",
193           "v128.load"#size#"_splat\t$off$p2align", simdop>;
194  }
195}
196
197defm "" : SIMDLoadSplat<8, 7>;
198defm "" : SIMDLoadSplat<16, 8>;
199defm "" : SIMDLoadSplat<32, 9>;
200defm "" : SIMDLoadSplat<64, 10>;
201
202def wasm_load_splat_t : SDTypeProfile<1, 1, [SDTCisPtrTy<1>]>;
203def wasm_load_splat : SDNode<"WebAssemblyISD::LOAD_SPLAT", wasm_load_splat_t,
204                             [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
205def load_splat : PatFrag<(ops node:$addr), (wasm_load_splat node:$addr)>;
206
207foreach vec = AllVecs in {
208defvar inst = "LOAD"#vec.lane_bits#"_SPLAT";
209defm : LoadPatNoOffset<vec.vt, load_splat, inst>;
210defm : LoadPatImmOff<vec.vt, load_splat, regPlusImm, inst>;
211defm : LoadPatImmOff<vec.vt, load_splat, or_is_add, inst>;
212defm : LoadPatOffsetOnly<vec.vt, load_splat, inst>;
213defm : LoadPatGlobalAddrOffOnly<vec.vt, load_splat, inst>;
214}
215
216// Load and extend
217multiclass SIMDLoadExtend<Vec vec, string loadPat, bits<32> simdop> {
218  defvar signed = vec.prefix#".load"#loadPat#"_s";
219  defvar unsigned = vec.prefix#".load"#loadPat#"_u";
220  let mayLoad = 1, UseNamedOperandTable = 1 in {
221  defm LOAD_EXTEND_S_#vec#_A32 :
222    SIMD_I<(outs V128:$dst),
223           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
224           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
225           signed#"\t$dst, ${off}(${addr})$p2align",
226           signed#"\t$off$p2align", simdop>;
227  defm LOAD_EXTEND_U_#vec#_A32 :
228    SIMD_I<(outs V128:$dst),
229           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
230           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
231           unsigned#"\t$dst, ${off}(${addr})$p2align",
232           unsigned#"\t$off$p2align", !add(simdop, 1)>;
233  defm LOAD_EXTEND_S_#vec#_A64 :
234    SIMD_I<(outs V128:$dst),
235           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
236           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
237           signed#"\t$dst, ${off}(${addr})$p2align",
238           signed#"\t$off$p2align", simdop>;
239  defm LOAD_EXTEND_U_#vec#_A64 :
240    SIMD_I<(outs V128:$dst),
241           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
242           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
243           unsigned#"\t$dst, ${off}(${addr})$p2align",
244           unsigned#"\t$off$p2align", !add(simdop, 1)>;
245  }
246}
247
248defm "" : SIMDLoadExtend<I16x8, "8x8", 1>;
249defm "" : SIMDLoadExtend<I32x4, "16x4", 3>;
250defm "" : SIMDLoadExtend<I64x2, "32x2", 5>;
251
252foreach vec = [I16x8, I32x4, I64x2] in
253foreach exts = [["sextloadvi", "_S"],
254                ["zextloadvi", "_U"],
255                ["extloadvi", "_U"]] in {
256defvar loadpat = !cast<PatFrag>(exts[0]#vec.split.lane_bits);
257defvar inst = "LOAD_EXTEND"#exts[1]#"_"#vec;
258defm : LoadPatNoOffset<vec.vt, loadpat, inst>;
259defm : LoadPatImmOff<vec.vt, loadpat, regPlusImm, inst>;
260defm : LoadPatImmOff<vec.vt, loadpat, or_is_add, inst>;
261defm : LoadPatOffsetOnly<vec.vt, loadpat, inst>;
262defm : LoadPatGlobalAddrOffOnly<vec.vt, loadpat, inst>;
263}
264
265// Load lane into zero vector
266multiclass SIMDLoadZero<Vec vec, bits<32> simdop> {
267  defvar name = "v128.load"#vec.lane_bits#"_zero";
268  let mayLoad = 1, UseNamedOperandTable = 1 in {
269  defm LOAD_ZERO_#vec#_A32 :
270    SIMD_I<(outs V128:$dst),
271           (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
272           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
273           name#"\t$dst, ${off}(${addr})$p2align",
274           name#"\t$off$p2align", simdop>;
275  defm LOAD_ZERO_#vec#_A64 :
276    SIMD_I<(outs V128:$dst),
277           (ins P2Align:$p2align, offset64_op:$off, I64:$addr),
278           (outs), (ins P2Align:$p2align, offset64_op:$off), [],
279           name#"\t$dst, ${off}(${addr})$p2align",
280           name#"\t$off$p2align", simdop>;
281  } // mayLoad = 1, UseNamedOperandTable = 1
282}
283
284defm "" : SIMDLoadZero<I32x4, 0x5c>;
285defm "" : SIMDLoadZero<I64x2, 0x5d>;
286
287// Use load_zero to load scalars into vectors as well where possible.
288// TODO: i32, i16, and i8 scalars
289def load_scalar :
290  PatFrag<(ops node:$addr), (scalar_to_vector (i64 (load $addr)))>;
291defm : LoadPatNoOffset<v2i64, load_scalar, "LOAD_ZERO_I64x2">;
292defm : LoadPatImmOff<v2i64, load_scalar, regPlusImm, "LOAD_ZERO_I64x2">;
293defm : LoadPatImmOff<v2i64, load_scalar, or_is_add, "LOAD_ZERO_I64x2">;
294defm : LoadPatOffsetOnly<v2i64, load_scalar, "LOAD_ZERO_I64x2">;
295defm : LoadPatGlobalAddrOffOnly<v2i64, load_scalar, "LOAD_ZERO_I64x2">;
296
297// TODO: f32x4 and f64x2 as well
298foreach vec = [I32x4, I64x2] in {
299  defvar inst = "LOAD_ZERO_"#vec;
300  defvar pat = PatFrag<(ops node:$ptr),
301    (vector_insert (vec.splat (vec.lane_vt 0)), (vec.lane_vt (load $ptr)), 0)>;
302  defm : LoadPatNoOffset<vec.vt, pat, inst>;
303  defm : LoadPatImmOff<vec.vt, pat, regPlusImm, inst>;
304  defm : LoadPatImmOff<vec.vt, pat, or_is_add, inst>;
305  defm : LoadPatOffsetOnly<vec.vt, pat, inst>;
306  defm : LoadPatGlobalAddrOffOnly<vec.vt, pat, inst>;
307}
308
309// Load lane
310multiclass SIMDLoadLane<Vec vec, bits<32> simdop> {
311  defvar name = "v128.load"#vec.lane_bits#"_lane";
312  let mayLoad = 1, UseNamedOperandTable = 1 in {
313  defm LOAD_LANE_#vec#_A32 :
314    SIMD_I<(outs V128:$dst),
315           (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx,
316                I32:$addr, V128:$vec),
317           (outs), (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx),
318           [], name#"\t$dst, ${off}(${addr})$p2align, $vec, $idx",
319           name#"\t$off$p2align, $idx", simdop>;
320  defm LOAD_LANE_#vec#_A64 :
321    SIMD_I<(outs V128:$dst),
322           (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx,
323                I64:$addr, V128:$vec),
324           (outs), (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx),
325           [], name#"\t$dst, ${off}(${addr})$p2align, $vec, $idx",
326           name#"\t$off$p2align, $idx", simdop>;
327  } // mayLoad = 1, UseNamedOperandTable = 1
328}
329
330defm "" : SIMDLoadLane<I8x16, 0x54>;
331defm "" : SIMDLoadLane<I16x8, 0x55>;
332defm "" : SIMDLoadLane<I32x4, 0x56>;
333defm "" : SIMDLoadLane<I64x2, 0x57>;
334
335// Select loads with no constant offset.
336multiclass LoadLanePatNoOffset<Vec vec, SDPatternOperator kind> {
337  defvar load_lane_a32 = !cast<NI>("LOAD_LANE_"#vec#"_A32");
338  defvar load_lane_a64 = !cast<NI>("LOAD_LANE_"#vec#"_A64");
339  def : Pat<(vec.vt (kind (i32 I32:$addr),
340              (vec.vt V128:$vec), (i32 vec.lane_idx:$idx))),
341            (load_lane_a32 0, 0, imm:$idx, $addr, $vec)>,
342        Requires<[HasAddr32]>;
343  def : Pat<(vec.vt (kind (i64 I64:$addr),
344              (vec.vt V128:$vec), (i32 vec.lane_idx:$idx))),
345            (load_lane_a64 0, 0, imm:$idx, $addr, $vec)>,
346        Requires<[HasAddr64]>;
347}
348
349def load8_lane :
350  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
351          (vector_insert $vec, (i32 (extloadi8 $ptr)), $idx)>;
352def load16_lane :
353  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
354          (vector_insert $vec, (i32 (extloadi16 $ptr)), $idx)>;
355def load32_lane :
356  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
357          (vector_insert $vec, (i32 (load $ptr)), $idx)>;
358def load64_lane :
359  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
360          (vector_insert $vec, (i64 (load $ptr)), $idx)>;
361// TODO: floating point lanes as well
362
363defm : LoadLanePatNoOffset<I8x16, load8_lane>;
364defm : LoadLanePatNoOffset<I16x8, load16_lane>;
365defm : LoadLanePatNoOffset<I32x4, load32_lane>;
366defm : LoadLanePatNoOffset<I64x2, load64_lane>;
367
368// TODO: Also support the other load patterns for load_lane once the instructions
369// are merged to the proposal.
370
371// Store: v128.store
372let mayStore = 1, UseNamedOperandTable = 1 in {
373defm STORE_V128_A32 :
374  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, V128:$vec),
375         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
376         "v128.store\t${off}(${addr})$p2align, $vec",
377         "v128.store\t$off$p2align", 11>;
378defm STORE_V128_A64 :
379  SIMD_I<(outs), (ins P2Align:$p2align, offset64_op:$off, I64:$addr, V128:$vec),
380         (outs), (ins P2Align:$p2align, offset64_op:$off), [],
381         "v128.store\t${off}(${addr})$p2align, $vec",
382         "v128.store\t$off$p2align", 11>;
383}
384
385// Def store patterns from WebAssemblyInstrMemory.td for vector types
386foreach vec = AllVecs in {
387defm : StorePatNoOffset<vec.vt, store, "STORE_V128">;
388defm : StorePatImmOff<vec.vt, store, regPlusImm, "STORE_V128">;
389defm : StorePatImmOff<vec.vt, store, or_is_add, "STORE_V128">;
390defm : StorePatOffsetOnly<vec.vt, store, "STORE_V128">;
391defm : StorePatGlobalAddrOffOnly<vec.vt, store, "STORE_V128">;
392}
393
394// Store lane
395multiclass SIMDStoreLane<Vec vec, bits<32> simdop> {
396  defvar name = "v128.store"#vec.lane_bits#"_lane";
397  let mayStore = 1, UseNamedOperandTable = 1 in {
398  defm STORE_LANE_#vec#_A32 :
399    SIMD_I<(outs),
400           (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx,
401                I32:$addr, V128:$vec),
402           (outs), (ins P2Align:$p2align, offset32_op:$off, vec_i8imm_op:$idx),
403           [], name#"\t${off}(${addr})$p2align, $vec, $idx",
404           name#"\t$off$p2align, $idx", simdop>;
405  defm STORE_LANE_#vec#_A64 :
406    SIMD_I<(outs V128:$dst),
407           (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx,
408                I64:$addr, V128:$vec),
409           (outs), (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx),
410           [], name#"\t${off}(${addr})$p2align, $vec, $idx",
411           name#"\t$off$p2align, $idx", simdop>;
412  } // mayStore = 1, UseNamedOperandTable = 1
413}
414
415defm "" : SIMDStoreLane<I8x16, 0x58>;
416defm "" : SIMDStoreLane<I16x8, 0x59>;
417defm "" : SIMDStoreLane<I32x4, 0x5a>;
418defm "" : SIMDStoreLane<I64x2, 0x5b>;
419
420// Select stores with no constant offset.
421multiclass StoreLanePatNoOffset<Vec vec, SDPatternOperator kind> {
422  def : Pat<(kind (i32 I32:$addr), (vec.vt V128:$vec), (i32 vec.lane_idx:$idx)),
423            (!cast<NI>("STORE_LANE_"#vec#"_A32") 0, 0, imm:$idx, $addr, $vec)>,
424        Requires<[HasAddr32]>;
425  def : Pat<(kind (i64 I64:$addr), (vec.vt V128:$vec), (i32 vec.lane_idx:$idx)),
426            (!cast<NI>("STORE_LANE_"#vec#"_A64") 0, 0, imm:$idx, $addr, $vec)>,
427        Requires<[HasAddr64]>;
428}
429
430def store8_lane :
431  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
432          (truncstorei8 (i32 (vector_extract $vec, $idx)), $ptr)>;
433def store16_lane :
434  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
435          (truncstorei16 (i32 (vector_extract $vec, $idx)), $ptr)>;
436def store32_lane :
437  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
438          (store (i32 (vector_extract $vec, $idx)), $ptr)>;
439def store64_lane :
440  PatFrag<(ops node:$ptr, node:$vec, node:$idx),
441          (store (i64 (vector_extract $vec, $idx)), $ptr)>;
442// TODO: floating point lanes as well
443
444let AddedComplexity = 1 in {
445defm : StoreLanePatNoOffset<I8x16, store8_lane>;
446defm : StoreLanePatNoOffset<I16x8, store16_lane>;
447defm : StoreLanePatNoOffset<I32x4, store32_lane>;
448defm : StoreLanePatNoOffset<I64x2, store64_lane>;
449}
450
451//===----------------------------------------------------------------------===//
452// Constructing SIMD values
453//===----------------------------------------------------------------------===//
454
455// Constant: v128.const
456multiclass ConstVec<Vec vec, dag ops, dag pat, string args> {
457  let isMoveImm = 1, isReMaterializable = 1 in
458  defm CONST_V128_#vec : SIMD_I<(outs V128:$dst), ops, (outs), ops,
459                                 [(set V128:$dst, (vec.vt pat))],
460                                 "v128.const\t$dst, "#args,
461                                 "v128.const\t"#args, 12>;
462}
463
464defm "" : ConstVec<I8x16,
465                   (ins vec_i8imm_op:$i0, vec_i8imm_op:$i1,
466                        vec_i8imm_op:$i2, vec_i8imm_op:$i3,
467                        vec_i8imm_op:$i4, vec_i8imm_op:$i5,
468                        vec_i8imm_op:$i6, vec_i8imm_op:$i7,
469                        vec_i8imm_op:$i8, vec_i8imm_op:$i9,
470                        vec_i8imm_op:$iA, vec_i8imm_op:$iB,
471                        vec_i8imm_op:$iC, vec_i8imm_op:$iD,
472                        vec_i8imm_op:$iE, vec_i8imm_op:$iF),
473                   (build_vector ImmI8:$i0, ImmI8:$i1, ImmI8:$i2, ImmI8:$i3,
474                                 ImmI8:$i4, ImmI8:$i5, ImmI8:$i6, ImmI8:$i7,
475                                 ImmI8:$i8, ImmI8:$i9, ImmI8:$iA, ImmI8:$iB,
476                                 ImmI8:$iC, ImmI8:$iD, ImmI8:$iE, ImmI8:$iF),
477                   !strconcat("$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7, ",
478                              "$i8, $i9, $iA, $iB, $iC, $iD, $iE, $iF")>;
479defm "" : ConstVec<I16x8,
480                   (ins vec_i16imm_op:$i0, vec_i16imm_op:$i1,
481                        vec_i16imm_op:$i2, vec_i16imm_op:$i3,
482                        vec_i16imm_op:$i4, vec_i16imm_op:$i5,
483                        vec_i16imm_op:$i6, vec_i16imm_op:$i7),
484                   (build_vector
485                     ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3,
486                     ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7),
487                   "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">;
488let IsCanonical = 1 in
489defm "" : ConstVec<I32x4,
490                   (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1,
491                        vec_i32imm_op:$i2, vec_i32imm_op:$i3),
492                   (build_vector (i32 imm:$i0), (i32 imm:$i1),
493                                 (i32 imm:$i2), (i32 imm:$i3)),
494                   "$i0, $i1, $i2, $i3">;
495defm "" : ConstVec<I64x2,
496                   (ins vec_i64imm_op:$i0, vec_i64imm_op:$i1),
497                   (build_vector (i64 imm:$i0), (i64 imm:$i1)),
498                   "$i0, $i1">;
499defm "" : ConstVec<F32x4,
500                   (ins f32imm_op:$i0, f32imm_op:$i1,
501                        f32imm_op:$i2, f32imm_op:$i3),
502                   (build_vector (f32 fpimm:$i0), (f32 fpimm:$i1),
503                                 (f32 fpimm:$i2), (f32 fpimm:$i3)),
504                   "$i0, $i1, $i2, $i3">;
505defm "" : ConstVec<F64x2,
506                  (ins f64imm_op:$i0, f64imm_op:$i1),
507                  (build_vector (f64 fpimm:$i0), (f64 fpimm:$i1)),
508                  "$i0, $i1">;
509
510// Shuffle lanes: shuffle
511defm SHUFFLE :
512  SIMD_I<(outs V128:$dst),
513         (ins V128:$x, V128:$y,
514           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
515           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
516           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
517           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
518           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
519           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
520           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
521           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
522         (outs),
523         (ins
524           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
525           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
526           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
527           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
528           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
529           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
530           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
531           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
532         [],
533         "i8x16.shuffle\t$dst, $x, $y, "#
534           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
535           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
536         "i8x16.shuffle\t"#
537           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
538           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
539         13>;
540
541// Shuffles after custom lowering
542def wasm_shuffle_t : SDTypeProfile<1, 18, []>;
543def wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>;
544foreach vec = AllVecs in {
545def : Pat<(vec.vt (wasm_shuffle (vec.vt V128:$x), (vec.vt V128:$y),
546            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
547            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
548            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
549            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
550            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
551            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
552            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
553            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF))),
554          (SHUFFLE $x, $y,
555            imm:$m0, imm:$m1, imm:$m2, imm:$m3,
556            imm:$m4, imm:$m5, imm:$m6, imm:$m7,
557            imm:$m8, imm:$m9, imm:$mA, imm:$mB,
558            imm:$mC, imm:$mD, imm:$mE, imm:$mF)>;
559}
560
561// Swizzle lanes: i8x16.swizzle
562def wasm_swizzle_t : SDTypeProfile<1, 2, []>;
563def wasm_swizzle : SDNode<"WebAssemblyISD::SWIZZLE", wasm_swizzle_t>;
564defm SWIZZLE :
565  SIMD_I<(outs V128:$dst), (ins V128:$src, V128:$mask), (outs), (ins),
566         [(set (v16i8 V128:$dst),
567           (wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)))],
568         "i8x16.swizzle\t$dst, $src, $mask", "i8x16.swizzle", 14>;
569
570def : Pat<(int_wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)),
571          (SWIZZLE $src, $mask)>;
572
573multiclass Splat<Vec vec, bits<32> simdop> {
574  defm SPLAT_#vec : SIMD_I<(outs V128:$dst), (ins vec.lane_rc:$x),
575                           (outs), (ins),
576                           [(set (vec.vt V128:$dst),
577                              (vec.splat vec.lane_rc:$x))],
578                           vec.prefix#".splat\t$dst, $x", vec.prefix#".splat",
579                           simdop>;
580}
581
582defm "" : Splat<I8x16, 15>;
583defm "" : Splat<I16x8, 16>;
584defm "" : Splat<I32x4, 17>;
585defm "" : Splat<I64x2, 18>;
586defm "" : Splat<F32x4, 19>;
587defm "" : Splat<F64x2, 20>;
588
589// scalar_to_vector leaves high lanes undefined, so can be a splat
590foreach vec = AllVecs in
591def : Pat<(vec.vt (scalar_to_vector (vec.lane_vt vec.lane_rc:$x))),
592          (!cast<Instruction>("SPLAT_"#vec) $x)>;
593
594//===----------------------------------------------------------------------===//
595// Accessing lanes
596//===----------------------------------------------------------------------===//
597
598// Extract lane as a scalar: extract_lane / extract_lane_s / extract_lane_u
599multiclass ExtractLane<Vec vec, bits<32> simdop, string suffix = ""> {
600  defm EXTRACT_LANE_#vec#suffix :
601      SIMD_I<(outs vec.lane_rc:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
602             (outs), (ins vec_i8imm_op:$idx), [],
603             vec.prefix#".extract_lane"#suffix#"\t$dst, $vec, $idx",
604             vec.prefix#".extract_lane"#suffix#"\t$idx", simdop>;
605}
606
607defm "" : ExtractLane<I8x16, 21, "_s">;
608defm "" : ExtractLane<I8x16, 22, "_u">;
609defm "" : ExtractLane<I16x8, 24, "_s">;
610defm "" : ExtractLane<I16x8, 25, "_u">;
611defm "" : ExtractLane<I32x4, 27>;
612defm "" : ExtractLane<I64x2, 29>;
613defm "" : ExtractLane<F32x4, 31>;
614defm "" : ExtractLane<F64x2, 33>;
615
616def : Pat<(vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx)),
617          (EXTRACT_LANE_I8x16_u $vec, imm:$idx)>;
618def : Pat<(vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx)),
619          (EXTRACT_LANE_I16x8_u $vec, imm:$idx)>;
620def : Pat<(vector_extract (v4i32 V128:$vec), (i32 LaneIdx4:$idx)),
621          (EXTRACT_LANE_I32x4 $vec, imm:$idx)>;
622def : Pat<(vector_extract (v4f32 V128:$vec), (i32 LaneIdx4:$idx)),
623          (EXTRACT_LANE_F32x4 $vec, imm:$idx)>;
624def : Pat<(vector_extract (v2i64 V128:$vec), (i32 LaneIdx2:$idx)),
625          (EXTRACT_LANE_I64x2 $vec, imm:$idx)>;
626def : Pat<(vector_extract (v2f64 V128:$vec), (i32 LaneIdx2:$idx)),
627          (EXTRACT_LANE_F64x2 $vec, imm:$idx)>;
628
629def : Pat<
630  (sext_inreg (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx)), i8),
631  (EXTRACT_LANE_I8x16_s $vec, imm:$idx)>;
632def : Pat<
633  (and (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx)), (i32 0xff)),
634  (EXTRACT_LANE_I8x16_u $vec, imm:$idx)>;
635def : Pat<
636  (sext_inreg (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx)), i16),
637  (EXTRACT_LANE_I16x8_s $vec, imm:$idx)>;
638def : Pat<
639  (and (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx)), (i32 0xffff)),
640  (EXTRACT_LANE_I16x8_u $vec, imm:$idx)>;
641
642// Replace lane value: replace_lane
643multiclass ReplaceLane<Vec vec, bits<32> simdop> {
644  defm REPLACE_LANE_#vec :
645    SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, vec.lane_rc:$x),
646           (outs), (ins vec_i8imm_op:$idx),
647           [(set V128:$dst, (vector_insert
648             (vec.vt V128:$vec),
649             (vec.lane_vt vec.lane_rc:$x),
650             (i32 vec.lane_idx:$idx)))],
651           vec.prefix#".replace_lane\t$dst, $vec, $idx, $x",
652           vec.prefix#".replace_lane\t$idx", simdop>;
653}
654
655defm "" : ReplaceLane<I8x16, 23>;
656defm "" : ReplaceLane<I16x8, 26>;
657defm "" : ReplaceLane<I32x4, 28>;
658defm "" : ReplaceLane<I64x2, 30>;
659defm "" : ReplaceLane<F32x4, 32>;
660defm "" : ReplaceLane<F64x2, 34>;
661
662// Lower undef lane indices to zero
663def : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef),
664          (REPLACE_LANE_I8x16 $vec, 0, $x)>;
665def : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef),
666          (REPLACE_LANE_I16x8 $vec, 0, $x)>;
667def : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef),
668          (REPLACE_LANE_I32x4 $vec, 0, $x)>;
669def : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef),
670          (REPLACE_LANE_I64x2 $vec, 0, $x)>;
671def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
672          (REPLACE_LANE_F32x4 $vec, 0, $x)>;
673def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
674          (REPLACE_LANE_F64x2 $vec, 0, $x)>;
675
676//===----------------------------------------------------------------------===//
677// Comparisons
678//===----------------------------------------------------------------------===//
679
680multiclass SIMDCondition<Vec vec, string name, CondCode cond, bits<32> simdop> {
681  defm _#vec :
682    SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
683           [(set (vec.int_vt V128:$dst),
684             (setcc (vec.vt V128:$lhs), (vec.vt V128:$rhs), cond))],
685           vec.prefix#"."#name#"\t$dst, $lhs, $rhs",
686           vec.prefix#"."#name, simdop>;
687}
688
689multiclass SIMDConditionInt<string name, CondCode cond, bits<32> baseInst> {
690  defm "" : SIMDCondition<I8x16, name, cond, baseInst>;
691  defm "" : SIMDCondition<I16x8, name, cond, !add(baseInst, 10)>;
692  defm "" : SIMDCondition<I32x4, name, cond, !add(baseInst, 20)>;
693}
694
695multiclass SIMDConditionFP<string name, CondCode cond, bits<32> baseInst> {
696  defm "" : SIMDCondition<F32x4, name, cond, baseInst>;
697  defm "" : SIMDCondition<F64x2, name, cond, !add(baseInst, 6)>;
698}
699
700// Equality: eq
701let isCommutable = 1 in {
702defm EQ : SIMDConditionInt<"eq", SETEQ, 35>;
703defm EQ : SIMDCondition<I64x2, "eq", SETEQ, 214>;
704defm EQ : SIMDConditionFP<"eq", SETOEQ, 65>;
705} // isCommutable = 1
706
707// Non-equality: ne
708let isCommutable = 1 in {
709defm NE : SIMDConditionInt<"ne", SETNE, 36>;
710defm NE : SIMDCondition<I64x2, "ne", SETNE, 215>;
711defm NE : SIMDConditionFP<"ne", SETUNE, 66>;
712} // isCommutable = 1
713
714// Less than: lt_s / lt_u / lt
715defm LT_S : SIMDConditionInt<"lt_s", SETLT, 37>;
716defm LT_S : SIMDCondition<I64x2, "lt_s", SETLT, 216>;
717defm LT_U : SIMDConditionInt<"lt_u", SETULT, 38>;
718defm LT : SIMDConditionFP<"lt", SETOLT, 67>;
719
720// Greater than: gt_s / gt_u / gt
721defm GT_S : SIMDConditionInt<"gt_s", SETGT, 39>;
722defm GT_S : SIMDCondition<I64x2, "gt_s", SETGT, 217>;
723defm GT_U : SIMDConditionInt<"gt_u", SETUGT, 40>;
724defm GT : SIMDConditionFP<"gt", SETOGT, 68>;
725
726// Less than or equal: le_s / le_u / le
727defm LE_S : SIMDConditionInt<"le_s", SETLE, 41>;
728defm LE_S : SIMDCondition<I64x2, "le_s", SETLE, 218>;
729defm LE_U : SIMDConditionInt<"le_u", SETULE, 42>;
730defm LE : SIMDConditionFP<"le", SETOLE, 69>;
731
732// Greater than or equal: ge_s / ge_u / ge
733defm GE_S : SIMDConditionInt<"ge_s", SETGE, 43>;
734defm GE_S : SIMDCondition<I64x2, "ge_s", SETGE, 219>;
735defm GE_U : SIMDConditionInt<"ge_u", SETUGE, 44>;
736defm GE : SIMDConditionFP<"ge", SETOGE, 70>;
737
738// Lower float comparisons that don't care about NaN to standard WebAssembly
739// float comparisons. These instructions are generated with nnan and in the
740// target-independent expansion of unordered comparisons and ordered ne.
741foreach nodes = [[seteq, EQ_F32x4], [setne, NE_F32x4], [setlt, LT_F32x4],
742                 [setgt, GT_F32x4], [setle, LE_F32x4], [setge, GE_F32x4]] in
743def : Pat<(v4i32 (nodes[0] (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
744          (nodes[1] $lhs, $rhs)>;
745
746foreach nodes = [[seteq, EQ_F64x2], [setne, NE_F64x2], [setlt, LT_F64x2],
747                 [setgt, GT_F64x2], [setle, LE_F64x2], [setge, GE_F64x2]] in
748def : Pat<(v2i64 (nodes[0] (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
749          (nodes[1] $lhs, $rhs)>;
750
751//===----------------------------------------------------------------------===//
752// Bitwise operations
753//===----------------------------------------------------------------------===//
754
755multiclass SIMDBinary<Vec vec, SDPatternOperator node, string name, bits<32> simdop> {
756  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
757                      (outs), (ins),
758                      [(set (vec.vt V128:$dst),
759                        (node (vec.vt V128:$lhs), (vec.vt V128:$rhs)))],
760                      vec.prefix#"."#name#"\t$dst, $lhs, $rhs",
761                      vec.prefix#"."#name, simdop>;
762}
763
764multiclass SIMDBitwise<SDPatternOperator node, string name, bits<32> simdop,
765                       bit commutable = false> {
766  let isCommutable = commutable in
767  defm "" : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
768                   (outs), (ins), [],
769                   "v128."#name#"\t$dst, $lhs, $rhs", "v128."#name, simdop>;
770  foreach vec = IntVecs in
771  def : Pat<(node (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
772            (!cast<NI>(NAME) $lhs, $rhs)>;
773}
774
775multiclass SIMDUnary<Vec vec, SDPatternOperator node, string name, bits<32> simdop> {
776  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$v), (outs), (ins),
777                      [(set (vec.vt V128:$dst),
778                        (vec.vt (node (vec.vt V128:$v))))],
779                      vec.prefix#"."#name#"\t$dst, $v",
780                      vec.prefix#"."#name, simdop>;
781}
782
783// Bitwise logic: v128.not
784defm NOT : SIMD_I<(outs V128:$dst), (ins V128:$v), (outs), (ins), [],
785                  "v128.not\t$dst, $v", "v128.not", 77>;
786foreach vec = IntVecs in
787def : Pat<(vnot (vec.vt V128:$v)), (NOT $v)>;
788
789// Bitwise logic: v128.and / v128.or / v128.xor
790defm AND : SIMDBitwise<and, "and", 78, true>;
791defm OR : SIMDBitwise<or, "or", 80, true>;
792defm XOR : SIMDBitwise<xor, "xor", 81, true>;
793
794// Bitwise logic: v128.andnot
795def andnot : PatFrag<(ops node:$left, node:$right), (and $left, (vnot $right))>;
796defm ANDNOT : SIMDBitwise<andnot, "andnot", 79>;
797
798// Bitwise select: v128.bitselect
799defm BITSELECT :
800  SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins), [],
801         "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 82>;
802
803foreach vec = AllVecs in
804def : Pat<(vec.vt (int_wasm_bitselect
805            (vec.vt V128:$v1), (vec.vt V128:$v2), (vec.vt V128:$c))),
806          (BITSELECT $v1, $v2, $c)>;
807
808// Bitselect is equivalent to (c & v1) | (~c & v2)
809foreach vec = IntVecs in
810def : Pat<(vec.vt (or (and (vec.vt V128:$c), (vec.vt V128:$v1)),
811            (and (vnot V128:$c), (vec.vt V128:$v2)))),
812          (BITSELECT $v1, $v2, $c)>;
813
814// Also implement vselect in terms of bitselect
815foreach vec = AllVecs in
816def : Pat<(vec.vt (vselect
817            (vec.int_vt V128:$c), (vec.vt V128:$v1), (vec.vt V128:$v2))),
818          (BITSELECT $v1, $v2, $c)>;
819
820// MVP select on v128 values
821defm SELECT_V128 :
822  I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs, I32:$cond), (outs), (ins), [],
823    "v128.select\t$dst, $lhs, $rhs, $cond", "v128.select", 0x1b>;
824
825foreach vec = AllVecs in {
826def : Pat<(select I32:$cond, (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
827          (SELECT_V128 $lhs, $rhs, $cond)>;
828
829// ISD::SELECT requires its operand to conform to getBooleanContents, but
830// WebAssembly's select interprets any non-zero value as true, so we can fold
831// a setne with 0 into a select.
832def : Pat<(select
833            (i32 (setne I32:$cond, 0)), (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
834          (SELECT_V128 $lhs, $rhs, $cond)>;
835
836// And again, this time with seteq instead of setne and the arms reversed.
837def : Pat<(select
838            (i32 (seteq I32:$cond, 0)), (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
839          (SELECT_V128 $rhs, $lhs, $cond)>;
840} // foreach vec
841
842//===----------------------------------------------------------------------===//
843// Integer unary arithmetic
844//===----------------------------------------------------------------------===//
845
846multiclass SIMDUnaryInt<SDPatternOperator node, string name, bits<32> baseInst> {
847  defm "" : SIMDUnary<I8x16, node, name, baseInst>;
848  defm "" : SIMDUnary<I16x8, node, name, !add(baseInst, 32)>;
849  defm "" : SIMDUnary<I32x4, node, name, !add(baseInst, 64)>;
850  defm "" : SIMDUnary<I64x2, node, name, !add(baseInst, 96)>;
851}
852
853// Integer vector negation
854def ivneg : PatFrag<(ops node:$in), (sub immAllZerosV, $in)>;
855
856// Integer absolute value: abs
857defm ABS : SIMDUnaryInt<abs, "abs", 96>;
858
859// Integer negation: neg
860defm NEG : SIMDUnaryInt<ivneg, "neg", 97>;
861
862// Population count: popcnt
863defm POPCNT : SIMDUnary<I8x16, ctpop, "popcnt", 0x62>;
864
865// Any lane true: any_true
866defm ANYTRUE : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins), [],
867                      "v128.any_true\t$dst, $vec", "v128.any_true", 0x53>;
868
869foreach vec = IntVecs in
870def : Pat<(int_wasm_anytrue (vec.vt V128:$vec)), (ANYTRUE V128:$vec)>;
871
872// All lanes true: all_true
873multiclass SIMDAllTrue<Vec vec, bits<32> simdop> {
874  defm ALLTRUE_#vec : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
875                             [(set I32:$dst,
876                               (i32 (int_wasm_alltrue (vec.vt V128:$vec))))],
877                             vec.prefix#".all_true\t$dst, $vec",
878                             vec.prefix#".all_true", simdop>;
879}
880
881defm "" : SIMDAllTrue<I8x16, 0x63>;
882defm "" : SIMDAllTrue<I16x8, 0x83>;
883defm "" : SIMDAllTrue<I32x4, 0xa3>;
884defm "" : SIMDAllTrue<I64x2, 0xc3>;
885
886// Reductions already return 0 or 1, so and 1, setne 0, and seteq 1
887// can be folded out
888foreach reduction =
889  [["int_wasm_anytrue", "ANYTRUE", "I8x16"],
890   ["int_wasm_anytrue", "ANYTRUE", "I16x8"],
891   ["int_wasm_anytrue", "ANYTRUE", "I32x4"],
892   ["int_wasm_anytrue", "ANYTRUE", "I64x2"],
893   ["int_wasm_alltrue", "ALLTRUE_I8x16", "I8x16"],
894   ["int_wasm_alltrue", "ALLTRUE_I16x8", "I16x8"],
895   ["int_wasm_alltrue", "ALLTRUE_I32x4", "I32x4"],
896   ["int_wasm_alltrue", "ALLTRUE_I64x2", "I64x2"]] in {
897defvar intrinsic = !cast<Intrinsic>(reduction[0]);
898defvar inst = !cast<NI>(reduction[1]);
899defvar vec = !cast<Vec>(reduction[2]);
900def : Pat<(i32 (and (i32 (intrinsic (vec.vt V128:$x))), (i32 1))), (inst $x)>;
901def : Pat<(i32 (setne (i32 (intrinsic (vec.vt V128:$x))), (i32 0))), (inst $x)>;
902def : Pat<(i32 (seteq (i32 (intrinsic (vec.vt V128:$x))), (i32 1))), (inst $x)>;
903}
904
905multiclass SIMDBitmask<Vec vec, bits<32> simdop> {
906  defm _#vec : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
907                      [(set I32:$dst,
908                         (i32 (int_wasm_bitmask (vec.vt V128:$vec))))],
909                      vec.prefix#".bitmask\t$dst, $vec", vec.prefix#".bitmask",
910                      simdop>;
911}
912
913defm BITMASK : SIMDBitmask<I8x16, 100>;
914defm BITMASK : SIMDBitmask<I16x8, 132>;
915defm BITMASK : SIMDBitmask<I32x4, 164>;
916defm BITMASK : SIMDBitmask<I64x2, 196>;
917
918//===----------------------------------------------------------------------===//
919// Bit shifts
920//===----------------------------------------------------------------------===//
921
922multiclass SIMDShift<Vec vec, SDNode node, string name, bits<32> simdop> {
923  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x), (outs), (ins),
924                      [(set (vec.vt V128:$dst), (node V128:$vec, I32:$x))],
925                      vec.prefix#"."#name#"\t$dst, $vec, $x",
926                      vec.prefix#"."#name, simdop>;
927}
928
929multiclass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> {
930  defm "" : SIMDShift<I8x16, node, name, baseInst>;
931  defm "" : SIMDShift<I16x8, node, name, !add(baseInst, 32)>;
932  defm "" : SIMDShift<I32x4, node, name, !add(baseInst, 64)>;
933  defm "" : SIMDShift<I64x2, node, name, !add(baseInst, 96)>;
934}
935
936// WebAssembly SIMD shifts are nonstandard in that the shift amount is
937// an i32 rather than a vector, so they need custom nodes.
938def wasm_shift_t :
939  SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]>;
940def wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>;
941def wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>;
942def wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>;
943
944// Left shift by scalar: shl
945defm SHL : SIMDShiftInt<wasm_shl, "shl", 107>;
946
947// Right shift by scalar: shr_s / shr_u
948defm SHR_S : SIMDShiftInt<wasm_shr_s, "shr_s", 108>;
949defm SHR_U : SIMDShiftInt<wasm_shr_u, "shr_u", 109>;
950
951// Optimize away an explicit mask on a shift count.
952def : Pat<(wasm_shl (v16i8 V128:$lhs), (and I32:$rhs, 7)),
953          (SHL_I8x16 V128:$lhs, I32:$rhs)>;
954def : Pat<(wasm_shr_s (v16i8 V128:$lhs), (and I32:$rhs, 7)),
955          (SHR_S_I8x16 V128:$lhs, I32:$rhs)>;
956def : Pat<(wasm_shr_u (v16i8 V128:$lhs), (and I32:$rhs, 7)),
957          (SHR_U_I8x16 V128:$lhs, I32:$rhs)>;
958
959def : Pat<(wasm_shl (v8i16 V128:$lhs), (and I32:$rhs, 15)),
960          (SHL_I16x8 V128:$lhs, I32:$rhs)>;
961def : Pat<(wasm_shr_s (v8i16 V128:$lhs), (and I32:$rhs, 15)),
962          (SHR_S_I16x8 V128:$lhs, I32:$rhs)>;
963def : Pat<(wasm_shr_u (v8i16 V128:$lhs), (and I32:$rhs, 15)),
964          (SHR_U_I16x8 V128:$lhs, I32:$rhs)>;
965
966def : Pat<(wasm_shl (v4i32 V128:$lhs), (and I32:$rhs, 31)),
967          (SHL_I32x4 V128:$lhs, I32:$rhs)>;
968def : Pat<(wasm_shr_s (v4i32 V128:$lhs), (and I32:$rhs, 31)),
969          (SHR_S_I32x4 V128:$lhs, I32:$rhs)>;
970def : Pat<(wasm_shr_u (v4i32 V128:$lhs), (and I32:$rhs, 31)),
971          (SHR_U_I32x4 V128:$lhs, I32:$rhs)>;
972
973def : Pat<(wasm_shl (v2i64 V128:$lhs), (trunc (and I64:$rhs, 63))),
974          (SHL_I64x2 V128:$lhs, (I32_WRAP_I64 I64:$rhs))>;
975def : Pat<(wasm_shr_s (v2i64 V128:$lhs), (trunc (and I64:$rhs, 63))),
976          (SHR_S_I64x2 V128:$lhs, (I32_WRAP_I64 I64:$rhs))>;
977def : Pat<(wasm_shr_u (v2i64 V128:$lhs), (trunc (and I64:$rhs, 63))),
978          (SHR_U_I64x2 V128:$lhs, (I32_WRAP_I64 I64:$rhs))>;
979
980//===----------------------------------------------------------------------===//
981// Integer binary arithmetic
982//===----------------------------------------------------------------------===//
983
984multiclass SIMDBinaryIntNoI8x16<SDPatternOperator node, string name, bits<32> baseInst> {
985  defm "" : SIMDBinary<I16x8, node, name, !add(baseInst, 32)>;
986  defm "" : SIMDBinary<I32x4, node, name, !add(baseInst, 64)>;
987  defm "" : SIMDBinary<I64x2, node, name, !add(baseInst, 96)>;
988}
989
990multiclass SIMDBinaryIntSmall<SDPatternOperator node, string name, bits<32> baseInst> {
991  defm "" : SIMDBinary<I8x16, node, name, baseInst>;
992  defm "" : SIMDBinary<I16x8, node, name, !add(baseInst, 32)>;
993}
994
995multiclass SIMDBinaryIntNoI64x2<SDPatternOperator node, string name, bits<32> baseInst> {
996  defm "" : SIMDBinaryIntSmall<node, name, baseInst>;
997  defm "" : SIMDBinary<I32x4, node, name, !add(baseInst, 64)>;
998}
999
1000multiclass SIMDBinaryInt<SDPatternOperator node, string name, bits<32> baseInst> {
1001  defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>;
1002  defm "" : SIMDBinary<I64x2, node, name, !add(baseInst, 96)>;
1003}
1004
1005// Integer addition: add / add_sat_s / add_sat_u
1006let isCommutable = 1 in {
1007defm ADD : SIMDBinaryInt<add, "add", 110>;
1008defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_sat_s", 111>;
1009defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_sat_u", 112>;
1010} // isCommutable = 1
1011
1012// Integer subtraction: sub / sub_sat_s / sub_sat_u
1013defm SUB : SIMDBinaryInt<sub, "sub", 113>;
1014defm SUB_SAT_S :
1015  SIMDBinaryIntSmall<int_wasm_sub_sat_signed, "sub_sat_s", 114>;
1016defm SUB_SAT_U :
1017  SIMDBinaryIntSmall<int_wasm_sub_sat_unsigned, "sub_sat_u", 115>;
1018
1019// Integer multiplication: mul
1020let isCommutable = 1 in
1021defm MUL : SIMDBinaryIntNoI8x16<mul, "mul", 117>;
1022
1023// Integer min_s / min_u / max_s / max_u
1024let isCommutable = 1 in {
1025defm MIN_S : SIMDBinaryIntNoI64x2<smin, "min_s", 118>;
1026defm MIN_U : SIMDBinaryIntNoI64x2<umin, "min_u", 119>;
1027defm MAX_S : SIMDBinaryIntNoI64x2<smax, "max_s", 120>;
1028defm MAX_U : SIMDBinaryIntNoI64x2<umax, "max_u", 121>;
1029} // isCommutable = 1
1030
1031// Integer unsigned rounding average: avgr_u
1032let isCommutable = 1 in {
1033defm AVGR_U : SIMDBinary<I8x16, int_wasm_avgr_unsigned, "avgr_u", 123>;
1034defm AVGR_U : SIMDBinary<I16x8, int_wasm_avgr_unsigned, "avgr_u", 155>;
1035}
1036
1037def add_nuw : PatFrag<(ops node:$lhs, node:$rhs), (add $lhs, $rhs),
1038                      "return N->getFlags().hasNoUnsignedWrap();">;
1039
1040foreach vec = [I8x16, I16x8] in {
1041defvar inst = !cast<NI>("AVGR_U_"#vec);
1042def : Pat<(wasm_shr_u
1043            (add_nuw
1044              (add_nuw (vec.vt V128:$lhs), (vec.vt V128:$rhs)),
1045              (vec.splat (i32 1))),
1046            (i32 1)),
1047          (inst $lhs, $rhs)>;
1048}
1049
1050// Widening dot product: i32x4.dot_i16x8_s
1051let isCommutable = 1 in
1052defm DOT : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
1053                  [(set V128:$dst, (int_wasm_dot V128:$lhs, V128:$rhs))],
1054                  "i32x4.dot_i16x8_s\t$dst, $lhs, $rhs", "i32x4.dot_i16x8_s",
1055                  186>;
1056
1057// Extending multiplication: extmul_{low,high}_P, extmul_high
1058def extend_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1059def extend_low_s : SDNode<"WebAssemblyISD::EXTEND_LOW_S", extend_t>;
1060def extend_high_s : SDNode<"WebAssemblyISD::EXTEND_HIGH_S", extend_t>;
1061def extend_low_u : SDNode<"WebAssemblyISD::EXTEND_LOW_U", extend_t>;
1062def extend_high_u : SDNode<"WebAssemblyISD::EXTEND_HIGH_U", extend_t>;
1063
1064multiclass SIMDExtBinary<Vec vec, SDPatternOperator node, string name,
1065                         bits<32> simdop> {
1066  defm _#vec : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
1067                      (outs), (ins),
1068                      [(set (vec.vt V128:$dst), (node
1069                         (vec.split.vt V128:$lhs),(vec.split.vt V128:$rhs)))],
1070                      vec.prefix#"."#name#"\t$dst, $lhs, $rhs",
1071                      vec.prefix#"."#name, simdop>;
1072}
1073
1074class ExtMulPat<SDNode extend> :
1075  PatFrag<(ops node:$lhs, node:$rhs),
1076          (mul (extend $lhs), (extend $rhs))> {}
1077
1078def extmul_low_s : ExtMulPat<extend_low_s>;
1079def extmul_high_s : ExtMulPat<extend_high_s>;
1080def extmul_low_u : ExtMulPat<extend_low_u>;
1081def extmul_high_u : ExtMulPat<extend_high_u>;
1082
1083defm EXTMUL_LOW_S :
1084  SIMDExtBinary<I16x8, extmul_low_s, "extmul_low_i8x16_s", 0x9c>;
1085defm EXTMUL_HIGH_S :
1086  SIMDExtBinary<I16x8, extmul_high_s, "extmul_high_i8x16_s", 0x9d>;
1087defm EXTMUL_LOW_U :
1088  SIMDExtBinary<I16x8, extmul_low_u, "extmul_low_i8x16_u", 0x9e>;
1089defm EXTMUL_HIGH_U :
1090  SIMDExtBinary<I16x8, extmul_high_u, "extmul_high_i8x16_u", 0x9f>;
1091
1092defm EXTMUL_LOW_S :
1093  SIMDExtBinary<I32x4, extmul_low_s, "extmul_low_i16x8_s", 0xbc>;
1094defm EXTMUL_HIGH_S :
1095  SIMDExtBinary<I32x4, extmul_high_s, "extmul_high_i16x8_s", 0xbd>;
1096defm EXTMUL_LOW_U :
1097  SIMDExtBinary<I32x4, extmul_low_u, "extmul_low_i16x8_u", 0xbe>;
1098defm EXTMUL_HIGH_U :
1099  SIMDExtBinary<I32x4, extmul_high_u, "extmul_high_i16x8_u", 0xbf>;
1100
1101defm EXTMUL_LOW_S :
1102  SIMDExtBinary<I64x2, extmul_low_s, "extmul_low_i32x4_s", 0xdc>;
1103defm EXTMUL_HIGH_S :
1104  SIMDExtBinary<I64x2, extmul_high_s, "extmul_high_i32x4_s", 0xdd>;
1105defm EXTMUL_LOW_U :
1106  SIMDExtBinary<I64x2, extmul_low_u, "extmul_low_i32x4_u", 0xde>;
1107defm EXTMUL_HIGH_U :
1108  SIMDExtBinary<I64x2, extmul_high_u, "extmul_high_i32x4_u", 0xdf>;
1109
1110//===----------------------------------------------------------------------===//
1111// Floating-point unary arithmetic
1112//===----------------------------------------------------------------------===//
1113
1114multiclass SIMDUnaryFP<SDNode node, string name, bits<32> baseInst> {
1115  defm "" : SIMDUnary<F32x4, node, name, baseInst>;
1116  defm "" : SIMDUnary<F64x2, node, name, !add(baseInst, 12)>;
1117}
1118
1119// Absolute value: abs
1120defm ABS : SIMDUnaryFP<fabs, "abs", 224>;
1121
1122// Negation: neg
1123defm NEG : SIMDUnaryFP<fneg, "neg", 225>;
1124
1125// Square root: sqrt
1126defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 227>;
1127
1128// Rounding: ceil, floor, trunc, nearest
1129defm CEIL : SIMDUnary<F32x4, fceil, "ceil", 0x67>;
1130defm FLOOR : SIMDUnary<F32x4, ffloor, "floor", 0x68>;
1131defm TRUNC: SIMDUnary<F32x4, ftrunc, "trunc", 0x69>;
1132defm NEAREST: SIMDUnary<F32x4, fnearbyint, "nearest", 0x6a>;
1133defm CEIL : SIMDUnary<F64x2, fceil, "ceil", 0x74>;
1134defm FLOOR : SIMDUnary<F64x2, ffloor, "floor", 0x75>;
1135defm TRUNC: SIMDUnary<F64x2, ftrunc, "trunc", 0x7a>;
1136defm NEAREST: SIMDUnary<F64x2, fnearbyint, "nearest", 0x94>;
1137
1138//===----------------------------------------------------------------------===//
1139// Floating-point binary arithmetic
1140//===----------------------------------------------------------------------===//
1141
1142multiclass SIMDBinaryFP<SDPatternOperator node, string name, bits<32> baseInst> {
1143  defm "" : SIMDBinary<F32x4, node, name, baseInst>;
1144  defm "" : SIMDBinary<F64x2, node, name, !add(baseInst, 12)>;
1145}
1146
1147// Addition: add
1148let isCommutable = 1 in
1149defm ADD : SIMDBinaryFP<fadd, "add", 228>;
1150
1151// Subtraction: sub
1152defm SUB : SIMDBinaryFP<fsub, "sub", 229>;
1153
1154// Multiplication: mul
1155let isCommutable = 1 in
1156defm MUL : SIMDBinaryFP<fmul, "mul", 230>;
1157
1158// Division: div
1159defm DIV : SIMDBinaryFP<fdiv, "div", 231>;
1160
1161// NaN-propagating minimum: min
1162defm MIN : SIMDBinaryFP<fminimum, "min", 232>;
1163
1164// NaN-propagating maximum: max
1165defm MAX : SIMDBinaryFP<fmaximum, "max", 233>;
1166
1167// Pseudo-minimum: pmin
1168def pmin : PatFrag<(ops node:$lhs, node:$rhs),
1169                   (vselect (setolt $rhs, $lhs), $rhs, $lhs)>;
1170defm PMIN : SIMDBinaryFP<pmin, "pmin", 234>;
1171
1172// Pseudo-maximum: pmax
1173def pmax : PatFrag<(ops node:$lhs, node:$rhs),
1174                   (vselect (setolt $lhs, $rhs), $rhs, $lhs)>;
1175defm PMAX : SIMDBinaryFP<pmax, "pmax", 235>;
1176
1177// Also match the pmin/pmax cases where the operands are int vectors (but the
1178// comparison is still a floating point comparison). This can happen when using
1179// the wasm_simd128.h intrinsics because v128_t is an integer vector.
1180foreach vec = [F32x4, F64x2] in {
1181defvar pmin = !cast<NI>("PMIN_"#vec);
1182defvar pmax = !cast<NI>("PMAX_"#vec);
1183def : Pat<(vec.int_vt (vselect
1184            (setolt (vec.vt (bitconvert V128:$rhs)),
1185                    (vec.vt (bitconvert V128:$lhs))),
1186            V128:$rhs, V128:$lhs)),
1187          (pmin $lhs, $rhs)>;
1188def : Pat<(vec.int_vt (vselect
1189            (setolt (vec.vt (bitconvert V128:$lhs)),
1190                    (vec.vt (bitconvert V128:$rhs))),
1191            V128:$rhs, V128:$lhs)),
1192          (pmax $lhs, $rhs)>;
1193}
1194
1195// And match the pmin/pmax LLVM intrinsics as well
1196def : Pat<(v4f32 (int_wasm_pmin (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
1197          (PMIN_F32x4 V128:$lhs, V128:$rhs)>;
1198def : Pat<(v4f32 (int_wasm_pmax (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
1199          (PMAX_F32x4 V128:$lhs, V128:$rhs)>;
1200def : Pat<(v2f64 (int_wasm_pmin (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
1201          (PMIN_F64x2 V128:$lhs, V128:$rhs)>;
1202def : Pat<(v2f64 (int_wasm_pmax (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
1203          (PMAX_F64x2 V128:$lhs, V128:$rhs)>;
1204
1205//===----------------------------------------------------------------------===//
1206// Conversions
1207//===----------------------------------------------------------------------===//
1208
1209multiclass SIMDConvert<Vec vec, Vec arg, SDPatternOperator op, string name,
1210                       bits<32> simdop> {
1211  defm op#_#vec :
1212    SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
1213           [(set (vec.vt V128:$dst), (vec.vt (op (arg.vt V128:$vec))))],
1214           vec.prefix#"."#name#"\t$dst, $vec", vec.prefix#"."#name, simdop>;
1215}
1216
1217// Floating point to integer with saturation: trunc_sat
1218defm "" : SIMDConvert<I32x4, F32x4, fp_to_sint, "trunc_sat_f32x4_s", 248>;
1219defm "" : SIMDConvert<I32x4, F32x4, fp_to_uint, "trunc_sat_f32x4_u", 249>;
1220
1221// Support the saturating variety as well.
1222def trunc_s_sat32 : PatFrag<(ops node:$x), (fp_to_sint_sat $x, i32)>;
1223def trunc_u_sat32 : PatFrag<(ops node:$x), (fp_to_uint_sat $x, i32)>;
1224def : Pat<(v4i32 (trunc_s_sat32 (v4f32 V128:$src))), (fp_to_sint_I32x4 $src)>;
1225def : Pat<(v4i32 (trunc_u_sat32 (v4f32 V128:$src))), (fp_to_uint_I32x4 $src)>;
1226
1227def trunc_sat_zero_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1228def trunc_sat_zero_s :
1229  SDNode<"WebAssemblyISD::TRUNC_SAT_ZERO_S", trunc_sat_zero_t>;
1230def trunc_sat_zero_u :
1231  SDNode<"WebAssemblyISD::TRUNC_SAT_ZERO_U", trunc_sat_zero_t>;
1232defm "" : SIMDConvert<I32x4, F64x2, trunc_sat_zero_s, "trunc_sat_zero_f64x2_s",
1233                      0xfc>;
1234defm "" : SIMDConvert<I32x4, F64x2, trunc_sat_zero_u, "trunc_sat_zero_f64x2_u",
1235                      0xfd>;
1236
1237// Integer to floating point: convert
1238def convert_low_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1239def convert_low_s : SDNode<"WebAssemblyISD::CONVERT_LOW_S", convert_low_t>;
1240def convert_low_u : SDNode<"WebAssemblyISD::CONVERT_LOW_U", convert_low_t>;
1241defm "" : SIMDConvert<F32x4, I32x4, sint_to_fp, "convert_i32x4_s", 250>;
1242defm "" : SIMDConvert<F32x4, I32x4, uint_to_fp, "convert_i32x4_u", 251>;
1243defm "" : SIMDConvert<F64x2, I32x4, convert_low_s, "convert_low_i32x4_s", 0xfe>;
1244defm "" : SIMDConvert<F64x2, I32x4, convert_low_u, "convert_low_i32x4_u", 0xff>;
1245
1246// Extending operations
1247// TODO: refactor this to be uniform for i64x2 if the numbering is not changed.
1248multiclass SIMDExtend<Vec vec, bits<32> baseInst> {
1249  defm "" : SIMDConvert<vec, vec.split, extend_low_s,
1250                        "extend_low_"#vec.split.prefix#"_s", baseInst>;
1251  defm "" : SIMDConvert<vec, vec.split, extend_high_s,
1252                        "extend_high_"#vec.split.prefix#"_s", !add(baseInst, 1)>;
1253  defm "" : SIMDConvert<vec, vec.split, extend_low_u,
1254                        "extend_low_"#vec.split.prefix#"_u", !add(baseInst, 2)>;
1255  defm "" : SIMDConvert<vec, vec.split, extend_high_u,
1256                        "extend_high_"#vec.split.prefix#"_u", !add(baseInst, 3)>;
1257}
1258
1259defm "" : SIMDExtend<I16x8, 0x87>;
1260defm "" : SIMDExtend<I32x4, 0xa7>;
1261defm "" : SIMDExtend<I64x2, 0xc7>;
1262
1263// Narrowing operations
1264multiclass SIMDNarrow<Vec vec, bits<32> baseInst> {
1265  defvar name = vec.split.prefix#".narrow_"#vec.prefix;
1266  defm NARROW_S_#vec.split :
1267    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
1268           [(set (vec.split.vt V128:$dst), (vec.split.vt (int_wasm_narrow_signed
1269             (vec.vt V128:$low), (vec.vt V128:$high))))],
1270           name#"_s\t$dst, $low, $high", name#"_s", baseInst>;
1271  defm NARROW_U_#vec.split :
1272    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
1273           [(set (vec.split.vt V128:$dst), (vec.split.vt (int_wasm_narrow_unsigned
1274             (vec.vt V128:$low), (vec.vt V128:$high))))],
1275           name#"_u\t$dst, $low, $high", name#"_u", !add(baseInst, 1)>;
1276}
1277
1278defm "" : SIMDNarrow<I16x8, 101>;
1279defm "" : SIMDNarrow<I32x4, 133>;
1280
1281// WebAssemblyISD::NARROW_U
1282def wasm_narrow_t : SDTypeProfile<1, 2, []>;
1283def wasm_narrow_u : SDNode<"WebAssemblyISD::NARROW_U", wasm_narrow_t>;
1284def : Pat<(v16i8 (wasm_narrow_u (v8i16 V128:$left), (v8i16 V128:$right))),
1285          (NARROW_U_I8x16 $left, $right)>;
1286def : Pat<(v8i16 (wasm_narrow_u (v4i32 V128:$left), (v4i32 V128:$right))),
1287          (NARROW_U_I16x8 $left, $right)>;
1288
1289// Bitcasts are nops
1290// Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
1291foreach t1 = AllVecs in
1292foreach t2 = AllVecs in
1293if !ne(t1, t2) then
1294def : Pat<(t1.vt (bitconvert (t2.vt V128:$v))), (t1.vt V128:$v)>;
1295
1296// Extended pairwise addition
1297defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_signed,
1298                      "extadd_pairwise_i8x16_s", 0x7c>;
1299defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_unsigned,
1300                      "extadd_pairwise_i8x16_u", 0x7d>;
1301defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_signed,
1302                      "extadd_pairwise_i16x8_s", 0x7e>;
1303defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_unsigned,
1304                      "extadd_pairwise_i16x8_u", 0x7f>;
1305
1306// f64x2 <-> f32x4 conversions
1307def demote_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1308def demote_zero : SDNode<"WebAssemblyISD::DEMOTE_ZERO", demote_t>;
1309defm "" : SIMDConvert<F32x4, F64x2, demote_zero,
1310                      "demote_zero_f64x2", 0x5e>;
1311
1312def promote_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
1313def promote_low : SDNode<"WebAssemblyISD::PROMOTE_LOW", promote_t>;
1314defm "" : SIMDConvert<F64x2, F32x4, promote_low, "promote_low_f32x4", 0x5f>;
1315
1316// Lower extending loads to load64_zero + promote_low
1317def extloadv2f32 : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
1318  let MemoryVT = v2f32;
1319}
1320// Adapted from the body of LoadPatNoOffset
1321// TODO: other addressing patterns
1322def : Pat<(v2f64 (extloadv2f32 (i32 I32:$addr))),
1323          (promote_low_F64x2 (LOAD_ZERO_I64x2_A32 0, 0, I32:$addr))>,
1324      Requires<[HasAddr32]>;
1325def : Pat<(v2f64 (extloadv2f32 (i64 I64:$addr))),
1326          (promote_low_F64x2 (LOAD_ZERO_I64x2_A64 0, 0, I64:$addr))>,
1327      Requires<[HasAddr64]>;
1328
1329//===----------------------------------------------------------------------===//
1330// Saturating Rounding Q-Format Multiplication
1331//===----------------------------------------------------------------------===//
1332
1333defm Q15MULR_SAT_S :
1334  SIMDBinary<I16x8, int_wasm_q15mulr_sat_signed, "q15mulr_sat_s", 0x82>;
1335
1336//===----------------------------------------------------------------------===//
1337// Fused Multiply- Add and Subtract (FMA/FMS)
1338//===----------------------------------------------------------------------===//
1339
1340multiclass SIMDFM<Vec vec, bits<32> simdopA, bits<32> simdopS> {
1341  defm FMA_#vec :
1342    RELAXED_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c), (outs), (ins),
1343              [(set (vec.vt V128:$dst), (int_wasm_fma
1344                (vec.vt V128:$a), (vec.vt V128:$b), (vec.vt V128:$c)))],
1345              vec.prefix#".fma\t$dst, $a, $b, $c", vec.prefix#".fma", simdopA>;
1346  defm FMS_#vec :
1347    RELAXED_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c), (outs), (ins),
1348              [(set (vec.vt V128:$dst), (int_wasm_fms
1349                (vec.vt V128:$a), (vec.vt V128:$b), (vec.vt V128:$c)))],
1350              vec.prefix#".fms\t$dst, $a, $b, $c", vec.prefix#".fms", simdopS>;
1351}
1352
1353defm "" : SIMDFM<F32x4, 0xaf, 0xb0>;
1354defm "" : SIMDFM<F64x2, 0xcf, 0xd0>;
1355
1356//===----------------------------------------------------------------------===//
1357// Laneselect
1358//===----------------------------------------------------------------------===//
1359
1360multiclass SIMDLANESELECT<Vec vec, bits<32> op> {
1361  defm LANESELECT_#vec :
1362    RELAXED_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c), (outs), (ins),
1363              [(set (vec.vt V128:$dst), (int_wasm_laneselect
1364                (vec.vt V128:$a), (vec.vt V128:$b), (vec.vt V128:$c)))],
1365              vec.prefix#".laneselect\t$dst, $a, $b, $c", vec.prefix#".laneselect", op>;
1366}
1367
1368defm "" : SIMDLANESELECT<I8x16, 0xb2>;
1369defm "" : SIMDLANESELECT<I16x8, 0xb3>;
1370defm "" : SIMDLANESELECT<I32x4, 0xd2>;
1371defm "" : SIMDLANESELECT<I64x2, 0xd3>;
1372
1373
1374//===----------------------------------------------------------------------===//
1375// Relaxed swizzle
1376//===----------------------------------------------------------------------===//
1377
1378defm RELAXED_SWIZZLE :
1379  RELAXED_I<(outs V128:$dst), (ins V128:$src, V128:$mask), (outs), (ins),
1380         [(set (v16i8 V128:$dst),
1381           (int_wasm_relaxed_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)))],
1382         "i8x16.relaxed_swizzle\t$dst, $src, $mask", "i8x16.relaxed_swizzle", 162>;
1383
1384//===----------------------------------------------------------------------===//
1385// Relaxed floating-point min and max.
1386//===----------------------------------------------------------------------===//
1387
1388multiclass SIMD_RELAXED_FMINMAX<Vec vec, bits<32> simdopMin, bits<32> simdopMax> {
1389  defm RELAXED_FMIN_#vec :
1390    RELAXED_I<(outs V128:$dst), (ins V128:$a, V128:$b), (outs), (ins),
1391              [(set (vec.vt V128:$dst), (int_wasm_relaxed_min
1392                (vec.vt V128:$a), (vec.vt V128:$b)))],
1393              vec.prefix#".relaxed_min\t$dst, $a, $b", vec.prefix#".relaxed_min", simdopMin>;
1394  defm RELAXED_FMAX_#vec :
1395    RELAXED_I<(outs V128:$dst), (ins V128:$a, V128:$b), (outs), (ins),
1396              [(set (vec.vt V128:$dst), (int_wasm_relaxed_max
1397                (vec.vt V128:$a), (vec.vt V128:$b)))],
1398              vec.prefix#".relaxed_max\t$dst, $a, $b", vec.prefix#".relaxed_max", simdopMax>;
1399}
1400
1401defm "" : SIMD_RELAXED_FMINMAX<F32x4, 0xb4, 0xe2>;
1402defm "" : SIMD_RELAXED_FMINMAX<F64x2, 0xd4, 0xee>;
1403
1404//===----------------------------------------------------------------------===//
1405// Relaxed floating-point to int conversions
1406//===----------------------------------------------------------------------===//
1407
1408multiclass SIMD_RELAXED_CONVERT<Vec vec, Vec arg, SDPatternOperator op, string name, bits<32> simdop> {
1409  defm op#_#vec :
1410    RELAXED_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
1411              [(set (vec.vt V128:$dst), (vec.vt (op (arg.vt V128:$vec))))],
1412              vec.prefix#"."#name#"\t$dst, $vec", vec.prefix#"."#name, simdop>;
1413}
1414
1415defm "" : SIMD_RELAXED_CONVERT<I32x4, F32x4, int_wasm_relaxed_trunc_signed, "relaxed_trunc_f32x4_s", 0xa5>;
1416defm "" : SIMD_RELAXED_CONVERT<I32x4, F32x4, int_wasm_relaxed_trunc_unsigned, "relaxed_trunc_f32x4_u", 0xa6>;
1417
1418defm "" : SIMD_RELAXED_CONVERT<I32x4, F64x2, int_wasm_relaxed_trunc_zero_signed, "relaxed_trunc_f64x2_s_zero", 0xc5>;
1419defm "" : SIMD_RELAXED_CONVERT<I32x4, F64x2, int_wasm_relaxed_trunc_zero_unsigned, "relaxed_trunc_f64x2_u_zero", 0xc6>;
1420