xref: /freebsd/contrib/llvm-project/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td (revision b4e38a41f584ad4391c04b8cfec81f46176b18b0)
1// WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// WebAssembly SIMD operand code-gen constructs.
11///
12//===----------------------------------------------------------------------===//
13
14// Instructions requiring HasSIMD128 and the simd128 prefix byte
15multiclass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
16                  list<dag> pattern_r, string asmstr_r = "",
17                  string asmstr_s = "", bits<32> simdop = -1> {
18  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
19              !or(0xfd00, !and(0xff, simdop))>,
20            Requires<[HasSIMD128]>;
21}
22
23defm "" : ARGUMENT<V128, v16i8>;
24defm "" : ARGUMENT<V128, v8i16>;
25defm "" : ARGUMENT<V128, v4i32>;
26defm "" : ARGUMENT<V128, v2i64>;
27defm "" : ARGUMENT<V128, v4f32>;
28defm "" : ARGUMENT<V128, v2f64>;
29
30// Constrained immediate argument types
31foreach SIZE = [8, 16] in
32def ImmI#SIZE : ImmLeaf<i32,
33  "return -(1 << ("#SIZE#" - 1)) <= Imm && Imm < (1 << ("#SIZE#" - 1));"
34>;
35foreach SIZE = [2, 4, 8, 16, 32] in
36def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
37
38//===----------------------------------------------------------------------===//
39// Load and store
40//===----------------------------------------------------------------------===//
41
42// Load: v128.load
43let mayLoad = 1, UseNamedOperandTable = 1 in
44defm LOAD_V128 :
45  SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
46         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
47         "v128.load\t$dst, ${off}(${addr})$p2align",
48         "v128.load\t$off$p2align", 0>;
49
50// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
51foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
52def : LoadPatNoOffset<vec_t, load, LOAD_V128>;
53def : LoadPatImmOff<vec_t, load, regPlusImm, LOAD_V128>;
54def : LoadPatImmOff<vec_t, load, or_is_add, LOAD_V128>;
55def : LoadPatOffsetOnly<vec_t, load, LOAD_V128>;
56def : LoadPatGlobalAddrOffOnly<vec_t, load, LOAD_V128>;
57}
58
59// vNxM.load_splat
60multiclass SIMDLoadSplat<string vec, bits<32> simdop> {
61  let mayLoad = 1, UseNamedOperandTable = 1,
62      Predicates = [HasUnimplementedSIMD128] in
63  defm LOAD_SPLAT_#vec :
64    SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
65           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
66           vec#".load_splat\t$dst, ${off}(${addr})$p2align",
67           vec#".load_splat\t$off$p2align", simdop>;
68}
69
70defm "" : SIMDLoadSplat<"v8x16", 194>;
71defm "" : SIMDLoadSplat<"v16x8", 195>;
72defm "" : SIMDLoadSplat<"v32x4", 196>;
73defm "" : SIMDLoadSplat<"v64x2", 197>;
74
75def wasm_load_splat_t : SDTypeProfile<1, 1, [SDTCisPtrTy<1>]>;
76def wasm_load_splat : SDNode<"WebAssemblyISD::LOAD_SPLAT", wasm_load_splat_t,
77                             [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
78def load_splat : PatFrag<(ops node:$addr), (wasm_load_splat node:$addr)>;
79
80let Predicates = [HasUnimplementedSIMD128] in
81foreach args = [["v16i8", "v8x16"], ["v8i16", "v16x8"], ["v4i32", "v32x4"],
82                ["v2i64", "v64x2"], ["v4f32", "v32x4"], ["v2f64", "v64x2"]] in {
83def : LoadPatNoOffset<!cast<ValueType>(args[0]),
84                      load_splat,
85                      !cast<NI>("LOAD_SPLAT_"#args[1])>;
86def : LoadPatImmOff<!cast<ValueType>(args[0]),
87                    load_splat,
88                    regPlusImm,
89                    !cast<NI>("LOAD_SPLAT_"#args[1])>;
90def : LoadPatImmOff<!cast<ValueType>(args[0]),
91                    load_splat,
92                    or_is_add,
93                    !cast<NI>("LOAD_SPLAT_"#args[1])>;
94def : LoadPatOffsetOnly<!cast<ValueType>(args[0]),
95                        load_splat,
96                        !cast<NI>("LOAD_SPLAT_"#args[1])>;
97def : LoadPatGlobalAddrOffOnly<!cast<ValueType>(args[0]),
98                               load_splat,
99                               !cast<NI>("LOAD_SPLAT_"#args[1])>;
100}
101
102// Load and extend
103multiclass SIMDLoadExtend<ValueType vec_t, string name, bits<32> simdop> {
104  let mayLoad = 1, UseNamedOperandTable = 1,
105      Predicates = [HasUnimplementedSIMD128] in {
106  defm LOAD_EXTEND_S_#vec_t :
107    SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
108           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
109           name#"_s\t$dst, ${off}(${addr})$p2align",
110           name#"_s\t$off$p2align", simdop>;
111  defm LOAD_EXTEND_U_#vec_t :
112    SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
113           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
114           name#"_u\t$dst, ${off}(${addr})$p2align",
115           name#"_u\t$off$p2align", !add(simdop, 1)>;
116  }
117}
118
119defm "" : SIMDLoadExtend<v8i16, "i16x8.load8x8", 210>;
120defm "" : SIMDLoadExtend<v4i32, "i32x4.load16x4", 212>;
121defm "" : SIMDLoadExtend<v2i64, "i64x2.load32x2", 214>;
122
123let Predicates = [HasUnimplementedSIMD128] in
124foreach types = [[v8i16, i8], [v4i32, i16], [v2i64, i32]] in
125foreach exts = [["sextloadv", "_S"],
126                ["zextloadv", "_U"],
127                ["extloadv", "_U"]] in {
128def : LoadPatNoOffset<types[0], !cast<PatFrag>(exts[0]#types[1]),
129                      !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
130def : LoadPatImmOff<types[0], !cast<PatFrag>(exts[0]#types[1]), regPlusImm,
131                    !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
132def : LoadPatImmOff<types[0], !cast<PatFrag>(exts[0]#types[1]), or_is_add,
133                    !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
134def : LoadPatOffsetOnly<types[0], !cast<PatFrag>(exts[0]#types[1]),
135                        !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
136def : LoadPatGlobalAddrOffOnly<types[0], !cast<PatFrag>(exts[0]#types[1]),
137                               !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
138}
139
140
141// Store: v128.store
142let mayStore = 1, UseNamedOperandTable = 1 in
143defm STORE_V128 :
144  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, V128:$vec),
145         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
146         "v128.store\t${off}(${addr})$p2align, $vec",
147         "v128.store\t$off$p2align", 1>;
148
149foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
150// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
151def : StorePatNoOffset<vec_t, store, STORE_V128>;
152def : StorePatImmOff<vec_t, store, regPlusImm, STORE_V128>;
153def : StorePatImmOff<vec_t, store, or_is_add, STORE_V128>;
154def : StorePatOffsetOnly<vec_t, store, STORE_V128>;
155def : StorePatGlobalAddrOffOnly<vec_t, store, STORE_V128>;
156}
157
158//===----------------------------------------------------------------------===//
159// Constructing SIMD values
160//===----------------------------------------------------------------------===//
161
162// Constant: v128.const
163multiclass ConstVec<ValueType vec_t, dag ops, dag pat, string args> {
164  let isMoveImm = 1, isReMaterializable = 1,
165      Predicates = [HasUnimplementedSIMD128] in
166  defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops,
167                                  [(set V128:$dst, (vec_t pat))],
168                                  "v128.const\t$dst, "#args,
169                                  "v128.const\t"#args, 2>;
170}
171
172defm "" : ConstVec<v16i8,
173                   (ins vec_i8imm_op:$i0, vec_i8imm_op:$i1,
174                        vec_i8imm_op:$i2, vec_i8imm_op:$i3,
175                        vec_i8imm_op:$i4, vec_i8imm_op:$i5,
176                        vec_i8imm_op:$i6, vec_i8imm_op:$i7,
177                        vec_i8imm_op:$i8, vec_i8imm_op:$i9,
178                        vec_i8imm_op:$iA, vec_i8imm_op:$iB,
179                        vec_i8imm_op:$iC, vec_i8imm_op:$iD,
180                        vec_i8imm_op:$iE, vec_i8imm_op:$iF),
181                   (build_vector ImmI8:$i0, ImmI8:$i1, ImmI8:$i2, ImmI8:$i3,
182                                 ImmI8:$i4, ImmI8:$i5, ImmI8:$i6, ImmI8:$i7,
183                                 ImmI8:$i8, ImmI8:$i9, ImmI8:$iA, ImmI8:$iB,
184                                 ImmI8:$iC, ImmI8:$iD, ImmI8:$iE, ImmI8:$iF),
185                   !strconcat("$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7, ",
186                              "$i8, $i9, $iA, $iB, $iC, $iD, $iE, $iF")>;
187defm "" : ConstVec<v8i16,
188                   (ins vec_i16imm_op:$i0, vec_i16imm_op:$i1,
189                        vec_i16imm_op:$i2, vec_i16imm_op:$i3,
190                        vec_i16imm_op:$i4, vec_i16imm_op:$i5,
191                        vec_i16imm_op:$i6, vec_i16imm_op:$i7),
192                   (build_vector
193                     ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3,
194                     ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7),
195                   "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">;
196let IsCanonical = 1 in
197defm "" : ConstVec<v4i32,
198                   (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1,
199                        vec_i32imm_op:$i2, vec_i32imm_op:$i3),
200                   (build_vector (i32 imm:$i0), (i32 imm:$i1),
201                                 (i32 imm:$i2), (i32 imm:$i3)),
202                   "$i0, $i1, $i2, $i3">;
203defm "" : ConstVec<v2i64,
204                   (ins vec_i64imm_op:$i0, vec_i64imm_op:$i1),
205                   (build_vector (i64 imm:$i0), (i64 imm:$i1)),
206                   "$i0, $i1">;
207defm "" : ConstVec<v4f32,
208                   (ins f32imm_op:$i0, f32imm_op:$i1,
209                        f32imm_op:$i2, f32imm_op:$i3),
210                   (build_vector (f32 fpimm:$i0), (f32 fpimm:$i1),
211                                 (f32 fpimm:$i2), (f32 fpimm:$i3)),
212                   "$i0, $i1, $i2, $i3">;
213defm "" : ConstVec<v2f64,
214                  (ins f64imm_op:$i0, f64imm_op:$i1),
215                  (build_vector (f64 fpimm:$i0), (f64 fpimm:$i1)),
216                  "$i0, $i1">;
217
218// Shuffle lanes: shuffle
219defm SHUFFLE :
220  SIMD_I<(outs V128:$dst),
221         (ins V128:$x, V128:$y,
222           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
223           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
224           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
225           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
226           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
227           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
228           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
229           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
230         (outs),
231         (ins
232           vec_i8imm_op:$m0, vec_i8imm_op:$m1,
233           vec_i8imm_op:$m2, vec_i8imm_op:$m3,
234           vec_i8imm_op:$m4, vec_i8imm_op:$m5,
235           vec_i8imm_op:$m6, vec_i8imm_op:$m7,
236           vec_i8imm_op:$m8, vec_i8imm_op:$m9,
237           vec_i8imm_op:$mA, vec_i8imm_op:$mB,
238           vec_i8imm_op:$mC, vec_i8imm_op:$mD,
239           vec_i8imm_op:$mE, vec_i8imm_op:$mF),
240         [],
241         "v8x16.shuffle\t$dst, $x, $y, "#
242           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
243           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
244         "v8x16.shuffle\t"#
245           "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "#
246           "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF",
247         3>;
248
249// Shuffles after custom lowering
250def wasm_shuffle_t : SDTypeProfile<1, 18, []>;
251def wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>;
252foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
253def : Pat<(vec_t (wasm_shuffle (vec_t V128:$x), (vec_t V128:$y),
254            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
255            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
256            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
257            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
258            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
259            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
260            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
261            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF))),
262          (vec_t (SHUFFLE (vec_t V128:$x), (vec_t V128:$y),
263            (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1),
264            (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3),
265            (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5),
266            (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7),
267            (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9),
268            (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB),
269            (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD),
270            (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF)))>;
271}
272
273// Swizzle lanes: v8x16.swizzle
274def wasm_swizzle_t : SDTypeProfile<1, 2, []>;
275def wasm_swizzle : SDNode<"WebAssemblyISD::SWIZZLE", wasm_swizzle_t>;
276let Predicates = [HasUnimplementedSIMD128] in
277defm SWIZZLE :
278  SIMD_I<(outs V128:$dst), (ins V128:$src, V128:$mask), (outs), (ins),
279         [(set (v16i8 V128:$dst),
280           (wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)))],
281         "v8x16.swizzle\t$dst, $src, $mask", "v8x16.swizzle", 192>;
282
283def : Pat<(int_wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)),
284          (SWIZZLE V128:$src, V128:$mask)>;
285
286// Create vector with identical lanes: splat
287def splat2 : PatFrag<(ops node:$x), (build_vector node:$x, node:$x)>;
288def splat4 : PatFrag<(ops node:$x), (build_vector
289                       node:$x, node:$x, node:$x, node:$x)>;
290def splat8 : PatFrag<(ops node:$x), (build_vector
291                       node:$x, node:$x, node:$x, node:$x,
292                       node:$x, node:$x, node:$x, node:$x)>;
293def splat16 : PatFrag<(ops node:$x), (build_vector
294                        node:$x, node:$x, node:$x, node:$x,
295                        node:$x, node:$x, node:$x, node:$x,
296                        node:$x, node:$x, node:$x, node:$x,
297                        node:$x, node:$x, node:$x, node:$x)>;
298
299multiclass Splat<ValueType vec_t, string vec, WebAssemblyRegClass reg_t,
300                 PatFrag splat_pat, bits<32> simdop> {
301  // Prefer splats over v128.const for const splats (65 is lowest that works)
302  let AddedComplexity = 65 in
303  defm SPLAT_#vec_t : SIMD_I<(outs V128:$dst), (ins reg_t:$x), (outs), (ins),
304                             [(set (vec_t V128:$dst), (splat_pat reg_t:$x))],
305                             vec#".splat\t$dst, $x", vec#".splat", simdop>;
306}
307
308defm "" : Splat<v16i8, "i8x16", I32, splat16, 4>;
309defm "" : Splat<v8i16, "i16x8", I32, splat8, 8>;
310defm "" : Splat<v4i32, "i32x4", I32, splat4, 12>;
311defm "" : Splat<v2i64, "i64x2", I64, splat2, 15>;
312defm "" : Splat<v4f32, "f32x4", F32, splat4, 18>;
313defm "" : Splat<v2f64, "f64x2", F64, splat2, 21>;
314
315// scalar_to_vector leaves high lanes undefined, so can be a splat
316class ScalarSplatPat<ValueType vec_t, ValueType lane_t,
317                     WebAssemblyRegClass reg_t> :
318  Pat<(vec_t (scalar_to_vector (lane_t reg_t:$x))),
319      (!cast<Instruction>("SPLAT_"#vec_t) reg_t:$x)>;
320
321def : ScalarSplatPat<v16i8, i32, I32>;
322def : ScalarSplatPat<v8i16, i32, I32>;
323def : ScalarSplatPat<v4i32, i32, I32>;
324def : ScalarSplatPat<v2i64, i64, I64>;
325def : ScalarSplatPat<v4f32, f32, F32>;
326def : ScalarSplatPat<v2f64, f64, F64>;
327
328//===----------------------------------------------------------------------===//
329// Accessing lanes
330//===----------------------------------------------------------------------===//
331
332// Extract lane as a scalar: extract_lane / extract_lane_s / extract_lane_u
333multiclass ExtractLane<ValueType vec_t, string vec, ImmLeaf imm_t,
334                       WebAssemblyRegClass reg_t, bits<32> simdop,
335                       string suffix = "", SDNode extract = vector_extract> {
336  defm EXTRACT_LANE_#vec_t#suffix :
337      SIMD_I<(outs reg_t:$dst), (ins V128:$vec, vec_i8imm_op:$idx),
338             (outs), (ins vec_i8imm_op:$idx),
339             [(set reg_t:$dst, (extract (vec_t V128:$vec), (i32 imm_t:$idx)))],
340             vec#".extract_lane"#suffix#"\t$dst, $vec, $idx",
341             vec#".extract_lane"#suffix#"\t$idx", simdop>;
342}
343
344multiclass ExtractPat<ValueType lane_t, int mask> {
345  def _s : PatFrag<(ops node:$vec, node:$idx),
346                   (i32 (sext_inreg
347                     (i32 (vector_extract
348                       node:$vec,
349                       node:$idx
350                     )),
351                     lane_t
352                   ))>;
353  def _u : PatFrag<(ops node:$vec, node:$idx),
354                   (i32 (and
355                     (i32 (vector_extract
356                       node:$vec,
357                       node:$idx
358                     )),
359                     (i32 mask)
360                   ))>;
361}
362
363defm extract_i8x16 : ExtractPat<i8, 0xff>;
364defm extract_i16x8 : ExtractPat<i16, 0xffff>;
365
366multiclass ExtractLaneExtended<string sign, bits<32> baseInst> {
367  defm "" : ExtractLane<v16i8, "i8x16", LaneIdx16, I32, baseInst, sign,
368                        !cast<PatFrag>("extract_i8x16"#sign)>;
369  defm "" : ExtractLane<v8i16, "i16x8", LaneIdx8, I32, !add(baseInst, 4), sign,
370                        !cast<PatFrag>("extract_i16x8"#sign)>;
371}
372
373defm "" : ExtractLaneExtended<"_s", 5>;
374let Predicates = [HasUnimplementedSIMD128] in
375defm "" : ExtractLaneExtended<"_u", 6>;
376defm "" : ExtractLane<v4i32, "i32x4", LaneIdx4, I32, 13>;
377defm "" : ExtractLane<v2i64, "i64x2", LaneIdx2, I64, 16>;
378defm "" : ExtractLane<v4f32, "f32x4", LaneIdx4, F32, 19>;
379defm "" : ExtractLane<v2f64, "f64x2", LaneIdx2, F64, 22>;
380
381// It would be more conventional to use unsigned extracts, but v8
382// doesn't implement them yet
383def : Pat<(i32 (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx))),
384          (EXTRACT_LANE_v16i8_s V128:$vec, (i32 LaneIdx16:$idx))>;
385def : Pat<(i32 (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx))),
386          (EXTRACT_LANE_v8i16_s V128:$vec, (i32 LaneIdx8:$idx))>;
387
388// Lower undef lane indices to zero
389def : Pat<(and (i32 (vector_extract (v16i8 V128:$vec), undef)), (i32 0xff)),
390          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
391def : Pat<(and (i32 (vector_extract (v8i16 V128:$vec), undef)), (i32 0xffff)),
392          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
393def : Pat<(i32 (vector_extract (v16i8 V128:$vec), undef)),
394          (EXTRACT_LANE_v16i8_u V128:$vec, 0)>;
395def : Pat<(i32 (vector_extract (v8i16 V128:$vec), undef)),
396          (EXTRACT_LANE_v8i16_u V128:$vec, 0)>;
397def : Pat<(sext_inreg (i32 (vector_extract (v16i8 V128:$vec), undef)), i8),
398          (EXTRACT_LANE_v16i8_s V128:$vec, 0)>;
399def : Pat<(sext_inreg (i32 (vector_extract (v8i16 V128:$vec), undef)), i16),
400          (EXTRACT_LANE_v8i16_s V128:$vec, 0)>;
401def : Pat<(vector_extract (v4i32 V128:$vec), undef),
402          (EXTRACT_LANE_v4i32 V128:$vec, 0)>;
403def : Pat<(vector_extract (v2i64 V128:$vec), undef),
404          (EXTRACT_LANE_v2i64 V128:$vec, 0)>;
405def : Pat<(vector_extract (v4f32 V128:$vec), undef),
406          (EXTRACT_LANE_v4f32 V128:$vec, 0)>;
407def : Pat<(vector_extract (v2f64 V128:$vec), undef),
408          (EXTRACT_LANE_v2f64 V128:$vec, 0)>;
409
410// Replace lane value: replace_lane
411multiclass ReplaceLane<ValueType vec_t, string vec, ImmLeaf imm_t,
412                       WebAssemblyRegClass reg_t, ValueType lane_t,
413                       bits<32> simdop> {
414  defm REPLACE_LANE_#vec_t :
415      SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, reg_t:$x),
416             (outs), (ins vec_i8imm_op:$idx),
417             [(set V128:$dst, (vector_insert
418               (vec_t V128:$vec), (lane_t reg_t:$x), (i32 imm_t:$idx)))],
419             vec#".replace_lane\t$dst, $vec, $idx, $x",
420             vec#".replace_lane\t$idx", simdop>;
421}
422
423defm "" : ReplaceLane<v16i8, "i8x16", LaneIdx16, I32, i32, 7>;
424defm "" : ReplaceLane<v8i16, "i16x8", LaneIdx8, I32, i32, 11>;
425defm "" : ReplaceLane<v4i32, "i32x4", LaneIdx4, I32, i32, 14>;
426defm "" : ReplaceLane<v2i64, "i64x2", LaneIdx2, I64, i64, 17>;
427defm "" : ReplaceLane<v4f32, "f32x4", LaneIdx4, F32, f32, 20>;
428defm "" : ReplaceLane<v2f64, "f64x2", LaneIdx2, F64, f64, 23>;
429
430// Lower undef lane indices to zero
431def : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef),
432          (REPLACE_LANE_v16i8 V128:$vec, 0, I32:$x)>;
433def : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef),
434          (REPLACE_LANE_v8i16 V128:$vec, 0, I32:$x)>;
435def : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef),
436          (REPLACE_LANE_v4i32 V128:$vec, 0, I32:$x)>;
437def : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef),
438          (REPLACE_LANE_v2i64 V128:$vec, 0, I64:$x)>;
439def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
440          (REPLACE_LANE_v4f32 V128:$vec, 0, F32:$x)>;
441def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
442          (REPLACE_LANE_v2f64 V128:$vec, 0, F64:$x)>;
443
444//===----------------------------------------------------------------------===//
445// Comparisons
446//===----------------------------------------------------------------------===//
447
448multiclass SIMDCondition<ValueType vec_t, ValueType out_t, string vec,
449                         string name, CondCode cond, bits<32> simdop> {
450  defm _#vec_t :
451    SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
452           [(set (out_t V128:$dst),
453             (setcc (vec_t V128:$lhs), (vec_t V128:$rhs), cond)
454           )],
455           vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name, simdop>;
456}
457
458multiclass SIMDConditionInt<string name, CondCode cond, bits<32> baseInst> {
459  defm "" : SIMDCondition<v16i8, v16i8, "i8x16", name, cond, baseInst>;
460  defm "" : SIMDCondition<v8i16, v8i16, "i16x8", name, cond,
461                          !add(baseInst, 10)>;
462  defm "" : SIMDCondition<v4i32, v4i32, "i32x4", name, cond,
463                          !add(baseInst, 20)>;
464}
465
466multiclass SIMDConditionFP<string name, CondCode cond, bits<32> baseInst> {
467  defm "" : SIMDCondition<v4f32, v4i32, "f32x4", name, cond, baseInst>;
468  defm "" : SIMDCondition<v2f64, v2i64, "f64x2", name, cond,
469                          !add(baseInst, 6)>;
470}
471
472// Equality: eq
473let isCommutable = 1 in {
474defm EQ : SIMDConditionInt<"eq", SETEQ, 24>;
475defm EQ : SIMDConditionFP<"eq", SETOEQ, 64>;
476} // isCommutable = 1
477
478// Non-equality: ne
479let isCommutable = 1 in {
480defm NE : SIMDConditionInt<"ne", SETNE, 25>;
481defm NE : SIMDConditionFP<"ne", SETUNE, 65>;
482} // isCommutable = 1
483
484// Less than: lt_s / lt_u / lt
485defm LT_S : SIMDConditionInt<"lt_s", SETLT, 26>;
486defm LT_U : SIMDConditionInt<"lt_u", SETULT, 27>;
487defm LT : SIMDConditionFP<"lt", SETOLT, 66>;
488
489// Greater than: gt_s / gt_u / gt
490defm GT_S : SIMDConditionInt<"gt_s", SETGT, 28>;
491defm GT_U : SIMDConditionInt<"gt_u", SETUGT, 29>;
492defm GT : SIMDConditionFP<"gt", SETOGT, 67>;
493
494// Less than or equal: le_s / le_u / le
495defm LE_S : SIMDConditionInt<"le_s", SETLE, 30>;
496defm LE_U : SIMDConditionInt<"le_u", SETULE, 31>;
497defm LE : SIMDConditionFP<"le", SETOLE, 68>;
498
499// Greater than or equal: ge_s / ge_u / ge
500defm GE_S : SIMDConditionInt<"ge_s", SETGE, 32>;
501defm GE_U : SIMDConditionInt<"ge_u", SETUGE, 33>;
502defm GE : SIMDConditionFP<"ge", SETOGE, 69>;
503
504// Lower float comparisons that don't care about NaN to standard WebAssembly
505// float comparisons. These instructions are generated with nnan and in the
506// target-independent expansion of unordered comparisons and ordered ne.
507foreach nodes = [[seteq, EQ_v4f32], [setne, NE_v4f32], [setlt, LT_v4f32],
508                 [setgt, GT_v4f32], [setle, LE_v4f32], [setge, GE_v4f32]] in
509def : Pat<(v4i32 (nodes[0] (v4f32 V128:$lhs), (v4f32 V128:$rhs))),
510          (v4i32 (nodes[1] (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>;
511
512foreach nodes = [[seteq, EQ_v2f64], [setne, NE_v2f64], [setlt, LT_v2f64],
513                 [setgt, GT_v2f64], [setle, LE_v2f64], [setge, GE_v2f64]] in
514def : Pat<(v2i64 (nodes[0] (v2f64 V128:$lhs), (v2f64 V128:$rhs))),
515          (v2i64 (nodes[1] (v2f64 V128:$lhs), (v2f64 V128:$rhs)))>;
516
517
518//===----------------------------------------------------------------------===//
519// Bitwise operations
520//===----------------------------------------------------------------------===//
521
522multiclass SIMDBinary<ValueType vec_t, string vec, SDNode node, string name,
523                      bits<32> simdop> {
524  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs),
525                        (outs), (ins),
526                        [(set (vec_t V128:$dst),
527                          (node (vec_t V128:$lhs), (vec_t V128:$rhs))
528                        )],
529                        vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name,
530                        simdop>;
531}
532
533multiclass SIMDBitwise<SDNode node, string name, bits<32> simdop> {
534  defm "" : SIMDBinary<v16i8, "v128", node, name, simdop>;
535  defm "" : SIMDBinary<v8i16, "v128", node, name, simdop>;
536  defm "" : SIMDBinary<v4i32, "v128", node, name, simdop>;
537  defm "" : SIMDBinary<v2i64, "v128", node, name, simdop>;
538}
539
540multiclass SIMDUnary<ValueType vec_t, string vec, SDNode node, string name,
541                     bits<32> simdop> {
542  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
543                        [(set (vec_t V128:$dst),
544                          (vec_t (node (vec_t V128:$vec)))
545                        )],
546                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
547}
548
549// Bitwise logic: v128.not
550foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
551defm NOT: SIMDUnary<vec_t, "v128", vnot, "not", 76>;
552
553// Bitwise logic: v128.and / v128.or / v128.xor
554let isCommutable = 1 in {
555defm AND : SIMDBitwise<and, "and", 77>;
556defm OR : SIMDBitwise<or, "or", 78>;
557defm XOR : SIMDBitwise<xor, "xor", 79>;
558} // isCommutable = 1
559
560// Bitwise logic: v128.andnot
561def andnot : PatFrag<(ops node:$left, node:$right), (and $left, (vnot $right))>;
562let Predicates = [HasUnimplementedSIMD128] in
563defm ANDNOT : SIMDBitwise<andnot, "andnot", 216>;
564
565// Bitwise select: v128.bitselect
566foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
567  defm BITSELECT_#vec_t :
568    SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins),
569           [(set (vec_t V128:$dst),
570             (vec_t (int_wasm_bitselect
571               (vec_t V128:$v1), (vec_t V128:$v2), (vec_t V128:$c)
572             ))
573           )],
574           "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 80>;
575
576// Bitselect is equivalent to (c & v1) | (~c & v2)
577foreach vec_t = [v16i8, v8i16, v4i32, v2i64] in
578  def : Pat<(vec_t (or (and (vec_t V128:$c), (vec_t V128:$v1)),
579              (and (vnot V128:$c), (vec_t V128:$v2)))),
580            (!cast<Instruction>("BITSELECT_"#vec_t)
581              V128:$v1, V128:$v2, V128:$c)>;
582
583//===----------------------------------------------------------------------===//
584// Integer unary arithmetic
585//===----------------------------------------------------------------------===//
586
587multiclass SIMDUnaryInt<SDNode node, string name, bits<32> baseInst> {
588  defm "" : SIMDUnary<v16i8, "i8x16", node, name, baseInst>;
589  defm "" : SIMDUnary<v8i16, "i16x8", node, name, !add(baseInst, 17)>;
590  defm "" : SIMDUnary<v4i32, "i32x4", node, name, !add(baseInst, 34)>;
591  defm "" : SIMDUnary<v2i64, "i64x2", node, name, !add(baseInst, 51)>;
592}
593
594multiclass SIMDReduceVec<ValueType vec_t, string vec, SDNode op, string name,
595                         bits<32> simdop> {
596  defm _#vec_t : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins),
597                        [(set I32:$dst, (i32 (op (vec_t V128:$vec))))],
598                        vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>;
599}
600
601multiclass SIMDReduce<SDNode op, string name, bits<32> baseInst> {
602  defm "" : SIMDReduceVec<v16i8, "i8x16", op, name, baseInst>;
603  defm "" : SIMDReduceVec<v8i16, "i16x8", op, name, !add(baseInst, 17)>;
604  defm "" : SIMDReduceVec<v4i32, "i32x4", op, name, !add(baseInst, 34)>;
605  defm "" : SIMDReduceVec<v2i64, "i64x2", op, name, !add(baseInst, 51)>;
606}
607
608// Integer vector negation
609def ivneg : PatFrag<(ops node:$in), (sub immAllZerosV, node:$in)>;
610
611// Integer negation: neg
612defm NEG : SIMDUnaryInt<ivneg, "neg", 81>;
613
614// Any lane true: any_true
615defm ANYTRUE : SIMDReduce<int_wasm_anytrue, "any_true", 82>;
616
617// All lanes true: all_true
618defm ALLTRUE : SIMDReduce<int_wasm_alltrue, "all_true", 83>;
619
620// Reductions already return 0 or 1, so and 1, setne 0, and seteq 1
621// can be folded out
622foreach reduction =
623  [["int_wasm_anytrue", "ANYTRUE"], ["int_wasm_alltrue", "ALLTRUE"]] in
624foreach ty = [v16i8, v8i16, v4i32, v2i64] in {
625def : Pat<(i32 (and
626            (i32 (!cast<Intrinsic>(reduction[0]) (ty V128:$x))),
627            (i32 1)
628          )),
629          (i32 (!cast<NI>(reduction[1]#"_"#ty) (ty V128:$x)))>;
630def : Pat<(i32 (setne
631            (i32 (!cast<Intrinsic>(reduction[0]) (ty V128:$x))),
632            (i32 0)
633          )),
634          (i32 (!cast<NI>(reduction[1]#"_"#ty) (ty V128:$x)))>;
635def : Pat<(i32 (seteq
636            (i32 (!cast<Intrinsic>(reduction[0]) (ty V128:$x))),
637            (i32 1)
638          )),
639          (i32 (!cast<NI>(reduction[1]#"_"#ty) (ty V128:$x)))>;
640}
641
642//===----------------------------------------------------------------------===//
643// Bit shifts
644//===----------------------------------------------------------------------===//
645
646multiclass SIMDShift<ValueType vec_t, string vec, SDNode node, dag shift_vec,
647                     string name, bits<32> simdop> {
648  defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x),
649                        (outs), (ins),
650                        [(set (vec_t V128:$dst),
651                          (node V128:$vec, (vec_t shift_vec)))],
652                        vec#"."#name#"\t$dst, $vec, $x", vec#"."#name, simdop>;
653}
654
655multiclass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> {
656  defm "" : SIMDShift<v16i8, "i8x16", node, (splat16 I32:$x), name, baseInst>;
657  defm "" : SIMDShift<v8i16, "i16x8", node, (splat8 I32:$x), name,
658                      !add(baseInst, 17)>;
659  defm "" : SIMDShift<v4i32, "i32x4", node, (splat4 I32:$x), name,
660                      !add(baseInst, 34)>;
661  defm "" : SIMDShift<v2i64, "i64x2", node, (splat2 (i64 (zext I32:$x))),
662                      name, !add(baseInst, 51)>;
663}
664
665// Left shift by scalar: shl
666defm SHL : SIMDShiftInt<shl, "shl", 84>;
667
668// Right shift by scalar: shr_s / shr_u
669defm SHR_S : SIMDShiftInt<sra, "shr_s", 85>;
670defm SHR_U : SIMDShiftInt<srl, "shr_u", 86>;
671
672// Truncate i64 shift operands to i32s, except if they are already i32s
673foreach shifts = [[shl, SHL_v2i64], [sra, SHR_S_v2i64], [srl, SHR_U_v2i64]] in {
674def : Pat<(v2i64 (shifts[0]
675            (v2i64 V128:$vec),
676            (v2i64 (splat2 (i64 (sext I32:$x))))
677          )),
678          (v2i64 (shifts[1] (v2i64 V128:$vec), (i32 I32:$x)))>;
679def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), (v2i64 (splat2 I64:$x)))),
680          (v2i64 (shifts[1] (v2i64 V128:$vec), (I32_WRAP_I64 I64:$x)))>;
681}
682
683// 2xi64 shifts with constant shift amounts are custom lowered to avoid wrapping
684def wasm_shift_t : SDTypeProfile<1, 2,
685  [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]
686>;
687def wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>;
688def wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>;
689def wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>;
690foreach shifts = [[wasm_shl, SHL_v2i64],
691                  [wasm_shr_s, SHR_S_v2i64],
692                  [wasm_shr_u, SHR_U_v2i64]] in
693def : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), I32:$x)),
694          (v2i64 (shifts[1] (v2i64 V128:$vec), I32:$x))>;
695
696//===----------------------------------------------------------------------===//
697// Integer binary arithmetic
698//===----------------------------------------------------------------------===//
699
700multiclass SIMDBinaryIntSmall<SDNode node, string name, bits<32> baseInst> {
701  defm "" : SIMDBinary<v16i8, "i8x16", node, name, baseInst>;
702  defm "" : SIMDBinary<v8i16, "i16x8", node, name, !add(baseInst, 17)>;
703}
704
705multiclass SIMDBinaryIntNoI64x2<SDNode node, string name, bits<32> baseInst> {
706  defm "" : SIMDBinaryIntSmall<node, name, baseInst>;
707  defm "" : SIMDBinary<v4i32, "i32x4", node, name, !add(baseInst, 34)>;
708}
709
710multiclass SIMDBinaryInt<SDNode node, string name, bits<32> baseInst> {
711  defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>;
712  defm "" : SIMDBinary<v2i64, "i64x2", node, name, !add(baseInst, 51)>;
713}
714
715// Integer addition: add / add_saturate_s / add_saturate_u
716let isCommutable = 1 in {
717defm ADD : SIMDBinaryInt<add, "add", 87>;
718defm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_saturate_s", 88>;
719defm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_saturate_u", 89>;
720} // isCommutable = 1
721
722// Integer subtraction: sub / sub_saturate_s / sub_saturate_u
723defm SUB : SIMDBinaryInt<sub, "sub", 90>;
724defm SUB_SAT_S :
725  SIMDBinaryIntSmall<int_wasm_sub_saturate_signed, "sub_saturate_s", 91>;
726defm SUB_SAT_U :
727  SIMDBinaryIntSmall<int_wasm_sub_saturate_unsigned, "sub_saturate_u", 92>;
728
729// Integer multiplication: mul
730let isCommutable = 1 in
731defm MUL : SIMDBinaryIntNoI64x2<mul, "mul", 93>;
732
733// Integer min_s / min_u / max_s / max_u
734let isCommutable = 1 in {
735defm MIN_S : SIMDBinaryIntNoI64x2<smin, "min_s", 94>;
736defm MIN_U : SIMDBinaryIntNoI64x2<umin, "min_u", 95>;
737defm MAX_S : SIMDBinaryIntNoI64x2<smax, "max_s", 96>;
738defm MAX_U : SIMDBinaryIntNoI64x2<umax, "max_u", 97>;
739} // isCommutable = 1
740
741// Integer unsigned rounding average: avgr_u
742let isCommutable = 1, Predicates = [HasUnimplementedSIMD128] in {
743defm AVGR_U : SIMDBinary<v16i8, "i8x16", int_wasm_avgr_unsigned, "avgr_u", 217>;
744defm AVGR_U : SIMDBinary<v8i16, "i16x8", int_wasm_avgr_unsigned, "avgr_u", 218>;
745}
746
747def add_nuw : PatFrag<(ops node:$lhs, node:$rhs),
748                      (add node:$lhs, node:$rhs),
749                      "return N->getFlags().hasNoUnsignedWrap();">;
750
751foreach nodes = [[v16i8, splat16], [v8i16, splat8]] in
752def : Pat<(srl
753            (add_nuw
754              (add_nuw (nodes[0] V128:$lhs), (nodes[0] V128:$rhs)),
755              (nodes[1] (i32 1))
756            ),
757            (nodes[0] (nodes[1] (i32 1)))
758          ),
759          (!cast<NI>("AVGR_U_"#nodes[0]) V128:$lhs, V128:$rhs)>;
760
761// Widening dot product: i32x4.dot_i16x8_s
762let isCommutable = 1 in
763defm DOT : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins),
764                  [(set V128:$dst, (int_wasm_dot V128:$lhs, V128:$rhs))],
765                  "i32x4.dot_i16x8_s\t$dst, $lhs, $rhs", "i32x4.dot_i16x8_s",
766                  219>;
767
768//===----------------------------------------------------------------------===//
769// Floating-point unary arithmetic
770//===----------------------------------------------------------------------===//
771
772multiclass SIMDUnaryFP<SDNode node, string name, bits<32> baseInst> {
773  defm "" : SIMDUnary<v4f32, "f32x4", node, name, baseInst>;
774  defm "" : SIMDUnary<v2f64, "f64x2", node, name, !add(baseInst, 11)>;
775}
776
777// Absolute value: abs
778defm ABS : SIMDUnaryFP<fabs, "abs", 149>;
779
780// Negation: neg
781defm NEG : SIMDUnaryFP<fneg, "neg", 150>;
782
783// Square root: sqrt
784let Predicates = [HasUnimplementedSIMD128] in
785defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 151>;
786
787//===----------------------------------------------------------------------===//
788// Floating-point binary arithmetic
789//===----------------------------------------------------------------------===//
790
791multiclass SIMDBinaryFP<SDNode node, string name, bits<32> baseInst> {
792  defm "" : SIMDBinary<v4f32, "f32x4", node, name, baseInst>;
793  defm "" : SIMDBinary<v2f64, "f64x2", node, name, !add(baseInst, 11)>;
794}
795
796// Addition: add
797let isCommutable = 1 in
798defm ADD : SIMDBinaryFP<fadd, "add", 154>;
799
800// Subtraction: sub
801defm SUB : SIMDBinaryFP<fsub, "sub", 155>;
802
803// Multiplication: mul
804let isCommutable = 1 in
805defm MUL : SIMDBinaryFP<fmul, "mul", 156>;
806
807// Division: div
808let Predicates = [HasUnimplementedSIMD128] in
809defm DIV : SIMDBinaryFP<fdiv, "div", 157>;
810
811// NaN-propagating minimum: min
812defm MIN : SIMDBinaryFP<fminimum, "min", 158>;
813
814// NaN-propagating maximum: max
815defm MAX : SIMDBinaryFP<fmaximum, "max", 159>;
816
817//===----------------------------------------------------------------------===//
818// Conversions
819//===----------------------------------------------------------------------===//
820
821multiclass SIMDConvert<ValueType vec_t, ValueType arg_t, SDNode op,
822                       string name, bits<32> simdop> {
823  defm op#_#vec_t#_#arg_t :
824    SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins),
825           [(set (vec_t V128:$dst), (vec_t (op (arg_t V128:$vec))))],
826           name#"\t$dst, $vec", name, simdop>;
827}
828
829// Integer to floating point: convert
830defm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_i32x4_s", 175>;
831defm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_i32x4_u", 176>;
832defm "" : SIMDConvert<v2f64, v2i64, sint_to_fp, "f64x2.convert_i64x2_s", 177>;
833defm "" : SIMDConvert<v2f64, v2i64, uint_to_fp, "f64x2.convert_i64x2_u", 178>;
834
835// Floating point to integer with saturation: trunc_sat
836defm "" : SIMDConvert<v4i32, v4f32, fp_to_sint, "i32x4.trunc_sat_f32x4_s", 171>;
837defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_f32x4_u", 172>;
838defm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_f64x2_s", 173>;
839defm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_f64x2_u", 174>;
840
841// Widening operations
842multiclass SIMDWiden<ValueType vec_t, string vec, ValueType arg_t, string arg,
843                     bits<32> baseInst> {
844  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_signed,
845                        vec#".widen_low_"#arg#"_s", baseInst>;
846  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_signed,
847                        vec#".widen_high_"#arg#"_s", !add(baseInst, 1)>;
848  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_unsigned,
849                        vec#".widen_low_"#arg#"_u", !add(baseInst, 2)>;
850  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_unsigned,
851                        vec#".widen_high_"#arg#"_u", !add(baseInst, 3)>;
852}
853
854defm "" : SIMDWiden<v8i16, "i16x8", v16i8, "i8x16", 202>;
855defm "" : SIMDWiden<v4i32, "i32x4", v8i16, "i16x8", 206>;
856
857// Narrowing operations
858multiclass SIMDNarrow<ValueType vec_t, string vec, ValueType arg_t, string arg,
859                      bits<32> baseInst> {
860  defm NARROW_S_#vec_t :
861    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
862           [(set (vec_t V128:$dst), (vec_t (int_wasm_narrow_signed
863             (arg_t V128:$low), (arg_t V128:$high))))],
864           vec#".narrow_"#arg#"_s\t$dst, $low, $high", vec#".narrow_"#arg#"_s",
865           baseInst>;
866  defm NARROW_U_#vec_t :
867    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
868           [(set (vec_t V128:$dst), (vec_t (int_wasm_narrow_unsigned
869             (arg_t V128:$low), (arg_t V128:$high))))],
870           vec#".narrow_"#arg#"_u\t$dst, $low, $high", vec#".narrow_"#arg#"_u",
871           !add(baseInst, 1)>;
872}
873
874defm "" : SIMDNarrow<v16i8, "i8x16", v8i16, "i16x8", 198>;
875defm "" : SIMDNarrow<v8i16, "i16x8", v4i32, "i32x4", 200>;
876
877// Lower llvm.wasm.trunc.saturate.* to saturating instructions
878def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
879          (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>;
880def : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))),
881          (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>;
882def : Pat<(v2i64 (int_wasm_trunc_saturate_signed (v2f64 V128:$src))),
883          (fp_to_sint_v2i64_v2f64 (v2f64 V128:$src))>;
884def : Pat<(v2i64 (int_wasm_trunc_saturate_unsigned (v2f64 V128:$src))),
885          (fp_to_uint_v2i64_v2f64 (v2f64 V128:$src))>;
886
887// Bitcasts are nops
888// Matching bitcast t1 to t1 causes strange errors, so avoid repeating types
889foreach t1 = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
890foreach t2 = !foldl(
891  []<ValueType>, [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
892  acc, cur, !if(!eq(!cast<string>(t1), !cast<string>(cur)),
893    acc, !listconcat(acc, [cur])
894  )
895) in
896def : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>;
897
898//===----------------------------------------------------------------------===//
899// Quasi-Fused Multiply- Add and Subtract (QFMA/QFMS)
900//===----------------------------------------------------------------------===//
901
902multiclass SIMDQFM<ValueType vec_t, string vec, bits<32> baseInst> {
903  defm QFMA_#vec_t :
904    SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
905           (outs), (ins),
906           [(set (vec_t V128:$dst),
907             (int_wasm_qfma (vec_t V128:$a), (vec_t V128:$b), (vec_t V128:$c)))],
908           vec#".qfma\t$dst, $a, $b, $c", vec#".qfma", baseInst>;
909  defm QFMS_#vec_t :
910    SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
911           (outs), (ins),
912           [(set (vec_t V128:$dst),
913             (int_wasm_qfms (vec_t V128:$a), (vec_t V128:$b), (vec_t V128:$c)))],
914           vec#".qfms\t$dst, $a, $b, $c", vec#".qfms", !add(baseInst, 1)>;
915}
916
917defm "" : SIMDQFM<v4f32, "f32x4", 0x98>;
918defm "" : SIMDQFM<v2f64, "f64x2", 0xa3>;
919