xref: /freebsd/contrib/llvm-project/llvm/lib/Target/SystemZ/SystemZInstrVector.td (revision b1879975794772ee51f0b4865753364c7d7626c3)
1//==- SystemZInstrVector.td - SystemZ Vector instructions ------*- tblgen-*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9//===----------------------------------------------------------------------===//
10// Move instructions
11//===----------------------------------------------------------------------===//
12
13let Predicates = [FeatureVector] in {
14  // Register move.
15  def VLR : UnaryVRRa<"vlr", 0xE756, null_frag, v128any, v128any>;
16  def VLR32 : UnaryAliasVRR<null_frag, v32sb, v32sb>;
17  def VLR64 : UnaryAliasVRR<null_frag, v64db, v64db>;
18
19  // Load GR from VR element.
20  def VLGV  : BinaryVRScGeneric<"vlgv", 0xE721>;
21  def VLGVB : BinaryVRSc<"vlgvb", 0xE721, null_frag, v128b, 0>;
22  def VLGVH : BinaryVRSc<"vlgvh", 0xE721, null_frag, v128h, 1>;
23  def VLGVF : BinaryVRSc<"vlgvf", 0xE721, null_frag, v128f, 2>;
24  def VLGVG : BinaryVRSc<"vlgvg", 0xE721, z_vector_extract, v128g, 3>;
25
26  // Load VR element from GR.
27  def VLVG  : TernaryVRSbGeneric<"vlvg", 0xE722>;
28  def VLVGB : TernaryVRSb<"vlvgb", 0xE722, z_vector_insert,
29                          v128b, v128b, GR32, 0>;
30  def VLVGH : TernaryVRSb<"vlvgh", 0xE722, z_vector_insert,
31                          v128h, v128h, GR32, 1>;
32  def VLVGF : TernaryVRSb<"vlvgf", 0xE722, z_vector_insert,
33                          v128f, v128f, GR32, 2>;
34  def VLVGG : TernaryVRSb<"vlvgg", 0xE722, z_vector_insert,
35                          v128g, v128g, GR64, 3>;
36
37  // Load VR from GRs disjoint.
38  def VLVGP : BinaryVRRf<"vlvgp", 0xE762, z_join_dwords, v128g>;
39  def VLVGP32 : BinaryAliasVRRf<GR32>;
40}
41
42// Extractions always assign to the full GR64, even if the element would
43// fit in the lower 32 bits.  Sub-i64 extracts therefore need to take a
44// subreg of the result.
45class VectorExtractSubreg<ValueType type, Instruction insn>
46  : Pat<(i32 (z_vector_extract (type VR128:$vec), shift12only:$index)),
47        (EXTRACT_SUBREG (insn VR128:$vec, shift12only:$index), subreg_l32)>;
48
49def : VectorExtractSubreg<v16i8, VLGVB>;
50def : VectorExtractSubreg<v8i16, VLGVH>;
51def : VectorExtractSubreg<v4i32, VLGVF>;
52
53//===----------------------------------------------------------------------===//
54// Immediate instructions
55//===----------------------------------------------------------------------===//
56
57let Predicates = [FeatureVector] in {
58  let isAsCheapAsAMove = 1, isMoveImm = 1, isReMaterializable = 1 in {
59
60    // Generate byte mask.
61    def VZERO : InherentVRIa<"vzero", 0xE744, 0>;
62    def VONE  : InherentVRIa<"vone", 0xE744, 0xffff>;
63    def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16_timm>;
64
65    // Generate mask.
66    def VGM  : BinaryVRIbGeneric<"vgm", 0xE746>;
67    def VGMB : BinaryVRIb<"vgmb", 0xE746, z_rotate_mask, v128b, 0>;
68    def VGMH : BinaryVRIb<"vgmh", 0xE746, z_rotate_mask, v128h, 1>;
69    def VGMF : BinaryVRIb<"vgmf", 0xE746, z_rotate_mask, v128f, 2>;
70    def VGMG : BinaryVRIb<"vgmg", 0xE746, z_rotate_mask, v128g, 3>;
71
72    // Replicate immediate.
73    def VREPI  : UnaryVRIaGeneric<"vrepi", 0xE745, imm32sx16>;
74    def VREPIB : UnaryVRIa<"vrepib", 0xE745, z_replicate, v128b, imm32sx16_timm, 0>;
75    def VREPIH : UnaryVRIa<"vrepih", 0xE745, z_replicate, v128h, imm32sx16_timm, 1>;
76    def VREPIF : UnaryVRIa<"vrepif", 0xE745, z_replicate, v128f, imm32sx16_timm, 2>;
77    def VREPIG : UnaryVRIa<"vrepig", 0xE745, z_replicate, v128g, imm32sx16_timm, 3>;
78  }
79
80  // Load element immediate.
81  //
82  // We want these instructions to be used ahead of VLVG* where possible.
83  // However, VLVG* takes a variable BD-format index whereas VLEI takes
84  // a plain immediate index.  This means that VLVG* has an extra "base"
85  // register operand and is 3 units more complex.  Bumping the complexity
86  // of the VLEI* instructions by 4 means that they are strictly better
87  // than VLVG* in cases where both forms match.
88  let AddedComplexity = 4 in {
89    def VLEIB : TernaryVRIa<"vleib", 0xE740, z_vector_insert,
90                            v128b, v128b, imm32sx16trunc, imm32zx4>;
91    def VLEIH : TernaryVRIa<"vleih", 0xE741, z_vector_insert,
92                            v128h, v128h, imm32sx16trunc, imm32zx3>;
93    def VLEIF : TernaryVRIa<"vleif", 0xE743, z_vector_insert,
94                            v128f, v128f, imm32sx16, imm32zx2>;
95    def VLEIG : TernaryVRIa<"vleig", 0xE742, z_vector_insert,
96                            v128g, v128g, imm64sx16, imm32zx1>;
97  }
98}
99
100//===----------------------------------------------------------------------===//
101// Loads
102//===----------------------------------------------------------------------===//
103
104let Predicates = [FeatureVector] in {
105  // Load.
106  defm VL : UnaryVRXAlign<"vl", 0xE706>;
107
108  // Load to block boundary.  The number of loaded bytes is only known
109  // at run time.  The instruction is really polymorphic, but v128b matches
110  // the return type of the associated intrinsic.
111  def VLBB : BinaryVRX<"vlbb", 0xE707, int_s390_vlbb, v128b, 0>;
112
113  // Load count to block boundary.
114  let Defs = [CC] in
115    def LCBB : InstRXE<0xE727, (outs GR32:$R1),
116                               (ins (bdxaddr12only $B2, $D2, $X2):$XBD2, imm32zx4:$M3),
117                       "lcbb\t$R1, $XBD2, $M3",
118                       [(set GR32:$R1, (int_s390_lcbb bdxaddr12only:$XBD2,
119                                                      imm32zx4_timm:$M3))]>;
120
121  // Load with length.  The number of loaded bytes is only known at run time.
122  def VLL : BinaryVRSb<"vll", 0xE737, int_s390_vll, 0>;
123
124  // Load multiple.
125  defm VLM : LoadMultipleVRSaAlign<"vlm", 0xE736>;
126
127  // Load and replicate
128  def VLREP  : UnaryVRXGeneric<"vlrep", 0xE705>;
129  def VLREPB : UnaryVRX<"vlrepb", 0xE705, z_replicate_loadi8,  v128b, 1, 0>;
130  def VLREPH : UnaryVRX<"vlreph", 0xE705, z_replicate_loadi16, v128h, 2, 1>;
131  def VLREPF : UnaryVRX<"vlrepf", 0xE705, z_replicate_loadi32, v128f, 4, 2>;
132  def VLREPG : UnaryVRX<"vlrepg", 0xE705, z_replicate_loadi64, v128g, 8, 3>;
133  def : Pat<(v4f32 (z_replicate_loadf32 bdxaddr12only:$addr)),
134            (VLREPF bdxaddr12only:$addr)>;
135  def : Pat<(v2f64 (z_replicate_loadf64 bdxaddr12only:$addr)),
136            (VLREPG bdxaddr12only:$addr)>;
137
138  // Use VLREP to load subvectors.  These patterns use "12pair" because
139  // LEY and LDY offer full 20-bit displacement fields.  It's often better
140  // to use those instructions rather than force a 20-bit displacement
141  // into a GPR temporary.
142  let mayLoad = 1, canFoldAsLoad = 1 in {
143    def VL32 : UnaryAliasVRX<z_load, v32sb, bdxaddr12pair>;
144    def VL64 : UnaryAliasVRX<z_load, v64db, bdxaddr12pair>;
145  }
146
147  // Load logical element and zero.
148  def VLLEZ  : UnaryVRXGeneric<"vllez", 0xE704>;
149  def VLLEZB : UnaryVRX<"vllezb", 0xE704, z_vllezi8,  v128b, 1, 0>;
150  def VLLEZH : UnaryVRX<"vllezh", 0xE704, z_vllezi16, v128h, 2, 1>;
151  def VLLEZF : UnaryVRX<"vllezf", 0xE704, z_vllezi32, v128f, 4, 2>;
152  def VLLEZG : UnaryVRX<"vllezg", 0xE704, z_vllezi64, v128g, 8, 3>;
153  def : Pat<(z_vllezf32 bdxaddr12only:$addr),
154            (VLLEZF bdxaddr12only:$addr)>;
155  def : Pat<(z_vllezf64 bdxaddr12only:$addr),
156            (VLLEZG bdxaddr12only:$addr)>;
157  let Predicates = [FeatureVectorEnhancements1] in {
158    def VLLEZLF : UnaryVRX<"vllezlf", 0xE704, z_vllezli32, v128f, 4, 6>;
159    def : Pat<(z_vllezlf32 bdxaddr12only:$addr),
160              (VLLEZLF bdxaddr12only:$addr)>;
161  }
162
163  // Load element.
164  def VLEB : TernaryVRX<"vleb", 0xE700, z_vlei8,  v128b, v128b, 1, imm32zx4>;
165  def VLEH : TernaryVRX<"vleh", 0xE701, z_vlei16, v128h, v128h, 2, imm32zx3>;
166  def VLEF : TernaryVRX<"vlef", 0xE703, z_vlei32, v128f, v128f, 4, imm32zx2>;
167  def VLEG : TernaryVRX<"vleg", 0xE702, z_vlei64, v128g, v128g, 8, imm32zx1>;
168  def : Pat<(z_vlef32 (v4f32 VR128:$val), bdxaddr12only:$addr, imm32zx2:$index),
169            (VLEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
170  def : Pat<(z_vlef64 (v2f64 VR128:$val), bdxaddr12only:$addr, imm32zx1:$index),
171            (VLEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;
172
173  // Gather element.
174  def VGEF : TernaryVRV<"vgef", 0xE713, 4, imm32zx2>;
175  def VGEG : TernaryVRV<"vgeg", 0xE712, 8, imm32zx1>;
176}
177
178let Predicates = [FeatureVectorPackedDecimal] in {
179  // Load rightmost with length.  The number of loaded bytes is only known
180  // at run time.  Note that while the instruction will accept immediate
181  // lengths larger that 15 at runtime, those will always result in a trap,
182  // so we never emit them here.
183  def VLRL : BinaryVSI<"vlrl", 0xE635, null_frag, 0>;
184  def VLRLR : BinaryVRSd<"vlrlr", 0xE637, int_s390_vlrl, 0>;
185  def : Pat<(int_s390_vlrl imm32zx4:$len, bdaddr12only:$addr),
186            (VLRL bdaddr12only:$addr, imm32zx4:$len)>;
187}
188
189// Use replicating loads if we're inserting a single element into an
190// undefined vector.  This avoids a false dependency on the previous
191// register contents.
192multiclass ReplicatePeephole<Instruction vlrep, ValueType vectype,
193                             SDPatternOperator load, ValueType scalartype> {
194  def : Pat<(vectype (z_vector_insert
195                      (undef), (scalartype (load bdxaddr12only:$addr)), 0)),
196            (vlrep bdxaddr12only:$addr)>;
197  def : Pat<(vectype (scalar_to_vector
198                      (scalartype (load bdxaddr12only:$addr)))),
199            (vlrep bdxaddr12only:$addr)>;
200}
201defm : ReplicatePeephole<VLREPB, v16i8, z_anyextloadi8, i32>;
202defm : ReplicatePeephole<VLREPH, v8i16, z_anyextloadi16, i32>;
203defm : ReplicatePeephole<VLREPF, v4i32, z_load, i32>;
204defm : ReplicatePeephole<VLREPG, v2i64, z_load, i64>;
205defm : ReplicatePeephole<VLREPF, v4f32, z_load, f32>;
206defm : ReplicatePeephole<VLREPG, v2f64, z_load, f64>;
207
208//===----------------------------------------------------------------------===//
209// Stores
210//===----------------------------------------------------------------------===//
211
212let Predicates = [FeatureVector] in {
213  // Store.
214  defm VST : StoreVRXAlign<"vst", 0xE70E>;
215
216  // Store with length.  The number of stored bytes is only known at run time.
217  def VSTL : StoreLengthVRSb<"vstl", 0xE73F, int_s390_vstl, 0>;
218
219  // Store multiple.
220  defm VSTM : StoreMultipleVRSaAlign<"vstm", 0xE73E>;
221
222  // Store element.
223  def VSTEB : StoreBinaryVRX<"vsteb", 0xE708, z_vstei8,  v128b, 1, imm32zx4>;
224  def VSTEH : StoreBinaryVRX<"vsteh", 0xE709, z_vstei16, v128h, 2, imm32zx3>;
225  def VSTEF : StoreBinaryVRX<"vstef", 0xE70B, z_vstei32, v128f, 4, imm32zx2>;
226  def VSTEG : StoreBinaryVRX<"vsteg", 0xE70A, z_vstei64, v128g, 8, imm32zx1>;
227  def : Pat<(z_vstef32 (v4f32 VR128:$val), bdxaddr12only:$addr,
228                       imm32zx2:$index),
229            (VSTEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
230  def : Pat<(z_vstef64 (v2f64 VR128:$val), bdxaddr12only:$addr,
231                       imm32zx1:$index),
232            (VSTEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;
233
234  // Use VSTE to store subvectors.  These patterns use "12pair" because
235  // STEY and STDY offer full 20-bit displacement fields.  It's often better
236  // to use those instructions rather than force a 20-bit displacement
237  // into a GPR temporary.
238  let mayStore = 1 in {
239    def VST32 : StoreAliasVRX<store, v32sb, bdxaddr12pair>;
240    def VST64 : StoreAliasVRX<store, v64db, bdxaddr12pair>;
241  }
242
243  // Scatter element.
244  def VSCEF : StoreBinaryVRV<"vscef", 0xE71B, 4, imm32zx2>;
245  def VSCEG : StoreBinaryVRV<"vsceg", 0xE71A, 8, imm32zx1>;
246}
247
248let Predicates = [FeatureVectorPackedDecimal] in {
249  // Store rightmost with length.  The number of stored bytes is only known
250  // at run time.  Note that while the instruction will accept immediate
251  // lengths larger that 15 at runtime, those will always result in a trap,
252  // so we never emit them here.
253  def VSTRL : StoreLengthVSI<"vstrl", 0xE63D, null_frag, 0>;
254  def VSTRLR : StoreLengthVRSd<"vstrlr", 0xE63F, int_s390_vstrl, 0>;
255  def : Pat<(int_s390_vstrl VR128:$val, imm32zx4:$len, bdaddr12only:$addr),
256            (VSTRL VR128:$val, bdaddr12only:$addr, imm32zx4:$len)>;
257}
258
259//===----------------------------------------------------------------------===//
260// Byte swaps
261//===----------------------------------------------------------------------===//
262
263let Predicates = [FeatureVectorEnhancements2] in {
264  // Load byte-reversed elements.
265  def VLBR  : UnaryVRXGeneric<"vlbr", 0xE606>;
266  def VLBRH : UnaryVRX<"vlbrh", 0xE606, z_loadbswap, v128h, 16, 1>;
267  def VLBRF : UnaryVRX<"vlbrf", 0xE606, z_loadbswap, v128f, 16, 2>;
268  def VLBRG : UnaryVRX<"vlbrg", 0xE606, z_loadbswap, v128g, 16, 3>;
269  def VLBRQ : UnaryVRX<"vlbrq", 0xE606, z_loadbswap, v128q, 16, 4>;
270
271  // Load elements reversed.
272  def VLER  : UnaryVRXGeneric<"vler", 0xE607>;
273  def VLERH : UnaryVRX<"vlerh", 0xE607, z_loadeswap, v128h, 16, 1>;
274  def VLERF : UnaryVRX<"vlerf", 0xE607, z_loadeswap, v128f, 16, 2>;
275  def VLERG : UnaryVRX<"vlerg", 0xE607, z_loadeswap, v128g, 16, 3>;
276  def : Pat<(v4f32 (z_loadeswap bdxaddr12only:$addr)),
277            (VLERF bdxaddr12only:$addr)>;
278  def : Pat<(v2f64 (z_loadeswap bdxaddr12only:$addr)),
279            (VLERG bdxaddr12only:$addr)>;
280  def : Pat<(v16i8 (z_loadeswap bdxaddr12only:$addr)),
281            (VLBRQ bdxaddr12only:$addr)>;
282
283  // Load byte-reversed element.
284  def VLEBRH : TernaryVRX<"vlebrh", 0xE601, z_vlebri16, v128h, v128h, 2, imm32zx3>;
285  def VLEBRF : TernaryVRX<"vlebrf", 0xE603, z_vlebri32, v128f, v128f, 4, imm32zx2>;
286  def VLEBRG : TernaryVRX<"vlebrg", 0xE602, z_vlebri64, v128g, v128g, 8, imm32zx1>;
287
288  // Load byte-reversed element and zero.
289  def VLLEBRZ  : UnaryVRXGeneric<"vllebrz", 0xE604>;
290  def VLLEBRZH : UnaryVRX<"vllebrzh", 0xE604, z_vllebrzi16, v128h, 2, 1>;
291  def VLLEBRZF : UnaryVRX<"vllebrzf", 0xE604, z_vllebrzi32, v128f, 4, 2>;
292  def VLLEBRZG : UnaryVRX<"vllebrzg", 0xE604, z_vllebrzi64, v128g, 8, 3>;
293  def VLLEBRZE : UnaryVRX<"vllebrze", 0xE604, z_vllebrzli32, v128f, 4, 6>;
294  def : InstAlias<"lerv\t$V1, $XBD2",
295                  (VLLEBRZE VR128:$V1, bdxaddr12only:$XBD2), 0>;
296  def : InstAlias<"ldrv\t$V1, $XBD2",
297                  (VLLEBRZG VR128:$V1, bdxaddr12only:$XBD2), 0>;
298
299  // Load byte-reversed element and replicate.
300  def VLBRREP  : UnaryVRXGeneric<"vlbrrep", 0xE605>;
301  def VLBRREPH : UnaryVRX<"vlbrreph", 0xE605, z_replicate_loadbswapi16, v128h, 2, 1>;
302  def VLBRREPF : UnaryVRX<"vlbrrepf", 0xE605, z_replicate_loadbswapi32, v128f, 4, 2>;
303  def VLBRREPG : UnaryVRX<"vlbrrepg", 0xE605, z_replicate_loadbswapi64, v128g, 8, 3>;
304
305  // Store byte-reversed elements.
306  def VSTBR  : StoreVRXGeneric<"vstbr", 0xE60E>;
307  def VSTBRH : StoreVRX<"vstbrh", 0xE60E, z_storebswap, v128h, 16, 1>;
308  def VSTBRF : StoreVRX<"vstbrf", 0xE60E, z_storebswap, v128f, 16, 2>;
309  def VSTBRG : StoreVRX<"vstbrg", 0xE60E, z_storebswap, v128g, 16, 3>;
310  def VSTBRQ : StoreVRX<"vstbrq", 0xE60E, z_storebswap, v128q, 16, 4>;
311
312  // Store elements reversed.
313  def VSTER  : StoreVRXGeneric<"vster", 0xE60F>;
314  def VSTERH : StoreVRX<"vsterh", 0xE60F, z_storeeswap, v128h, 16, 1>;
315  def VSTERF : StoreVRX<"vsterf", 0xE60F, z_storeeswap, v128f, 16, 2>;
316  def VSTERG : StoreVRX<"vsterg", 0xE60F, z_storeeswap, v128g, 16, 3>;
317  def : Pat<(z_storeeswap (v4f32 VR128:$val), bdxaddr12only:$addr),
318            (VSTERF VR128:$val, bdxaddr12only:$addr)>;
319  def : Pat<(z_storeeswap (v2f64 VR128:$val), bdxaddr12only:$addr),
320            (VSTERG VR128:$val, bdxaddr12only:$addr)>;
321  def : Pat<(z_storeeswap (v16i8 VR128:$val), bdxaddr12only:$addr),
322            (VSTBRQ VR128:$val, bdxaddr12only:$addr)>;
323
324  // Store byte-reversed element.
325  def VSTEBRH : StoreBinaryVRX<"vstebrh", 0xE609, z_vstebri16, v128h, 2, imm32zx3>;
326  def VSTEBRF : StoreBinaryVRX<"vstebrf", 0xE60B, z_vstebri32, v128f, 4, imm32zx2>;
327  def VSTEBRG : StoreBinaryVRX<"vstebrg", 0xE60A, z_vstebri64, v128g, 8, imm32zx1>;
328  def : InstAlias<"sterv\t$V1, $XBD2",
329                  (VSTEBRF VR128:$V1, bdxaddr12only:$XBD2, 0), 0>;
330  def : InstAlias<"stdrv\t$V1, $XBD2",
331                  (VSTEBRG VR128:$V1, bdxaddr12only:$XBD2, 0), 0>;
332}
333
334//===----------------------------------------------------------------------===//
335// Selects and permutes
336//===----------------------------------------------------------------------===//
337
338let Predicates = [FeatureVector] in {
339  // Merge high.
340  def VMRH:   BinaryVRRcGeneric<"vmrh", 0xE761>;
341  def VMRHB : BinaryVRRc<"vmrhb", 0xE761, z_merge_high, v128b, v128b, 0>;
342  def VMRHH : BinaryVRRc<"vmrhh", 0xE761, z_merge_high, v128h, v128h, 1>;
343  def VMRHF : BinaryVRRc<"vmrhf", 0xE761, z_merge_high, v128f, v128f, 2>;
344  def VMRHG : BinaryVRRc<"vmrhg", 0xE761, z_merge_high, v128g, v128g, 3>;
345  def : BinaryRRWithType<VMRHF, VR128, z_merge_high, v4f32>;
346  def : BinaryRRWithType<VMRHG, VR128, z_merge_high, v2f64>;
347
348  // Merge low.
349  def VMRL:   BinaryVRRcGeneric<"vmrl", 0xE760>;
350  def VMRLB : BinaryVRRc<"vmrlb", 0xE760, z_merge_low, v128b, v128b, 0>;
351  def VMRLH : BinaryVRRc<"vmrlh", 0xE760, z_merge_low, v128h, v128h, 1>;
352  def VMRLF : BinaryVRRc<"vmrlf", 0xE760, z_merge_low, v128f, v128f, 2>;
353  def VMRLG : BinaryVRRc<"vmrlg", 0xE760, z_merge_low, v128g, v128g, 3>;
354  def : BinaryRRWithType<VMRLF, VR128, z_merge_low, v4f32>;
355  def : BinaryRRWithType<VMRLG, VR128, z_merge_low, v2f64>;
356
357  // Permute.
358  def VPERM : TernaryVRRe<"vperm", 0xE78C, z_permute, v128b, v128b>;
359
360  // Permute doubleword immediate.
361  def VPDI : TernaryVRRc<"vpdi", 0xE784, z_permute_dwords, v128g, v128g>;
362
363  // Bit Permute.
364  let Predicates = [FeatureVectorEnhancements1] in
365    def VBPERM : BinaryVRRc<"vbperm", 0xE785, int_s390_vbperm, v128g, v128b>;
366
367  // Replicate.
368  def VREP:   BinaryVRIcGeneric<"vrep", 0xE74D>;
369  def VREPB : BinaryVRIc<"vrepb", 0xE74D, z_splat, v128b, v128b, 0>;
370  def VREPH : BinaryVRIc<"vreph", 0xE74D, z_splat, v128h, v128h, 1>;
371  def VREPF : BinaryVRIc<"vrepf", 0xE74D, z_splat, v128f, v128f, 2>;
372  def VREPG : BinaryVRIc<"vrepg", 0xE74D, z_splat, v128g, v128g, 3>;
373  def : Pat<(v4f32 (z_splat VR128:$vec, imm32zx16_timm:$index)),
374            (VREPF VR128:$vec, imm32zx16:$index)>;
375  def : Pat<(v2f64 (z_splat VR128:$vec, imm32zx16_timm:$index)),
376            (VREPG VR128:$vec, imm32zx16:$index)>;
377
378  // Select.
379  def VSEL : TernaryVRRe<"vsel", 0xE78D, null_frag, v128any, v128any>;
380}
381
382//===----------------------------------------------------------------------===//
383// Widening and narrowing
384//===----------------------------------------------------------------------===//
385
386let Predicates = [FeatureVector] in {
387  // Pack
388  def VPK  : BinaryVRRcGeneric<"vpk", 0xE794>;
389  def VPKH : BinaryVRRc<"vpkh", 0xE794, z_pack, v128b, v128h, 1>;
390  def VPKF : BinaryVRRc<"vpkf", 0xE794, z_pack, v128h, v128f, 2>;
391  def VPKG : BinaryVRRc<"vpkg", 0xE794, z_pack, v128f, v128g, 3>;
392
393  // Pack saturate.
394  def  VPKS  : BinaryVRRbSPairGeneric<"vpks", 0xE797>;
395  defm VPKSH : BinaryVRRbSPair<"vpksh", 0xE797, int_s390_vpksh, z_packs_cc,
396                               v128b, v128h, 1>;
397  defm VPKSF : BinaryVRRbSPair<"vpksf", 0xE797, int_s390_vpksf, z_packs_cc,
398                               v128h, v128f, 2>;
399  defm VPKSG : BinaryVRRbSPair<"vpksg", 0xE797, int_s390_vpksg, z_packs_cc,
400                               v128f, v128g, 3>;
401
402  // Pack saturate logical.
403  def  VPKLS  : BinaryVRRbSPairGeneric<"vpkls", 0xE795>;
404  defm VPKLSH : BinaryVRRbSPair<"vpklsh", 0xE795, int_s390_vpklsh, z_packls_cc,
405                                v128b, v128h, 1>;
406  defm VPKLSF : BinaryVRRbSPair<"vpklsf", 0xE795, int_s390_vpklsf, z_packls_cc,
407                                v128h, v128f, 2>;
408  defm VPKLSG : BinaryVRRbSPair<"vpklsg", 0xE795, int_s390_vpklsg, z_packls_cc,
409                                v128f, v128g, 3>;
410
411  // Sign-extend to doubleword.
412  def VSEG  : UnaryVRRaGeneric<"vseg", 0xE75F>;
413  def VSEGB : UnaryVRRa<"vsegb", 0xE75F, z_vsei8,  v128g, v128g, 0>;
414  def VSEGH : UnaryVRRa<"vsegh", 0xE75F, z_vsei16, v128g, v128g, 1>;
415  def VSEGF : UnaryVRRa<"vsegf", 0xE75F, z_vsei32, v128g, v128g, 2>;
416  def : Pat<(z_vsei8_by_parts  (v16i8 VR128:$src)), (VSEGB VR128:$src)>;
417  def : Pat<(z_vsei16_by_parts (v8i16 VR128:$src)), (VSEGH VR128:$src)>;
418  def : Pat<(z_vsei32_by_parts (v4i32 VR128:$src)), (VSEGF VR128:$src)>;
419
420  // Unpack high.
421  def VUPH  : UnaryVRRaGeneric<"vuph", 0xE7D7>;
422  def VUPHB : UnaryVRRa<"vuphb", 0xE7D7, z_unpack_high, v128h, v128b, 0>;
423  def VUPHH : UnaryVRRa<"vuphh", 0xE7D7, z_unpack_high, v128f, v128h, 1>;
424  def VUPHF : UnaryVRRa<"vuphf", 0xE7D7, z_unpack_high, v128g, v128f, 2>;
425
426  // Unpack logical high.
427  def VUPLH  : UnaryVRRaGeneric<"vuplh", 0xE7D5>;
428  def VUPLHB : UnaryVRRa<"vuplhb", 0xE7D5, z_unpackl_high, v128h, v128b, 0>;
429  def VUPLHH : UnaryVRRa<"vuplhh", 0xE7D5, z_unpackl_high, v128f, v128h, 1>;
430  def VUPLHF : UnaryVRRa<"vuplhf", 0xE7D5, z_unpackl_high, v128g, v128f, 2>;
431
432  // Unpack low.
433  def VUPL   : UnaryVRRaGeneric<"vupl", 0xE7D6>;
434  def VUPLB  : UnaryVRRa<"vuplb",  0xE7D6, z_unpack_low, v128h, v128b, 0>;
435  def VUPLHW : UnaryVRRa<"vuplhw", 0xE7D6, z_unpack_low, v128f, v128h, 1>;
436  def VUPLF  : UnaryVRRa<"vuplf",  0xE7D6, z_unpack_low, v128g, v128f, 2>;
437
438  // Unpack logical low.
439  def VUPLL  : UnaryVRRaGeneric<"vupll", 0xE7D4>;
440  def VUPLLB : UnaryVRRa<"vupllb", 0xE7D4, z_unpackl_low, v128h, v128b, 0>;
441  def VUPLLH : UnaryVRRa<"vupllh", 0xE7D4, z_unpackl_low, v128f, v128h, 1>;
442  def VUPLLF : UnaryVRRa<"vupllf", 0xE7D4, z_unpackl_low, v128g, v128f, 2>;
443}
444
445//===----------------------------------------------------------------------===//
446// Instantiating generic operations for specific types.
447//===----------------------------------------------------------------------===//
448
449multiclass GenericVectorOps<ValueType type, ValueType inttype> {
450  let Predicates = [FeatureVector] in {
451    def : Pat<(type (load bdxaddr12only:$addr)),
452              (VL bdxaddr12only:$addr)>;
453    def : Pat<(store (type VR128:$src), bdxaddr12only:$addr),
454              (VST VR128:$src, bdxaddr12only:$addr)>;
455    def : Pat<(type (vselect (inttype VR128:$x), VR128:$y, VR128:$z)),
456              (VSEL VR128:$y, VR128:$z, VR128:$x)>;
457    def : Pat<(type (vselect (inttype (z_vnot VR128:$x)), VR128:$y, VR128:$z)),
458              (VSEL VR128:$z, VR128:$y, VR128:$x)>;
459  }
460}
461
462defm : GenericVectorOps<v16i8, v16i8>;
463defm : GenericVectorOps<v8i16, v8i16>;
464defm : GenericVectorOps<v4i32, v4i32>;
465defm : GenericVectorOps<v2i64, v2i64>;
466defm : GenericVectorOps<v4f32, v4i32>;
467defm : GenericVectorOps<v2f64, v2i64>;
468
469//===----------------------------------------------------------------------===//
470// Integer arithmetic
471//===----------------------------------------------------------------------===//
472
473let Predicates = [FeatureVector] in {
474  let isCommutable = 1 in {
475    // Add.
476    def VA  : BinaryVRRcGeneric<"va", 0xE7F3>;
477    def VAB : BinaryVRRc<"vab", 0xE7F3, add, v128b, v128b, 0>;
478    def VAH : BinaryVRRc<"vah", 0xE7F3, add, v128h, v128h, 1>;
479    def VAF : BinaryVRRc<"vaf", 0xE7F3, add, v128f, v128f, 2>;
480    def VAG : BinaryVRRc<"vag", 0xE7F3, add, v128g, v128g, 3>;
481    def VAQ : BinaryVRRc<"vaq", 0xE7F3, add, v128q, v128q, 4>;
482  }
483
484  let isCommutable = 1 in {
485    // Add compute carry.
486    def VACC  : BinaryVRRcGeneric<"vacc", 0xE7F1>;
487    def VACCB : BinaryVRRc<"vaccb", 0xE7F1, z_vacc, v128b, v128b, 0>;
488    def VACCH : BinaryVRRc<"vacch", 0xE7F1, z_vacc, v128h, v128h, 1>;
489    def VACCF : BinaryVRRc<"vaccf", 0xE7F1, z_vacc, v128f, v128f, 2>;
490    def VACCG : BinaryVRRc<"vaccg", 0xE7F1, z_vacc, v128g, v128g, 3>;
491    def VACCQ : BinaryVRRc<"vaccq", 0xE7F1, z_vacc, v128q, v128q, 4>;
492
493    // Add with carry.
494    def VAC  : TernaryVRRdGeneric<"vac", 0xE7BB>;
495    def VACQ : TernaryVRRd<"vacq", 0xE7BB, z_vac, v128q, v128q, 4>;
496
497    // Add with carry compute carry.
498    def VACCC  : TernaryVRRdGeneric<"vaccc", 0xE7B9>;
499    def VACCCQ : TernaryVRRd<"vacccq", 0xE7B9, z_vaccc, v128q, v128q, 4>;
500  }
501
502  // And.
503  let isCommutable = 1 in
504    def VN : BinaryVRRc<"vn", 0xE768, null_frag, v128any, v128any>;
505
506  // And with complement.
507  def VNC : BinaryVRRc<"vnc", 0xE769, null_frag, v128any, v128any>;
508
509  let isCommutable = 1 in {
510    // Average.
511    def VAVG  : BinaryVRRcGeneric<"vavg", 0xE7F2>;
512    def VAVGB : BinaryVRRc<"vavgb", 0xE7F2, int_s390_vavgb, v128b, v128b, 0>;
513    def VAVGH : BinaryVRRc<"vavgh", 0xE7F2, int_s390_vavgh, v128h, v128h, 1>;
514    def VAVGF : BinaryVRRc<"vavgf", 0xE7F2, int_s390_vavgf, v128f, v128f, 2>;
515    def VAVGG : BinaryVRRc<"vavgg", 0xE7F2, int_s390_vavgg, v128g, v128g, 3>;
516
517    // Average logical.
518    def VAVGL  : BinaryVRRcGeneric<"vavgl", 0xE7F0>;
519    def VAVGLB : BinaryVRRc<"vavglb", 0xE7F0, int_s390_vavglb, v128b, v128b, 0>;
520    def VAVGLH : BinaryVRRc<"vavglh", 0xE7F0, int_s390_vavglh, v128h, v128h, 1>;
521    def VAVGLF : BinaryVRRc<"vavglf", 0xE7F0, int_s390_vavglf, v128f, v128f, 2>;
522    def VAVGLG : BinaryVRRc<"vavglg", 0xE7F0, int_s390_vavglg, v128g, v128g, 3>;
523  }
524
525  // Checksum.
526  def VCKSM : BinaryVRRc<"vcksm", 0xE766, int_s390_vcksm, v128f, v128f>;
527
528  // Count leading zeros.
529  def VCLZ  : UnaryVRRaGeneric<"vclz", 0xE753>;
530  def VCLZB : UnaryVRRa<"vclzb", 0xE753, ctlz, v128b, v128b, 0>;
531  def VCLZH : UnaryVRRa<"vclzh", 0xE753, ctlz, v128h, v128h, 1>;
532  def VCLZF : UnaryVRRa<"vclzf", 0xE753, ctlz, v128f, v128f, 2>;
533  def VCLZG : UnaryVRRa<"vclzg", 0xE753, ctlz, v128g, v128g, 3>;
534
535  // Count trailing zeros.
536  def VCTZ  : UnaryVRRaGeneric<"vctz", 0xE752>;
537  def VCTZB : UnaryVRRa<"vctzb", 0xE752, cttz, v128b, v128b, 0>;
538  def VCTZH : UnaryVRRa<"vctzh", 0xE752, cttz, v128h, v128h, 1>;
539  def VCTZF : UnaryVRRa<"vctzf", 0xE752, cttz, v128f, v128f, 2>;
540  def VCTZG : UnaryVRRa<"vctzg", 0xE752, cttz, v128g, v128g, 3>;
541
542  let isCommutable = 1 in {
543    // Not exclusive or.
544    let Predicates = [FeatureVectorEnhancements1] in
545      def VNX : BinaryVRRc<"vnx", 0xE76C, null_frag, v128any, v128any>;
546
547    // Exclusive or.
548    def VX : BinaryVRRc<"vx", 0xE76D, null_frag, v128any, v128any>;
549  }
550
551  // Galois field multiply sum.
552  def VGFM  : BinaryVRRcGeneric<"vgfm", 0xE7B4>;
553  def VGFMB : BinaryVRRc<"vgfmb", 0xE7B4, int_s390_vgfmb, v128h, v128b, 0>;
554  def VGFMH : BinaryVRRc<"vgfmh", 0xE7B4, int_s390_vgfmh, v128f, v128h, 1>;
555  def VGFMF : BinaryVRRc<"vgfmf", 0xE7B4, int_s390_vgfmf, v128g, v128f, 2>;
556  def VGFMG : BinaryVRRc<"vgfmg", 0xE7B4, int_s390_vgfmg, v128q, v128g, 3>;
557
558  // Galois field multiply sum and accumulate.
559  def VGFMA  : TernaryVRRdGeneric<"vgfma", 0xE7BC>;
560  def VGFMAB : TernaryVRRd<"vgfmab", 0xE7BC, int_s390_vgfmab, v128h, v128b, 0>;
561  def VGFMAH : TernaryVRRd<"vgfmah", 0xE7BC, int_s390_vgfmah, v128f, v128h, 1>;
562  def VGFMAF : TernaryVRRd<"vgfmaf", 0xE7BC, int_s390_vgfmaf, v128g, v128f, 2>;
563  def VGFMAG : TernaryVRRd<"vgfmag", 0xE7BC, int_s390_vgfmag, v128q, v128g, 3>;
564
565  // Load complement.
566  def VLC  : UnaryVRRaGeneric<"vlc", 0xE7DE>;
567  def VLCB : UnaryVRRa<"vlcb", 0xE7DE, z_vneg, v128b, v128b, 0>;
568  def VLCH : UnaryVRRa<"vlch", 0xE7DE, z_vneg, v128h, v128h, 1>;
569  def VLCF : UnaryVRRa<"vlcf", 0xE7DE, z_vneg, v128f, v128f, 2>;
570  def VLCG : UnaryVRRa<"vlcg", 0xE7DE, z_vneg, v128g, v128g, 3>;
571
572  // Load positive.
573  def VLP  : UnaryVRRaGeneric<"vlp", 0xE7DF>;
574  def VLPB : UnaryVRRa<"vlpb", 0xE7DF, abs, v128b, v128b, 0>;
575  def VLPH : UnaryVRRa<"vlph", 0xE7DF, abs, v128h, v128h, 1>;
576  def VLPF : UnaryVRRa<"vlpf", 0xE7DF, abs, v128f, v128f, 2>;
577  def VLPG : UnaryVRRa<"vlpg", 0xE7DF, abs, v128g, v128g, 3>;
578
579  let isCommutable = 1 in {
580    // Maximum.
581    def VMX  : BinaryVRRcGeneric<"vmx", 0xE7FF>;
582    def VMXB : BinaryVRRc<"vmxb", 0xE7FF, null_frag, v128b, v128b, 0>;
583    def VMXH : BinaryVRRc<"vmxh", 0xE7FF, null_frag, v128h, v128h, 1>;
584    def VMXF : BinaryVRRc<"vmxf", 0xE7FF, null_frag, v128f, v128f, 2>;
585    def VMXG : BinaryVRRc<"vmxg", 0xE7FF, null_frag, v128g, v128g, 3>;
586
587    // Maximum logical.
588    def VMXL  : BinaryVRRcGeneric<"vmxl", 0xE7FD>;
589    def VMXLB : BinaryVRRc<"vmxlb", 0xE7FD, null_frag, v128b, v128b, 0>;
590    def VMXLH : BinaryVRRc<"vmxlh", 0xE7FD, null_frag, v128h, v128h, 1>;
591    def VMXLF : BinaryVRRc<"vmxlf", 0xE7FD, null_frag, v128f, v128f, 2>;
592    def VMXLG : BinaryVRRc<"vmxlg", 0xE7FD, null_frag, v128g, v128g, 3>;
593  }
594
595  let isCommutable = 1 in {
596    // Minimum.
597    def VMN  : BinaryVRRcGeneric<"vmn", 0xE7FE>;
598    def VMNB : BinaryVRRc<"vmnb", 0xE7FE, null_frag, v128b, v128b, 0>;
599    def VMNH : BinaryVRRc<"vmnh", 0xE7FE, null_frag, v128h, v128h, 1>;
600    def VMNF : BinaryVRRc<"vmnf", 0xE7FE, null_frag, v128f, v128f, 2>;
601    def VMNG : BinaryVRRc<"vmng", 0xE7FE, null_frag, v128g, v128g, 3>;
602
603    // Minimum logical.
604    def VMNL  : BinaryVRRcGeneric<"vmnl", 0xE7FC>;
605    def VMNLB : BinaryVRRc<"vmnlb", 0xE7FC, null_frag, v128b, v128b, 0>;
606    def VMNLH : BinaryVRRc<"vmnlh", 0xE7FC, null_frag, v128h, v128h, 1>;
607    def VMNLF : BinaryVRRc<"vmnlf", 0xE7FC, null_frag, v128f, v128f, 2>;
608    def VMNLG : BinaryVRRc<"vmnlg", 0xE7FC, null_frag, v128g, v128g, 3>;
609  }
610
611  let isCommutable = 1 in {
612    // Multiply and add low.
613    def VMAL   : TernaryVRRdGeneric<"vmal", 0xE7AA>;
614    def VMALB  : TernaryVRRd<"vmalb",  0xE7AA, z_muladd, v128b, v128b, 0>;
615    def VMALHW : TernaryVRRd<"vmalhw", 0xE7AA, z_muladd, v128h, v128h, 1>;
616    def VMALF  : TernaryVRRd<"vmalf",  0xE7AA, z_muladd, v128f, v128f, 2>;
617
618    // Multiply and add high.
619    def VMAH  : TernaryVRRdGeneric<"vmah", 0xE7AB>;
620    def VMAHB : TernaryVRRd<"vmahb", 0xE7AB, int_s390_vmahb, v128b, v128b, 0>;
621    def VMAHH : TernaryVRRd<"vmahh", 0xE7AB, int_s390_vmahh, v128h, v128h, 1>;
622    def VMAHF : TernaryVRRd<"vmahf", 0xE7AB, int_s390_vmahf, v128f, v128f, 2>;
623
624    // Multiply and add logical high.
625    def VMALH  : TernaryVRRdGeneric<"vmalh", 0xE7A9>;
626    def VMALHB : TernaryVRRd<"vmalhb", 0xE7A9, int_s390_vmalhb, v128b, v128b, 0>;
627    def VMALHH : TernaryVRRd<"vmalhh", 0xE7A9, int_s390_vmalhh, v128h, v128h, 1>;
628    def VMALHF : TernaryVRRd<"vmalhf", 0xE7A9, int_s390_vmalhf, v128f, v128f, 2>;
629
630    // Multiply and add even.
631    def VMAE  : TernaryVRRdGeneric<"vmae", 0xE7AE>;
632    def VMAEB : TernaryVRRd<"vmaeb", 0xE7AE, int_s390_vmaeb, v128h, v128b, 0>;
633    def VMAEH : TernaryVRRd<"vmaeh", 0xE7AE, int_s390_vmaeh, v128f, v128h, 1>;
634    def VMAEF : TernaryVRRd<"vmaef", 0xE7AE, int_s390_vmaef, v128g, v128f, 2>;
635
636    // Multiply and add logical even.
637    def VMALE  : TernaryVRRdGeneric<"vmale", 0xE7AC>;
638    def VMALEB : TernaryVRRd<"vmaleb", 0xE7AC, int_s390_vmaleb, v128h, v128b, 0>;
639    def VMALEH : TernaryVRRd<"vmaleh", 0xE7AC, int_s390_vmaleh, v128f, v128h, 1>;
640    def VMALEF : TernaryVRRd<"vmalef", 0xE7AC, int_s390_vmalef, v128g, v128f, 2>;
641
642    // Multiply and add odd.
643    def VMAO  : TernaryVRRdGeneric<"vmao", 0xE7AF>;
644    def VMAOB : TernaryVRRd<"vmaob", 0xE7AF, int_s390_vmaob, v128h, v128b, 0>;
645    def VMAOH : TernaryVRRd<"vmaoh", 0xE7AF, int_s390_vmaoh, v128f, v128h, 1>;
646    def VMAOF : TernaryVRRd<"vmaof", 0xE7AF, int_s390_vmaof, v128g, v128f, 2>;
647
648    // Multiply and add logical odd.
649    def VMALO  : TernaryVRRdGeneric<"vmalo", 0xE7AD>;
650    def VMALOB : TernaryVRRd<"vmalob", 0xE7AD, int_s390_vmalob, v128h, v128b, 0>;
651    def VMALOH : TernaryVRRd<"vmaloh", 0xE7AD, int_s390_vmaloh, v128f, v128h, 1>;
652    def VMALOF : TernaryVRRd<"vmalof", 0xE7AD, int_s390_vmalof, v128g, v128f, 2>;
653  }
654
655  let isCommutable = 1 in {
656    // Multiply high.
657    def VMH  : BinaryVRRcGeneric<"vmh", 0xE7A3>;
658    def VMHB : BinaryVRRc<"vmhb", 0xE7A3, int_s390_vmhb, v128b, v128b, 0>;
659    def VMHH : BinaryVRRc<"vmhh", 0xE7A3, int_s390_vmhh, v128h, v128h, 1>;
660    def VMHF : BinaryVRRc<"vmhf", 0xE7A3, int_s390_vmhf, v128f, v128f, 2>;
661
662    // Multiply logical high.
663    def VMLH  : BinaryVRRcGeneric<"vmlh", 0xE7A1>;
664    def VMLHB : BinaryVRRc<"vmlhb", 0xE7A1, int_s390_vmlhb, v128b, v128b, 0>;
665    def VMLHH : BinaryVRRc<"vmlhh", 0xE7A1, int_s390_vmlhh, v128h, v128h, 1>;
666    def VMLHF : BinaryVRRc<"vmlhf", 0xE7A1, int_s390_vmlhf, v128f, v128f, 2>;
667
668    // Multiply low.
669    def VML   : BinaryVRRcGeneric<"vml", 0xE7A2>;
670    def VMLB  : BinaryVRRc<"vmlb",  0xE7A2, mul, v128b, v128b, 0>;
671    def VMLHW : BinaryVRRc<"vmlhw", 0xE7A2, mul, v128h, v128h, 1>;
672    def VMLF  : BinaryVRRc<"vmlf",  0xE7A2, mul, v128f, v128f, 2>;
673
674    // Multiply even.
675    def VME  : BinaryVRRcGeneric<"vme", 0xE7A6>;
676    def VMEB : BinaryVRRc<"vmeb", 0xE7A6, int_s390_vmeb, v128h, v128b, 0>;
677    def VMEH : BinaryVRRc<"vmeh", 0xE7A6, int_s390_vmeh, v128f, v128h, 1>;
678    def VMEF : BinaryVRRc<"vmef", 0xE7A6, int_s390_vmef, v128g, v128f, 2>;
679
680    // Multiply logical even.
681    def VMLE  : BinaryVRRcGeneric<"vmle", 0xE7A4>;
682    def VMLEB : BinaryVRRc<"vmleb", 0xE7A4, int_s390_vmleb, v128h, v128b, 0>;
683    def VMLEH : BinaryVRRc<"vmleh", 0xE7A4, int_s390_vmleh, v128f, v128h, 1>;
684    def VMLEF : BinaryVRRc<"vmlef", 0xE7A4, int_s390_vmlef, v128g, v128f, 2>;
685
686    // Multiply odd.
687    def VMO  : BinaryVRRcGeneric<"vmo", 0xE7A7>;
688    def VMOB : BinaryVRRc<"vmob", 0xE7A7, int_s390_vmob, v128h, v128b, 0>;
689    def VMOH : BinaryVRRc<"vmoh", 0xE7A7, int_s390_vmoh, v128f, v128h, 1>;
690    def VMOF : BinaryVRRc<"vmof", 0xE7A7, int_s390_vmof, v128g, v128f, 2>;
691
692    // Multiply logical odd.
693    def VMLO  : BinaryVRRcGeneric<"vmlo", 0xE7A5>;
694    def VMLOB : BinaryVRRc<"vmlob", 0xE7A5, int_s390_vmlob, v128h, v128b, 0>;
695    def VMLOH : BinaryVRRc<"vmloh", 0xE7A5, int_s390_vmloh, v128f, v128h, 1>;
696    def VMLOF : BinaryVRRc<"vmlof", 0xE7A5, int_s390_vmlof, v128g, v128f, 2>;
697  }
698
699  // Multiply sum logical.
700  let Predicates = [FeatureVectorEnhancements1], isCommutable = 1 in {
701    def VMSL  : QuaternaryVRRdGeneric<"vmsl", 0xE7B8>;
702    def VMSLG : QuaternaryVRRd<"vmslg", 0xE7B8, int_s390_vmslg,
703                               v128q, v128g, v128g, v128q, 3>;
704  }
705
706  // Nand.
707  let Predicates = [FeatureVectorEnhancements1], isCommutable = 1 in
708    def VNN : BinaryVRRc<"vnn", 0xE76E, null_frag, v128any, v128any>;
709
710  // Nor.
711  let isCommutable = 1 in
712    def VNO : BinaryVRRc<"vno", 0xE76B, null_frag, v128any, v128any>;
713  def : InstAlias<"vnot\t$V1, $V2", (VNO VR128:$V1, VR128:$V2, VR128:$V2), 0>;
714
715  // Or.
716  let isCommutable = 1 in
717    def VO : BinaryVRRc<"vo", 0xE76A, null_frag, v128any, v128any>;
718
719  // Or with complement.
720  let Predicates = [FeatureVectorEnhancements1] in
721    def VOC : BinaryVRRc<"voc", 0xE76F, null_frag, v128any, v128any>;
722
723  // Population count.
724  def VPOPCT : UnaryVRRaGeneric<"vpopct", 0xE750>;
725  def : Pat<(v16i8 (z_popcnt VR128:$x)), (VPOPCT VR128:$x, 0)>;
726  let Predicates = [FeatureVectorEnhancements1] in {
727    def VPOPCTB : UnaryVRRa<"vpopctb", 0xE750, ctpop, v128b, v128b, 0>;
728    def VPOPCTH : UnaryVRRa<"vpopcth", 0xE750, ctpop, v128h, v128h, 1>;
729    def VPOPCTF : UnaryVRRa<"vpopctf", 0xE750, ctpop, v128f, v128f, 2>;
730    def VPOPCTG : UnaryVRRa<"vpopctg", 0xE750, ctpop, v128g, v128g, 3>;
731  }
732
733  // Element rotate left logical (with vector shift amount).
734  def VERLLV  : BinaryVRRcGeneric<"verllv", 0xE773>;
735  def VERLLVB : BinaryVRRc<"verllvb", 0xE773, rotl, v128b, v128b, 0>;
736  def VERLLVH : BinaryVRRc<"verllvh", 0xE773, rotl, v128h, v128h, 1>;
737  def VERLLVF : BinaryVRRc<"verllvf", 0xE773, rotl, v128f, v128f, 2>;
738  def VERLLVG : BinaryVRRc<"verllvg", 0xE773, rotl, v128g, v128g, 3>;
739
740  // Element rotate left logical (with scalar shift amount).
741  def VERLL  : BinaryVRSaGeneric<"verll", 0xE733>;
742  def VERLLB : BinaryVRSa<"verllb", 0xE733, z_vrotl_by_scalar, v128b, v128b, 0>;
743  def VERLLH : BinaryVRSa<"verllh", 0xE733, z_vrotl_by_scalar, v128h, v128h, 1>;
744  def VERLLF : BinaryVRSa<"verllf", 0xE733, z_vrotl_by_scalar, v128f, v128f, 2>;
745  def VERLLG : BinaryVRSa<"verllg", 0xE733, z_vrotl_by_scalar, v128g, v128g, 3>;
746
747  // Element rotate and insert under mask.
748  def VERIM  : QuaternaryVRIdGeneric<"verim", 0xE772>;
749  def VERIMB : QuaternaryVRId<"verimb", 0xE772, int_s390_verimb, v128b, v128b, 0>;
750  def VERIMH : QuaternaryVRId<"verimh", 0xE772, int_s390_verimh, v128h, v128h, 1>;
751  def VERIMF : QuaternaryVRId<"verimf", 0xE772, int_s390_verimf, v128f, v128f, 2>;
752  def VERIMG : QuaternaryVRId<"verimg", 0xE772, int_s390_verimg, v128g, v128g, 3>;
753
754  // Element shift left (with vector shift amount).
755  def VESLV  : BinaryVRRcGeneric<"veslv", 0xE770>;
756  def VESLVB : BinaryVRRc<"veslvb", 0xE770, z_vshl, v128b, v128b, 0>;
757  def VESLVH : BinaryVRRc<"veslvh", 0xE770, z_vshl, v128h, v128h, 1>;
758  def VESLVF : BinaryVRRc<"veslvf", 0xE770, z_vshl, v128f, v128f, 2>;
759  def VESLVG : BinaryVRRc<"veslvg", 0xE770, z_vshl, v128g, v128g, 3>;
760
761  // Element shift left (with scalar shift amount).
762  def VESL  : BinaryVRSaGeneric<"vesl", 0xE730>;
763  def VESLB : BinaryVRSa<"veslb", 0xE730, z_vshl_by_scalar, v128b, v128b, 0>;
764  def VESLH : BinaryVRSa<"veslh", 0xE730, z_vshl_by_scalar, v128h, v128h, 1>;
765  def VESLF : BinaryVRSa<"veslf", 0xE730, z_vshl_by_scalar, v128f, v128f, 2>;
766  def VESLG : BinaryVRSa<"veslg", 0xE730, z_vshl_by_scalar, v128g, v128g, 3>;
767
768  // Element shift right arithmetic (with vector shift amount).
769  def VESRAV  : BinaryVRRcGeneric<"vesrav", 0xE77A>;
770  def VESRAVB : BinaryVRRc<"vesravb", 0xE77A, z_vsra, v128b, v128b, 0>;
771  def VESRAVH : BinaryVRRc<"vesravh", 0xE77A, z_vsra, v128h, v128h, 1>;
772  def VESRAVF : BinaryVRRc<"vesravf", 0xE77A, z_vsra, v128f, v128f, 2>;
773  def VESRAVG : BinaryVRRc<"vesravg", 0xE77A, z_vsra, v128g, v128g, 3>;
774
775  // Element shift right arithmetic (with scalar shift amount).
776  def VESRA  : BinaryVRSaGeneric<"vesra", 0xE73A>;
777  def VESRAB : BinaryVRSa<"vesrab", 0xE73A, z_vsra_by_scalar, v128b, v128b, 0>;
778  def VESRAH : BinaryVRSa<"vesrah", 0xE73A, z_vsra_by_scalar, v128h, v128h, 1>;
779  def VESRAF : BinaryVRSa<"vesraf", 0xE73A, z_vsra_by_scalar, v128f, v128f, 2>;
780  def VESRAG : BinaryVRSa<"vesrag", 0xE73A, z_vsra_by_scalar, v128g, v128g, 3>;
781
782  // Element shift right logical (with vector shift amount).
783  def VESRLV  : BinaryVRRcGeneric<"vesrlv", 0xE778>;
784  def VESRLVB : BinaryVRRc<"vesrlvb", 0xE778, z_vsrl, v128b, v128b, 0>;
785  def VESRLVH : BinaryVRRc<"vesrlvh", 0xE778, z_vsrl, v128h, v128h, 1>;
786  def VESRLVF : BinaryVRRc<"vesrlvf", 0xE778, z_vsrl, v128f, v128f, 2>;
787  def VESRLVG : BinaryVRRc<"vesrlvg", 0xE778, z_vsrl, v128g, v128g, 3>;
788
789  // Element shift right logical (with scalar shift amount).
790  def VESRL  : BinaryVRSaGeneric<"vesrl", 0xE738>;
791  def VESRLB : BinaryVRSa<"vesrlb", 0xE738, z_vsrl_by_scalar, v128b, v128b, 0>;
792  def VESRLH : BinaryVRSa<"vesrlh", 0xE738, z_vsrl_by_scalar, v128h, v128h, 1>;
793  def VESRLF : BinaryVRSa<"vesrlf", 0xE738, z_vsrl_by_scalar, v128f, v128f, 2>;
794  def VESRLG : BinaryVRSa<"vesrlg", 0xE738, z_vsrl_by_scalar, v128g, v128g, 3>;
795
796  // Shift left.
797  def VSL : BinaryVRRc<"vsl", 0xE774, int_s390_vsl, v128b, v128b>;
798
799  // Shift left by byte.
800  def VSLB : BinaryVRRc<"vslb", 0xE775, int_s390_vslb, v128b, v128b>;
801
802  // Shift left double by byte.
803  def VSLDB : TernaryVRId<"vsldb", 0xE777, z_shl_double, v128b, v128b, 0>;
804  def : Pat<(int_s390_vsldb VR128:$x, VR128:$y, imm32zx8_timm:$z),
805            (VSLDB VR128:$x, VR128:$y, imm32zx8:$z)>;
806
807  // Shift left double by bit.
808  let Predicates = [FeatureVectorEnhancements2] in
809    def VSLD : TernaryVRId<"vsld", 0xE786, int_s390_vsld, v128b, v128b, 0>;
810
811  // Shift right arithmetic.
812  def VSRA : BinaryVRRc<"vsra", 0xE77E, int_s390_vsra, v128b, v128b>;
813
814  // Shift right arithmetic by byte.
815  def VSRAB : BinaryVRRc<"vsrab", 0xE77F, int_s390_vsrab, v128b, v128b>;
816
817  // Shift right logical.
818  def VSRL : BinaryVRRc<"vsrl", 0xE77C, int_s390_vsrl, v128b, v128b>;
819
820  // Shift right logical by byte.
821  def VSRLB : BinaryVRRc<"vsrlb", 0xE77D, int_s390_vsrlb, v128b, v128b>;
822
823  // Shift right double by bit.
824  let Predicates = [FeatureVectorEnhancements2] in
825    def VSRD : TernaryVRId<"vsrd", 0xE787, int_s390_vsrd, v128b, v128b, 0>;
826
827  // Subtract.
828  def VS  : BinaryVRRcGeneric<"vs", 0xE7F7>;
829  def VSB : BinaryVRRc<"vsb", 0xE7F7, sub, v128b, v128b, 0>;
830  def VSH : BinaryVRRc<"vsh", 0xE7F7, sub, v128h, v128h, 1>;
831  def VSF : BinaryVRRc<"vsf", 0xE7F7, sub, v128f, v128f, 2>;
832  def VSG : BinaryVRRc<"vsg", 0xE7F7, sub, v128g, v128g, 3>;
833  def VSQ : BinaryVRRc<"vsq", 0xE7F7, sub, v128q, v128q, 4>;
834
835  // Subtract compute borrow indication.
836  def VSCBI  : BinaryVRRcGeneric<"vscbi", 0xE7F5>;
837  def VSCBIB : BinaryVRRc<"vscbib", 0xE7F5, z_vscbi, v128b, v128b, 0>;
838  def VSCBIH : BinaryVRRc<"vscbih", 0xE7F5, z_vscbi, v128h, v128h, 1>;
839  def VSCBIF : BinaryVRRc<"vscbif", 0xE7F5, z_vscbi, v128f, v128f, 2>;
840  def VSCBIG : BinaryVRRc<"vscbig", 0xE7F5, z_vscbi, v128g, v128g, 3>;
841  def VSCBIQ : BinaryVRRc<"vscbiq", 0xE7F5, z_vscbi, v128q, v128q, 4>;
842
843  // Subtract with borrow indication.
844  def VSBI  : TernaryVRRdGeneric<"vsbi", 0xE7BF>;
845  def VSBIQ : TernaryVRRd<"vsbiq", 0xE7BF, z_vsbi, v128q, v128q, 4>;
846
847  // Subtract with borrow compute borrow indication.
848  def VSBCBI  : TernaryVRRdGeneric<"vsbcbi", 0xE7BD>;
849  def VSBCBIQ : TernaryVRRd<"vsbcbiq", 0xE7BD, z_vsbcbi, v128q, v128q, 4>;
850
851  // Sum across doubleword.
852  def VSUMG  : BinaryVRRcGeneric<"vsumg", 0xE765>;
853  def VSUMGH : BinaryVRRc<"vsumgh", 0xE765, z_vsum, v128g, v128h, 1>;
854  def VSUMGF : BinaryVRRc<"vsumgf", 0xE765, z_vsum, v128g, v128f, 2>;
855
856  // Sum across quadword.
857  def VSUMQ  : BinaryVRRcGeneric<"vsumq", 0xE767>;
858  def VSUMQF : BinaryVRRc<"vsumqf", 0xE767, z_vsum, v128q, v128f, 2>;
859  def VSUMQG : BinaryVRRc<"vsumqg", 0xE767, z_vsum, v128q, v128g, 3>;
860
861  // Sum across word.
862  def VSUM  : BinaryVRRcGeneric<"vsum", 0xE764>;
863  def VSUMB : BinaryVRRc<"vsumb", 0xE764, z_vsum, v128f, v128b, 0>;
864  def VSUMH : BinaryVRRc<"vsumh", 0xE764, z_vsum, v128f, v128h, 1>;
865}
866
867// Instantiate the bitwise ops for type TYPE.
868multiclass BitwiseVectorOps<ValueType type, SDPatternOperator not_op> {
869  let Predicates = [FeatureVector] in {
870    def : Pat<(type (and VR128:$x, VR128:$y)), (VN VR128:$x, VR128:$y)>;
871    def : Pat<(type (and VR128:$x, (not_op VR128:$y))),
872              (VNC VR128:$x, VR128:$y)>;
873    def : Pat<(type (or VR128:$x, VR128:$y)), (VO VR128:$x, VR128:$y)>;
874    def : Pat<(type (xor VR128:$x, VR128:$y)), (VX VR128:$x, VR128:$y)>;
875    def : Pat<(type (or (and VR128:$x, VR128:$z),
876                        (and VR128:$y, (not_op VR128:$z)))),
877              (VSEL VR128:$x, VR128:$y, VR128:$z)>;
878    def : Pat<(type (not_op (or VR128:$x, VR128:$y))),
879              (VNO VR128:$x, VR128:$y)>;
880    def : Pat<(type (not_op VR128:$x)), (VNO VR128:$x, VR128:$x)>;
881  }
882  let Predicates = [FeatureVectorEnhancements1] in {
883    def : Pat<(type (not_op (xor VR128:$x, VR128:$y))),
884              (VNX VR128:$x, VR128:$y)>;
885    def : Pat<(type (not_op (and VR128:$x, VR128:$y))),
886              (VNN VR128:$x, VR128:$y)>;
887    def : Pat<(type (or VR128:$x, (not_op VR128:$y))),
888              (VOC VR128:$x, VR128:$y)>;
889  }
890}
891
892defm : BitwiseVectorOps<v16i8, z_vnot>;
893defm : BitwiseVectorOps<v8i16, z_vnot>;
894defm : BitwiseVectorOps<v4i32, z_vnot>;
895defm : BitwiseVectorOps<v2i64, z_vnot>;
896defm : BitwiseVectorOps<i128, not>;
897
898// Instantiate additional patterns for absolute-related expressions on
899// type TYPE.  LC is the negate instruction for TYPE and LP is the absolute
900// instruction.
901multiclass IntegerAbsoluteVectorOps<ValueType type, Instruction lc,
902                                    Instruction lp, int shift> {
903  let Predicates = [FeatureVector] in {
904    def : Pat<(type (vselect (type (z_vicmph_zero VR128:$x)),
905                             (z_vneg VR128:$x), VR128:$x)),
906              (lc (lp VR128:$x))>;
907    def : Pat<(type (vselect (type (z_vnot (z_vicmph_zero VR128:$x))),
908                             VR128:$x, (z_vneg VR128:$x))),
909              (lc (lp VR128:$x))>;
910    def : Pat<(type (vselect (type (z_vicmpl_zero VR128:$x)),
911                             VR128:$x, (z_vneg VR128:$x))),
912              (lc (lp VR128:$x))>;
913    def : Pat<(type (vselect (type (z_vnot (z_vicmpl_zero VR128:$x))),
914                             (z_vneg VR128:$x), VR128:$x)),
915              (lc (lp VR128:$x))>;
916    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
917                             (z_vneg VR128:$x)),
918                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
919                             VR128:$x))),
920              (lp VR128:$x)>;
921    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
922                             VR128:$x),
923                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
924                             (z_vneg VR128:$x)))),
925              (lc (lp VR128:$x))>;
926  }
927}
928
929defm : IntegerAbsoluteVectorOps<v16i8, VLCB, VLPB, 7>;
930defm : IntegerAbsoluteVectorOps<v8i16, VLCH, VLPH, 15>;
931defm : IntegerAbsoluteVectorOps<v4i32, VLCF, VLPF, 31>;
932defm : IntegerAbsoluteVectorOps<v2i64, VLCG, VLPG, 63>;
933
934// Instantiate minimum- and maximum-related patterns for TYPE.  CMPH is the
935// signed or unsigned "set if greater than" comparison instruction and
936// MIN and MAX are the associated minimum and maximum instructions.
937multiclass IntegerMinMaxVectorOps<ValueType type, SDPatternOperator cmph,
938                                  Instruction min, Instruction max> {
939  let Predicates = [FeatureVector] in {
940    def : Pat<(type (vselect (cmph VR128:$x, VR128:$y), VR128:$x, VR128:$y)),
941              (max VR128:$x, VR128:$y)>;
942    def : Pat<(type (vselect (cmph VR128:$x, VR128:$y), VR128:$y, VR128:$x)),
943              (min VR128:$x, VR128:$y)>;
944    def : Pat<(type (vselect (z_vnot (cmph VR128:$x, VR128:$y)),
945                             VR128:$x, VR128:$y)),
946              (min VR128:$x, VR128:$y)>;
947    def : Pat<(type (vselect (z_vnot (cmph VR128:$x, VR128:$y)),
948                             VR128:$y, VR128:$x)),
949              (max VR128:$x, VR128:$y)>;
950  }
951}
952
953// Signed min/max.
954defm : IntegerMinMaxVectorOps<v16i8, z_vicmph, VMNB, VMXB>;
955defm : IntegerMinMaxVectorOps<v8i16, z_vicmph, VMNH, VMXH>;
956defm : IntegerMinMaxVectorOps<v4i32, z_vicmph, VMNF, VMXF>;
957defm : IntegerMinMaxVectorOps<v2i64, z_vicmph, VMNG, VMXG>;
958
959// Unsigned min/max.
960defm : IntegerMinMaxVectorOps<v16i8, z_vicmphl, VMNLB, VMXLB>;
961defm : IntegerMinMaxVectorOps<v8i16, z_vicmphl, VMNLH, VMXLH>;
962defm : IntegerMinMaxVectorOps<v4i32, z_vicmphl, VMNLF, VMXLF>;
963defm : IntegerMinMaxVectorOps<v2i64, z_vicmphl, VMNLG, VMXLG>;
964
965// Instantiate full-vector shifts.
966multiclass FullVectorShiftOps<SDPatternOperator shift,
967                              Instruction sbit, Instruction sbyte> {
968  let Predicates = [FeatureVector] in {
969    def : Pat<(shift (i128 VR128:$x), imm32nobytes:$amt),
970              (sbit VR128:$x, (VREPIB (UIMM8 imm:$amt)))>;
971    def : Pat<(shift (i128 VR128:$x), imm32nobits:$amt),
972              (sbyte VR128:$x, (VREPIB (UIMM8 imm:$amt)))>;
973    def : Pat<(shift (i128 VR128:$x), imm32:$amt),
974              (sbit (sbyte VR128:$x, (VREPIB (UIMM8 imm:$amt))),
975                    (VREPIB (UIMM8 imm:$amt)))>;
976    def : Pat<(shift (i128 VR128:$x), GR32:$amt),
977              (sbit (sbyte VR128:$x, (VREPB (VLVGP32 GR32:$amt, GR32:$amt), 15)),
978                    (VREPB (VLVGP32 GR32:$amt, GR32:$amt), 15))>;
979  }
980}
981defm : FullVectorShiftOps<vshiftop<shl>, VSL, VSLB>;
982defm : FullVectorShiftOps<vshiftop<srl>, VSRL, VSRLB>;
983defm : FullVectorShiftOps<vshiftop<sra>, VSRA, VSRAB>;
984
985//===----------------------------------------------------------------------===//
986// Integer comparison
987//===----------------------------------------------------------------------===//
988
989let Predicates = [FeatureVector] in {
990  // Element compare.
991  let Defs = [CC] in {
992    def VEC  : CompareVRRaGeneric<"vec", 0xE7DB>;
993    def VECB : CompareVRRa<"vecb", 0xE7DB, null_frag, v128b, 0>;
994    def VECH : CompareVRRa<"vech", 0xE7DB, null_frag, v128h, 1>;
995    def VECF : CompareVRRa<"vecf", 0xE7DB, null_frag, v128f, 2>;
996    def VECG : CompareVRRa<"vecg", 0xE7DB, null_frag, v128g, 3>;
997  }
998
999  // Element compare logical.
1000  let Defs = [CC] in {
1001    def VECL  : CompareVRRaGeneric<"vecl", 0xE7D9>;
1002    def VECLB : CompareVRRa<"veclb", 0xE7D9, null_frag, v128b, 0>;
1003    def VECLH : CompareVRRa<"veclh", 0xE7D9, null_frag, v128h, 1>;
1004    def VECLF : CompareVRRa<"veclf", 0xE7D9, null_frag, v128f, 2>;
1005    def VECLG : CompareVRRa<"veclg", 0xE7D9, null_frag, v128g, 3>;
1006  }
1007
1008  // Compare equal.
1009  def  VCEQ  : BinaryVRRbSPairGeneric<"vceq", 0xE7F8>;
1010  defm VCEQB : BinaryVRRbSPair<"vceqb", 0xE7F8, z_vicmpe, z_vicmpes,
1011                               v128b, v128b, 0>;
1012  defm VCEQH : BinaryVRRbSPair<"vceqh", 0xE7F8, z_vicmpe, z_vicmpes,
1013                               v128h, v128h, 1>;
1014  defm VCEQF : BinaryVRRbSPair<"vceqf", 0xE7F8, z_vicmpe, z_vicmpes,
1015                               v128f, v128f, 2>;
1016  defm VCEQG : BinaryVRRbSPair<"vceqg", 0xE7F8, z_vicmpe, z_vicmpes,
1017                               v128g, v128g, 3>;
1018
1019  // Compare high.
1020  def  VCH  : BinaryVRRbSPairGeneric<"vch", 0xE7FB>;
1021  defm VCHB : BinaryVRRbSPair<"vchb", 0xE7FB, z_vicmph, z_vicmphs,
1022                              v128b, v128b, 0>;
1023  defm VCHH : BinaryVRRbSPair<"vchh", 0xE7FB, z_vicmph, z_vicmphs,
1024                              v128h, v128h, 1>;
1025  defm VCHF : BinaryVRRbSPair<"vchf", 0xE7FB, z_vicmph, z_vicmphs,
1026                              v128f, v128f, 2>;
1027  defm VCHG : BinaryVRRbSPair<"vchg", 0xE7FB, z_vicmph, z_vicmphs,
1028                              v128g, v128g, 3>;
1029
1030  // Compare high logical.
1031  def  VCHL  : BinaryVRRbSPairGeneric<"vchl", 0xE7F9>;
1032  defm VCHLB : BinaryVRRbSPair<"vchlb", 0xE7F9, z_vicmphl, z_vicmphls,
1033                               v128b, v128b, 0>;
1034  defm VCHLH : BinaryVRRbSPair<"vchlh", 0xE7F9, z_vicmphl, z_vicmphls,
1035                               v128h, v128h, 1>;
1036  defm VCHLF : BinaryVRRbSPair<"vchlf", 0xE7F9, z_vicmphl, z_vicmphls,
1037                               v128f, v128f, 2>;
1038  defm VCHLG : BinaryVRRbSPair<"vchlg", 0xE7F9, z_vicmphl, z_vicmphls,
1039                               v128g, v128g, 3>;
1040
1041  // Test under mask.
1042  let Defs = [CC] in
1043    def VTM : CompareVRRa<"vtm", 0xE7D8, z_vtm, v128b, 0>;
1044}
1045
1046//===----------------------------------------------------------------------===//
1047// Floating-point arithmetic
1048//===----------------------------------------------------------------------===//
1049
1050// See comments in SystemZInstrFP.td for the suppression flags and
1051// rounding modes.
1052multiclass VectorRounding<Instruction insn, TypedReg tr> {
1053  def : FPConversion<insn, any_frint,      tr, tr, 0, 0>;
1054  def : FPConversion<insn, any_fnearbyint, tr, tr, 4, 0>;
1055  def : FPConversion<insn, any_ffloor,     tr, tr, 4, 7>;
1056  def : FPConversion<insn, any_fceil,      tr, tr, 4, 6>;
1057  def : FPConversion<insn, any_ftrunc,     tr, tr, 4, 5>;
1058  def : FPConversion<insn, any_fround,     tr, tr, 4, 1>;
1059}
1060
1061let Predicates = [FeatureVector] in {
1062  // Add.
1063  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
1064    def VFA   : BinaryVRRcFloatGeneric<"vfa", 0xE7E3>;
1065    def VFADB : BinaryVRRc<"vfadb", 0xE7E3, any_fadd, v128db, v128db, 3, 0>;
1066    def WFADB : BinaryVRRc<"wfadb", 0xE7E3, any_fadd, v64db, v64db, 3, 8, 0,
1067                           "adbr">;
1068    let Predicates = [FeatureVectorEnhancements1] in {
1069      def VFASB : BinaryVRRc<"vfasb", 0xE7E3, any_fadd, v128sb, v128sb, 2, 0>;
1070      def WFASB : BinaryVRRc<"wfasb", 0xE7E3, any_fadd, v32sb, v32sb, 2, 8, 0,
1071                             "aebr">;
1072      def WFAXB : BinaryVRRc<"wfaxb", 0xE7E3, any_fadd, v128xb, v128xb, 4, 8>;
1073    }
1074  }
1075
1076  // Convert from fixed.
1077  let Uses = [FPC], mayRaiseFPException = 1 in {
1078    def VCDG  : TernaryVRRaFloatGeneric<"vcdg", 0xE7C3>;
1079    def VCDGB : TernaryVRRa<"vcdgb", 0xE7C3, null_frag, v128db, v128g, 3, 0>;
1080    def WCDGB : TernaryVRRa<"wcdgb", 0xE7C3, null_frag, v64db, v64g, 3, 8>;
1081  }
1082  def : FPConversion<VCDGB, any_sint_to_fp, v128db, v128g, 0, 0>;
1083  let Predicates = [FeatureVectorEnhancements2] in {
1084    let Uses = [FPC], mayRaiseFPException = 1 in {
1085      let isAsmParserOnly = 1 in
1086        def VCFPS  : TernaryVRRaFloatGeneric<"vcfps", 0xE7C3>;
1087      def VCEFB : TernaryVRRa<"vcefb", 0xE7C3, null_frag, v128sb, v128g, 2, 0>;
1088      def WCEFB : TernaryVRRa<"wcefb", 0xE7C3, null_frag, v32sb, v32f, 2, 8>;
1089    }
1090    def : FPConversion<VCEFB, any_sint_to_fp, v128sb, v128f, 0, 0>;
1091  }
1092
1093  // Convert from logical.
1094  let Uses = [FPC], mayRaiseFPException = 1 in {
1095    def VCDLG  : TernaryVRRaFloatGeneric<"vcdlg", 0xE7C1>;
1096    def VCDLGB : TernaryVRRa<"vcdlgb", 0xE7C1, null_frag, v128db, v128g, 3, 0>;
1097    def WCDLGB : TernaryVRRa<"wcdlgb", 0xE7C1, null_frag, v64db, v64g, 3, 8>;
1098  }
1099  def : FPConversion<VCDLGB, any_uint_to_fp, v128db, v128g, 0, 0>;
1100  let Predicates = [FeatureVectorEnhancements2] in {
1101    let Uses = [FPC], mayRaiseFPException = 1 in {
1102      let isAsmParserOnly = 1 in
1103        def VCFPL  : TernaryVRRaFloatGeneric<"vcfpl", 0xE7C1>;
1104      def VCELFB : TernaryVRRa<"vcelfb", 0xE7C1, null_frag, v128sb, v128g, 2, 0>;
1105      def WCELFB : TernaryVRRa<"wcelfb", 0xE7C1, null_frag, v32sb, v32f, 2, 8>;
1106    }
1107    def : FPConversion<VCELFB, any_uint_to_fp, v128sb, v128f, 0, 0>;
1108  }
1109
1110  // Convert to fixed.
1111  let Uses = [FPC], mayRaiseFPException = 1 in {
1112    def VCGD  : TernaryVRRaFloatGeneric<"vcgd", 0xE7C2>;
1113    def VCGDB : TernaryVRRa<"vcgdb", 0xE7C2, null_frag, v128g, v128db, 3, 0>;
1114    def WCGDB : TernaryVRRa<"wcgdb", 0xE7C2, null_frag, v64g, v64db, 3, 8>;
1115  }
1116  // Rounding mode should agree with SystemZInstrFP.td.
1117  def : FPConversion<VCGDB, any_fp_to_sint, v128g, v128db, 0, 5>;
1118  let Predicates = [FeatureVectorEnhancements2] in {
1119    let Uses = [FPC], mayRaiseFPException = 1 in {
1120      let isAsmParserOnly = 1 in
1121        def VCSFP  : TernaryVRRaFloatGeneric<"vcsfp", 0xE7C2>;
1122      def VCFEB : TernaryVRRa<"vcfeb", 0xE7C2, null_frag, v128sb, v128g, 2, 0>;
1123      def WCFEB : TernaryVRRa<"wcfeb", 0xE7C2, null_frag, v32sb, v32f, 2, 8>;
1124    }
1125    // Rounding mode should agree with SystemZInstrFP.td.
1126    def : FPConversion<VCFEB, any_fp_to_sint, v128f, v128sb, 0, 5>;
1127  }
1128
1129  // Convert to logical.
1130  let Uses = [FPC], mayRaiseFPException = 1 in {
1131    def VCLGD  : TernaryVRRaFloatGeneric<"vclgd", 0xE7C0>;
1132    def VCLGDB : TernaryVRRa<"vclgdb", 0xE7C0, null_frag, v128g, v128db, 3, 0>;
1133    def WCLGDB : TernaryVRRa<"wclgdb", 0xE7C0, null_frag, v64g, v64db, 3, 8>;
1134  }
1135  // Rounding mode should agree with SystemZInstrFP.td.
1136  def : FPConversion<VCLGDB, any_fp_to_uint, v128g, v128db, 0, 5>;
1137  let Predicates = [FeatureVectorEnhancements2] in {
1138    let Uses = [FPC], mayRaiseFPException = 1 in {
1139      let isAsmParserOnly = 1 in
1140        def VCLFP  : TernaryVRRaFloatGeneric<"vclfp", 0xE7C0>;
1141      def VCLFEB : TernaryVRRa<"vclfeb", 0xE7C0, null_frag, v128sb, v128g, 2, 0>;
1142      def WCLFEB : TernaryVRRa<"wclfeb", 0xE7C0, null_frag, v32sb, v32f, 2, 8>;
1143    }
1144    // Rounding mode should agree with SystemZInstrFP.td.
1145    def : FPConversion<VCLFEB, any_fp_to_uint, v128f, v128sb, 0, 5>;
1146  }
1147
1148  // Divide.
1149  let Uses = [FPC], mayRaiseFPException = 1 in {
1150    def VFD   : BinaryVRRcFloatGeneric<"vfd", 0xE7E5>;
1151    def VFDDB : BinaryVRRc<"vfddb", 0xE7E5, any_fdiv, v128db, v128db, 3, 0>;
1152    def WFDDB : BinaryVRRc<"wfddb", 0xE7E5, any_fdiv, v64db, v64db, 3, 8, 0,
1153                           "ddbr">;
1154    let Predicates = [FeatureVectorEnhancements1] in {
1155      def VFDSB : BinaryVRRc<"vfdsb", 0xE7E5, any_fdiv, v128sb, v128sb, 2, 0>;
1156      def WFDSB : BinaryVRRc<"wfdsb", 0xE7E5, any_fdiv, v32sb, v32sb, 2, 8, 0,
1157                             "debr">;
1158      def WFDXB : BinaryVRRc<"wfdxb", 0xE7E5, any_fdiv, v128xb, v128xb, 4, 8>;
1159    }
1160  }
1161
1162  // Load FP integer.
1163  let Uses = [FPC], mayRaiseFPException = 1 in {
1164    def VFI   : TernaryVRRaFloatGeneric<"vfi", 0xE7C7>;
1165    def VFIDB : TernaryVRRa<"vfidb", 0xE7C7, int_s390_vfidb, v128db, v128db, 3, 0>;
1166    def WFIDB : TernaryVRRa<"wfidb", 0xE7C7, null_frag, v64db, v64db, 3, 8>;
1167  }
1168  defm : VectorRounding<VFIDB, v128db>;
1169  defm : VectorRounding<WFIDB, v64db>;
1170  let Predicates = [FeatureVectorEnhancements1] in {
1171    let Uses = [FPC], mayRaiseFPException = 1 in {
1172      def VFISB : TernaryVRRa<"vfisb", 0xE7C7, int_s390_vfisb, v128sb, v128sb, 2, 0>;
1173      def WFISB : TernaryVRRa<"wfisb", 0xE7C7, null_frag, v32sb, v32sb, 2, 8>;
1174      def WFIXB : TernaryVRRa<"wfixb", 0xE7C7, null_frag, v128xb, v128xb, 4, 8>;
1175    }
1176    defm : VectorRounding<VFISB, v128sb>;
1177    defm : VectorRounding<WFISB, v32sb>;
1178    defm : VectorRounding<WFIXB, v128xb>;
1179  }
1180
1181  // Load lengthened.
1182  let Uses = [FPC], mayRaiseFPException = 1 in {
1183    def VLDE  : UnaryVRRaFloatGeneric<"vlde", 0xE7C4>;
1184    def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_any_vextend, v128db, v128sb, 2, 0>;
1185    def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, any_fpextend, v64db, v32sb, 2, 8, 0,
1186                          "ldebr">;
1187  }
1188  let Predicates = [FeatureVectorEnhancements1] in {
1189    let Uses = [FPC], mayRaiseFPException = 1 in {
1190      let isAsmParserOnly = 1 in {
1191        def VFLL  : UnaryVRRaFloatGeneric<"vfll", 0xE7C4>;
1192        def VFLLS : UnaryVRRa<"vflls", 0xE7C4, null_frag, v128db, v128sb, 2, 0>;
1193        def WFLLS : UnaryVRRa<"wflls", 0xE7C4, null_frag, v64db, v32sb, 2, 8>;
1194      }
1195      def WFLLD : UnaryVRRa<"wflld", 0xE7C4, any_fpextend, v128xb, v64db, 3, 8>;
1196    }
1197    def : Pat<(f128 (any_fpextend (f32 VR32:$src))),
1198              (WFLLD (WLDEB VR32:$src))>;
1199  }
1200
1201  // Load rounded.
1202  let Uses = [FPC], mayRaiseFPException = 1 in {
1203    def VLED  : TernaryVRRaFloatGeneric<"vled", 0xE7C5>;
1204    def VLEDB : TernaryVRRa<"vledb", 0xE7C5, null_frag, v128sb, v128db, 3, 0>;
1205    def WLEDB : TernaryVRRa<"wledb", 0xE7C5, null_frag, v32sb, v64db, 3, 8>;
1206  }
1207  def : Pat<(v4f32 (z_any_vround (v2f64 VR128:$src))), (VLEDB VR128:$src, 0, 0)>;
1208  def : FPConversion<WLEDB, any_fpround, v32sb, v64db, 0, 0>;
1209  let Predicates = [FeatureVectorEnhancements1] in {
1210    let Uses = [FPC], mayRaiseFPException = 1 in {
1211      let isAsmParserOnly = 1 in {
1212        def VFLR  : TernaryVRRaFloatGeneric<"vflr", 0xE7C5>;
1213        def VFLRD : TernaryVRRa<"vflrd", 0xE7C5, null_frag, v128sb, v128db, 3, 0>;
1214        def WFLRD : TernaryVRRa<"wflrd", 0xE7C5, null_frag, v32sb, v64db, 3, 8>;
1215      }
1216      def WFLRX : TernaryVRRa<"wflrx", 0xE7C5, null_frag, v64db, v128xb, 4, 8>;
1217    }
1218    def : FPConversion<WFLRX, any_fpround, v64db, v128xb, 0, 0>;
1219    def : Pat<(f32 (any_fpround (f128 VR128:$src))),
1220              (WLEDB (WFLRX VR128:$src, 0, 3), 0, 0)>;
1221  }
1222
1223  // Maximum.
1224  multiclass VectorMax<Instruction insn, TypedReg tr> {
1225    def : FPMinMax<insn, any_fmaxnum, tr, 4>;
1226    def : FPMinMax<insn, any_fmaximum, tr, 1>;
1227  }
1228  let Predicates = [FeatureVectorEnhancements1] in {
1229    let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
1230      def VFMAX   : TernaryVRRcFloatGeneric<"vfmax", 0xE7EF>;
1231      def VFMAXDB : TernaryVRRcFloat<"vfmaxdb", 0xE7EF, int_s390_vfmaxdb,
1232                                     v128db, v128db, 3, 0>;
1233      def WFMAXDB : TernaryVRRcFloat<"wfmaxdb", 0xE7EF, null_frag,
1234                                     v64db, v64db, 3, 8>;
1235      def VFMAXSB : TernaryVRRcFloat<"vfmaxsb", 0xE7EF, int_s390_vfmaxsb,
1236                                     v128sb, v128sb, 2, 0>;
1237      def WFMAXSB : TernaryVRRcFloat<"wfmaxsb", 0xE7EF, null_frag,
1238                                     v32sb, v32sb, 2, 8>;
1239      def WFMAXXB : TernaryVRRcFloat<"wfmaxxb", 0xE7EF, null_frag,
1240                                     v128xb, v128xb, 4, 8>;
1241    }
1242    defm : VectorMax<VFMAXDB, v128db>;
1243    defm : VectorMax<WFMAXDB, v64db>;
1244    defm : VectorMax<VFMAXSB, v128sb>;
1245    defm : VectorMax<WFMAXSB, v32sb>;
1246    defm : VectorMax<WFMAXXB, v128xb>;
1247  }
1248
1249  // Minimum.
1250  multiclass VectorMin<Instruction insn, TypedReg tr> {
1251    def : FPMinMax<insn, any_fminnum, tr, 4>;
1252    def : FPMinMax<insn, any_fminimum, tr, 1>;
1253  }
1254  let Predicates = [FeatureVectorEnhancements1] in {
1255    let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
1256      def VFMIN   : TernaryVRRcFloatGeneric<"vfmin", 0xE7EE>;
1257      def VFMINDB : TernaryVRRcFloat<"vfmindb", 0xE7EE, int_s390_vfmindb,
1258                                     v128db, v128db, 3, 0>;
1259      def WFMINDB : TernaryVRRcFloat<"wfmindb", 0xE7EE, null_frag,
1260                                     v64db, v64db, 3, 8>;
1261      def VFMINSB : TernaryVRRcFloat<"vfminsb", 0xE7EE, int_s390_vfminsb,
1262                                     v128sb, v128sb, 2, 0>;
1263      def WFMINSB : TernaryVRRcFloat<"wfminsb", 0xE7EE, null_frag,
1264                                     v32sb, v32sb, 2, 8>;
1265      def WFMINXB : TernaryVRRcFloat<"wfminxb", 0xE7EE, null_frag,
1266                                     v128xb, v128xb, 4, 8>;
1267    }
1268    defm : VectorMin<VFMINDB, v128db>;
1269    defm : VectorMin<WFMINDB, v64db>;
1270    defm : VectorMin<VFMINSB, v128sb>;
1271    defm : VectorMin<WFMINSB, v32sb>;
1272    defm : VectorMin<WFMINXB, v128xb>;
1273  }
1274
1275  // Multiply.
1276  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
1277    def VFM   : BinaryVRRcFloatGeneric<"vfm", 0xE7E7>;
1278    def VFMDB : BinaryVRRc<"vfmdb", 0xE7E7, any_fmul, v128db, v128db, 3, 0>;
1279    def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, any_fmul, v64db, v64db, 3, 8, 0,
1280                           "mdbr">;
1281    let Predicates = [FeatureVectorEnhancements1] in {
1282      def VFMSB : BinaryVRRc<"vfmsb", 0xE7E7, any_fmul, v128sb, v128sb, 2, 0>;
1283      def WFMSB : BinaryVRRc<"wfmsb", 0xE7E7, any_fmul, v32sb, v32sb, 2, 8, 0,
1284                             "meebr">;
1285      def WFMXB : BinaryVRRc<"wfmxb", 0xE7E7, any_fmul, v128xb, v128xb, 4, 8>;
1286    }
1287  }
1288
1289  // Multiply and add.
1290  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
1291    def VFMA   : TernaryVRReFloatGeneric<"vfma", 0xE78F>;
1292    def VFMADB : TernaryVRRe<"vfmadb", 0xE78F, any_fma, v128db, v128db, 0, 3>;
1293    def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, any_fma, v64db, v64db, 8, 3,
1294                             "madbr">;
1295    let Predicates = [FeatureVectorEnhancements1] in {
1296      def VFMASB : TernaryVRRe<"vfmasb", 0xE78F, any_fma, v128sb, v128sb, 0, 2>;
1297      def WFMASB : TernaryVRRe<"wfmasb", 0xE78F, any_fma, v32sb, v32sb, 8, 2,
1298                               "maebr">;
1299      def WFMAXB : TernaryVRRe<"wfmaxb", 0xE78F, any_fma, v128xb, v128xb, 8, 4>;
1300    }
1301  }
1302
1303  // Multiply and subtract.
1304  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
1305    def VFMS   : TernaryVRReFloatGeneric<"vfms", 0xE78E>;
1306    def VFMSDB : TernaryVRRe<"vfmsdb", 0xE78E, any_fms, v128db, v128db, 0, 3>;
1307    def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, any_fms, v64db, v64db, 8, 3,
1308                             "msdbr">;
1309    let Predicates = [FeatureVectorEnhancements1] in {
1310      def VFMSSB : TernaryVRRe<"vfmssb", 0xE78E, any_fms, v128sb, v128sb, 0, 2>;
1311      def WFMSSB : TernaryVRRe<"wfmssb", 0xE78E, any_fms, v32sb, v32sb, 8, 2,
1312                               "msebr">;
1313      def WFMSXB : TernaryVRRe<"wfmsxb", 0xE78E, any_fms, v128xb, v128xb, 8, 4>;
1314    }
1315  }
1316
1317  // Negative multiply and add.
1318  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1,
1319      Predicates = [FeatureVectorEnhancements1] in {
1320    def VFNMA   : TernaryVRReFloatGeneric<"vfnma", 0xE79F>;
1321    def VFNMADB : TernaryVRRe<"vfnmadb", 0xE79F, any_fnma, v128db, v128db, 0, 3>;
1322    def WFNMADB : TernaryVRRe<"wfnmadb", 0xE79F, any_fnma, v64db, v64db, 8, 3>;
1323    def VFNMASB : TernaryVRRe<"vfnmasb", 0xE79F, any_fnma, v128sb, v128sb, 0, 2>;
1324    def WFNMASB : TernaryVRRe<"wfnmasb", 0xE79F, any_fnma, v32sb, v32sb, 8, 2>;
1325    def WFNMAXB : TernaryVRRe<"wfnmaxb", 0xE79F, any_fnma, v128xb, v128xb, 8, 4>;
1326  }
1327
1328  // Negative multiply and subtract.
1329  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1,
1330      Predicates = [FeatureVectorEnhancements1] in {
1331    def VFNMS   : TernaryVRReFloatGeneric<"vfnms", 0xE79E>;
1332    def VFNMSDB : TernaryVRRe<"vfnmsdb", 0xE79E, any_fnms, v128db, v128db, 0, 3>;
1333    def WFNMSDB : TernaryVRRe<"wfnmsdb", 0xE79E, any_fnms, v64db, v64db, 8, 3>;
1334    def VFNMSSB : TernaryVRRe<"vfnmssb", 0xE79E, any_fnms, v128sb, v128sb, 0, 2>;
1335    def WFNMSSB : TernaryVRRe<"wfnmssb", 0xE79E, any_fnms, v32sb, v32sb, 8, 2>;
1336    def WFNMSXB : TernaryVRRe<"wfnmsxb", 0xE79E, any_fnms, v128xb, v128xb, 8, 4>;
1337  }
1338
1339  // Perform sign operation.
1340  def VFPSO   : BinaryVRRaFloatGeneric<"vfpso", 0xE7CC>;
1341  def VFPSODB : BinaryVRRa<"vfpsodb", 0xE7CC, null_frag, v128db, v128db, 3, 0>;
1342  def WFPSODB : BinaryVRRa<"wfpsodb", 0xE7CC, null_frag, v64db, v64db, 3, 8>;
1343  let Predicates = [FeatureVectorEnhancements1] in {
1344    def VFPSOSB : BinaryVRRa<"vfpsosb", 0xE7CC, null_frag, v128sb, v128sb, 2, 0>;
1345    def WFPSOSB : BinaryVRRa<"wfpsosb", 0xE7CC, null_frag, v32sb, v32sb, 2, 8>;
1346    def WFPSOXB : BinaryVRRa<"wfpsoxb", 0xE7CC, null_frag, v128xb, v128xb, 4, 8>;
1347  }
1348
1349  // Load complement.
1350  def VFLCDB : UnaryVRRa<"vflcdb", 0xE7CC, fneg, v128db, v128db, 3, 0, 0>;
1351  def WFLCDB : UnaryVRRa<"wflcdb", 0xE7CC, fneg, v64db, v64db, 3, 8, 0>;
1352  let Predicates = [FeatureVectorEnhancements1] in {
1353    def VFLCSB : UnaryVRRa<"vflcsb", 0xE7CC, fneg, v128sb, v128sb, 2, 0, 0>;
1354    def WFLCSB : UnaryVRRa<"wflcsb", 0xE7CC, fneg, v32sb, v32sb, 2, 8, 0>;
1355    def WFLCXB : UnaryVRRa<"wflcxb", 0xE7CC, fneg, v128xb, v128xb, 4, 8, 0>;
1356  }
1357
1358  // Load negative.
1359  def VFLNDB : UnaryVRRa<"vflndb", 0xE7CC, fnabs, v128db, v128db, 3, 0, 1>;
1360  def WFLNDB : UnaryVRRa<"wflndb", 0xE7CC, fnabs, v64db, v64db, 3, 8, 1>;
1361  let Predicates = [FeatureVectorEnhancements1] in {
1362    def VFLNSB : UnaryVRRa<"vflnsb", 0xE7CC, fnabs, v128sb, v128sb, 2, 0, 1>;
1363    def WFLNSB : UnaryVRRa<"wflnsb", 0xE7CC, fnabs, v32sb, v32sb, 2, 8, 1>;
1364    def WFLNXB : UnaryVRRa<"wflnxb", 0xE7CC, fnabs, v128xb, v128xb, 4, 8, 1>;
1365  }
1366
1367  // Load positive.
1368  def VFLPDB : UnaryVRRa<"vflpdb", 0xE7CC, fabs, v128db, v128db, 3, 0, 2>;
1369  def WFLPDB : UnaryVRRa<"wflpdb", 0xE7CC, fabs, v64db, v64db, 3, 8, 2>;
1370  let Predicates = [FeatureVectorEnhancements1] in {
1371    def VFLPSB : UnaryVRRa<"vflpsb", 0xE7CC, fabs, v128sb, v128sb, 2, 0, 2>;
1372    def WFLPSB : UnaryVRRa<"wflpsb", 0xE7CC, fabs, v32sb, v32sb, 2, 8, 2>;
1373    def WFLPXB : UnaryVRRa<"wflpxb", 0xE7CC, fabs, v128xb, v128xb, 4, 8, 2>;
1374  }
1375
1376  // Square root.
1377  let Uses = [FPC], mayRaiseFPException = 1 in {
1378    def VFSQ   : UnaryVRRaFloatGeneric<"vfsq", 0xE7CE>;
1379    def VFSQDB : UnaryVRRa<"vfsqdb", 0xE7CE, any_fsqrt, v128db, v128db, 3, 0>;
1380    def WFSQDB : UnaryVRRa<"wfsqdb", 0xE7CE, any_fsqrt, v64db, v64db, 3, 8, 0,
1381                           "sqdbr">;
1382    let Predicates = [FeatureVectorEnhancements1] in {
1383      def VFSQSB : UnaryVRRa<"vfsqsb", 0xE7CE, any_fsqrt, v128sb, v128sb, 2, 0>;
1384      def WFSQSB : UnaryVRRa<"wfsqsb", 0xE7CE, any_fsqrt, v32sb, v32sb, 2, 8, 0,
1385                             "sqebr">;
1386      def WFSQXB : UnaryVRRa<"wfsqxb", 0xE7CE, any_fsqrt, v128xb, v128xb, 4, 8>;
1387    }
1388  }
1389
1390  // Subtract.
1391  let Uses = [FPC], mayRaiseFPException = 1 in {
1392    def VFS   : BinaryVRRcFloatGeneric<"vfs", 0xE7E2>;
1393    def VFSDB : BinaryVRRc<"vfsdb", 0xE7E2, any_fsub, v128db, v128db, 3, 0>;
1394    def WFSDB : BinaryVRRc<"wfsdb", 0xE7E2, any_fsub, v64db, v64db, 3, 8, 0,
1395                           "sdbr">;
1396    let Predicates = [FeatureVectorEnhancements1] in {
1397      def VFSSB : BinaryVRRc<"vfssb", 0xE7E2, any_fsub, v128sb, v128sb, 2, 0>;
1398      def WFSSB : BinaryVRRc<"wfssb", 0xE7E2, any_fsub, v32sb, v32sb, 2, 8, 0,
1399                             "sebr">;
1400      def WFSXB : BinaryVRRc<"wfsxb", 0xE7E2, any_fsub, v128xb, v128xb, 4, 8>;
1401    }
1402  }
1403
1404  // Test data class immediate.
1405  let Defs = [CC] in {
1406    def VFTCI   : BinaryVRIeFloatGeneric<"vftci", 0xE74A>;
1407    def VFTCIDB : BinaryVRIe<"vftcidb", 0xE74A, z_vftci, v128g, v128db, 3, 0>;
1408    def WFTCIDB : BinaryVRIe<"wftcidb", 0xE74A, null_frag, v64g, v64db, 3, 8>;
1409    let Predicates = [FeatureVectorEnhancements1] in {
1410      def VFTCISB : BinaryVRIe<"vftcisb", 0xE74A, z_vftci, v128f, v128sb, 2, 0>;
1411      def WFTCISB : BinaryVRIe<"wftcisb", 0xE74A, null_frag, v32f, v32sb, 2, 8>;
1412      def WFTCIXB : BinaryVRIe<"wftcixb", 0xE74A, null_frag, v128q, v128xb, 4, 8>;
1413    }
1414  }
1415}
1416
1417//===----------------------------------------------------------------------===//
1418// Floating-point comparison
1419//===----------------------------------------------------------------------===//
1420
1421let Predicates = [FeatureVector] in {
1422  // Compare scalar.
1423  let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
1424    def WFC   : CompareVRRaFloatGeneric<"wfc", 0xE7CB>;
1425    def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_any_fcmp, v64db, 3, "cdbr">;
1426    let Predicates = [FeatureVectorEnhancements1] in {
1427      def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_any_fcmp, v32sb, 2, "cebr">;
1428      def WFCXB : CompareVRRa<"wfcxb", 0xE7CB, z_any_fcmp, v128xb, 4>;
1429    }
1430  }
1431
1432  // Compare and signal scalar.
1433  let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
1434    def WFK   : CompareVRRaFloatGeneric<"wfk", 0xE7CA>;
1435    def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, z_strict_fcmps, v64db, 3, "kdbr">;
1436    let Predicates = [FeatureVectorEnhancements1] in {
1437      def WFKSB : CompareVRRa<"wfksb", 0xE7CA, z_strict_fcmps, v32sb, 2, "kebr">;
1438      def WFKXB : CompareVRRa<"wfkxb", 0xE7CA, z_strict_fcmps, v128xb, 4>;
1439    }
1440  }
1441
1442  // Compare equal.
1443  let Uses = [FPC], mayRaiseFPException = 1 in {
1444    def  VFCE   : BinaryVRRcSPairFloatGeneric<"vfce", 0xE7E8>;
1445    defm VFCEDB : BinaryVRRcSPair<"vfcedb", 0xE7E8, z_any_vfcmpe, z_vfcmpes,
1446                                  v128g, v128db, 3, 0>;
1447    defm WFCEDB : BinaryVRRcSPair<"wfcedb", 0xE7E8, null_frag, null_frag,
1448                                  v64g, v64db, 3, 8>;
1449    let Predicates = [FeatureVectorEnhancements1] in {
1450      defm VFCESB : BinaryVRRcSPair<"vfcesb", 0xE7E8, z_any_vfcmpe, z_vfcmpes,
1451                                    v128f, v128sb, 2, 0>;
1452      defm WFCESB : BinaryVRRcSPair<"wfcesb", 0xE7E8, null_frag, null_frag,
1453                                    v32f, v32sb, 2, 8>;
1454      defm WFCEXB : BinaryVRRcSPair<"wfcexb", 0xE7E8, null_frag, null_frag,
1455                                    v128q, v128xb, 4, 8>;
1456    }
1457  }
1458
1459  // Compare and signal equal.
1460  let Uses = [FPC], mayRaiseFPException = 1,
1461      Predicates = [FeatureVectorEnhancements1] in {
1462    defm VFKEDB : BinaryVRRcSPair<"vfkedb", 0xE7E8, z_strict_vfcmpes, null_frag,
1463                                  v128g, v128db, 3, 4>;
1464    defm WFKEDB : BinaryVRRcSPair<"wfkedb", 0xE7E8, null_frag, null_frag,
1465                                  v64g, v64db, 3, 12>;
1466    defm VFKESB : BinaryVRRcSPair<"vfkesb", 0xE7E8, z_strict_vfcmpes, null_frag,
1467                                  v128f, v128sb, 2, 4>;
1468    defm WFKESB : BinaryVRRcSPair<"wfkesb", 0xE7E8, null_frag, null_frag,
1469                                  v32f, v32sb, 2, 12>;
1470    defm WFKEXB : BinaryVRRcSPair<"wfkexb", 0xE7E8, null_frag, null_frag,
1471                                  v128q, v128xb, 4, 12>;
1472  }
1473
1474  // Compare high.
1475  let Uses = [FPC], mayRaiseFPException = 1 in {
1476    def  VFCH   : BinaryVRRcSPairFloatGeneric<"vfch", 0xE7EB>;
1477    defm VFCHDB : BinaryVRRcSPair<"vfchdb", 0xE7EB, z_any_vfcmph, z_vfcmphs,
1478                                  v128g, v128db, 3, 0>;
1479    defm WFCHDB : BinaryVRRcSPair<"wfchdb", 0xE7EB, null_frag, null_frag,
1480                                  v64g, v64db, 3, 8>;
1481    let Predicates = [FeatureVectorEnhancements1] in {
1482      defm VFCHSB : BinaryVRRcSPair<"vfchsb", 0xE7EB, z_any_vfcmph, z_vfcmphs,
1483                                    v128f, v128sb, 2, 0>;
1484      defm WFCHSB : BinaryVRRcSPair<"wfchsb", 0xE7EB, null_frag, null_frag,
1485                                    v32f, v32sb, 2, 8>;
1486      defm WFCHXB : BinaryVRRcSPair<"wfchxb", 0xE7EB, null_frag, null_frag,
1487                                    v128q, v128xb, 4, 8>;
1488    }
1489  }
1490
1491  // Compare and signal high.
1492  let Uses = [FPC], mayRaiseFPException = 1,
1493      Predicates = [FeatureVectorEnhancements1] in {
1494    defm VFKHDB : BinaryVRRcSPair<"vfkhdb", 0xE7EB, z_strict_vfcmphs, null_frag,
1495                                  v128g, v128db, 3, 4>;
1496    defm WFKHDB : BinaryVRRcSPair<"wfkhdb", 0xE7EB, null_frag, null_frag,
1497                                  v64g, v64db, 3, 12>;
1498    defm VFKHSB : BinaryVRRcSPair<"vfkhsb", 0xE7EB, z_strict_vfcmphs, null_frag,
1499                                  v128f, v128sb, 2, 4>;
1500    defm WFKHSB : BinaryVRRcSPair<"wfkhsb", 0xE7EB, null_frag, null_frag,
1501                                  v32f, v32sb, 2, 12>;
1502    defm WFKHXB : BinaryVRRcSPair<"wfkhxb", 0xE7EB, null_frag, null_frag,
1503                                  v128q, v128xb, 4, 12>;
1504  }
1505
1506  // Compare high or equal.
1507  let Uses = [FPC], mayRaiseFPException = 1 in {
1508    def  VFCHE   : BinaryVRRcSPairFloatGeneric<"vfche", 0xE7EA>;
1509    defm VFCHEDB : BinaryVRRcSPair<"vfchedb", 0xE7EA, z_any_vfcmphe, z_vfcmphes,
1510                                   v128g, v128db, 3, 0>;
1511    defm WFCHEDB : BinaryVRRcSPair<"wfchedb", 0xE7EA, null_frag, null_frag,
1512                                   v64g, v64db, 3, 8>;
1513    let Predicates = [FeatureVectorEnhancements1] in {
1514      defm VFCHESB : BinaryVRRcSPair<"vfchesb", 0xE7EA, z_any_vfcmphe, z_vfcmphes,
1515                                     v128f, v128sb, 2, 0>;
1516      defm WFCHESB : BinaryVRRcSPair<"wfchesb", 0xE7EA, null_frag, null_frag,
1517                                     v32f, v32sb, 2, 8>;
1518      defm WFCHEXB : BinaryVRRcSPair<"wfchexb", 0xE7EA, null_frag, null_frag,
1519                                     v128q, v128xb, 4, 8>;
1520    }
1521  }
1522
1523  // Compare and signal high or equal.
1524  let Uses = [FPC], mayRaiseFPException = 1,
1525      Predicates = [FeatureVectorEnhancements1] in {
1526    defm VFKHEDB : BinaryVRRcSPair<"vfkhedb", 0xE7EA, z_strict_vfcmphes, null_frag,
1527                                   v128g, v128db, 3, 4>;
1528    defm WFKHEDB : BinaryVRRcSPair<"wfkhedb", 0xE7EA, null_frag, null_frag,
1529                                   v64g, v64db, 3, 12>;
1530    defm VFKHESB : BinaryVRRcSPair<"vfkhesb", 0xE7EA, z_strict_vfcmphes, null_frag,
1531                                   v128f, v128sb, 2, 4>;
1532    defm WFKHESB : BinaryVRRcSPair<"wfkhesb", 0xE7EA, null_frag, null_frag,
1533                                   v32f, v32sb, 2, 12>;
1534    defm WFKHEXB : BinaryVRRcSPair<"wfkhexb", 0xE7EA, null_frag, null_frag,
1535                                   v128q, v128xb, 4, 12>;
1536  }
1537}
1538
1539//===----------------------------------------------------------------------===//
1540// Support for 128-bit integer values in vector registers
1541//===----------------------------------------------------------------------===//
1542
1543// Loads and stores.
1544let Predicates = [FeatureVector] in {
1545  def : Pat<(i128 (load bdxaddr12only:$addr)),
1546            (VL bdxaddr12only:$addr)>;
1547  def : Pat<(store (i128 VR128:$src), bdxaddr12only:$addr),
1548            (VST VR128:$src, bdxaddr12only:$addr)>;
1549}
1550
1551// Full i128 move from GPR pair.
1552let Predicates = [FeatureVector] in
1553  def : Pat<(i128 (or (zext GR64:$x), (shl (anyext GR64:$y), (i32 64)))),
1554            (VLVGP GR64:$y, GR64:$x)>;
1555
1556// Any-extensions from GPR to i128.
1557let Predicates = [FeatureVector] in {
1558  def : Pat<(i128 (anyext GR32:$x)), (VLVGP32 GR32:$x, GR32:$x)>;
1559  def : Pat<(i128 (anyext GR64:$x)), (VLVGP GR64:$x, GR64:$x)>;
1560}
1561
1562// Any-extending loads into i128.
1563let Predicates = [FeatureVector] in {
1564  def : Pat<(i128 (z_extloadi8 bdxaddr12only:$addr)),
1565            (VLREPB bdxaddr12only:$addr)>;
1566  def : Pat<(i128 (z_extloadi16 bdxaddr12only:$addr)),
1567            (VLREPH bdxaddr12only:$addr)>;
1568  def : Pat<(i128 (z_extloadi32 bdxaddr12only:$addr)),
1569            (VLREPF bdxaddr12only:$addr)>;
1570  def : Pat<(i128 (z_extloadi64 bdxaddr12only:$addr)),
1571            (VLREPG bdxaddr12only:$addr)>;
1572}
1573
1574// Truncations from i128 to GPR.
1575let Predicates = [FeatureVector] in {
1576  def : Pat<(i32 (trunc (i128 VR128:$vec))),
1577            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 3), subreg_l32)>;
1578  def : Pat<(i32 (trunc (srl (i128 VR128:$vec), (i32 32)))),
1579            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 2), subreg_l32)>;
1580  def : Pat<(i32 (trunc (srl (i128 VR128:$vec), (i32 64)))),
1581            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 1), subreg_l32)>;
1582  def : Pat<(i32 (trunc (srl (i128 VR128:$vec), (i32 96)))),
1583            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 0), subreg_l32)>;
1584  def : Pat<(i64 (trunc (i128 VR128:$vec))),
1585            (VLGVG VR128:$vec, zero_reg, 1)>;
1586  def : Pat<(i64 (trunc (srl (i128 VR128:$vec), (i32 64)))),
1587            (VLGVG VR128:$vec, zero_reg, 0)>;
1588}
1589
1590// Truncating stores from i128.
1591let Predicates = [FeatureVector] in {
1592  def : Pat<(truncstorei8 (i128 VR128:$x), bdxaddr12only:$addr),
1593            (VSTEB VR128:$x, bdxaddr12only:$addr, 15)>;
1594  def : Pat<(truncstorei16 (i128 VR128:$x), bdxaddr12only:$addr),
1595            (VSTEH VR128:$x, bdxaddr12only:$addr, 7)>;
1596  def : Pat<(truncstorei32 (i128 VR128:$x), bdxaddr12only:$addr),
1597            (VSTEF VR128:$x, bdxaddr12only:$addr, 3)>;
1598  def : Pat<(truncstorei32 (srl (i128 VR128:$x), (i32 32)), bdxaddr12only:$addr),
1599            (VSTEF VR128:$x, bdxaddr12only:$addr, 2)>;
1600  def : Pat<(truncstorei32 (srl (i128 VR128:$x), (i32 64)), bdxaddr12only:$addr),
1601            (VSTEF VR128:$x, bdxaddr12only:$addr, 1)>;
1602  def : Pat<(truncstorei32 (srl (i128 VR128:$x), (i32 96)), bdxaddr12only:$addr),
1603            (VSTEF VR128:$x, bdxaddr12only:$addr, 0)>;
1604  def : Pat<(truncstorei64 (i128 VR128:$x), bdxaddr12only:$addr),
1605            (VSTEG VR128:$x, bdxaddr12only:$addr, 1)>;
1606  def : Pat<(truncstorei64 (srl (i128 VR128:$x), (i32 64)), bdxaddr12only:$addr),
1607            (VSTEG VR128:$x, bdxaddr12only:$addr, 0)>;
1608}
1609
1610// Zero-extensions from GPR to i128.
1611let Predicates = [FeatureVector] in {
1612  def : Pat<(i128 (zext8 (anyext GR32:$x))),
1613            (VLVGB (VGBM 0), GR32:$x, zero_reg, 15)>;
1614  def : Pat<(i128 (zext16 (anyext GR32:$x))),
1615            (VLVGH (VGBM 0), GR32:$x, zero_reg, 7)>;
1616  def : Pat<(i128 (zext GR32:$x)),
1617            (VLVGF (VGBM 0), GR32:$x, zero_reg, 3)>;
1618  def : Pat<(i128 (zext GR64:$x)),
1619            (VLVGG (VGBM 0), GR64:$x, zero_reg, 1)>;
1620}
1621
1622// Zero-extending loads into i128.
1623let Predicates = [FeatureVector] in {
1624  def : Pat<(i128 (z_zextloadi8 bdxaddr12only:$addr)),
1625            (VLEB (VGBM 0), bdxaddr12only:$addr, 15)>;
1626  def : Pat<(i128 (z_zextloadi16 bdxaddr12only:$addr)),
1627            (VLEH (VGBM 0), bdxaddr12only:$addr, 7)>;
1628  def : Pat<(i128 (z_zextloadi32 bdxaddr12only:$addr)),
1629            (VLEF (VGBM 0), bdxaddr12only:$addr, 3)>;
1630  def : Pat<(i128 (z_zextloadi64 bdxaddr12only:$addr)),
1631            (VLEG (VGBM 0), bdxaddr12only:$addr, 1)>;
1632}
1633
1634// In-register i128 sign-extensions.
1635let Predicates = [FeatureVector] in {
1636  def : Pat<(i128 (sext_inreg VR128:$x, i8)),
1637            (VSRAB (VREPB VR128:$x, 15), (VREPIB 120))>;
1638  def : Pat<(i128 (sext_inreg VR128:$x, i16)),
1639            (VSRAB (VREPH VR128:$x, 7), (VREPIB 112))>;
1640  def : Pat<(i128 (sext_inreg VR128:$x, i32)),
1641            (VSRAB (VREPF VR128:$x, 3), (VREPIB 96))>;
1642  def : Pat<(i128 (sext_inreg VR128:$x, i64)),
1643            (VSRAB (VREPG VR128:$x, 1), (VREPIB 64))>;
1644}
1645
1646// Sign-extensions from GPR to i128.
1647let Predicates = [FeatureVector] in {
1648  def : Pat<(i128 (sext_inreg (anyext GR32:$x), i8)),
1649            (VLVGP (SRAG (LGBR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
1650                                 GR32:$x, subreg_l32)), zero_reg, 63),
1651                   (LGBR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
1652                            GR32:$x, subreg_l32)))>;
1653  def : Pat<(i128 (sext_inreg (anyext GR32:$x), i16)),
1654            (VLVGP (SRAG (LGHR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
1655                                  GR32:$x, subreg_l32)), zero_reg, 63),
1656                   (LGHR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
1657                            GR32:$x, subreg_l32)))>;
1658  def : Pat<(i128 (sext GR32:$x)),
1659            (VLVGP (SRAG (LGFR GR32:$x), zero_reg, 63), (LGFR GR32:$x))>;
1660  def : Pat<(i128 (sext GR64:$x)),
1661            (VLVGP (SRAG GR64:$x, zero_reg, 63), GR64:$x)>;
1662}
1663
1664// Sign-extending loads into i128.
1665let Predicates = [FeatureVector] in {
1666  def : Pat<(i128 (z_sextloadi8 bdxaddr12only:$addr)),
1667            (VSRAB (VLREPB bdxaddr12only:$addr), (VREPIB 120))>;
1668  def : Pat<(i128 (z_sextloadi16 bdxaddr12only:$addr)),
1669            (VSRAB (VLREPH bdxaddr12only:$addr), (VREPIB 112))>;
1670  def : Pat<(i128 (z_sextloadi32 bdxaddr12only:$addr)),
1671            (VSRAB (VLREPF bdxaddr12only:$addr), (VREPIB 96))>;
1672  def : Pat<(i128 (z_sextloadi64 bdxaddr12only:$addr)),
1673            (VSRAB (VLREPG bdxaddr12only:$addr), (VREPIB 64))>;
1674}
1675
1676// i128 comparison pseudo-instructions.
1677let Predicates = [FeatureVector], Defs = [CC],
1678    usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
1679  def SCmp128Hi : Pseudo<(outs), (ins VR128:$src1, VR128:$src2),
1680                         [(set CC, (z_scmp128hi (i128 VR128:$src1),
1681                                                (i128 VR128:$src2)))]>;
1682  def UCmp128Hi : Pseudo<(outs), (ins VR128:$src1, VR128:$src2),
1683                         [(set CC, (z_ucmp128hi (i128 VR128:$src1),
1684                                                (i128 VR128:$src2)))]>;
1685}
1686
1687// i128 select pseudo-instructions.
1688let Predicates = [FeatureVector] in
1689  def Select128 : SelectWrapper<i128, VR128>;
1690
1691//===----------------------------------------------------------------------===//
1692// Conversions
1693//===----------------------------------------------------------------------===//
1694
1695let Predicates = [FeatureVector] in {
1696def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>;
1697def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>;
1698def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>;
1699def : Pat<(v16i8 (bitconvert (i128  VR128:$src))), (v16i8 VR128:$src)>;
1700def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>;
1701def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>;
1702def : Pat<(v16i8 (bitconvert (f128  VR128:$src))), (v16i8 VR128:$src)>;
1703
1704def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>;
1705def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>;
1706def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>;
1707def : Pat<(v8i16 (bitconvert (i128  VR128:$src))), (v8i16 VR128:$src)>;
1708def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>;
1709def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>;
1710def : Pat<(v8i16 (bitconvert (f128  VR128:$src))), (v8i16 VR128:$src)>;
1711
1712def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>;
1713def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>;
1714def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>;
1715def : Pat<(v4i32 (bitconvert (i128  VR128:$src))), (v4i32 VR128:$src)>;
1716def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>;
1717def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>;
1718def : Pat<(v4i32 (bitconvert (f128  VR128:$src))), (v4i32 VR128:$src)>;
1719
1720def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>;
1721def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>;
1722def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>;
1723def : Pat<(v2i64 (bitconvert (i128  VR128:$src))), (v2i64 VR128:$src)>;
1724def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>;
1725def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>;
1726def : Pat<(v2i64 (bitconvert (f128  VR128:$src))), (v2i64 VR128:$src)>;
1727
1728def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>;
1729def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>;
1730def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>;
1731def : Pat<(v4f32 (bitconvert (i128  VR128:$src))), (v4f32 VR128:$src)>;
1732def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>;
1733def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>;
1734def : Pat<(v4f32 (bitconvert (f128  VR128:$src))), (v4f32 VR128:$src)>;
1735
1736def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>;
1737def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>;
1738def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>;
1739def : Pat<(v2f64 (bitconvert (i128  VR128:$src))), (v2f64 VR128:$src)>;
1740def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>;
1741def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>;
1742def : Pat<(v2f64 (bitconvert (f128  VR128:$src))), (v2f64 VR128:$src)>;
1743
1744def : Pat<(f128  (bitconvert (v16i8 VR128:$src))), (f128  VR128:$src)>;
1745def : Pat<(f128  (bitconvert (v8i16 VR128:$src))), (f128  VR128:$src)>;
1746def : Pat<(f128  (bitconvert (v4i32 VR128:$src))), (f128  VR128:$src)>;
1747def : Pat<(f128  (bitconvert (v2i64 VR128:$src))), (f128  VR128:$src)>;
1748def : Pat<(f128  (bitconvert (i128  VR128:$src))), (f128  VR128:$src)>;
1749def : Pat<(f128  (bitconvert (v4f32 VR128:$src))), (f128  VR128:$src)>;
1750def : Pat<(f128  (bitconvert (v2f64 VR128:$src))), (f128  VR128:$src)>;
1751
1752def : Pat<(i128  (bitconvert (v16i8 VR128:$src))), (i128  VR128:$src)>;
1753def : Pat<(i128  (bitconvert (v8i16 VR128:$src))), (i128  VR128:$src)>;
1754def : Pat<(i128  (bitconvert (v4i32 VR128:$src))), (i128  VR128:$src)>;
1755def : Pat<(i128  (bitconvert (v2i64 VR128:$src))), (i128  VR128:$src)>;
1756def : Pat<(i128  (bitconvert (v4f32 VR128:$src))), (i128  VR128:$src)>;
1757def : Pat<(i128  (bitconvert (v2f64 VR128:$src))), (i128  VR128:$src)>;
1758def : Pat<(i128  (bitconvert (f128  VR128:$src))), (i128  VR128:$src)>;
1759} // End Predicates = [FeatureVector]
1760
1761//===----------------------------------------------------------------------===//
1762// Replicating scalars
1763//===----------------------------------------------------------------------===//
1764
1765// Define patterns for replicating a scalar GR32 into a vector of type TYPE.
1766// INDEX is 8 minus the element size in bytes.
1767class VectorReplicateScalar<ValueType type, Instruction insn, bits<16> index>
1768  : Pat<(type (z_replicate GR32:$scalar)),
1769        (insn (VLVGP32 GR32:$scalar, GR32:$scalar), index)>;
1770
1771def : VectorReplicateScalar<v16i8, VREPB, 7>;
1772def : VectorReplicateScalar<v8i16, VREPH, 3>;
1773def : VectorReplicateScalar<v4i32, VREPF, 1>;
1774
1775// i64 replications are just a single instruction.
1776def : Pat<(v2i64 (z_replicate GR64:$scalar)),
1777          (VLVGP GR64:$scalar, GR64:$scalar)>;
1778
1779//===----------------------------------------------------------------------===//
1780// Floating-point insertion and extraction
1781//===----------------------------------------------------------------------===//
1782
1783// Moving 32-bit values between GPRs and FPRs can be done using VLVGF
1784// and VLGVF.
1785let Predicates = [FeatureVector] in {
1786  def LEFR : UnaryAliasVRS<VR32, GR32>;
1787  def LFER : UnaryAliasVRS<GR64, VR32>;
1788  def : Pat<(f32 (bitconvert (i32 GR32:$src))), (LEFR GR32:$src)>;
1789  def : Pat<(i32 (bitconvert (f32 VR32:$src))),
1790            (EXTRACT_SUBREG (LFER VR32:$src), subreg_l32)>;
1791}
1792
1793// Floating-point values are stored in element 0 of the corresponding
1794// vector register.  Scalar to vector conversion is just a subreg and
1795// scalar replication can just replicate element 0 of the vector register.
1796multiclass ScalarToVectorFP<Instruction vrep, ValueType vt, RegisterOperand cls,
1797                            SubRegIndex subreg> {
1798  def : Pat<(vt (scalar_to_vector cls:$scalar)),
1799            (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar, subreg)>;
1800  def : Pat<(vt (z_replicate cls:$scalar)),
1801            (vrep (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar,
1802                                 subreg), 0)>;
1803}
1804defm : ScalarToVectorFP<VREPF, v4f32, FP32, subreg_h32>;
1805defm : ScalarToVectorFP<VREPG, v2f64, FP64, subreg_h64>;
1806
1807// Match v2f64 insertions.  The AddedComplexity counters the 3 added by
1808// TableGen for the base register operand in VLVG-based integer insertions
1809// and ensures that this version is strictly better.
1810let AddedComplexity = 4 in {
1811  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 0),
1812            (VPDI (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
1813                                 subreg_h64), VR128:$vec, 1)>;
1814  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 1),
1815            (VPDI VR128:$vec, (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
1816                                             subreg_h64), 0)>;
1817}
1818
1819// We extract floating-point element X by replicating (for elements other
1820// than 0) and then taking a high subreg.  The AddedComplexity counters the
1821// 3 added by TableGen for the base register operand in VLGV-based integer
1822// extractions and ensures that this version is strictly better.
1823let AddedComplexity = 4 in {
1824  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), 0)),
1825            (EXTRACT_SUBREG VR128:$vec, subreg_h32)>;
1826  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), imm32zx2:$index)),
1827            (EXTRACT_SUBREG (VREPF VR128:$vec, imm32zx2:$index), subreg_h32)>;
1828
1829  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), 0)),
1830            (EXTRACT_SUBREG VR128:$vec, subreg_h64)>;
1831  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), imm32zx1:$index)),
1832            (EXTRACT_SUBREG (VREPG VR128:$vec, imm32zx1:$index), subreg_h64)>;
1833}
1834
1835//===----------------------------------------------------------------------===//
1836// Support for 128-bit floating-point values in vector registers
1837//===----------------------------------------------------------------------===//
1838
1839let Predicates = [FeatureVectorEnhancements1] in {
1840  def : Pat<(f128 (load bdxaddr12only:$addr)),
1841            (VL bdxaddr12only:$addr)>;
1842  def : Pat<(store (f128 VR128:$src), bdxaddr12only:$addr),
1843            (VST VR128:$src, bdxaddr12only:$addr)>;
1844
1845  def : Pat<(f128 fpimm0), (VZERO)>;
1846  def : Pat<(f128 fpimmneg0), (WFLNXB (VZERO))>;
1847}
1848
1849//===----------------------------------------------------------------------===//
1850// String instructions
1851//===----------------------------------------------------------------------===//
1852
1853let Predicates = [FeatureVector] in {
1854  defm VFAE  : TernaryOptVRRbSPairGeneric<"vfae", 0xE782>;
1855  defm VFAEB : TernaryOptVRRbSPair<"vfaeb", 0xE782, int_s390_vfaeb,
1856                                   z_vfae_cc, v128b, v128b, 0>;
1857  defm VFAEH : TernaryOptVRRbSPair<"vfaeh", 0xE782, int_s390_vfaeh,
1858                                   z_vfae_cc, v128h, v128h, 1>;
1859  defm VFAEF : TernaryOptVRRbSPair<"vfaef", 0xE782, int_s390_vfaef,
1860                                   z_vfae_cc, v128f, v128f, 2>;
1861  defm VFAEZB : TernaryOptVRRbSPair<"vfaezb", 0xE782, int_s390_vfaezb,
1862                                    z_vfaez_cc, v128b, v128b, 0, 2>;
1863  defm VFAEZH : TernaryOptVRRbSPair<"vfaezh", 0xE782, int_s390_vfaezh,
1864                                    z_vfaez_cc, v128h, v128h, 1, 2>;
1865  defm VFAEZF : TernaryOptVRRbSPair<"vfaezf", 0xE782, int_s390_vfaezf,
1866                                    z_vfaez_cc, v128f, v128f, 2, 2>;
1867
1868  defm VFEE  : BinaryExtraVRRbSPairGeneric<"vfee", 0xE780>;
1869  defm VFEEB : BinaryExtraVRRbSPair<"vfeeb", 0xE780, int_s390_vfeeb,
1870                                    z_vfee_cc, v128b, v128b, 0>;
1871  defm VFEEH : BinaryExtraVRRbSPair<"vfeeh", 0xE780, int_s390_vfeeh,
1872                                    z_vfee_cc, v128h, v128h, 1>;
1873  defm VFEEF : BinaryExtraVRRbSPair<"vfeef", 0xE780, int_s390_vfeef,
1874                                    z_vfee_cc, v128f, v128f, 2>;
1875  defm VFEEZB : BinaryVRRbSPair<"vfeezb", 0xE780, int_s390_vfeezb,
1876                                z_vfeez_cc, v128b, v128b, 0, 2>;
1877  defm VFEEZH : BinaryVRRbSPair<"vfeezh", 0xE780, int_s390_vfeezh,
1878                                z_vfeez_cc, v128h, v128h, 1, 2>;
1879  defm VFEEZF : BinaryVRRbSPair<"vfeezf", 0xE780, int_s390_vfeezf,
1880                                z_vfeez_cc, v128f, v128f, 2, 2>;
1881
1882  defm VFENE  : BinaryExtraVRRbSPairGeneric<"vfene", 0xE781>;
1883  defm VFENEB : BinaryExtraVRRbSPair<"vfeneb", 0xE781, int_s390_vfeneb,
1884                                     z_vfene_cc, v128b, v128b, 0>;
1885  defm VFENEH : BinaryExtraVRRbSPair<"vfeneh", 0xE781, int_s390_vfeneh,
1886                                     z_vfene_cc, v128h, v128h, 1>;
1887  defm VFENEF : BinaryExtraVRRbSPair<"vfenef", 0xE781, int_s390_vfenef,
1888                                     z_vfene_cc, v128f, v128f, 2>;
1889  defm VFENEZB : BinaryVRRbSPair<"vfenezb", 0xE781, int_s390_vfenezb,
1890                                 z_vfenez_cc, v128b, v128b, 0, 2>;
1891  defm VFENEZH : BinaryVRRbSPair<"vfenezh", 0xE781, int_s390_vfenezh,
1892                                 z_vfenez_cc, v128h, v128h, 1, 2>;
1893  defm VFENEZF : BinaryVRRbSPair<"vfenezf", 0xE781, int_s390_vfenezf,
1894                                 z_vfenez_cc, v128f, v128f, 2, 2>;
1895
1896  defm VISTR  : UnaryExtraVRRaSPairGeneric<"vistr", 0xE75C>;
1897  defm VISTRB : UnaryExtraVRRaSPair<"vistrb", 0xE75C, int_s390_vistrb,
1898                                    z_vistr_cc, v128b, v128b, 0>;
1899  defm VISTRH : UnaryExtraVRRaSPair<"vistrh", 0xE75C, int_s390_vistrh,
1900                                    z_vistr_cc, v128h, v128h, 1>;
1901  defm VISTRF : UnaryExtraVRRaSPair<"vistrf", 0xE75C, int_s390_vistrf,
1902                                    z_vistr_cc, v128f, v128f, 2>;
1903
1904  defm VSTRC  : QuaternaryOptVRRdSPairGeneric<"vstrc", 0xE78A>;
1905  defm VSTRCB : QuaternaryOptVRRdSPair<"vstrcb", 0xE78A, int_s390_vstrcb,
1906                                       z_vstrc_cc, v128b, v128b, 0>;
1907  defm VSTRCH : QuaternaryOptVRRdSPair<"vstrch", 0xE78A, int_s390_vstrch,
1908                                       z_vstrc_cc, v128h, v128h, 1>;
1909  defm VSTRCF : QuaternaryOptVRRdSPair<"vstrcf", 0xE78A, int_s390_vstrcf,
1910                                       z_vstrc_cc, v128f, v128f, 2>;
1911  defm VSTRCZB : QuaternaryOptVRRdSPair<"vstrczb", 0xE78A, int_s390_vstrczb,
1912                                        z_vstrcz_cc, v128b, v128b, 0, 2>;
1913  defm VSTRCZH : QuaternaryOptVRRdSPair<"vstrczh", 0xE78A, int_s390_vstrczh,
1914                                        z_vstrcz_cc, v128h, v128h, 1, 2>;
1915  defm VSTRCZF : QuaternaryOptVRRdSPair<"vstrczf", 0xE78A, int_s390_vstrczf,
1916                                        z_vstrcz_cc, v128f, v128f, 2, 2>;
1917}
1918
1919let Predicates = [FeatureVectorEnhancements2] in {
1920  defm VSTRS  : TernaryExtraVRRdGeneric<"vstrs", 0xE78B>;
1921  defm VSTRSB : TernaryExtraVRRd<"vstrsb", 0xE78B,
1922                                 z_vstrs_cc, v128b, v128b, 0>;
1923  defm VSTRSH : TernaryExtraVRRd<"vstrsh", 0xE78B,
1924                                 z_vstrs_cc, v128b, v128h, 1>;
1925  defm VSTRSF : TernaryExtraVRRd<"vstrsf", 0xE78B,
1926                                 z_vstrs_cc, v128b, v128f, 2>;
1927  let Defs = [CC] in {
1928    def VSTRSZB : TernaryVRRd<"vstrszb", 0xE78B,
1929                              z_vstrsz_cc, v128b, v128b, 0, 2>;
1930    def VSTRSZH : TernaryVRRd<"vstrszh", 0xE78B,
1931                              z_vstrsz_cc, v128b, v128h, 1, 2>;
1932    def VSTRSZF : TernaryVRRd<"vstrszf", 0xE78B,
1933                              z_vstrsz_cc, v128b, v128f, 2, 2>;
1934  }
1935}
1936
1937//===----------------------------------------------------------------------===//
1938// NNP assist instructions
1939//===----------------------------------------------------------------------===//
1940
1941let Predicates = [FeatureVector, FeatureNNPAssist] in {
1942  let Uses = [FPC], mayRaiseFPException = 1 in
1943    def VCFN : UnaryVRRaFloatGeneric<"vcfn", 0xE65D>;
1944  def : Pat<(int_s390_vcfn VR128:$x, imm32zx4_timm:$m),
1945            (VCFN VR128:$x, 1, imm32zx4:$m)>;
1946
1947  let Uses = [FPC], mayRaiseFPException = 1 in
1948    def VCLFNL : UnaryVRRaFloatGeneric<"vclfnl", 0xE65E>;
1949  def : Pat<(int_s390_vclfnls VR128:$x, imm32zx4_timm:$m),
1950            (VCLFNL VR128:$x, 2, imm32zx4:$m)>;
1951
1952  let Uses = [FPC], mayRaiseFPException = 1 in
1953    def VCLFNH : UnaryVRRaFloatGeneric<"vclfnh", 0xE656>;
1954  def : Pat<(int_s390_vclfnhs VR128:$x, imm32zx4_timm:$m),
1955            (VCLFNH VR128:$x, 2, imm32zx4:$m)>;
1956
1957  let Uses = [FPC], mayRaiseFPException = 1 in
1958    def VCNF : UnaryVRRaFloatGeneric<"vcnf", 0xE655>;
1959  def : Pat<(int_s390_vcnf VR128:$x, imm32zx4_timm:$m),
1960            (VCNF VR128:$x, imm32zx4:$m, 1)>;
1961
1962  let Uses = [FPC], mayRaiseFPException = 1 in
1963    def VCRNF : BinaryVRRcFloatGeneric<"vcrnf", 0xE675>;
1964  def : Pat<(int_s390_vcrnfs VR128:$x, VR128:$y, imm32zx4_timm:$m),
1965            (VCRNF VR128:$x, VR128:$y, imm32zx4:$m, 2)>;
1966}
1967
1968//===----------------------------------------------------------------------===//
1969// Packed-decimal instructions
1970//===----------------------------------------------------------------------===//
1971
1972let Predicates = [FeatureVectorPackedDecimal] in {
1973  def VLIP : BinaryVRIh<"vlip", 0xE649>;
1974
1975  def VPKZ : BinaryVSI<"vpkz", 0xE634, null_frag, 0>;
1976  def VUPKZ : StoreLengthVSI<"vupkz", 0xE63C, null_frag, 0>;
1977
1978  let Defs = [CC] in {
1979    let Predicates = [FeatureVectorPackedDecimalEnhancement] in {
1980      def VCVBOpt : TernaryVRRi<"vcvb", 0xE650, GR32>;
1981      def VCVBGOpt : TernaryVRRi<"vcvbg", 0xE652, GR64>;
1982    }
1983    def VCVB : BinaryVRRi<"vcvb", 0xE650, GR32>;
1984    def VCVBG : BinaryVRRi<"vcvbg", 0xE652, GR64>;
1985    def VCVD : TernaryVRIi<"vcvd", 0xE658, GR32>;
1986    def VCVDG : TernaryVRIi<"vcvdg", 0xE65A, GR64>;
1987
1988    def VAP : QuaternaryVRIf<"vap", 0xE671>;
1989    def VSP : QuaternaryVRIf<"vsp", 0xE673>;
1990
1991    def VMP : QuaternaryVRIf<"vmp", 0xE678>;
1992    def VMSP : QuaternaryVRIf<"vmsp", 0xE679>;
1993
1994    def VDP : QuaternaryVRIf<"vdp", 0xE67A>;
1995    def VRP : QuaternaryVRIf<"vrp", 0xE67B>;
1996    def VSDP : QuaternaryVRIf<"vsdp", 0xE67E>;
1997
1998    def VSRP : QuaternaryVRIg<"vsrp", 0xE659>;
1999    def VPSOP : QuaternaryVRIg<"vpsop", 0xE65B>;
2000
2001    def VTP : TestVRRg<"vtp", 0xE65F>;
2002    def VCP : CompareVRRh<"vcp", 0xE677>;
2003  }
2004}
2005
2006let Predicates = [FeatureVectorPackedDecimalEnhancement2] in {
2007  def VSCHP : BinaryExtraVRRbGeneric<"vschp", 0xE674>;
2008  def VSCHSP : BinaryExtraVRRb<"vschsp", 0xE674, 2>;
2009  def VSCHDP : BinaryExtraVRRb<"vschdp", 0xE674, 3>;
2010  def VSCHXP : BinaryExtraVRRb<"vschxp", 0xE674, 4>;
2011
2012  def VSCSHP : BinaryVRRb<"vscshp", 0xE67C, null_frag, v128b, v128b>;
2013
2014  def VCSPH : TernaryVRRj<"vcsph", 0xE67D>;
2015
2016  let Defs = [CC] in
2017    def VCLZDP : BinaryVRRk<"vclzdp", 0xE651>;
2018
2019  let Defs = [CC] in
2020    def VSRPR : QuaternaryVRIf<"vsrpr", 0xE672>;
2021
2022  let Defs = [CC] in {
2023    def VPKZR : QuaternaryVRIf<"vpkzr", 0xE670>;
2024    def VUPKZH : BinaryVRRk<"vupkzh", 0xE654>;
2025    def VUPKZL : BinaryVRRk<"vupkzl", 0xE65C>;
2026  }
2027}
2028