xref: /freebsd/contrib/llvm-project/llvm/lib/IR/AutoUpgrade.cpp (revision e9e8876a4d6afc1ad5315faaa191b25121a813d7)
1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the auto-upgrade helper functions.
10 // This is where deprecated IR intrinsics and other IR features are updated to
11 // current specifications.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/IR/AutoUpgrade.h"
16 #include "llvm/ADT/StringSwitch.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/InstVisitor.h"
24 #include "llvm/IR/Instruction.h"
25 #include "llvm/IR/IntrinsicInst.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include "llvm/IR/IntrinsicsAArch64.h"
28 #include "llvm/IR/IntrinsicsARM.h"
29 #include "llvm/IR/IntrinsicsX86.h"
30 #include "llvm/IR/LLVMContext.h"
31 #include "llvm/IR/Module.h"
32 #include "llvm/IR/Verifier.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/Regex.h"
35 #include <cstring>
36 using namespace llvm;
37 
38 static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
39 
40 // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
41 // changed their type from v4f32 to v2i64.
42 static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
43                                   Function *&NewFn) {
44   // Check whether this is an old version of the function, which received
45   // v4f32 arguments.
46   Type *Arg0Type = F->getFunctionType()->getParamType(0);
47   if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
48     return false;
49 
50   // Yes, it's old, replace it with new version.
51   rename(F);
52   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
53   return true;
54 }
55 
56 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
57 // arguments have changed their type from i32 to i8.
58 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
59                                              Function *&NewFn) {
60   // Check that the last argument is an i32.
61   Type *LastArgType = F->getFunctionType()->getParamType(
62      F->getFunctionType()->getNumParams() - 1);
63   if (!LastArgType->isIntegerTy(32))
64     return false;
65 
66   // Move this function aside and map down.
67   rename(F);
68   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
69   return true;
70 }
71 
72 // Upgrade the declaration of fp compare intrinsics that change return type
73 // from scalar to vXi1 mask.
74 static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
75                                       Function *&NewFn) {
76   // Check if the return type is a vector.
77   if (F->getReturnType()->isVectorTy())
78     return false;
79 
80   rename(F);
81   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
82   return true;
83 }
84 
85 static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
86   // All of the intrinsics matches below should be marked with which llvm
87   // version started autoupgrading them. At some point in the future we would
88   // like to use this information to remove upgrade code for some older
89   // intrinsics. It is currently undecided how we will determine that future
90   // point.
91   if (Name == "addcarryx.u32" || // Added in 8.0
92       Name == "addcarryx.u64" || // Added in 8.0
93       Name == "addcarry.u32" || // Added in 8.0
94       Name == "addcarry.u64" || // Added in 8.0
95       Name == "subborrow.u32" || // Added in 8.0
96       Name == "subborrow.u64" || // Added in 8.0
97       Name.startswith("sse2.padds.") || // Added in 8.0
98       Name.startswith("sse2.psubs.") || // Added in 8.0
99       Name.startswith("sse2.paddus.") || // Added in 8.0
100       Name.startswith("sse2.psubus.") || // Added in 8.0
101       Name.startswith("avx2.padds.") || // Added in 8.0
102       Name.startswith("avx2.psubs.") || // Added in 8.0
103       Name.startswith("avx2.paddus.") || // Added in 8.0
104       Name.startswith("avx2.psubus.") || // Added in 8.0
105       Name.startswith("avx512.padds.") || // Added in 8.0
106       Name.startswith("avx512.psubs.") || // Added in 8.0
107       Name.startswith("avx512.mask.padds.") || // Added in 8.0
108       Name.startswith("avx512.mask.psubs.") || // Added in 8.0
109       Name.startswith("avx512.mask.paddus.") || // Added in 8.0
110       Name.startswith("avx512.mask.psubus.") || // Added in 8.0
111       Name=="ssse3.pabs.b.128" || // Added in 6.0
112       Name=="ssse3.pabs.w.128" || // Added in 6.0
113       Name=="ssse3.pabs.d.128" || // Added in 6.0
114       Name.startswith("fma4.vfmadd.s") || // Added in 7.0
115       Name.startswith("fma.vfmadd.") || // Added in 7.0
116       Name.startswith("fma.vfmsub.") || // Added in 7.0
117       Name.startswith("fma.vfmsubadd.") || // Added in 7.0
118       Name.startswith("fma.vfnmadd.") || // Added in 7.0
119       Name.startswith("fma.vfnmsub.") || // Added in 7.0
120       Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
121       Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
122       Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
123       Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
124       Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
125       Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
126       Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
127       Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
128       Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
129       Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
130       Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
131       Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
132       Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
133       Name.startswith("avx512.kunpck") || //added in 6.0
134       Name.startswith("avx2.pabs.") || // Added in 6.0
135       Name.startswith("avx512.mask.pabs.") || // Added in 6.0
136       Name.startswith("avx512.broadcastm") || // Added in 6.0
137       Name == "sse.sqrt.ss" || // Added in 7.0
138       Name == "sse2.sqrt.sd" || // Added in 7.0
139       Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
140       Name.startswith("avx.sqrt.p") || // Added in 7.0
141       Name.startswith("sse2.sqrt.p") || // Added in 7.0
142       Name.startswith("sse.sqrt.p") || // Added in 7.0
143       Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
144       Name.startswith("sse2.pcmpeq.") || // Added in 3.1
145       Name.startswith("sse2.pcmpgt.") || // Added in 3.1
146       Name.startswith("avx2.pcmpeq.") || // Added in 3.1
147       Name.startswith("avx2.pcmpgt.") || // Added in 3.1
148       Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
149       Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
150       Name.startswith("avx.vperm2f128.") || // Added in 6.0
151       Name == "avx2.vperm2i128" || // Added in 6.0
152       Name == "sse.add.ss" || // Added in 4.0
153       Name == "sse2.add.sd" || // Added in 4.0
154       Name == "sse.sub.ss" || // Added in 4.0
155       Name == "sse2.sub.sd" || // Added in 4.0
156       Name == "sse.mul.ss" || // Added in 4.0
157       Name == "sse2.mul.sd" || // Added in 4.0
158       Name == "sse.div.ss" || // Added in 4.0
159       Name == "sse2.div.sd" || // Added in 4.0
160       Name == "sse41.pmaxsb" || // Added in 3.9
161       Name == "sse2.pmaxs.w" || // Added in 3.9
162       Name == "sse41.pmaxsd" || // Added in 3.9
163       Name == "sse2.pmaxu.b" || // Added in 3.9
164       Name == "sse41.pmaxuw" || // Added in 3.9
165       Name == "sse41.pmaxud" || // Added in 3.9
166       Name == "sse41.pminsb" || // Added in 3.9
167       Name == "sse2.pmins.w" || // Added in 3.9
168       Name == "sse41.pminsd" || // Added in 3.9
169       Name == "sse2.pminu.b" || // Added in 3.9
170       Name == "sse41.pminuw" || // Added in 3.9
171       Name == "sse41.pminud" || // Added in 3.9
172       Name == "avx512.kand.w" || // Added in 7.0
173       Name == "avx512.kandn.w" || // Added in 7.0
174       Name == "avx512.knot.w" || // Added in 7.0
175       Name == "avx512.kor.w" || // Added in 7.0
176       Name == "avx512.kxor.w" || // Added in 7.0
177       Name == "avx512.kxnor.w" || // Added in 7.0
178       Name == "avx512.kortestc.w" || // Added in 7.0
179       Name == "avx512.kortestz.w" || // Added in 7.0
180       Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
181       Name.startswith("avx2.pmax") || // Added in 3.9
182       Name.startswith("avx2.pmin") || // Added in 3.9
183       Name.startswith("avx512.mask.pmax") || // Added in 4.0
184       Name.startswith("avx512.mask.pmin") || // Added in 4.0
185       Name.startswith("avx2.vbroadcast") || // Added in 3.8
186       Name.startswith("avx2.pbroadcast") || // Added in 3.8
187       Name.startswith("avx.vpermil.") || // Added in 3.1
188       Name.startswith("sse2.pshuf") || // Added in 3.9
189       Name.startswith("avx512.pbroadcast") || // Added in 3.9
190       Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
191       Name.startswith("avx512.mask.movddup") || // Added in 3.9
192       Name.startswith("avx512.mask.movshdup") || // Added in 3.9
193       Name.startswith("avx512.mask.movsldup") || // Added in 3.9
194       Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
195       Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
196       Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
197       Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
198       Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
199       Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
200       Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
201       Name.startswith("avx512.mask.punpckl") || // Added in 3.9
202       Name.startswith("avx512.mask.punpckh") || // Added in 3.9
203       Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
204       Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
205       Name.startswith("avx512.mask.pand.") || // Added in 3.9
206       Name.startswith("avx512.mask.pandn.") || // Added in 3.9
207       Name.startswith("avx512.mask.por.") || // Added in 3.9
208       Name.startswith("avx512.mask.pxor.") || // Added in 3.9
209       Name.startswith("avx512.mask.and.") || // Added in 3.9
210       Name.startswith("avx512.mask.andn.") || // Added in 3.9
211       Name.startswith("avx512.mask.or.") || // Added in 3.9
212       Name.startswith("avx512.mask.xor.") || // Added in 3.9
213       Name.startswith("avx512.mask.padd.") || // Added in 4.0
214       Name.startswith("avx512.mask.psub.") || // Added in 4.0
215       Name.startswith("avx512.mask.pmull.") || // Added in 4.0
216       Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
217       Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
218       Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
219       Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
220       Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
221       Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
222       Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0
223       Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0
224       Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
225       Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
226       Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
227       Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
228       Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
229       Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
230       Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
231       Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
232       Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
233       Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
234       Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
235       Name == "avx512.cvtusi2sd" || // Added in 7.0
236       Name.startswith("avx512.mask.permvar.") || // Added in 7.0
237       Name == "sse2.pmulu.dq" || // Added in 7.0
238       Name == "sse41.pmuldq" || // Added in 7.0
239       Name == "avx2.pmulu.dq" || // Added in 7.0
240       Name == "avx2.pmul.dq" || // Added in 7.0
241       Name == "avx512.pmulu.dq.512" || // Added in 7.0
242       Name == "avx512.pmul.dq.512" || // Added in 7.0
243       Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
244       Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
245       Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
246       Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
247       Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
248       Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
249       Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
250       Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
251       Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
252       Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
253       Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
254       Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
255       Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
256       Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
257       Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
258       Name.startswith("avx512.cmp.p") || // Added in 12.0
259       Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
260       Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
261       Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
262       Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
263       Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
264       Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
265       Name.startswith("avx512.mask.psll.d") || // Added in 4.0
266       Name.startswith("avx512.mask.psll.q") || // Added in 4.0
267       Name.startswith("avx512.mask.psll.w") || // Added in 4.0
268       Name.startswith("avx512.mask.psra.d") || // Added in 4.0
269       Name.startswith("avx512.mask.psra.q") || // Added in 4.0
270       Name.startswith("avx512.mask.psra.w") || // Added in 4.0
271       Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
272       Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
273       Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
274       Name.startswith("avx512.mask.pslli") || // Added in 4.0
275       Name.startswith("avx512.mask.psrai") || // Added in 4.0
276       Name.startswith("avx512.mask.psrli") || // Added in 4.0
277       Name.startswith("avx512.mask.psllv") || // Added in 4.0
278       Name.startswith("avx512.mask.psrav") || // Added in 4.0
279       Name.startswith("avx512.mask.psrlv") || // Added in 4.0
280       Name.startswith("sse41.pmovsx") || // Added in 3.8
281       Name.startswith("sse41.pmovzx") || // Added in 3.9
282       Name.startswith("avx2.pmovsx") || // Added in 3.9
283       Name.startswith("avx2.pmovzx") || // Added in 3.9
284       Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
285       Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
286       Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
287       Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
288       Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
289       Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
290       Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
291       Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
292       Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
293       Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
294       Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
295       Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
296       Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
297       Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
298       Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
299       Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
300       Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
301       Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
302       Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
303       Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
304       Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
305       Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
306       Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
307       Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
308       Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
309       Name.startswith("avx512.vpshld.") || // Added in 8.0
310       Name.startswith("avx512.vpshrd.") || // Added in 8.0
311       Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
312       Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
313       Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
314       Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
315       Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
316       Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
317       Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
318       Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
319       Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
320       Name.startswith("avx512.mask.conflict.") || // Added in 9.0
321       Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
322       Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
323       Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
324       Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
325       Name == "sse.cvtsi2ss" || // Added in 7.0
326       Name == "sse.cvtsi642ss" || // Added in 7.0
327       Name == "sse2.cvtsi2sd" || // Added in 7.0
328       Name == "sse2.cvtsi642sd" || // Added in 7.0
329       Name == "sse2.cvtss2sd" || // Added in 7.0
330       Name == "sse2.cvtdq2pd" || // Added in 3.9
331       Name == "sse2.cvtdq2ps" || // Added in 7.0
332       Name == "sse2.cvtps2pd" || // Added in 3.9
333       Name == "avx.cvtdq2.pd.256" || // Added in 3.9
334       Name == "avx.cvtdq2.ps.256" || // Added in 7.0
335       Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
336       Name.startswith("vcvtph2ps.") || // Added in 11.0
337       Name.startswith("avx.vinsertf128.") || // Added in 3.7
338       Name == "avx2.vinserti128" || // Added in 3.7
339       Name.startswith("avx512.mask.insert") || // Added in 4.0
340       Name.startswith("avx.vextractf128.") || // Added in 3.7
341       Name == "avx2.vextracti128" || // Added in 3.7
342       Name.startswith("avx512.mask.vextract") || // Added in 4.0
343       Name.startswith("sse4a.movnt.") || // Added in 3.9
344       Name.startswith("avx.movnt.") || // Added in 3.2
345       Name.startswith("avx512.storent.") || // Added in 3.9
346       Name == "sse41.movntdqa" || // Added in 5.0
347       Name == "avx2.movntdqa" || // Added in 5.0
348       Name == "avx512.movntdqa" || // Added in 5.0
349       Name == "sse2.storel.dq" || // Added in 3.9
350       Name.startswith("sse.storeu.") || // Added in 3.9
351       Name.startswith("sse2.storeu.") || // Added in 3.9
352       Name.startswith("avx.storeu.") || // Added in 3.9
353       Name.startswith("avx512.mask.storeu.") || // Added in 3.9
354       Name.startswith("avx512.mask.store.p") || // Added in 3.9
355       Name.startswith("avx512.mask.store.b.") || // Added in 3.9
356       Name.startswith("avx512.mask.store.w.") || // Added in 3.9
357       Name.startswith("avx512.mask.store.d.") || // Added in 3.9
358       Name.startswith("avx512.mask.store.q.") || // Added in 3.9
359       Name == "avx512.mask.store.ss" || // Added in 7.0
360       Name.startswith("avx512.mask.loadu.") || // Added in 3.9
361       Name.startswith("avx512.mask.load.") || // Added in 3.9
362       Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
363       Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
364       Name.startswith("avx512.mask.expand.b") || // Added in 9.0
365       Name.startswith("avx512.mask.expand.w") || // Added in 9.0
366       Name.startswith("avx512.mask.expand.d") || // Added in 9.0
367       Name.startswith("avx512.mask.expand.q") || // Added in 9.0
368       Name.startswith("avx512.mask.expand.p") || // Added in 9.0
369       Name.startswith("avx512.mask.compress.b") || // Added in 9.0
370       Name.startswith("avx512.mask.compress.w") || // Added in 9.0
371       Name.startswith("avx512.mask.compress.d") || // Added in 9.0
372       Name.startswith("avx512.mask.compress.q") || // Added in 9.0
373       Name.startswith("avx512.mask.compress.p") || // Added in 9.0
374       Name == "sse42.crc32.64.8" || // Added in 3.4
375       Name.startswith("avx.vbroadcast.s") || // Added in 3.5
376       Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
377       Name.startswith("avx512.mask.palignr.") || // Added in 3.9
378       Name.startswith("avx512.mask.valign.") || // Added in 4.0
379       Name.startswith("sse2.psll.dq") || // Added in 3.7
380       Name.startswith("sse2.psrl.dq") || // Added in 3.7
381       Name.startswith("avx2.psll.dq") || // Added in 3.7
382       Name.startswith("avx2.psrl.dq") || // Added in 3.7
383       Name.startswith("avx512.psll.dq") || // Added in 3.9
384       Name.startswith("avx512.psrl.dq") || // Added in 3.9
385       Name == "sse41.pblendw" || // Added in 3.7
386       Name.startswith("sse41.blendp") || // Added in 3.7
387       Name.startswith("avx.blend.p") || // Added in 3.7
388       Name == "avx2.pblendw" || // Added in 3.7
389       Name.startswith("avx2.pblendd.") || // Added in 3.7
390       Name.startswith("avx.vbroadcastf128") || // Added in 4.0
391       Name == "avx2.vbroadcasti128" || // Added in 3.7
392       Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0
393       Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0
394       Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0
395       Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0
396       Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0
397       Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0
398       Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0
399       Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0
400       Name == "xop.vpcmov" || // Added in 3.8
401       Name == "xop.vpcmov.256" || // Added in 5.0
402       Name.startswith("avx512.mask.move.s") || // Added in 4.0
403       Name.startswith("avx512.cvtmask2") || // Added in 5.0
404       Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
405       Name.startswith("xop.vprot") || // Added in 8.0
406       Name.startswith("avx512.prol") || // Added in 8.0
407       Name.startswith("avx512.pror") || // Added in 8.0
408       Name.startswith("avx512.mask.prorv.") || // Added in 8.0
409       Name.startswith("avx512.mask.pror.") ||  // Added in 8.0
410       Name.startswith("avx512.mask.prolv.") || // Added in 8.0
411       Name.startswith("avx512.mask.prol.") ||  // Added in 8.0
412       Name.startswith("avx512.ptestm") || //Added in 6.0
413       Name.startswith("avx512.ptestnm") || //Added in 6.0
414       Name.startswith("avx512.mask.pavg")) // Added in 6.0
415     return true;
416 
417   return false;
418 }
419 
420 static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
421                                         Function *&NewFn) {
422   // Only handle intrinsics that start with "x86.".
423   if (!Name.startswith("x86."))
424     return false;
425   // Remove "x86." prefix.
426   Name = Name.substr(4);
427 
428   if (ShouldUpgradeX86Intrinsic(F, Name)) {
429     NewFn = nullptr;
430     return true;
431   }
432 
433   if (Name == "rdtscp") { // Added in 8.0
434     // If this intrinsic has 0 operands, it's the new version.
435     if (F->getFunctionType()->getNumParams() == 0)
436       return false;
437 
438     rename(F);
439     NewFn = Intrinsic::getDeclaration(F->getParent(),
440                                       Intrinsic::x86_rdtscp);
441     return true;
442   }
443 
444   // SSE4.1 ptest functions may have an old signature.
445   if (Name.startswith("sse41.ptest")) { // Added in 3.2
446     if (Name.substr(11) == "c")
447       return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
448     if (Name.substr(11) == "z")
449       return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
450     if (Name.substr(11) == "nzc")
451       return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
452   }
453   // Several blend and other instructions with masks used the wrong number of
454   // bits.
455   if (Name == "sse41.insertps") // Added in 3.6
456     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
457                                             NewFn);
458   if (Name == "sse41.dppd") // Added in 3.6
459     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
460                                             NewFn);
461   if (Name == "sse41.dpps") // Added in 3.6
462     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
463                                             NewFn);
464   if (Name == "sse41.mpsadbw") // Added in 3.6
465     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
466                                             NewFn);
467   if (Name == "avx.dp.ps.256") // Added in 3.6
468     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
469                                             NewFn);
470   if (Name == "avx2.mpsadbw") // Added in 3.6
471     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
472                                             NewFn);
473   if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
474     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
475                                      NewFn);
476   if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
477     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
478                                      NewFn);
479   if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
480     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
481                                      NewFn);
482   if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
483     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
484                                      NewFn);
485   if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
486     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
487                                      NewFn);
488   if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
489     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
490                                      NewFn);
491 
492   // frcz.ss/sd may need to have an argument dropped. Added in 3.2
493   if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
494     rename(F);
495     NewFn = Intrinsic::getDeclaration(F->getParent(),
496                                       Intrinsic::x86_xop_vfrcz_ss);
497     return true;
498   }
499   if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
500     rename(F);
501     NewFn = Intrinsic::getDeclaration(F->getParent(),
502                                       Intrinsic::x86_xop_vfrcz_sd);
503     return true;
504   }
505   // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
506   if (Name.startswith("xop.vpermil2")) { // Added in 3.9
507     auto Idx = F->getFunctionType()->getParamType(2);
508     if (Idx->isFPOrFPVectorTy()) {
509       rename(F);
510       unsigned IdxSize = Idx->getPrimitiveSizeInBits();
511       unsigned EltSize = Idx->getScalarSizeInBits();
512       Intrinsic::ID Permil2ID;
513       if (EltSize == 64 && IdxSize == 128)
514         Permil2ID = Intrinsic::x86_xop_vpermil2pd;
515       else if (EltSize == 32 && IdxSize == 128)
516         Permil2ID = Intrinsic::x86_xop_vpermil2ps;
517       else if (EltSize == 64 && IdxSize == 256)
518         Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
519       else
520         Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
521       NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
522       return true;
523     }
524   }
525 
526   if (Name == "seh.recoverfp") {
527     NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
528     return true;
529   }
530 
531   return false;
532 }
533 
534 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
535   assert(F && "Illegal to upgrade a non-existent Function.");
536 
537   // Quickly eliminate it, if it's not a candidate.
538   StringRef Name = F->getName();
539   if (Name.size() <= 8 || !Name.startswith("llvm."))
540     return false;
541   Name = Name.substr(5); // Strip off "llvm."
542 
543   switch (Name[0]) {
544   default: break;
545   case 'a': {
546     if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
547       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
548                                         F->arg_begin()->getType());
549       return true;
550     }
551     if (Name.startswith("aarch64.neon.frintn")) {
552       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,
553                                         F->arg_begin()->getType());
554       return true;
555     }
556     if (Name.startswith("aarch64.neon.rbit")) {
557       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
558                                         F->arg_begin()->getType());
559       return true;
560     }
561     if (Name.startswith("arm.neon.vclz")) {
562       Type* args[2] = {
563         F->arg_begin()->getType(),
564         Type::getInt1Ty(F->getContext())
565       };
566       // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
567       // the end of the name. Change name from llvm.arm.neon.vclz.* to
568       //  llvm.ctlz.*
569       FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
570       NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
571                                "llvm.ctlz." + Name.substr(14), F->getParent());
572       return true;
573     }
574     if (Name.startswith("arm.neon.vcnt")) {
575       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
576                                         F->arg_begin()->getType());
577       return true;
578     }
579     static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
580     if (vldRegex.match(Name)) {
581       auto fArgs = F->getFunctionType()->params();
582       SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
583       // Can't use Intrinsic::getDeclaration here as the return types might
584       // then only be structurally equal.
585       FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);
586       NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
587                                "llvm." + Name + ".p0i8", F->getParent());
588       return true;
589     }
590     static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
591     if (vstRegex.match(Name)) {
592       static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
593                                                 Intrinsic::arm_neon_vst2,
594                                                 Intrinsic::arm_neon_vst3,
595                                                 Intrinsic::arm_neon_vst4};
596 
597       static const Intrinsic::ID StoreLaneInts[] = {
598         Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
599         Intrinsic::arm_neon_vst4lane
600       };
601 
602       auto fArgs = F->getFunctionType()->params();
603       Type *Tys[] = {fArgs[0], fArgs[1]};
604       if (Name.find("lane") == StringRef::npos)
605         NewFn = Intrinsic::getDeclaration(F->getParent(),
606                                           StoreInts[fArgs.size() - 3], Tys);
607       else
608         NewFn = Intrinsic::getDeclaration(F->getParent(),
609                                           StoreLaneInts[fArgs.size() - 5], Tys);
610       return true;
611     }
612     if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
613       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
614       return true;
615     }
616     if (Name.startswith("arm.neon.vqadds.")) {
617       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
618                                         F->arg_begin()->getType());
619       return true;
620     }
621     if (Name.startswith("arm.neon.vqaddu.")) {
622       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
623                                         F->arg_begin()->getType());
624       return true;
625     }
626     if (Name.startswith("arm.neon.vqsubs.")) {
627       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
628                                         F->arg_begin()->getType());
629       return true;
630     }
631     if (Name.startswith("arm.neon.vqsubu.")) {
632       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
633                                         F->arg_begin()->getType());
634       return true;
635     }
636     if (Name.startswith("aarch64.neon.addp")) {
637       if (F->arg_size() != 2)
638         break; // Invalid IR.
639       VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
640       if (Ty && Ty->getElementType()->isFloatingPointTy()) {
641         NewFn = Intrinsic::getDeclaration(F->getParent(),
642                                           Intrinsic::aarch64_neon_faddp, Ty);
643         return true;
644       }
645     }
646 
647     // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
648     // respectively
649     if ((Name.startswith("arm.neon.bfdot.") ||
650          Name.startswith("aarch64.neon.bfdot.")) &&
651         Name.endswith("i8")) {
652       Intrinsic::ID IID =
653           StringSwitch<Intrinsic::ID>(Name)
654               .Cases("arm.neon.bfdot.v2f32.v8i8",
655                      "arm.neon.bfdot.v4f32.v16i8",
656                      Intrinsic::arm_neon_bfdot)
657               .Cases("aarch64.neon.bfdot.v2f32.v8i8",
658                      "aarch64.neon.bfdot.v4f32.v16i8",
659                      Intrinsic::aarch64_neon_bfdot)
660               .Default(Intrinsic::not_intrinsic);
661       if (IID == Intrinsic::not_intrinsic)
662         break;
663 
664       size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
665       assert((OperandWidth == 64 || OperandWidth == 128) &&
666              "Unexpected operand width");
667       LLVMContext &Ctx = F->getParent()->getContext();
668       std::array<Type *, 2> Tys {{
669         F->getReturnType(),
670         FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
671       }};
672       NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
673       return true;
674     }
675 
676     // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
677     // and accept v8bf16 instead of v16i8
678     if ((Name.startswith("arm.neon.bfm") ||
679          Name.startswith("aarch64.neon.bfm")) &&
680         Name.endswith(".v4f32.v16i8")) {
681       Intrinsic::ID IID =
682           StringSwitch<Intrinsic::ID>(Name)
683               .Case("arm.neon.bfmmla.v4f32.v16i8",
684                     Intrinsic::arm_neon_bfmmla)
685               .Case("arm.neon.bfmlalb.v4f32.v16i8",
686                     Intrinsic::arm_neon_bfmlalb)
687               .Case("arm.neon.bfmlalt.v4f32.v16i8",
688                     Intrinsic::arm_neon_bfmlalt)
689               .Case("aarch64.neon.bfmmla.v4f32.v16i8",
690                     Intrinsic::aarch64_neon_bfmmla)
691               .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
692                     Intrinsic::aarch64_neon_bfmlalb)
693               .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
694                     Intrinsic::aarch64_neon_bfmlalt)
695               .Default(Intrinsic::not_intrinsic);
696       if (IID == Intrinsic::not_intrinsic)
697         break;
698 
699       std::array<Type *, 0> Tys;
700       NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
701       return true;
702     }
703     break;
704   }
705 
706   case 'c': {
707     if (Name.startswith("ctlz.") && F->arg_size() == 1) {
708       rename(F);
709       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
710                                         F->arg_begin()->getType());
711       return true;
712     }
713     if (Name.startswith("cttz.") && F->arg_size() == 1) {
714       rename(F);
715       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
716                                         F->arg_begin()->getType());
717       return true;
718     }
719     break;
720   }
721   case 'd': {
722     if (Name == "dbg.value" && F->arg_size() == 4) {
723       rename(F);
724       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
725       return true;
726     }
727     break;
728   }
729   case 'e': {
730     SmallVector<StringRef, 2> Groups;
731     static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
732     if (R.match(Name, &Groups)) {
733       Intrinsic::ID ID;
734       ID = StringSwitch<Intrinsic::ID>(Groups[1])
735                .Case("add", Intrinsic::vector_reduce_add)
736                .Case("mul", Intrinsic::vector_reduce_mul)
737                .Case("and", Intrinsic::vector_reduce_and)
738                .Case("or", Intrinsic::vector_reduce_or)
739                .Case("xor", Intrinsic::vector_reduce_xor)
740                .Case("smax", Intrinsic::vector_reduce_smax)
741                .Case("smin", Intrinsic::vector_reduce_smin)
742                .Case("umax", Intrinsic::vector_reduce_umax)
743                .Case("umin", Intrinsic::vector_reduce_umin)
744                .Case("fmax", Intrinsic::vector_reduce_fmax)
745                .Case("fmin", Intrinsic::vector_reduce_fmin)
746                .Default(Intrinsic::not_intrinsic);
747       if (ID != Intrinsic::not_intrinsic) {
748         rename(F);
749         auto Args = F->getFunctionType()->params();
750         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});
751         return true;
752       }
753     }
754     static const Regex R2(
755         "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
756     Groups.clear();
757     if (R2.match(Name, &Groups)) {
758       Intrinsic::ID ID = Intrinsic::not_intrinsic;
759       if (Groups[1] == "fadd")
760         ID = Intrinsic::vector_reduce_fadd;
761       if (Groups[1] == "fmul")
762         ID = Intrinsic::vector_reduce_fmul;
763       if (ID != Intrinsic::not_intrinsic) {
764         rename(F);
765         auto Args = F->getFunctionType()->params();
766         Type *Tys[] = {Args[1]};
767         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
768         return true;
769       }
770     }
771     break;
772   }
773   case 'i':
774   case 'l': {
775     bool IsLifetimeStart = Name.startswith("lifetime.start");
776     if (IsLifetimeStart || Name.startswith("invariant.start")) {
777       Intrinsic::ID ID = IsLifetimeStart ?
778         Intrinsic::lifetime_start : Intrinsic::invariant_start;
779       auto Args = F->getFunctionType()->params();
780       Type* ObjectPtr[1] = {Args[1]};
781       if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {
782         rename(F);
783         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
784         return true;
785       }
786     }
787 
788     bool IsLifetimeEnd = Name.startswith("lifetime.end");
789     if (IsLifetimeEnd || Name.startswith("invariant.end")) {
790       Intrinsic::ID ID = IsLifetimeEnd ?
791         Intrinsic::lifetime_end : Intrinsic::invariant_end;
792 
793       auto Args = F->getFunctionType()->params();
794       Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
795       if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {
796         rename(F);
797         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
798         return true;
799       }
800     }
801     if (Name.startswith("invariant.group.barrier")) {
802       // Rename invariant.group.barrier to launder.invariant.group
803       auto Args = F->getFunctionType()->params();
804       Type* ObjectPtr[1] = {Args[0]};
805       rename(F);
806       NewFn = Intrinsic::getDeclaration(F->getParent(),
807           Intrinsic::launder_invariant_group, ObjectPtr);
808       return true;
809 
810     }
811 
812     break;
813   }
814   case 'm': {
815     if (Name.startswith("masked.load.")) {
816       Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
817       if (F->getName() !=
818           Intrinsic::getName(Intrinsic::masked_load, Tys, F->getParent())) {
819         rename(F);
820         NewFn = Intrinsic::getDeclaration(F->getParent(),
821                                           Intrinsic::masked_load,
822                                           Tys);
823         return true;
824       }
825     }
826     if (Name.startswith("masked.store.")) {
827       auto Args = F->getFunctionType()->params();
828       Type *Tys[] = { Args[0], Args[1] };
829       if (F->getName() !=
830           Intrinsic::getName(Intrinsic::masked_store, Tys, F->getParent())) {
831         rename(F);
832         NewFn = Intrinsic::getDeclaration(F->getParent(),
833                                           Intrinsic::masked_store,
834                                           Tys);
835         return true;
836       }
837     }
838     // Renaming gather/scatter intrinsics with no address space overloading
839     // to the new overload which includes an address space
840     if (Name.startswith("masked.gather.")) {
841       Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
842       if (F->getName() !=
843           Intrinsic::getName(Intrinsic::masked_gather, Tys, F->getParent())) {
844         rename(F);
845         NewFn = Intrinsic::getDeclaration(F->getParent(),
846                                           Intrinsic::masked_gather, Tys);
847         return true;
848       }
849     }
850     if (Name.startswith("masked.scatter.")) {
851       auto Args = F->getFunctionType()->params();
852       Type *Tys[] = {Args[0], Args[1]};
853       if (F->getName() !=
854           Intrinsic::getName(Intrinsic::masked_scatter, Tys, F->getParent())) {
855         rename(F);
856         NewFn = Intrinsic::getDeclaration(F->getParent(),
857                                           Intrinsic::masked_scatter, Tys);
858         return true;
859       }
860     }
861     // Updating the memory intrinsics (memcpy/memmove/memset) that have an
862     // alignment parameter to embedding the alignment as an attribute of
863     // the pointer args.
864     if (Name.startswith("memcpy.") && F->arg_size() == 5) {
865       rename(F);
866       // Get the types of dest, src, and len
867       ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
868       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
869                                         ParamTypes);
870       return true;
871     }
872     if (Name.startswith("memmove.") && F->arg_size() == 5) {
873       rename(F);
874       // Get the types of dest, src, and len
875       ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
876       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
877                                         ParamTypes);
878       return true;
879     }
880     if (Name.startswith("memset.") && F->arg_size() == 5) {
881       rename(F);
882       // Get the types of dest, and len
883       const auto *FT = F->getFunctionType();
884       Type *ParamTypes[2] = {
885           FT->getParamType(0), // Dest
886           FT->getParamType(2)  // len
887       };
888       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
889                                         ParamTypes);
890       return true;
891     }
892     break;
893   }
894   case 'n': {
895     if (Name.startswith("nvvm.")) {
896       Name = Name.substr(5);
897 
898       // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
899       Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
900                               .Cases("brev32", "brev64", Intrinsic::bitreverse)
901                               .Case("clz.i", Intrinsic::ctlz)
902                               .Case("popc.i", Intrinsic::ctpop)
903                               .Default(Intrinsic::not_intrinsic);
904       if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
905         NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
906                                           {F->getReturnType()});
907         return true;
908       }
909 
910       // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
911       // not to an intrinsic alone.  We expand them in UpgradeIntrinsicCall.
912       //
913       // TODO: We could add lohi.i2d.
914       bool Expand = StringSwitch<bool>(Name)
915                         .Cases("abs.i", "abs.ll", true)
916                         .Cases("clz.ll", "popc.ll", "h2f", true)
917                         .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
918                         .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
919                         .StartsWith("atomic.load.add.f32.p", true)
920                         .StartsWith("atomic.load.add.f64.p", true)
921                         .Default(false);
922       if (Expand) {
923         NewFn = nullptr;
924         return true;
925       }
926     }
927     break;
928   }
929   case 'o':
930     // We only need to change the name to match the mangling including the
931     // address space.
932     if (Name.startswith("objectsize.")) {
933       Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
934       if (F->arg_size() == 2 || F->arg_size() == 3 ||
935           F->getName() !=
936               Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) {
937         rename(F);
938         NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
939                                           Tys);
940         return true;
941       }
942     }
943     break;
944 
945   case 'p':
946     if (Name == "prefetch") {
947       // Handle address space overloading.
948       Type *Tys[] = {F->arg_begin()->getType()};
949       if (F->getName() !=
950           Intrinsic::getName(Intrinsic::prefetch, Tys, F->getParent())) {
951         rename(F);
952         NewFn =
953             Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
954         return true;
955       }
956     } else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {
957       rename(F);
958       NewFn = Intrinsic::getDeclaration(F->getParent(),
959                                         Intrinsic::ptr_annotation,
960                                         F->arg_begin()->getType());
961       return true;
962     }
963     break;
964 
965   case 's':
966     if (Name == "stackprotectorcheck") {
967       NewFn = nullptr;
968       return true;
969     }
970     break;
971 
972   case 'v': {
973     if (Name == "var.annotation" && F->arg_size() == 4) {
974       rename(F);
975       NewFn = Intrinsic::getDeclaration(F->getParent(),
976                                         Intrinsic::var_annotation);
977       return true;
978     }
979     break;
980   }
981 
982   case 'x':
983     if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
984       return true;
985   }
986   // Remangle our intrinsic since we upgrade the mangling
987   auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
988   if (Result != None) {
989     NewFn = Result.getValue();
990     return true;
991   }
992 
993   //  This may not belong here. This function is effectively being overloaded
994   //  to both detect an intrinsic which needs upgrading, and to provide the
995   //  upgraded form of the intrinsic. We should perhaps have two separate
996   //  functions for this.
997   return false;
998 }
999 
1000 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
1001   NewFn = nullptr;
1002   bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
1003   assert(F != NewFn && "Intrinsic function upgraded to the same function");
1004 
1005   // Upgrade intrinsic attributes.  This does not change the function.
1006   if (NewFn)
1007     F = NewFn;
1008   if (Intrinsic::ID id = F->getIntrinsicID())
1009     F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
1010   return Upgraded;
1011 }
1012 
1013 GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
1014   if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
1015                           GV->getName() == "llvm.global_dtors")) ||
1016       !GV->hasInitializer())
1017     return nullptr;
1018   ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
1019   if (!ATy)
1020     return nullptr;
1021   StructType *STy = dyn_cast<StructType>(ATy->getElementType());
1022   if (!STy || STy->getNumElements() != 2)
1023     return nullptr;
1024 
1025   LLVMContext &C = GV->getContext();
1026   IRBuilder<> IRB(C);
1027   auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
1028                                IRB.getInt8PtrTy());
1029   Constant *Init = GV->getInitializer();
1030   unsigned N = Init->getNumOperands();
1031   std::vector<Constant *> NewCtors(N);
1032   for (unsigned i = 0; i != N; ++i) {
1033     auto Ctor = cast<Constant>(Init->getOperand(i));
1034     NewCtors[i] = ConstantStruct::get(
1035         EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1036         Constant::getNullValue(IRB.getInt8PtrTy()));
1037   }
1038   Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
1039 
1040   return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
1041                             NewInit, GV->getName());
1042 }
1043 
1044 // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
1045 // to byte shuffles.
1046 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
1047                                          Value *Op, unsigned Shift) {
1048   auto *ResultTy = cast<FixedVectorType>(Op->getType());
1049   unsigned NumElts = ResultTy->getNumElements() * 8;
1050 
1051   // Bitcast from a 64-bit element type to a byte element type.
1052   Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1053   Op = Builder.CreateBitCast(Op, VecTy, "cast");
1054 
1055   // We'll be shuffling in zeroes.
1056   Value *Res = Constant::getNullValue(VecTy);
1057 
1058   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1059   // we'll just return the zero vector.
1060   if (Shift < 16) {
1061     int Idxs[64];
1062     // 256/512-bit version is split into 2/4 16-byte lanes.
1063     for (unsigned l = 0; l != NumElts; l += 16)
1064       for (unsigned i = 0; i != 16; ++i) {
1065         unsigned Idx = NumElts + i - Shift;
1066         if (Idx < NumElts)
1067           Idx -= NumElts - 16; // end of lane, switch operand.
1068         Idxs[l + i] = Idx + l;
1069       }
1070 
1071     Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
1072   }
1073 
1074   // Bitcast back to a 64-bit element type.
1075   return Builder.CreateBitCast(Res, ResultTy, "cast");
1076 }
1077 
1078 // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
1079 // to byte shuffles.
1080 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1081                                          unsigned Shift) {
1082   auto *ResultTy = cast<FixedVectorType>(Op->getType());
1083   unsigned NumElts = ResultTy->getNumElements() * 8;
1084 
1085   // Bitcast from a 64-bit element type to a byte element type.
1086   Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1087   Op = Builder.CreateBitCast(Op, VecTy, "cast");
1088 
1089   // We'll be shuffling in zeroes.
1090   Value *Res = Constant::getNullValue(VecTy);
1091 
1092   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1093   // we'll just return the zero vector.
1094   if (Shift < 16) {
1095     int Idxs[64];
1096     // 256/512-bit version is split into 2/4 16-byte lanes.
1097     for (unsigned l = 0; l != NumElts; l += 16)
1098       for (unsigned i = 0; i != 16; ++i) {
1099         unsigned Idx = i + Shift;
1100         if (Idx >= 16)
1101           Idx += NumElts - 16; // end of lane, switch operand.
1102         Idxs[l + i] = Idx + l;
1103       }
1104 
1105     Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
1106   }
1107 
1108   // Bitcast back to a 64-bit element type.
1109   return Builder.CreateBitCast(Res, ResultTy, "cast");
1110 }
1111 
1112 static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
1113                             unsigned NumElts) {
1114   assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
1115   llvm::VectorType *MaskTy = FixedVectorType::get(
1116       Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
1117   Mask = Builder.CreateBitCast(Mask, MaskTy);
1118 
1119   // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
1120   // i8 and we need to extract down to the right number of elements.
1121   if (NumElts <= 4) {
1122     int Indices[4];
1123     for (unsigned i = 0; i != NumElts; ++i)
1124       Indices[i] = i;
1125     Mask = Builder.CreateShuffleVector(
1126         Mask, Mask, makeArrayRef(Indices, NumElts), "extract");
1127   }
1128 
1129   return Mask;
1130 }
1131 
1132 static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
1133                             Value *Op0, Value *Op1) {
1134   // If the mask is all ones just emit the first operation.
1135   if (const auto *C = dyn_cast<Constant>(Mask))
1136     if (C->isAllOnesValue())
1137       return Op0;
1138 
1139   Mask = getX86MaskVec(Builder, Mask,
1140                        cast<FixedVectorType>(Op0->getType())->getNumElements());
1141   return Builder.CreateSelect(Mask, Op0, Op1);
1142 }
1143 
1144 static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
1145                                   Value *Op0, Value *Op1) {
1146   // If the mask is all ones just emit the first operation.
1147   if (const auto *C = dyn_cast<Constant>(Mask))
1148     if (C->isAllOnesValue())
1149       return Op0;
1150 
1151   auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
1152                                       Mask->getType()->getIntegerBitWidth());
1153   Mask = Builder.CreateBitCast(Mask, MaskTy);
1154   Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
1155   return Builder.CreateSelect(Mask, Op0, Op1);
1156 }
1157 
1158 // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
1159 // PALIGNR handles large immediates by shifting while VALIGN masks the immediate
1160 // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
1161 static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
1162                                         Value *Op1, Value *Shift,
1163                                         Value *Passthru, Value *Mask,
1164                                         bool IsVALIGN) {
1165   unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1166 
1167   unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1168   assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
1169   assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
1170   assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
1171 
1172   // Mask the immediate for VALIGN.
1173   if (IsVALIGN)
1174     ShiftVal &= (NumElts - 1);
1175 
1176   // If palignr is shifting the pair of vectors more than the size of two
1177   // lanes, emit zero.
1178   if (ShiftVal >= 32)
1179     return llvm::Constant::getNullValue(Op0->getType());
1180 
1181   // If palignr is shifting the pair of input vectors more than one lane,
1182   // but less than two lanes, convert to shifting in zeroes.
1183   if (ShiftVal > 16) {
1184     ShiftVal -= 16;
1185     Op1 = Op0;
1186     Op0 = llvm::Constant::getNullValue(Op0->getType());
1187   }
1188 
1189   int Indices[64];
1190   // 256-bit palignr operates on 128-bit lanes so we need to handle that
1191   for (unsigned l = 0; l < NumElts; l += 16) {
1192     for (unsigned i = 0; i != 16; ++i) {
1193       unsigned Idx = ShiftVal + i;
1194       if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
1195         Idx += NumElts - 16; // End of lane, switch operand.
1196       Indices[l + i] = Idx + l;
1197     }
1198   }
1199 
1200   Value *Align = Builder.CreateShuffleVector(Op1, Op0,
1201                                              makeArrayRef(Indices, NumElts),
1202                                              "palignr");
1203 
1204   return EmitX86Select(Builder, Mask, Align, Passthru);
1205 }
1206 
1207 static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
1208                                           bool ZeroMask, bool IndexForm) {
1209   Type *Ty = CI.getType();
1210   unsigned VecWidth = Ty->getPrimitiveSizeInBits();
1211   unsigned EltWidth = Ty->getScalarSizeInBits();
1212   bool IsFloat = Ty->isFPOrFPVectorTy();
1213   Intrinsic::ID IID;
1214   if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1215     IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1216   else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1217     IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1218   else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1219     IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1220   else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1221     IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1222   else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1223     IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1224   else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1225     IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1226   else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1227     IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1228   else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1229     IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1230   else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1231     IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1232   else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1233     IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1234   else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1235     IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1236   else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1237     IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1238   else if (VecWidth == 128 && EltWidth == 16)
1239     IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1240   else if (VecWidth == 256 && EltWidth == 16)
1241     IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1242   else if (VecWidth == 512 && EltWidth == 16)
1243     IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1244   else if (VecWidth == 128 && EltWidth == 8)
1245     IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1246   else if (VecWidth == 256 && EltWidth == 8)
1247     IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1248   else if (VecWidth == 512 && EltWidth == 8)
1249     IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1250   else
1251     llvm_unreachable("Unexpected intrinsic");
1252 
1253   Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
1254                     CI.getArgOperand(2) };
1255 
1256   // If this isn't index form we need to swap operand 0 and 1.
1257   if (!IndexForm)
1258     std::swap(Args[0], Args[1]);
1259 
1260   Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1261                                 Args);
1262   Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
1263                              : Builder.CreateBitCast(CI.getArgOperand(1),
1264                                                      Ty);
1265   return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1266 }
1267 
1268 static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI,
1269                                          Intrinsic::ID IID) {
1270   Type *Ty = CI.getType();
1271   Value *Op0 = CI.getOperand(0);
1272   Value *Op1 = CI.getOperand(1);
1273   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1274   Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
1275 
1276   if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
1277     Value *VecSrc = CI.getOperand(2);
1278     Value *Mask = CI.getOperand(3);
1279     Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1280   }
1281   return Res;
1282 }
1283 
1284 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
1285                                bool IsRotateRight) {
1286   Type *Ty = CI.getType();
1287   Value *Src = CI.getArgOperand(0);
1288   Value *Amt = CI.getArgOperand(1);
1289 
1290   // Amount may be scalar immediate, in which case create a splat vector.
1291   // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1292   // we only care about the lowest log2 bits anyway.
1293   if (Amt->getType() != Ty) {
1294     unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1295     Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1296     Amt = Builder.CreateVectorSplat(NumElts, Amt);
1297   }
1298 
1299   Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1300   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1301   Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
1302 
1303   if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
1304     Value *VecSrc = CI.getOperand(2);
1305     Value *Mask = CI.getOperand(3);
1306     Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1307   }
1308   return Res;
1309 }
1310 
1311 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
1312                               bool IsSigned) {
1313   Type *Ty = CI.getType();
1314   Value *LHS = CI.getArgOperand(0);
1315   Value *RHS = CI.getArgOperand(1);
1316 
1317   CmpInst::Predicate Pred;
1318   switch (Imm) {
1319   case 0x0:
1320     Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1321     break;
1322   case 0x1:
1323     Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1324     break;
1325   case 0x2:
1326     Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1327     break;
1328   case 0x3:
1329     Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1330     break;
1331   case 0x4:
1332     Pred = ICmpInst::ICMP_EQ;
1333     break;
1334   case 0x5:
1335     Pred = ICmpInst::ICMP_NE;
1336     break;
1337   case 0x6:
1338     return Constant::getNullValue(Ty); // FALSE
1339   case 0x7:
1340     return Constant::getAllOnesValue(Ty); // TRUE
1341   default:
1342     llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
1343   }
1344 
1345   Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
1346   Value *Ext = Builder.CreateSExt(Cmp, Ty);
1347   return Ext;
1348 }
1349 
1350 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
1351                                     bool IsShiftRight, bool ZeroMask) {
1352   Type *Ty = CI.getType();
1353   Value *Op0 = CI.getArgOperand(0);
1354   Value *Op1 = CI.getArgOperand(1);
1355   Value *Amt = CI.getArgOperand(2);
1356 
1357   if (IsShiftRight)
1358     std::swap(Op0, Op1);
1359 
1360   // Amount may be scalar immediate, in which case create a splat vector.
1361   // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1362   // we only care about the lowest log2 bits anyway.
1363   if (Amt->getType() != Ty) {
1364     unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1365     Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1366     Amt = Builder.CreateVectorSplat(NumElts, Amt);
1367   }
1368 
1369   Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1370   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1371   Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1372 
1373   unsigned NumArgs = CI.getNumArgOperands();
1374   if (NumArgs >= 4) { // For masked intrinsics.
1375     Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
1376                     ZeroMask     ? ConstantAggregateZero::get(CI.getType()) :
1377                                    CI.getArgOperand(0);
1378     Value *Mask = CI.getOperand(NumArgs - 1);
1379     Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1380   }
1381   return Res;
1382 }
1383 
1384 static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
1385                                  Value *Ptr, Value *Data, Value *Mask,
1386                                  bool Aligned) {
1387   // Cast the pointer to the right type.
1388   Ptr = Builder.CreateBitCast(Ptr,
1389                               llvm::PointerType::getUnqual(Data->getType()));
1390   const Align Alignment =
1391       Aligned
1392           ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)
1393           : Align(1);
1394 
1395   // If the mask is all ones just emit a regular store.
1396   if (const auto *C = dyn_cast<Constant>(Mask))
1397     if (C->isAllOnesValue())
1398       return Builder.CreateAlignedStore(Data, Ptr, Alignment);
1399 
1400   // Convert the mask from an integer type to a vector of i1.
1401   unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1402   Mask = getX86MaskVec(Builder, Mask, NumElts);
1403   return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
1404 }
1405 
1406 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
1407                                 Value *Ptr, Value *Passthru, Value *Mask,
1408                                 bool Aligned) {
1409   Type *ValTy = Passthru->getType();
1410   // Cast the pointer to the right type.
1411   Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
1412   const Align Alignment =
1413       Aligned
1414           ? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() /
1415                   8)
1416           : Align(1);
1417 
1418   // If the mask is all ones just emit a regular store.
1419   if (const auto *C = dyn_cast<Constant>(Mask))
1420     if (C->isAllOnesValue())
1421       return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1422 
1423   // Convert the mask from an integer type to a vector of i1.
1424   unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1425   Mask = getX86MaskVec(Builder, Mask, NumElts);
1426   return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
1427 }
1428 
1429 static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
1430   Type *Ty = CI.getType();
1431   Value *Op0 = CI.getArgOperand(0);
1432   Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
1433   Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
1434   if (CI.getNumArgOperands() == 3)
1435     Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
1436   return Res;
1437 }
1438 
1439 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
1440   Type *Ty = CI.getType();
1441 
1442   // Arguments have a vXi32 type so cast to vXi64.
1443   Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1444   Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1445 
1446   if (IsSigned) {
1447     // Shift left then arithmetic shift right.
1448     Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1449     LHS = Builder.CreateShl(LHS, ShiftAmt);
1450     LHS = Builder.CreateAShr(LHS, ShiftAmt);
1451     RHS = Builder.CreateShl(RHS, ShiftAmt);
1452     RHS = Builder.CreateAShr(RHS, ShiftAmt);
1453   } else {
1454     // Clear the upper bits.
1455     Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1456     LHS = Builder.CreateAnd(LHS, Mask);
1457     RHS = Builder.CreateAnd(RHS, Mask);
1458   }
1459 
1460   Value *Res = Builder.CreateMul(LHS, RHS);
1461 
1462   if (CI.getNumArgOperands() == 4)
1463     Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1464 
1465   return Res;
1466 }
1467 
1468 // Applying mask on vector of i1's and make sure result is at least 8 bits wide.
1469 static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
1470                                      Value *Mask) {
1471   unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
1472   if (Mask) {
1473     const auto *C = dyn_cast<Constant>(Mask);
1474     if (!C || !C->isAllOnesValue())
1475       Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
1476   }
1477 
1478   if (NumElts < 8) {
1479     int Indices[8];
1480     for (unsigned i = 0; i != NumElts; ++i)
1481       Indices[i] = i;
1482     for (unsigned i = NumElts; i != 8; ++i)
1483       Indices[i] = NumElts + i % NumElts;
1484     Vec = Builder.CreateShuffleVector(Vec,
1485                                       Constant::getNullValue(Vec->getType()),
1486                                       Indices);
1487   }
1488   return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
1489 }
1490 
1491 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
1492                                    unsigned CC, bool Signed) {
1493   Value *Op0 = CI.getArgOperand(0);
1494   unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1495 
1496   Value *Cmp;
1497   if (CC == 3) {
1498     Cmp = Constant::getNullValue(
1499         FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1500   } else if (CC == 7) {
1501     Cmp = Constant::getAllOnesValue(
1502         FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1503   } else {
1504     ICmpInst::Predicate Pred;
1505     switch (CC) {
1506     default: llvm_unreachable("Unknown condition code");
1507     case 0: Pred = ICmpInst::ICMP_EQ;  break;
1508     case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
1509     case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
1510     case 4: Pred = ICmpInst::ICMP_NE;  break;
1511     case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
1512     case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
1513     }
1514     Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1515   }
1516 
1517   Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
1518 
1519   return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
1520 }
1521 
1522 // Replace a masked intrinsic with an older unmasked intrinsic.
1523 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
1524                                     Intrinsic::ID IID) {
1525   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1526   Value *Rep = Builder.CreateCall(Intrin,
1527                                  { CI.getArgOperand(0), CI.getArgOperand(1) });
1528   return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1529 }
1530 
1531 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
1532   Value* A = CI.getArgOperand(0);
1533   Value* B = CI.getArgOperand(1);
1534   Value* Src = CI.getArgOperand(2);
1535   Value* Mask = CI.getArgOperand(3);
1536 
1537   Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
1538   Value* Cmp = Builder.CreateIsNotNull(AndNode);
1539   Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
1540   Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
1541   Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
1542   return Builder.CreateInsertElement(A, Select, (uint64_t)0);
1543 }
1544 
1545 
1546 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
1547   Value* Op = CI.getArgOperand(0);
1548   Type* ReturnOp = CI.getType();
1549   unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
1550   Value *Mask = getX86MaskVec(Builder, Op, NumElts);
1551   return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
1552 }
1553 
1554 // Replace intrinsic with unmasked version and a select.
1555 static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
1556                                       CallInst &CI, Value *&Rep) {
1557   Name = Name.substr(12); // Remove avx512.mask.
1558 
1559   unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
1560   unsigned EltWidth = CI.getType()->getScalarSizeInBits();
1561   Intrinsic::ID IID;
1562   if (Name.startswith("max.p")) {
1563     if (VecWidth == 128 && EltWidth == 32)
1564       IID = Intrinsic::x86_sse_max_ps;
1565     else if (VecWidth == 128 && EltWidth == 64)
1566       IID = Intrinsic::x86_sse2_max_pd;
1567     else if (VecWidth == 256 && EltWidth == 32)
1568       IID = Intrinsic::x86_avx_max_ps_256;
1569     else if (VecWidth == 256 && EltWidth == 64)
1570       IID = Intrinsic::x86_avx_max_pd_256;
1571     else
1572       llvm_unreachable("Unexpected intrinsic");
1573   } else if (Name.startswith("min.p")) {
1574     if (VecWidth == 128 && EltWidth == 32)
1575       IID = Intrinsic::x86_sse_min_ps;
1576     else if (VecWidth == 128 && EltWidth == 64)
1577       IID = Intrinsic::x86_sse2_min_pd;
1578     else if (VecWidth == 256 && EltWidth == 32)
1579       IID = Intrinsic::x86_avx_min_ps_256;
1580     else if (VecWidth == 256 && EltWidth == 64)
1581       IID = Intrinsic::x86_avx_min_pd_256;
1582     else
1583       llvm_unreachable("Unexpected intrinsic");
1584   } else if (Name.startswith("pshuf.b.")) {
1585     if (VecWidth == 128)
1586       IID = Intrinsic::x86_ssse3_pshuf_b_128;
1587     else if (VecWidth == 256)
1588       IID = Intrinsic::x86_avx2_pshuf_b;
1589     else if (VecWidth == 512)
1590       IID = Intrinsic::x86_avx512_pshuf_b_512;
1591     else
1592       llvm_unreachable("Unexpected intrinsic");
1593   } else if (Name.startswith("pmul.hr.sw.")) {
1594     if (VecWidth == 128)
1595       IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1596     else if (VecWidth == 256)
1597       IID = Intrinsic::x86_avx2_pmul_hr_sw;
1598     else if (VecWidth == 512)
1599       IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1600     else
1601       llvm_unreachable("Unexpected intrinsic");
1602   } else if (Name.startswith("pmulh.w.")) {
1603     if (VecWidth == 128)
1604       IID = Intrinsic::x86_sse2_pmulh_w;
1605     else if (VecWidth == 256)
1606       IID = Intrinsic::x86_avx2_pmulh_w;
1607     else if (VecWidth == 512)
1608       IID = Intrinsic::x86_avx512_pmulh_w_512;
1609     else
1610       llvm_unreachable("Unexpected intrinsic");
1611   } else if (Name.startswith("pmulhu.w.")) {
1612     if (VecWidth == 128)
1613       IID = Intrinsic::x86_sse2_pmulhu_w;
1614     else if (VecWidth == 256)
1615       IID = Intrinsic::x86_avx2_pmulhu_w;
1616     else if (VecWidth == 512)
1617       IID = Intrinsic::x86_avx512_pmulhu_w_512;
1618     else
1619       llvm_unreachable("Unexpected intrinsic");
1620   } else if (Name.startswith("pmaddw.d.")) {
1621     if (VecWidth == 128)
1622       IID = Intrinsic::x86_sse2_pmadd_wd;
1623     else if (VecWidth == 256)
1624       IID = Intrinsic::x86_avx2_pmadd_wd;
1625     else if (VecWidth == 512)
1626       IID = Intrinsic::x86_avx512_pmaddw_d_512;
1627     else
1628       llvm_unreachable("Unexpected intrinsic");
1629   } else if (Name.startswith("pmaddubs.w.")) {
1630     if (VecWidth == 128)
1631       IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1632     else if (VecWidth == 256)
1633       IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1634     else if (VecWidth == 512)
1635       IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1636     else
1637       llvm_unreachable("Unexpected intrinsic");
1638   } else if (Name.startswith("packsswb.")) {
1639     if (VecWidth == 128)
1640       IID = Intrinsic::x86_sse2_packsswb_128;
1641     else if (VecWidth == 256)
1642       IID = Intrinsic::x86_avx2_packsswb;
1643     else if (VecWidth == 512)
1644       IID = Intrinsic::x86_avx512_packsswb_512;
1645     else
1646       llvm_unreachable("Unexpected intrinsic");
1647   } else if (Name.startswith("packssdw.")) {
1648     if (VecWidth == 128)
1649       IID = Intrinsic::x86_sse2_packssdw_128;
1650     else if (VecWidth == 256)
1651       IID = Intrinsic::x86_avx2_packssdw;
1652     else if (VecWidth == 512)
1653       IID = Intrinsic::x86_avx512_packssdw_512;
1654     else
1655       llvm_unreachable("Unexpected intrinsic");
1656   } else if (Name.startswith("packuswb.")) {
1657     if (VecWidth == 128)
1658       IID = Intrinsic::x86_sse2_packuswb_128;
1659     else if (VecWidth == 256)
1660       IID = Intrinsic::x86_avx2_packuswb;
1661     else if (VecWidth == 512)
1662       IID = Intrinsic::x86_avx512_packuswb_512;
1663     else
1664       llvm_unreachable("Unexpected intrinsic");
1665   } else if (Name.startswith("packusdw.")) {
1666     if (VecWidth == 128)
1667       IID = Intrinsic::x86_sse41_packusdw;
1668     else if (VecWidth == 256)
1669       IID = Intrinsic::x86_avx2_packusdw;
1670     else if (VecWidth == 512)
1671       IID = Intrinsic::x86_avx512_packusdw_512;
1672     else
1673       llvm_unreachable("Unexpected intrinsic");
1674   } else if (Name.startswith("vpermilvar.")) {
1675     if (VecWidth == 128 && EltWidth == 32)
1676       IID = Intrinsic::x86_avx_vpermilvar_ps;
1677     else if (VecWidth == 128 && EltWidth == 64)
1678       IID = Intrinsic::x86_avx_vpermilvar_pd;
1679     else if (VecWidth == 256 && EltWidth == 32)
1680       IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1681     else if (VecWidth == 256 && EltWidth == 64)
1682       IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1683     else if (VecWidth == 512 && EltWidth == 32)
1684       IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1685     else if (VecWidth == 512 && EltWidth == 64)
1686       IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1687     else
1688       llvm_unreachable("Unexpected intrinsic");
1689   } else if (Name == "cvtpd2dq.256") {
1690     IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1691   } else if (Name == "cvtpd2ps.256") {
1692     IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1693   } else if (Name == "cvttpd2dq.256") {
1694     IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1695   } else if (Name == "cvttps2dq.128") {
1696     IID = Intrinsic::x86_sse2_cvttps2dq;
1697   } else if (Name == "cvttps2dq.256") {
1698     IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1699   } else if (Name.startswith("permvar.")) {
1700     bool IsFloat = CI.getType()->isFPOrFPVectorTy();
1701     if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1702       IID = Intrinsic::x86_avx2_permps;
1703     else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1704       IID = Intrinsic::x86_avx2_permd;
1705     else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1706       IID = Intrinsic::x86_avx512_permvar_df_256;
1707     else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1708       IID = Intrinsic::x86_avx512_permvar_di_256;
1709     else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1710       IID = Intrinsic::x86_avx512_permvar_sf_512;
1711     else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1712       IID = Intrinsic::x86_avx512_permvar_si_512;
1713     else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1714       IID = Intrinsic::x86_avx512_permvar_df_512;
1715     else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1716       IID = Intrinsic::x86_avx512_permvar_di_512;
1717     else if (VecWidth == 128 && EltWidth == 16)
1718       IID = Intrinsic::x86_avx512_permvar_hi_128;
1719     else if (VecWidth == 256 && EltWidth == 16)
1720       IID = Intrinsic::x86_avx512_permvar_hi_256;
1721     else if (VecWidth == 512 && EltWidth == 16)
1722       IID = Intrinsic::x86_avx512_permvar_hi_512;
1723     else if (VecWidth == 128 && EltWidth == 8)
1724       IID = Intrinsic::x86_avx512_permvar_qi_128;
1725     else if (VecWidth == 256 && EltWidth == 8)
1726       IID = Intrinsic::x86_avx512_permvar_qi_256;
1727     else if (VecWidth == 512 && EltWidth == 8)
1728       IID = Intrinsic::x86_avx512_permvar_qi_512;
1729     else
1730       llvm_unreachable("Unexpected intrinsic");
1731   } else if (Name.startswith("dbpsadbw.")) {
1732     if (VecWidth == 128)
1733       IID = Intrinsic::x86_avx512_dbpsadbw_128;
1734     else if (VecWidth == 256)
1735       IID = Intrinsic::x86_avx512_dbpsadbw_256;
1736     else if (VecWidth == 512)
1737       IID = Intrinsic::x86_avx512_dbpsadbw_512;
1738     else
1739       llvm_unreachable("Unexpected intrinsic");
1740   } else if (Name.startswith("pmultishift.qb.")) {
1741     if (VecWidth == 128)
1742       IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1743     else if (VecWidth == 256)
1744       IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1745     else if (VecWidth == 512)
1746       IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1747     else
1748       llvm_unreachable("Unexpected intrinsic");
1749   } else if (Name.startswith("conflict.")) {
1750     if (Name[9] == 'd' && VecWidth == 128)
1751       IID = Intrinsic::x86_avx512_conflict_d_128;
1752     else if (Name[9] == 'd' && VecWidth == 256)
1753       IID = Intrinsic::x86_avx512_conflict_d_256;
1754     else if (Name[9] == 'd' && VecWidth == 512)
1755       IID = Intrinsic::x86_avx512_conflict_d_512;
1756     else if (Name[9] == 'q' && VecWidth == 128)
1757       IID = Intrinsic::x86_avx512_conflict_q_128;
1758     else if (Name[9] == 'q' && VecWidth == 256)
1759       IID = Intrinsic::x86_avx512_conflict_q_256;
1760     else if (Name[9] == 'q' && VecWidth == 512)
1761       IID = Intrinsic::x86_avx512_conflict_q_512;
1762     else
1763       llvm_unreachable("Unexpected intrinsic");
1764   } else if (Name.startswith("pavg.")) {
1765     if (Name[5] == 'b' && VecWidth == 128)
1766       IID = Intrinsic::x86_sse2_pavg_b;
1767     else if (Name[5] == 'b' && VecWidth == 256)
1768       IID = Intrinsic::x86_avx2_pavg_b;
1769     else if (Name[5] == 'b' && VecWidth == 512)
1770       IID = Intrinsic::x86_avx512_pavg_b_512;
1771     else if (Name[5] == 'w' && VecWidth == 128)
1772       IID = Intrinsic::x86_sse2_pavg_w;
1773     else if (Name[5] == 'w' && VecWidth == 256)
1774       IID = Intrinsic::x86_avx2_pavg_w;
1775     else if (Name[5] == 'w' && VecWidth == 512)
1776       IID = Intrinsic::x86_avx512_pavg_w_512;
1777     else
1778       llvm_unreachable("Unexpected intrinsic");
1779   } else
1780     return false;
1781 
1782   SmallVector<Value *, 4> Args(CI.arg_operands().begin(),
1783                                CI.arg_operands().end());
1784   Args.pop_back();
1785   Args.pop_back();
1786   Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1787                            Args);
1788   unsigned NumArgs = CI.getNumArgOperands();
1789   Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
1790                       CI.getArgOperand(NumArgs - 2));
1791   return true;
1792 }
1793 
1794 /// Upgrade comment in call to inline asm that represents an objc retain release
1795 /// marker.
1796 void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
1797   size_t Pos;
1798   if (AsmStr->find("mov\tfp") == 0 &&
1799       AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
1800       (Pos = AsmStr->find("# marker")) != std::string::npos) {
1801     AsmStr->replace(Pos, 1, ";");
1802   }
1803 }
1804 
1805 /// Upgrade a call to an old intrinsic. All argument and return casting must be
1806 /// provided to seamlessly integrate with existing context.
1807 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
1808   Function *F = CI->getCalledFunction();
1809   LLVMContext &C = CI->getContext();
1810   IRBuilder<> Builder(C);
1811   Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
1812 
1813   assert(F && "Intrinsic call is not direct?");
1814 
1815   if (!NewFn) {
1816     // Get the Function's name.
1817     StringRef Name = F->getName();
1818 
1819     assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
1820     Name = Name.substr(5);
1821 
1822     bool IsX86 = Name.startswith("x86.");
1823     if (IsX86)
1824       Name = Name.substr(4);
1825     bool IsNVVM = Name.startswith("nvvm.");
1826     if (IsNVVM)
1827       Name = Name.substr(5);
1828 
1829     if (IsX86 && Name.startswith("sse4a.movnt.")) {
1830       Module *M = F->getParent();
1831       SmallVector<Metadata *, 1> Elts;
1832       Elts.push_back(
1833           ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
1834       MDNode *Node = MDNode::get(C, Elts);
1835 
1836       Value *Arg0 = CI->getArgOperand(0);
1837       Value *Arg1 = CI->getArgOperand(1);
1838 
1839       // Nontemporal (unaligned) store of the 0'th element of the float/double
1840       // vector.
1841       Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
1842       PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
1843       Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
1844       Value *Extract =
1845           Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
1846 
1847       StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
1848       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1849 
1850       // Remove intrinsic.
1851       CI->eraseFromParent();
1852       return;
1853     }
1854 
1855     if (IsX86 && (Name.startswith("avx.movnt.") ||
1856                   Name.startswith("avx512.storent."))) {
1857       Module *M = F->getParent();
1858       SmallVector<Metadata *, 1> Elts;
1859       Elts.push_back(
1860           ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
1861       MDNode *Node = MDNode::get(C, Elts);
1862 
1863       Value *Arg0 = CI->getArgOperand(0);
1864       Value *Arg1 = CI->getArgOperand(1);
1865 
1866       // Convert the type of the pointer to a pointer to the stored type.
1867       Value *BC = Builder.CreateBitCast(Arg0,
1868                                         PointerType::getUnqual(Arg1->getType()),
1869                                         "cast");
1870       StoreInst *SI = Builder.CreateAlignedStore(
1871           Arg1, BC,
1872           Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
1873       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1874 
1875       // Remove intrinsic.
1876       CI->eraseFromParent();
1877       return;
1878     }
1879 
1880     if (IsX86 && Name == "sse2.storel.dq") {
1881       Value *Arg0 = CI->getArgOperand(0);
1882       Value *Arg1 = CI->getArgOperand(1);
1883 
1884       auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
1885       Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
1886       Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
1887       Value *BC = Builder.CreateBitCast(Arg0,
1888                                         PointerType::getUnqual(Elt->getType()),
1889                                         "cast");
1890       Builder.CreateAlignedStore(Elt, BC, Align(1));
1891 
1892       // Remove intrinsic.
1893       CI->eraseFromParent();
1894       return;
1895     }
1896 
1897     if (IsX86 && (Name.startswith("sse.storeu.") ||
1898                   Name.startswith("sse2.storeu.") ||
1899                   Name.startswith("avx.storeu."))) {
1900       Value *Arg0 = CI->getArgOperand(0);
1901       Value *Arg1 = CI->getArgOperand(1);
1902 
1903       Arg0 = Builder.CreateBitCast(Arg0,
1904                                    PointerType::getUnqual(Arg1->getType()),
1905                                    "cast");
1906       Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
1907 
1908       // Remove intrinsic.
1909       CI->eraseFromParent();
1910       return;
1911     }
1912 
1913     if (IsX86 && Name == "avx512.mask.store.ss") {
1914       Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
1915       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1916                          Mask, false);
1917 
1918       // Remove intrinsic.
1919       CI->eraseFromParent();
1920       return;
1921     }
1922 
1923     if (IsX86 && (Name.startswith("avx512.mask.store"))) {
1924       // "avx512.mask.storeu." or "avx512.mask.store."
1925       bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
1926       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1927                          CI->getArgOperand(2), Aligned);
1928 
1929       // Remove intrinsic.
1930       CI->eraseFromParent();
1931       return;
1932     }
1933 
1934     Value *Rep;
1935     // Upgrade packed integer vector compare intrinsics to compare instructions.
1936     if (IsX86 && (Name.startswith("sse2.pcmp") ||
1937                   Name.startswith("avx2.pcmp"))) {
1938       // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
1939       bool CmpEq = Name[9] == 'e';
1940       Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
1941                                CI->getArgOperand(0), CI->getArgOperand(1));
1942       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
1943     } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
1944       Type *ExtTy = Type::getInt32Ty(C);
1945       if (CI->getOperand(0)->getType()->isIntegerTy(8))
1946         ExtTy = Type::getInt64Ty(C);
1947       unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
1948                          ExtTy->getPrimitiveSizeInBits();
1949       Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
1950       Rep = Builder.CreateVectorSplat(NumElts, Rep);
1951     } else if (IsX86 && (Name == "sse.sqrt.ss" ||
1952                          Name == "sse2.sqrt.sd")) {
1953       Value *Vec = CI->getArgOperand(0);
1954       Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
1955       Function *Intr = Intrinsic::getDeclaration(F->getParent(),
1956                                                  Intrinsic::sqrt, Elt0->getType());
1957       Elt0 = Builder.CreateCall(Intr, Elt0);
1958       Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
1959     } else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
1960                          Name.startswith("sse2.sqrt.p") ||
1961                          Name.startswith("sse.sqrt.p"))) {
1962       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1963                                                          Intrinsic::sqrt,
1964                                                          CI->getType()),
1965                                {CI->getArgOperand(0)});
1966     } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
1967       if (CI->getNumArgOperands() == 4 &&
1968           (!isa<ConstantInt>(CI->getArgOperand(3)) ||
1969            cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
1970         Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
1971                                             : Intrinsic::x86_avx512_sqrt_pd_512;
1972 
1973         Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
1974         Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
1975                                                            IID), Args);
1976       } else {
1977         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1978                                                            Intrinsic::sqrt,
1979                                                            CI->getType()),
1980                                  {CI->getArgOperand(0)});
1981       }
1982       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1983                           CI->getArgOperand(1));
1984     } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
1985                          Name.startswith("avx512.ptestnm"))) {
1986       Value *Op0 = CI->getArgOperand(0);
1987       Value *Op1 = CI->getArgOperand(1);
1988       Value *Mask = CI->getArgOperand(2);
1989       Rep = Builder.CreateAnd(Op0, Op1);
1990       llvm::Type *Ty = Op0->getType();
1991       Value *Zero = llvm::Constant::getNullValue(Ty);
1992       ICmpInst::Predicate Pred =
1993         Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
1994       Rep = Builder.CreateICmp(Pred, Rep, Zero);
1995       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
1996     } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
1997       unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
1998                              ->getNumElements();
1999       Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
2000       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2001                           CI->getArgOperand(1));
2002     } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
2003       unsigned NumElts = CI->getType()->getScalarSizeInBits();
2004       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
2005       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
2006       int Indices[64];
2007       for (unsigned i = 0; i != NumElts; ++i)
2008         Indices[i] = i;
2009 
2010       // First extract half of each vector. This gives better codegen than
2011       // doing it in a single shuffle.
2012       LHS = Builder.CreateShuffleVector(LHS, LHS,
2013                                         makeArrayRef(Indices, NumElts / 2));
2014       RHS = Builder.CreateShuffleVector(RHS, RHS,
2015                                         makeArrayRef(Indices, NumElts / 2));
2016       // Concat the vectors.
2017       // NOTE: Operands have to be swapped to match intrinsic definition.
2018       Rep = Builder.CreateShuffleVector(RHS, LHS,
2019                                         makeArrayRef(Indices, NumElts));
2020       Rep = Builder.CreateBitCast(Rep, CI->getType());
2021     } else if (IsX86 && Name == "avx512.kand.w") {
2022       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2023       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2024       Rep = Builder.CreateAnd(LHS, RHS);
2025       Rep = Builder.CreateBitCast(Rep, CI->getType());
2026     } else if (IsX86 && Name == "avx512.kandn.w") {
2027       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2028       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2029       LHS = Builder.CreateNot(LHS);
2030       Rep = Builder.CreateAnd(LHS, RHS);
2031       Rep = Builder.CreateBitCast(Rep, CI->getType());
2032     } else if (IsX86 && Name == "avx512.kor.w") {
2033       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2034       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2035       Rep = Builder.CreateOr(LHS, RHS);
2036       Rep = Builder.CreateBitCast(Rep, CI->getType());
2037     } else if (IsX86 && Name == "avx512.kxor.w") {
2038       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2039       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2040       Rep = Builder.CreateXor(LHS, RHS);
2041       Rep = Builder.CreateBitCast(Rep, CI->getType());
2042     } else if (IsX86 && Name == "avx512.kxnor.w") {
2043       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2044       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2045       LHS = Builder.CreateNot(LHS);
2046       Rep = Builder.CreateXor(LHS, RHS);
2047       Rep = Builder.CreateBitCast(Rep, CI->getType());
2048     } else if (IsX86 && Name == "avx512.knot.w") {
2049       Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2050       Rep = Builder.CreateNot(Rep);
2051       Rep = Builder.CreateBitCast(Rep, CI->getType());
2052     } else if (IsX86 &&
2053                (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {
2054       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2055       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2056       Rep = Builder.CreateOr(LHS, RHS);
2057       Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
2058       Value *C;
2059       if (Name[14] == 'c')
2060         C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
2061       else
2062         C = ConstantInt::getNullValue(Builder.getInt16Ty());
2063       Rep = Builder.CreateICmpEQ(Rep, C);
2064       Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
2065     } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
2066                          Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
2067                          Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
2068                          Name == "sse.div.ss" || Name == "sse2.div.sd")) {
2069       Type *I32Ty = Type::getInt32Ty(C);
2070       Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
2071                                                  ConstantInt::get(I32Ty, 0));
2072       Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
2073                                                  ConstantInt::get(I32Ty, 0));
2074       Value *EltOp;
2075       if (Name.contains(".add."))
2076         EltOp = Builder.CreateFAdd(Elt0, Elt1);
2077       else if (Name.contains(".sub."))
2078         EltOp = Builder.CreateFSub(Elt0, Elt1);
2079       else if (Name.contains(".mul."))
2080         EltOp = Builder.CreateFMul(Elt0, Elt1);
2081       else
2082         EltOp = Builder.CreateFDiv(Elt0, Elt1);
2083       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
2084                                         ConstantInt::get(I32Ty, 0));
2085     } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
2086       // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
2087       bool CmpEq = Name[16] == 'e';
2088       Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
2089     } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
2090       Type *OpTy = CI->getArgOperand(0)->getType();
2091       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2092       Intrinsic::ID IID;
2093       switch (VecWidth) {
2094       default: llvm_unreachable("Unexpected intrinsic");
2095       case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
2096       case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
2097       case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
2098       }
2099 
2100       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2101                                { CI->getOperand(0), CI->getArgOperand(1) });
2102       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2103     } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
2104       Type *OpTy = CI->getArgOperand(0)->getType();
2105       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2106       unsigned EltWidth = OpTy->getScalarSizeInBits();
2107       Intrinsic::ID IID;
2108       if (VecWidth == 128 && EltWidth == 32)
2109         IID = Intrinsic::x86_avx512_fpclass_ps_128;
2110       else if (VecWidth == 256 && EltWidth == 32)
2111         IID = Intrinsic::x86_avx512_fpclass_ps_256;
2112       else if (VecWidth == 512 && EltWidth == 32)
2113         IID = Intrinsic::x86_avx512_fpclass_ps_512;
2114       else if (VecWidth == 128 && EltWidth == 64)
2115         IID = Intrinsic::x86_avx512_fpclass_pd_128;
2116       else if (VecWidth == 256 && EltWidth == 64)
2117         IID = Intrinsic::x86_avx512_fpclass_pd_256;
2118       else if (VecWidth == 512 && EltWidth == 64)
2119         IID = Intrinsic::x86_avx512_fpclass_pd_512;
2120       else
2121         llvm_unreachable("Unexpected intrinsic");
2122 
2123       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2124                                { CI->getOperand(0), CI->getArgOperand(1) });
2125       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2126     } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
2127       SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2128                                    CI->arg_operands().end());
2129       Type *OpTy = Args[0]->getType();
2130       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2131       unsigned EltWidth = OpTy->getScalarSizeInBits();
2132       Intrinsic::ID IID;
2133       if (VecWidth == 128 && EltWidth == 32)
2134         IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2135       else if (VecWidth == 256 && EltWidth == 32)
2136         IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2137       else if (VecWidth == 512 && EltWidth == 32)
2138         IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2139       else if (VecWidth == 128 && EltWidth == 64)
2140         IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2141       else if (VecWidth == 256 && EltWidth == 64)
2142         IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2143       else if (VecWidth == 512 && EltWidth == 64)
2144         IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2145       else
2146         llvm_unreachable("Unexpected intrinsic");
2147 
2148       Value *Mask = Constant::getAllOnesValue(CI->getType());
2149       if (VecWidth == 512)
2150         std::swap(Mask, Args.back());
2151       Args.push_back(Mask);
2152 
2153       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2154                                Args);
2155     } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
2156       // Integer compare intrinsics.
2157       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2158       Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2159     } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
2160       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2161       Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2162     } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
2163                          Name.startswith("avx512.cvtw2mask.") ||
2164                          Name.startswith("avx512.cvtd2mask.") ||
2165                          Name.startswith("avx512.cvtq2mask."))) {
2166       Value *Op = CI->getArgOperand(0);
2167       Value *Zero = llvm::Constant::getNullValue(Op->getType());
2168       Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2169       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
2170     } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
2171                         Name == "ssse3.pabs.w.128" ||
2172                         Name == "ssse3.pabs.d.128" ||
2173                         Name.startswith("avx2.pabs") ||
2174                         Name.startswith("avx512.mask.pabs"))) {
2175       Rep = upgradeAbs(Builder, *CI);
2176     } else if (IsX86 && (Name == "sse41.pmaxsb" ||
2177                          Name == "sse2.pmaxs.w" ||
2178                          Name == "sse41.pmaxsd" ||
2179                          Name.startswith("avx2.pmaxs") ||
2180                          Name.startswith("avx512.mask.pmaxs"))) {
2181       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
2182     } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
2183                          Name == "sse41.pmaxuw" ||
2184                          Name == "sse41.pmaxud" ||
2185                          Name.startswith("avx2.pmaxu") ||
2186                          Name.startswith("avx512.mask.pmaxu"))) {
2187       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
2188     } else if (IsX86 && (Name == "sse41.pminsb" ||
2189                          Name == "sse2.pmins.w" ||
2190                          Name == "sse41.pminsd" ||
2191                          Name.startswith("avx2.pmins") ||
2192                          Name.startswith("avx512.mask.pmins"))) {
2193       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
2194     } else if (IsX86 && (Name == "sse2.pminu.b" ||
2195                          Name == "sse41.pminuw" ||
2196                          Name == "sse41.pminud" ||
2197                          Name.startswith("avx2.pminu") ||
2198                          Name.startswith("avx512.mask.pminu"))) {
2199       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
2200     } else if (IsX86 && (Name == "sse2.pmulu.dq" ||
2201                          Name == "avx2.pmulu.dq" ||
2202                          Name == "avx512.pmulu.dq.512" ||
2203                          Name.startswith("avx512.mask.pmulu.dq."))) {
2204       Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2205     } else if (IsX86 && (Name == "sse41.pmuldq" ||
2206                          Name == "avx2.pmul.dq" ||
2207                          Name == "avx512.pmul.dq.512" ||
2208                          Name.startswith("avx512.mask.pmul.dq."))) {
2209       Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2210     } else if (IsX86 && (Name == "sse.cvtsi2ss" ||
2211                          Name == "sse2.cvtsi2sd" ||
2212                          Name == "sse.cvtsi642ss" ||
2213                          Name == "sse2.cvtsi642sd")) {
2214       Rep = Builder.CreateSIToFP(
2215           CI->getArgOperand(1),
2216           cast<VectorType>(CI->getType())->getElementType());
2217       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2218     } else if (IsX86 && Name == "avx512.cvtusi2sd") {
2219       Rep = Builder.CreateUIToFP(
2220           CI->getArgOperand(1),
2221           cast<VectorType>(CI->getType())->getElementType());
2222       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2223     } else if (IsX86 && Name == "sse2.cvtss2sd") {
2224       Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2225       Rep = Builder.CreateFPExt(
2226           Rep, cast<VectorType>(CI->getType())->getElementType());
2227       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2228     } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
2229                          Name == "sse2.cvtdq2ps" ||
2230                          Name == "avx.cvtdq2.pd.256" ||
2231                          Name == "avx.cvtdq2.ps.256" ||
2232                          Name.startswith("avx512.mask.cvtdq2pd.") ||
2233                          Name.startswith("avx512.mask.cvtudq2pd.") ||
2234                          Name.startswith("avx512.mask.cvtdq2ps.") ||
2235                          Name.startswith("avx512.mask.cvtudq2ps.") ||
2236                          Name.startswith("avx512.mask.cvtqq2pd.") ||
2237                          Name.startswith("avx512.mask.cvtuqq2pd.") ||
2238                          Name == "avx512.mask.cvtqq2ps.256" ||
2239                          Name == "avx512.mask.cvtqq2ps.512" ||
2240                          Name == "avx512.mask.cvtuqq2ps.256" ||
2241                          Name == "avx512.mask.cvtuqq2ps.512" ||
2242                          Name == "sse2.cvtps2pd" ||
2243                          Name == "avx.cvt.ps2.pd.256" ||
2244                          Name == "avx512.mask.cvtps2pd.128" ||
2245                          Name == "avx512.mask.cvtps2pd.256")) {
2246       auto *DstTy = cast<FixedVectorType>(CI->getType());
2247       Rep = CI->getArgOperand(0);
2248       auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2249 
2250       unsigned NumDstElts = DstTy->getNumElements();
2251       if (NumDstElts < SrcTy->getNumElements()) {
2252         assert(NumDstElts == 2 && "Unexpected vector size");
2253         Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
2254       }
2255 
2256       bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2257       bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
2258       if (IsPS2PD)
2259         Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2260       else if (CI->getNumArgOperands() == 4 &&
2261                (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2262                 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2263         Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2264                                        : Intrinsic::x86_avx512_sitofp_round;
2265         Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2266                                                 { DstTy, SrcTy });
2267         Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2268       } else {
2269         Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2270                          : Builder.CreateSIToFP(Rep, DstTy, "cvt");
2271       }
2272 
2273       if (CI->getNumArgOperands() >= 3)
2274         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2275                             CI->getArgOperand(1));
2276     } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
2277                          Name.startswith("vcvtph2ps."))) {
2278       auto *DstTy = cast<FixedVectorType>(CI->getType());
2279       Rep = CI->getArgOperand(0);
2280       auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2281       unsigned NumDstElts = DstTy->getNumElements();
2282       if (NumDstElts != SrcTy->getNumElements()) {
2283         assert(NumDstElts == 4 && "Unexpected vector size");
2284         Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
2285       }
2286       Rep = Builder.CreateBitCast(
2287           Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
2288       Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2289       if (CI->getNumArgOperands() >= 3)
2290         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2291                             CI->getArgOperand(1));
2292     } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
2293       Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2294                               CI->getArgOperand(1), CI->getArgOperand(2),
2295                               /*Aligned*/false);
2296     } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
2297       Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2298                               CI->getArgOperand(1),CI->getArgOperand(2),
2299                               /*Aligned*/true);
2300     } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
2301       auto *ResultTy = cast<FixedVectorType>(CI->getType());
2302       Type *PtrTy = ResultTy->getElementType();
2303 
2304       // Cast the pointer to element type.
2305       Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2306                                          llvm::PointerType::getUnqual(PtrTy));
2307 
2308       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2309                                      ResultTy->getNumElements());
2310 
2311       Function *ELd = Intrinsic::getDeclaration(F->getParent(),
2312                                                 Intrinsic::masked_expandload,
2313                                                 ResultTy);
2314       Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2315     } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
2316       auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
2317       Type *PtrTy = ResultTy->getElementType();
2318 
2319       // Cast the pointer to element type.
2320       Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2321                                          llvm::PointerType::getUnqual(PtrTy));
2322 
2323       Value *MaskVec =
2324           getX86MaskVec(Builder, CI->getArgOperand(2),
2325                         cast<FixedVectorType>(ResultTy)->getNumElements());
2326 
2327       Function *CSt = Intrinsic::getDeclaration(F->getParent(),
2328                                                 Intrinsic::masked_compressstore,
2329                                                 ResultTy);
2330       Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2331     } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
2332                          Name.startswith("avx512.mask.expand."))) {
2333       auto *ResultTy = cast<FixedVectorType>(CI->getType());
2334 
2335       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2336                                      ResultTy->getNumElements());
2337 
2338       bool IsCompress = Name[12] == 'c';
2339       Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2340                                      : Intrinsic::x86_avx512_mask_expand;
2341       Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
2342       Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2343                                        MaskVec });
2344     } else if (IsX86 && Name.startswith("xop.vpcom")) {
2345       bool IsSigned;
2346       if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
2347           Name.endswith("uq"))
2348         IsSigned = false;
2349       else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
2350                Name.endswith("q"))
2351         IsSigned = true;
2352       else
2353         llvm_unreachable("Unknown suffix");
2354 
2355       unsigned Imm;
2356       if (CI->getNumArgOperands() == 3) {
2357         Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2358       } else {
2359         Name = Name.substr(9); // strip off "xop.vpcom"
2360         if (Name.startswith("lt"))
2361           Imm = 0;
2362         else if (Name.startswith("le"))
2363           Imm = 1;
2364         else if (Name.startswith("gt"))
2365           Imm = 2;
2366         else if (Name.startswith("ge"))
2367           Imm = 3;
2368         else if (Name.startswith("eq"))
2369           Imm = 4;
2370         else if (Name.startswith("ne"))
2371           Imm = 5;
2372         else if (Name.startswith("false"))
2373           Imm = 6;
2374         else if (Name.startswith("true"))
2375           Imm = 7;
2376         else
2377           llvm_unreachable("Unknown condition");
2378       }
2379 
2380       Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2381     } else if (IsX86 && Name.startswith("xop.vpcmov")) {
2382       Value *Sel = CI->getArgOperand(2);
2383       Value *NotSel = Builder.CreateNot(Sel);
2384       Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2385       Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2386       Rep = Builder.CreateOr(Sel0, Sel1);
2387     } else if (IsX86 && (Name.startswith("xop.vprot") ||
2388                          Name.startswith("avx512.prol") ||
2389                          Name.startswith("avx512.mask.prol"))) {
2390       Rep = upgradeX86Rotate(Builder, *CI, false);
2391     } else if (IsX86 && (Name.startswith("avx512.pror") ||
2392                          Name.startswith("avx512.mask.pror"))) {
2393       Rep = upgradeX86Rotate(Builder, *CI, true);
2394     } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
2395                          Name.startswith("avx512.mask.vpshld") ||
2396                          Name.startswith("avx512.maskz.vpshld"))) {
2397       bool ZeroMask = Name[11] == 'z';
2398       Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2399     } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
2400                          Name.startswith("avx512.mask.vpshrd") ||
2401                          Name.startswith("avx512.maskz.vpshrd"))) {
2402       bool ZeroMask = Name[11] == 'z';
2403       Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2404     } else if (IsX86 && Name == "sse42.crc32.64.8") {
2405       Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
2406                                                Intrinsic::x86_sse42_crc32_32_8);
2407       Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2408       Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2409       Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2410     } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
2411                          Name.startswith("avx512.vbroadcast.s"))) {
2412       // Replace broadcasts with a series of insertelements.
2413       auto *VecTy = cast<FixedVectorType>(CI->getType());
2414       Type *EltTy = VecTy->getElementType();
2415       unsigned EltNum = VecTy->getNumElements();
2416       Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2417                                           EltTy->getPointerTo());
2418       Value *Load = Builder.CreateLoad(EltTy, Cast);
2419       Type *I32Ty = Type::getInt32Ty(C);
2420       Rep = UndefValue::get(VecTy);
2421       for (unsigned I = 0; I < EltNum; ++I)
2422         Rep = Builder.CreateInsertElement(Rep, Load,
2423                                           ConstantInt::get(I32Ty, I));
2424     } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
2425                          Name.startswith("sse41.pmovzx") ||
2426                          Name.startswith("avx2.pmovsx") ||
2427                          Name.startswith("avx2.pmovzx") ||
2428                          Name.startswith("avx512.mask.pmovsx") ||
2429                          Name.startswith("avx512.mask.pmovzx"))) {
2430       auto *DstTy = cast<FixedVectorType>(CI->getType());
2431       unsigned NumDstElts = DstTy->getNumElements();
2432 
2433       // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
2434       SmallVector<int, 8> ShuffleMask(NumDstElts);
2435       for (unsigned i = 0; i != NumDstElts; ++i)
2436         ShuffleMask[i] = i;
2437 
2438       Value *SV =
2439           Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
2440 
2441       bool DoSext = (StringRef::npos != Name.find("pmovsx"));
2442       Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
2443                    : Builder.CreateZExt(SV, DstTy);
2444       // If there are 3 arguments, it's a masked intrinsic so we need a select.
2445       if (CI->getNumArgOperands() == 3)
2446         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2447                             CI->getArgOperand(1));
2448     } else if (Name == "avx512.mask.pmov.qd.256" ||
2449                Name == "avx512.mask.pmov.qd.512" ||
2450                Name == "avx512.mask.pmov.wb.256" ||
2451                Name == "avx512.mask.pmov.wb.512") {
2452       Type *Ty = CI->getArgOperand(1)->getType();
2453       Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2454       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2455                           CI->getArgOperand(1));
2456     } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
2457                          Name == "avx2.vbroadcasti128")) {
2458       // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
2459       Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
2460       unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
2461       auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
2462       Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2463                                             PointerType::getUnqual(VT));
2464       Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
2465       if (NumSrcElts == 2)
2466         Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
2467       else
2468         Rep = Builder.CreateShuffleVector(
2469             Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
2470     } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
2471                          Name.startswith("avx512.mask.shuf.f"))) {
2472       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2473       Type *VT = CI->getType();
2474       unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
2475       unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
2476       unsigned ControlBitsMask = NumLanes - 1;
2477       unsigned NumControlBits = NumLanes / 2;
2478       SmallVector<int, 8> ShuffleMask(0);
2479 
2480       for (unsigned l = 0; l != NumLanes; ++l) {
2481         unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2482         // We actually need the other source.
2483         if (l >= NumLanes / 2)
2484           LaneMask += NumLanes;
2485         for (unsigned i = 0; i != NumElementsInLane; ++i)
2486           ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
2487       }
2488       Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2489                                         CI->getArgOperand(1), ShuffleMask);
2490       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2491                           CI->getArgOperand(3));
2492     }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
2493                          Name.startswith("avx512.mask.broadcasti"))) {
2494       unsigned NumSrcElts =
2495           cast<FixedVectorType>(CI->getArgOperand(0)->getType())
2496               ->getNumElements();
2497       unsigned NumDstElts =
2498           cast<FixedVectorType>(CI->getType())->getNumElements();
2499 
2500       SmallVector<int, 8> ShuffleMask(NumDstElts);
2501       for (unsigned i = 0; i != NumDstElts; ++i)
2502         ShuffleMask[i] = i % NumSrcElts;
2503 
2504       Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2505                                         CI->getArgOperand(0),
2506                                         ShuffleMask);
2507       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2508                           CI->getArgOperand(1));
2509     } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
2510                          Name.startswith("avx2.vbroadcast") ||
2511                          Name.startswith("avx512.pbroadcast") ||
2512                          Name.startswith("avx512.mask.broadcast.s"))) {
2513       // Replace vp?broadcasts with a vector shuffle.
2514       Value *Op = CI->getArgOperand(0);
2515       ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
2516       Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
2517       SmallVector<int, 8> M;
2518       ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
2519       Rep = Builder.CreateShuffleVector(Op, M);
2520 
2521       if (CI->getNumArgOperands() == 3)
2522         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2523                             CI->getArgOperand(1));
2524     } else if (IsX86 && (Name.startswith("sse2.padds.") ||
2525                          Name.startswith("avx2.padds.") ||
2526                          Name.startswith("avx512.padds.") ||
2527                          Name.startswith("avx512.mask.padds."))) {
2528       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
2529     } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
2530                          Name.startswith("avx2.psubs.") ||
2531                          Name.startswith("avx512.psubs.") ||
2532                          Name.startswith("avx512.mask.psubs."))) {
2533       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
2534     } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
2535                          Name.startswith("avx2.paddus.") ||
2536                          Name.startswith("avx512.mask.paddus."))) {
2537       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
2538     } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
2539                          Name.startswith("avx2.psubus.") ||
2540                          Name.startswith("avx512.mask.psubus."))) {
2541       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
2542     } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
2543       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2544                                       CI->getArgOperand(1),
2545                                       CI->getArgOperand(2),
2546                                       CI->getArgOperand(3),
2547                                       CI->getArgOperand(4),
2548                                       false);
2549     } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
2550       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2551                                       CI->getArgOperand(1),
2552                                       CI->getArgOperand(2),
2553                                       CI->getArgOperand(3),
2554                                       CI->getArgOperand(4),
2555                                       true);
2556     } else if (IsX86 && (Name == "sse2.psll.dq" ||
2557                          Name == "avx2.psll.dq")) {
2558       // 128/256-bit shift left specified in bits.
2559       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2560       Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2561                                        Shift / 8); // Shift is in bits.
2562     } else if (IsX86 && (Name == "sse2.psrl.dq" ||
2563                          Name == "avx2.psrl.dq")) {
2564       // 128/256-bit shift right specified in bits.
2565       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2566       Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2567                                        Shift / 8); // Shift is in bits.
2568     } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
2569                          Name == "avx2.psll.dq.bs" ||
2570                          Name == "avx512.psll.dq.512")) {
2571       // 128/256/512-bit shift left specified in bytes.
2572       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2573       Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2574     } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
2575                          Name == "avx2.psrl.dq.bs" ||
2576                          Name == "avx512.psrl.dq.512")) {
2577       // 128/256/512-bit shift right specified in bytes.
2578       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2579       Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2580     } else if (IsX86 && (Name == "sse41.pblendw" ||
2581                          Name.startswith("sse41.blendp") ||
2582                          Name.startswith("avx.blend.p") ||
2583                          Name == "avx2.pblendw" ||
2584                          Name.startswith("avx2.pblendd."))) {
2585       Value *Op0 = CI->getArgOperand(0);
2586       Value *Op1 = CI->getArgOperand(1);
2587       unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2588       auto *VecTy = cast<FixedVectorType>(CI->getType());
2589       unsigned NumElts = VecTy->getNumElements();
2590 
2591       SmallVector<int, 16> Idxs(NumElts);
2592       for (unsigned i = 0; i != NumElts; ++i)
2593         Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
2594 
2595       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2596     } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
2597                          Name == "avx2.vinserti128" ||
2598                          Name.startswith("avx512.mask.insert"))) {
2599       Value *Op0 = CI->getArgOperand(0);
2600       Value *Op1 = CI->getArgOperand(1);
2601       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2602       unsigned DstNumElts =
2603           cast<FixedVectorType>(CI->getType())->getNumElements();
2604       unsigned SrcNumElts =
2605           cast<FixedVectorType>(Op1->getType())->getNumElements();
2606       unsigned Scale = DstNumElts / SrcNumElts;
2607 
2608       // Mask off the high bits of the immediate value; hardware ignores those.
2609       Imm = Imm % Scale;
2610 
2611       // Extend the second operand into a vector the size of the destination.
2612       SmallVector<int, 8> Idxs(DstNumElts);
2613       for (unsigned i = 0; i != SrcNumElts; ++i)
2614         Idxs[i] = i;
2615       for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
2616         Idxs[i] = SrcNumElts;
2617       Rep = Builder.CreateShuffleVector(Op1, Idxs);
2618 
2619       // Insert the second operand into the first operand.
2620 
2621       // Note that there is no guarantee that instruction lowering will actually
2622       // produce a vinsertf128 instruction for the created shuffles. In
2623       // particular, the 0 immediate case involves no lane changes, so it can
2624       // be handled as a blend.
2625 
2626       // Example of shuffle mask for 32-bit elements:
2627       // Imm = 1  <i32 0, i32 1, i32 2,  i32 3,  i32 8, i32 9, i32 10, i32 11>
2628       // Imm = 0  <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6,  i32 7 >
2629 
2630       // First fill with identify mask.
2631       for (unsigned i = 0; i != DstNumElts; ++i)
2632         Idxs[i] = i;
2633       // Then replace the elements where we need to insert.
2634       for (unsigned i = 0; i != SrcNumElts; ++i)
2635         Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
2636       Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
2637 
2638       // If the intrinsic has a mask operand, handle that.
2639       if (CI->getNumArgOperands() == 5)
2640         Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2641                             CI->getArgOperand(3));
2642     } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
2643                          Name == "avx2.vextracti128" ||
2644                          Name.startswith("avx512.mask.vextract"))) {
2645       Value *Op0 = CI->getArgOperand(0);
2646       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2647       unsigned DstNumElts =
2648           cast<FixedVectorType>(CI->getType())->getNumElements();
2649       unsigned SrcNumElts =
2650           cast<FixedVectorType>(Op0->getType())->getNumElements();
2651       unsigned Scale = SrcNumElts / DstNumElts;
2652 
2653       // Mask off the high bits of the immediate value; hardware ignores those.
2654       Imm = Imm % Scale;
2655 
2656       // Get indexes for the subvector of the input vector.
2657       SmallVector<int, 8> Idxs(DstNumElts);
2658       for (unsigned i = 0; i != DstNumElts; ++i) {
2659         Idxs[i] = i + (Imm * DstNumElts);
2660       }
2661       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2662 
2663       // If the intrinsic has a mask operand, handle that.
2664       if (CI->getNumArgOperands() == 4)
2665         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2666                             CI->getArgOperand(2));
2667     } else if (!IsX86 && Name == "stackprotectorcheck") {
2668       Rep = nullptr;
2669     } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
2670                          Name.startswith("avx512.mask.perm.di."))) {
2671       Value *Op0 = CI->getArgOperand(0);
2672       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2673       auto *VecTy = cast<FixedVectorType>(CI->getType());
2674       unsigned NumElts = VecTy->getNumElements();
2675 
2676       SmallVector<int, 8> Idxs(NumElts);
2677       for (unsigned i = 0; i != NumElts; ++i)
2678         Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
2679 
2680       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2681 
2682       if (CI->getNumArgOperands() == 4)
2683         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2684                             CI->getArgOperand(2));
2685     } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
2686                          Name == "avx2.vperm2i128")) {
2687       // The immediate permute control byte looks like this:
2688       //    [1:0] - select 128 bits from sources for low half of destination
2689       //    [2]   - ignore
2690       //    [3]   - zero low half of destination
2691       //    [5:4] - select 128 bits from sources for high half of destination
2692       //    [6]   - ignore
2693       //    [7]   - zero high half of destination
2694 
2695       uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2696 
2697       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2698       unsigned HalfSize = NumElts / 2;
2699       SmallVector<int, 8> ShuffleMask(NumElts);
2700 
2701       // Determine which operand(s) are actually in use for this instruction.
2702       Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2703       Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2704 
2705       // If needed, replace operands based on zero mask.
2706       V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
2707       V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
2708 
2709       // Permute low half of result.
2710       unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
2711       for (unsigned i = 0; i < HalfSize; ++i)
2712         ShuffleMask[i] = StartIndex + i;
2713 
2714       // Permute high half of result.
2715       StartIndex = (Imm & 0x10) ? HalfSize : 0;
2716       for (unsigned i = 0; i < HalfSize; ++i)
2717         ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
2718 
2719       Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2720 
2721     } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
2722                          Name == "sse2.pshuf.d" ||
2723                          Name.startswith("avx512.mask.vpermil.p") ||
2724                          Name.startswith("avx512.mask.pshuf.d."))) {
2725       Value *Op0 = CI->getArgOperand(0);
2726       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2727       auto *VecTy = cast<FixedVectorType>(CI->getType());
2728       unsigned NumElts = VecTy->getNumElements();
2729       // Calculate the size of each index in the immediate.
2730       unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
2731       unsigned IdxMask = ((1 << IdxSize) - 1);
2732 
2733       SmallVector<int, 8> Idxs(NumElts);
2734       // Lookup the bits for this element, wrapping around the immediate every
2735       // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
2736       // to offset by the first index of each group.
2737       for (unsigned i = 0; i != NumElts; ++i)
2738         Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
2739 
2740       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2741 
2742       if (CI->getNumArgOperands() == 4)
2743         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2744                             CI->getArgOperand(2));
2745     } else if (IsX86 && (Name == "sse2.pshufl.w" ||
2746                          Name.startswith("avx512.mask.pshufl.w."))) {
2747       Value *Op0 = CI->getArgOperand(0);
2748       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2749       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2750 
2751       SmallVector<int, 16> Idxs(NumElts);
2752       for (unsigned l = 0; l != NumElts; l += 8) {
2753         for (unsigned i = 0; i != 4; ++i)
2754           Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
2755         for (unsigned i = 4; i != 8; ++i)
2756           Idxs[i + l] = i + l;
2757       }
2758 
2759       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2760 
2761       if (CI->getNumArgOperands() == 4)
2762         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2763                             CI->getArgOperand(2));
2764     } else if (IsX86 && (Name == "sse2.pshufh.w" ||
2765                          Name.startswith("avx512.mask.pshufh.w."))) {
2766       Value *Op0 = CI->getArgOperand(0);
2767       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2768       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2769 
2770       SmallVector<int, 16> Idxs(NumElts);
2771       for (unsigned l = 0; l != NumElts; l += 8) {
2772         for (unsigned i = 0; i != 4; ++i)
2773           Idxs[i + l] = i + l;
2774         for (unsigned i = 0; i != 4; ++i)
2775           Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
2776       }
2777 
2778       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2779 
2780       if (CI->getNumArgOperands() == 4)
2781         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2782                             CI->getArgOperand(2));
2783     } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
2784       Value *Op0 = CI->getArgOperand(0);
2785       Value *Op1 = CI->getArgOperand(1);
2786       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2787       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2788 
2789       unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2790       unsigned HalfLaneElts = NumLaneElts / 2;
2791 
2792       SmallVector<int, 16> Idxs(NumElts);
2793       for (unsigned i = 0; i != NumElts; ++i) {
2794         // Base index is the starting element of the lane.
2795         Idxs[i] = i - (i % NumLaneElts);
2796         // If we are half way through the lane switch to the other source.
2797         if ((i % NumLaneElts) >= HalfLaneElts)
2798           Idxs[i] += NumElts;
2799         // Now select the specific element. By adding HalfLaneElts bits from
2800         // the immediate. Wrapping around the immediate every 8-bits.
2801         Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
2802       }
2803 
2804       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2805 
2806       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2807                           CI->getArgOperand(3));
2808     } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
2809                          Name.startswith("avx512.mask.movshdup") ||
2810                          Name.startswith("avx512.mask.movsldup"))) {
2811       Value *Op0 = CI->getArgOperand(0);
2812       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2813       unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2814 
2815       unsigned Offset = 0;
2816       if (Name.startswith("avx512.mask.movshdup."))
2817         Offset = 1;
2818 
2819       SmallVector<int, 16> Idxs(NumElts);
2820       for (unsigned l = 0; l != NumElts; l += NumLaneElts)
2821         for (unsigned i = 0; i != NumLaneElts; i += 2) {
2822           Idxs[i + l + 0] = i + l + Offset;
2823           Idxs[i + l + 1] = i + l + Offset;
2824         }
2825 
2826       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2827 
2828       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2829                           CI->getArgOperand(1));
2830     } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
2831                          Name.startswith("avx512.mask.unpckl."))) {
2832       Value *Op0 = CI->getArgOperand(0);
2833       Value *Op1 = CI->getArgOperand(1);
2834       int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2835       int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2836 
2837       SmallVector<int, 64> Idxs(NumElts);
2838       for (int l = 0; l != NumElts; l += NumLaneElts)
2839         for (int i = 0; i != NumLaneElts; ++i)
2840           Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
2841 
2842       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2843 
2844       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2845                           CI->getArgOperand(2));
2846     } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
2847                          Name.startswith("avx512.mask.unpckh."))) {
2848       Value *Op0 = CI->getArgOperand(0);
2849       Value *Op1 = CI->getArgOperand(1);
2850       int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2851       int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2852 
2853       SmallVector<int, 64> Idxs(NumElts);
2854       for (int l = 0; l != NumElts; l += NumLaneElts)
2855         for (int i = 0; i != NumLaneElts; ++i)
2856           Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
2857 
2858       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2859 
2860       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2861                           CI->getArgOperand(2));
2862     } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
2863                          Name.startswith("avx512.mask.pand."))) {
2864       VectorType *FTy = cast<VectorType>(CI->getType());
2865       VectorType *ITy = VectorType::getInteger(FTy);
2866       Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2867                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2868       Rep = Builder.CreateBitCast(Rep, FTy);
2869       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2870                           CI->getArgOperand(2));
2871     } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
2872                          Name.startswith("avx512.mask.pandn."))) {
2873       VectorType *FTy = cast<VectorType>(CI->getType());
2874       VectorType *ITy = VectorType::getInteger(FTy);
2875       Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
2876       Rep = Builder.CreateAnd(Rep,
2877                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2878       Rep = Builder.CreateBitCast(Rep, FTy);
2879       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2880                           CI->getArgOperand(2));
2881     } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
2882                          Name.startswith("avx512.mask.por."))) {
2883       VectorType *FTy = cast<VectorType>(CI->getType());
2884       VectorType *ITy = VectorType::getInteger(FTy);
2885       Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2886                              Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2887       Rep = Builder.CreateBitCast(Rep, FTy);
2888       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2889                           CI->getArgOperand(2));
2890     } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
2891                          Name.startswith("avx512.mask.pxor."))) {
2892       VectorType *FTy = cast<VectorType>(CI->getType());
2893       VectorType *ITy = VectorType::getInteger(FTy);
2894       Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2895                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2896       Rep = Builder.CreateBitCast(Rep, FTy);
2897       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2898                           CI->getArgOperand(2));
2899     } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
2900       Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2901       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2902                           CI->getArgOperand(2));
2903     } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
2904       Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
2905       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2906                           CI->getArgOperand(2));
2907     } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
2908       Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
2909       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2910                           CI->getArgOperand(2));
2911     } else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
2912       if (Name.endswith(".512")) {
2913         Intrinsic::ID IID;
2914         if (Name[17] == 's')
2915           IID = Intrinsic::x86_avx512_add_ps_512;
2916         else
2917           IID = Intrinsic::x86_avx512_add_pd_512;
2918 
2919         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2920                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2921                                    CI->getArgOperand(4) });
2922       } else {
2923         Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2924       }
2925       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2926                           CI->getArgOperand(2));
2927     } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
2928       if (Name.endswith(".512")) {
2929         Intrinsic::ID IID;
2930         if (Name[17] == 's')
2931           IID = Intrinsic::x86_avx512_div_ps_512;
2932         else
2933           IID = Intrinsic::x86_avx512_div_pd_512;
2934 
2935         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2936                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2937                                    CI->getArgOperand(4) });
2938       } else {
2939         Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
2940       }
2941       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2942                           CI->getArgOperand(2));
2943     } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
2944       if (Name.endswith(".512")) {
2945         Intrinsic::ID IID;
2946         if (Name[17] == 's')
2947           IID = Intrinsic::x86_avx512_mul_ps_512;
2948         else
2949           IID = Intrinsic::x86_avx512_mul_pd_512;
2950 
2951         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2952                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2953                                    CI->getArgOperand(4) });
2954       } else {
2955         Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
2956       }
2957       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2958                           CI->getArgOperand(2));
2959     } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
2960       if (Name.endswith(".512")) {
2961         Intrinsic::ID IID;
2962         if (Name[17] == 's')
2963           IID = Intrinsic::x86_avx512_sub_ps_512;
2964         else
2965           IID = Intrinsic::x86_avx512_sub_pd_512;
2966 
2967         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2968                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2969                                    CI->getArgOperand(4) });
2970       } else {
2971         Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
2972       }
2973       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2974                           CI->getArgOperand(2));
2975     } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
2976                          Name.startswith("avx512.mask.min.p")) &&
2977                Name.drop_front(18) == ".512") {
2978       bool IsDouble = Name[17] == 'd';
2979       bool IsMin = Name[13] == 'i';
2980       static const Intrinsic::ID MinMaxTbl[2][2] = {
2981         { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
2982         { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
2983       };
2984       Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
2985 
2986       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2987                                { CI->getArgOperand(0), CI->getArgOperand(1),
2988                                  CI->getArgOperand(4) });
2989       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2990                           CI->getArgOperand(2));
2991     } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
2992       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2993                                                          Intrinsic::ctlz,
2994                                                          CI->getType()),
2995                                { CI->getArgOperand(0), Builder.getInt1(false) });
2996       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2997                           CI->getArgOperand(1));
2998     } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
2999       bool IsImmediate = Name[16] == 'i' ||
3000                          (Name.size() > 18 && Name[18] == 'i');
3001       bool IsVariable = Name[16] == 'v';
3002       char Size = Name[16] == '.' ? Name[17] :
3003                   Name[17] == '.' ? Name[18] :
3004                   Name[18] == '.' ? Name[19] :
3005                                     Name[20];
3006 
3007       Intrinsic::ID IID;
3008       if (IsVariable && Name[17] != '.') {
3009         if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
3010           IID = Intrinsic::x86_avx2_psllv_q;
3011         else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
3012           IID = Intrinsic::x86_avx2_psllv_q_256;
3013         else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
3014           IID = Intrinsic::x86_avx2_psllv_d;
3015         else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
3016           IID = Intrinsic::x86_avx2_psllv_d_256;
3017         else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
3018           IID = Intrinsic::x86_avx512_psllv_w_128;
3019         else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
3020           IID = Intrinsic::x86_avx512_psllv_w_256;
3021         else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
3022           IID = Intrinsic::x86_avx512_psllv_w_512;
3023         else
3024           llvm_unreachable("Unexpected size");
3025       } else if (Name.endswith(".128")) {
3026         if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
3027           IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3028                             : Intrinsic::x86_sse2_psll_d;
3029         else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
3030           IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3031                             : Intrinsic::x86_sse2_psll_q;
3032         else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
3033           IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3034                             : Intrinsic::x86_sse2_psll_w;
3035         else
3036           llvm_unreachable("Unexpected size");
3037       } else if (Name.endswith(".256")) {
3038         if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
3039           IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3040                             : Intrinsic::x86_avx2_psll_d;
3041         else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
3042           IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3043                             : Intrinsic::x86_avx2_psll_q;
3044         else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
3045           IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3046                             : Intrinsic::x86_avx2_psll_w;
3047         else
3048           llvm_unreachable("Unexpected size");
3049       } else {
3050         if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
3051           IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3052                 IsVariable  ? Intrinsic::x86_avx512_psllv_d_512 :
3053                               Intrinsic::x86_avx512_psll_d_512;
3054         else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
3055           IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3056                 IsVariable  ? Intrinsic::x86_avx512_psllv_q_512 :
3057                               Intrinsic::x86_avx512_psll_q_512;
3058         else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
3059           IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3060                             : Intrinsic::x86_avx512_psll_w_512;
3061         else
3062           llvm_unreachable("Unexpected size");
3063       }
3064 
3065       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3066     } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
3067       bool IsImmediate = Name[16] == 'i' ||
3068                          (Name.size() > 18 && Name[18] == 'i');
3069       bool IsVariable = Name[16] == 'v';
3070       char Size = Name[16] == '.' ? Name[17] :
3071                   Name[17] == '.' ? Name[18] :
3072                   Name[18] == '.' ? Name[19] :
3073                                     Name[20];
3074 
3075       Intrinsic::ID IID;
3076       if (IsVariable && Name[17] != '.') {
3077         if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
3078           IID = Intrinsic::x86_avx2_psrlv_q;
3079         else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
3080           IID = Intrinsic::x86_avx2_psrlv_q_256;
3081         else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
3082           IID = Intrinsic::x86_avx2_psrlv_d;
3083         else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
3084           IID = Intrinsic::x86_avx2_psrlv_d_256;
3085         else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
3086           IID = Intrinsic::x86_avx512_psrlv_w_128;
3087         else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
3088           IID = Intrinsic::x86_avx512_psrlv_w_256;
3089         else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
3090           IID = Intrinsic::x86_avx512_psrlv_w_512;
3091         else
3092           llvm_unreachable("Unexpected size");
3093       } else if (Name.endswith(".128")) {
3094         if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
3095           IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3096                             : Intrinsic::x86_sse2_psrl_d;
3097         else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
3098           IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3099                             : Intrinsic::x86_sse2_psrl_q;
3100         else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
3101           IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3102                             : Intrinsic::x86_sse2_psrl_w;
3103         else
3104           llvm_unreachable("Unexpected size");
3105       } else if (Name.endswith(".256")) {
3106         if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
3107           IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3108                             : Intrinsic::x86_avx2_psrl_d;
3109         else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
3110           IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3111                             : Intrinsic::x86_avx2_psrl_q;
3112         else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
3113           IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3114                             : Intrinsic::x86_avx2_psrl_w;
3115         else
3116           llvm_unreachable("Unexpected size");
3117       } else {
3118         if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
3119           IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3120                 IsVariable  ? Intrinsic::x86_avx512_psrlv_d_512 :
3121                               Intrinsic::x86_avx512_psrl_d_512;
3122         else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
3123           IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3124                 IsVariable  ? Intrinsic::x86_avx512_psrlv_q_512 :
3125                               Intrinsic::x86_avx512_psrl_q_512;
3126         else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
3127           IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3128                             : Intrinsic::x86_avx512_psrl_w_512;
3129         else
3130           llvm_unreachable("Unexpected size");
3131       }
3132 
3133       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3134     } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
3135       bool IsImmediate = Name[16] == 'i' ||
3136                          (Name.size() > 18 && Name[18] == 'i');
3137       bool IsVariable = Name[16] == 'v';
3138       char Size = Name[16] == '.' ? Name[17] :
3139                   Name[17] == '.' ? Name[18] :
3140                   Name[18] == '.' ? Name[19] :
3141                                     Name[20];
3142 
3143       Intrinsic::ID IID;
3144       if (IsVariable && Name[17] != '.') {
3145         if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
3146           IID = Intrinsic::x86_avx2_psrav_d;
3147         else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
3148           IID = Intrinsic::x86_avx2_psrav_d_256;
3149         else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
3150           IID = Intrinsic::x86_avx512_psrav_w_128;
3151         else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
3152           IID = Intrinsic::x86_avx512_psrav_w_256;
3153         else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
3154           IID = Intrinsic::x86_avx512_psrav_w_512;
3155         else
3156           llvm_unreachable("Unexpected size");
3157       } else if (Name.endswith(".128")) {
3158         if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
3159           IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3160                             : Intrinsic::x86_sse2_psra_d;
3161         else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
3162           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3163                 IsVariable  ? Intrinsic::x86_avx512_psrav_q_128 :
3164                               Intrinsic::x86_avx512_psra_q_128;
3165         else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
3166           IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3167                             : Intrinsic::x86_sse2_psra_w;
3168         else
3169           llvm_unreachable("Unexpected size");
3170       } else if (Name.endswith(".256")) {
3171         if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
3172           IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3173                             : Intrinsic::x86_avx2_psra_d;
3174         else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
3175           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3176                 IsVariable  ? Intrinsic::x86_avx512_psrav_q_256 :
3177                               Intrinsic::x86_avx512_psra_q_256;
3178         else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
3179           IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3180                             : Intrinsic::x86_avx2_psra_w;
3181         else
3182           llvm_unreachable("Unexpected size");
3183       } else {
3184         if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
3185           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3186                 IsVariable  ? Intrinsic::x86_avx512_psrav_d_512 :
3187                               Intrinsic::x86_avx512_psra_d_512;
3188         else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
3189           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3190                 IsVariable  ? Intrinsic::x86_avx512_psrav_q_512 :
3191                               Intrinsic::x86_avx512_psra_q_512;
3192         else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
3193           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3194                             : Intrinsic::x86_avx512_psra_w_512;
3195         else
3196           llvm_unreachable("Unexpected size");
3197       }
3198 
3199       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3200     } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
3201       Rep = upgradeMaskedMove(Builder, *CI);
3202     } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
3203       Rep = UpgradeMaskToInt(Builder, *CI);
3204     } else if (IsX86 && Name.endswith(".movntdqa")) {
3205       Module *M = F->getParent();
3206       MDNode *Node = MDNode::get(
3207           C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3208 
3209       Value *Ptr = CI->getArgOperand(0);
3210 
3211       // Convert the type of the pointer to a pointer to the stored type.
3212       Value *BC = Builder.CreateBitCast(
3213           Ptr, PointerType::getUnqual(CI->getType()), "cast");
3214       LoadInst *LI = Builder.CreateAlignedLoad(
3215           CI->getType(), BC,
3216           Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
3217       LI->setMetadata(M->getMDKindID("nontemporal"), Node);
3218       Rep = LI;
3219     } else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
3220                          Name.startswith("fma.vfmsub.") ||
3221                          Name.startswith("fma.vfnmadd.") ||
3222                          Name.startswith("fma.vfnmsub."))) {
3223       bool NegMul = Name[6] == 'n';
3224       bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
3225       bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
3226 
3227       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3228                        CI->getArgOperand(2) };
3229 
3230       if (IsScalar) {
3231         Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3232         Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3233         Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3234       }
3235 
3236       if (NegMul && !IsScalar)
3237         Ops[0] = Builder.CreateFNeg(Ops[0]);
3238       if (NegMul && IsScalar)
3239         Ops[1] = Builder.CreateFNeg(Ops[1]);
3240       if (NegAcc)
3241         Ops[2] = Builder.CreateFNeg(Ops[2]);
3242 
3243       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3244                                                          Intrinsic::fma,
3245                                                          Ops[0]->getType()),
3246                                Ops);
3247 
3248       if (IsScalar)
3249         Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3250                                           (uint64_t)0);
3251     } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
3252       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3253                        CI->getArgOperand(2) };
3254 
3255       Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3256       Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3257       Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3258 
3259       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3260                                                          Intrinsic::fma,
3261                                                          Ops[0]->getType()),
3262                                Ops);
3263 
3264       Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3265                                         Rep, (uint64_t)0);
3266     } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
3267                          Name.startswith("avx512.maskz.vfmadd.s") ||
3268                          Name.startswith("avx512.mask3.vfmadd.s") ||
3269                          Name.startswith("avx512.mask3.vfmsub.s") ||
3270                          Name.startswith("avx512.mask3.vfnmsub.s"))) {
3271       bool IsMask3 = Name[11] == '3';
3272       bool IsMaskZ = Name[11] == 'z';
3273       // Drop the "avx512.mask." to make it easier.
3274       Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3275       bool NegMul = Name[2] == 'n';
3276       bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3277 
3278       Value *A = CI->getArgOperand(0);
3279       Value *B = CI->getArgOperand(1);
3280       Value *C = CI->getArgOperand(2);
3281 
3282       if (NegMul && (IsMask3 || IsMaskZ))
3283         A = Builder.CreateFNeg(A);
3284       if (NegMul && !(IsMask3 || IsMaskZ))
3285         B = Builder.CreateFNeg(B);
3286       if (NegAcc)
3287         C = Builder.CreateFNeg(C);
3288 
3289       A = Builder.CreateExtractElement(A, (uint64_t)0);
3290       B = Builder.CreateExtractElement(B, (uint64_t)0);
3291       C = Builder.CreateExtractElement(C, (uint64_t)0);
3292 
3293       if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3294           cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3295         Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
3296 
3297         Intrinsic::ID IID;
3298         if (Name.back() == 'd')
3299           IID = Intrinsic::x86_avx512_vfmadd_f64;
3300         else
3301           IID = Intrinsic::x86_avx512_vfmadd_f32;
3302         Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3303         Rep = Builder.CreateCall(FMA, Ops);
3304       } else {
3305         Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3306                                                   Intrinsic::fma,
3307                                                   A->getType());
3308         Rep = Builder.CreateCall(FMA, { A, B, C });
3309       }
3310 
3311       Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
3312                         IsMask3 ? C : A;
3313 
3314       // For Mask3 with NegAcc, we need to create a new extractelement that
3315       // avoids the negation above.
3316       if (NegAcc && IsMask3)
3317         PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3318                                                 (uint64_t)0);
3319 
3320       Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3321                                 Rep, PassThru);
3322       Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3323                                         Rep, (uint64_t)0);
3324     } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
3325                          Name.startswith("avx512.mask.vfnmadd.p") ||
3326                          Name.startswith("avx512.mask.vfnmsub.p") ||
3327                          Name.startswith("avx512.mask3.vfmadd.p") ||
3328                          Name.startswith("avx512.mask3.vfmsub.p") ||
3329                          Name.startswith("avx512.mask3.vfnmsub.p") ||
3330                          Name.startswith("avx512.maskz.vfmadd.p"))) {
3331       bool IsMask3 = Name[11] == '3';
3332       bool IsMaskZ = Name[11] == 'z';
3333       // Drop the "avx512.mask." to make it easier.
3334       Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3335       bool NegMul = Name[2] == 'n';
3336       bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3337 
3338       Value *A = CI->getArgOperand(0);
3339       Value *B = CI->getArgOperand(1);
3340       Value *C = CI->getArgOperand(2);
3341 
3342       if (NegMul && (IsMask3 || IsMaskZ))
3343         A = Builder.CreateFNeg(A);
3344       if (NegMul && !(IsMask3 || IsMaskZ))
3345         B = Builder.CreateFNeg(B);
3346       if (NegAcc)
3347         C = Builder.CreateFNeg(C);
3348 
3349       if (CI->getNumArgOperands() == 5 &&
3350           (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3351            cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3352         Intrinsic::ID IID;
3353         // Check the character before ".512" in string.
3354         if (Name[Name.size()-5] == 's')
3355           IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3356         else
3357           IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3358 
3359         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3360                                  { A, B, C, CI->getArgOperand(4) });
3361       } else {
3362         Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3363                                                   Intrinsic::fma,
3364                                                   A->getType());
3365         Rep = Builder.CreateCall(FMA, { A, B, C });
3366       }
3367 
3368       Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3369                         IsMask3 ? CI->getArgOperand(2) :
3370                                   CI->getArgOperand(0);
3371 
3372       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3373     } else if (IsX86 &&  Name.startswith("fma.vfmsubadd.p")) {
3374       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3375       unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3376       Intrinsic::ID IID;
3377       if (VecWidth == 128 && EltWidth == 32)
3378         IID = Intrinsic::x86_fma_vfmaddsub_ps;
3379       else if (VecWidth == 256 && EltWidth == 32)
3380         IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3381       else if (VecWidth == 128 && EltWidth == 64)
3382         IID = Intrinsic::x86_fma_vfmaddsub_pd;
3383       else if (VecWidth == 256 && EltWidth == 64)
3384         IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3385       else
3386         llvm_unreachable("Unexpected intrinsic");
3387 
3388       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3389                        CI->getArgOperand(2) };
3390       Ops[2] = Builder.CreateFNeg(Ops[2]);
3391       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3392                                Ops);
3393     } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
3394                          Name.startswith("avx512.mask3.vfmaddsub.p") ||
3395                          Name.startswith("avx512.maskz.vfmaddsub.p") ||
3396                          Name.startswith("avx512.mask3.vfmsubadd.p"))) {
3397       bool IsMask3 = Name[11] == '3';
3398       bool IsMaskZ = Name[11] == 'z';
3399       // Drop the "avx512.mask." to make it easier.
3400       Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3401       bool IsSubAdd = Name[3] == 's';
3402       if (CI->getNumArgOperands() == 5) {
3403         Intrinsic::ID IID;
3404         // Check the character before ".512" in string.
3405         if (Name[Name.size()-5] == 's')
3406           IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3407         else
3408           IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3409 
3410         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3411                          CI->getArgOperand(2), CI->getArgOperand(4) };
3412         if (IsSubAdd)
3413           Ops[2] = Builder.CreateFNeg(Ops[2]);
3414 
3415         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3416                                  Ops);
3417       } else {
3418         int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3419 
3420         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3421                          CI->getArgOperand(2) };
3422 
3423         Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3424                                                   Ops[0]->getType());
3425         Value *Odd = Builder.CreateCall(FMA, Ops);
3426         Ops[2] = Builder.CreateFNeg(Ops[2]);
3427         Value *Even = Builder.CreateCall(FMA, Ops);
3428 
3429         if (IsSubAdd)
3430           std::swap(Even, Odd);
3431 
3432         SmallVector<int, 32> Idxs(NumElts);
3433         for (int i = 0; i != NumElts; ++i)
3434           Idxs[i] = i + (i % 2) * NumElts;
3435 
3436         Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
3437       }
3438 
3439       Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3440                         IsMask3 ? CI->getArgOperand(2) :
3441                                   CI->getArgOperand(0);
3442 
3443       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3444     } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
3445                          Name.startswith("avx512.maskz.pternlog."))) {
3446       bool ZeroMask = Name[11] == 'z';
3447       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3448       unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3449       Intrinsic::ID IID;
3450       if (VecWidth == 128 && EltWidth == 32)
3451         IID = Intrinsic::x86_avx512_pternlog_d_128;
3452       else if (VecWidth == 256 && EltWidth == 32)
3453         IID = Intrinsic::x86_avx512_pternlog_d_256;
3454       else if (VecWidth == 512 && EltWidth == 32)
3455         IID = Intrinsic::x86_avx512_pternlog_d_512;
3456       else if (VecWidth == 128 && EltWidth == 64)
3457         IID = Intrinsic::x86_avx512_pternlog_q_128;
3458       else if (VecWidth == 256 && EltWidth == 64)
3459         IID = Intrinsic::x86_avx512_pternlog_q_256;
3460       else if (VecWidth == 512 && EltWidth == 64)
3461         IID = Intrinsic::x86_avx512_pternlog_q_512;
3462       else
3463         llvm_unreachable("Unexpected intrinsic");
3464 
3465       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3466                         CI->getArgOperand(2), CI->getArgOperand(3) };
3467       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3468                                Args);
3469       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3470                                  : CI->getArgOperand(0);
3471       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3472     } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
3473                          Name.startswith("avx512.maskz.vpmadd52"))) {
3474       bool ZeroMask = Name[11] == 'z';
3475       bool High = Name[20] == 'h' || Name[21] == 'h';
3476       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3477       Intrinsic::ID IID;
3478       if (VecWidth == 128 && !High)
3479         IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3480       else if (VecWidth == 256 && !High)
3481         IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3482       else if (VecWidth == 512 && !High)
3483         IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3484       else if (VecWidth == 128 && High)
3485         IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3486       else if (VecWidth == 256 && High)
3487         IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3488       else if (VecWidth == 512 && High)
3489         IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3490       else
3491         llvm_unreachable("Unexpected intrinsic");
3492 
3493       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3494                         CI->getArgOperand(2) };
3495       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3496                                Args);
3497       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3498                                  : CI->getArgOperand(0);
3499       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3500     } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
3501                          Name.startswith("avx512.mask.vpermt2var.") ||
3502                          Name.startswith("avx512.maskz.vpermt2var."))) {
3503       bool ZeroMask = Name[11] == 'z';
3504       bool IndexForm = Name[17] == 'i';
3505       Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3506     } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
3507                          Name.startswith("avx512.maskz.vpdpbusd.") ||
3508                          Name.startswith("avx512.mask.vpdpbusds.") ||
3509                          Name.startswith("avx512.maskz.vpdpbusds."))) {
3510       bool ZeroMask = Name[11] == 'z';
3511       bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3512       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3513       Intrinsic::ID IID;
3514       if (VecWidth == 128 && !IsSaturating)
3515         IID = Intrinsic::x86_avx512_vpdpbusd_128;
3516       else if (VecWidth == 256 && !IsSaturating)
3517         IID = Intrinsic::x86_avx512_vpdpbusd_256;
3518       else if (VecWidth == 512 && !IsSaturating)
3519         IID = Intrinsic::x86_avx512_vpdpbusd_512;
3520       else if (VecWidth == 128 && IsSaturating)
3521         IID = Intrinsic::x86_avx512_vpdpbusds_128;
3522       else if (VecWidth == 256 && IsSaturating)
3523         IID = Intrinsic::x86_avx512_vpdpbusds_256;
3524       else if (VecWidth == 512 && IsSaturating)
3525         IID = Intrinsic::x86_avx512_vpdpbusds_512;
3526       else
3527         llvm_unreachable("Unexpected intrinsic");
3528 
3529       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3530                         CI->getArgOperand(2)  };
3531       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3532                                Args);
3533       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3534                                  : CI->getArgOperand(0);
3535       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3536     } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
3537                          Name.startswith("avx512.maskz.vpdpwssd.") ||
3538                          Name.startswith("avx512.mask.vpdpwssds.") ||
3539                          Name.startswith("avx512.maskz.vpdpwssds."))) {
3540       bool ZeroMask = Name[11] == 'z';
3541       bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3542       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3543       Intrinsic::ID IID;
3544       if (VecWidth == 128 && !IsSaturating)
3545         IID = Intrinsic::x86_avx512_vpdpwssd_128;
3546       else if (VecWidth == 256 && !IsSaturating)
3547         IID = Intrinsic::x86_avx512_vpdpwssd_256;
3548       else if (VecWidth == 512 && !IsSaturating)
3549         IID = Intrinsic::x86_avx512_vpdpwssd_512;
3550       else if (VecWidth == 128 && IsSaturating)
3551         IID = Intrinsic::x86_avx512_vpdpwssds_128;
3552       else if (VecWidth == 256 && IsSaturating)
3553         IID = Intrinsic::x86_avx512_vpdpwssds_256;
3554       else if (VecWidth == 512 && IsSaturating)
3555         IID = Intrinsic::x86_avx512_vpdpwssds_512;
3556       else
3557         llvm_unreachable("Unexpected intrinsic");
3558 
3559       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3560                         CI->getArgOperand(2)  };
3561       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3562                                Args);
3563       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3564                                  : CI->getArgOperand(0);
3565       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3566     } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
3567                          Name == "addcarry.u32" || Name == "addcarry.u64" ||
3568                          Name == "subborrow.u32" || Name == "subborrow.u64")) {
3569       Intrinsic::ID IID;
3570       if (Name[0] == 'a' && Name.back() == '2')
3571         IID = Intrinsic::x86_addcarry_32;
3572       else if (Name[0] == 'a' && Name.back() == '4')
3573         IID = Intrinsic::x86_addcarry_64;
3574       else if (Name[0] == 's' && Name.back() == '2')
3575         IID = Intrinsic::x86_subborrow_32;
3576       else if (Name[0] == 's' && Name.back() == '4')
3577         IID = Intrinsic::x86_subborrow_64;
3578       else
3579         llvm_unreachable("Unexpected intrinsic");
3580 
3581       // Make a call with 3 operands.
3582       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3583                         CI->getArgOperand(2)};
3584       Value *NewCall = Builder.CreateCall(
3585                                 Intrinsic::getDeclaration(CI->getModule(), IID),
3586                                 Args);
3587 
3588       // Extract the second result and store it.
3589       Value *Data = Builder.CreateExtractValue(NewCall, 1);
3590       // Cast the pointer to the right type.
3591       Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3592                                  llvm::PointerType::getUnqual(Data->getType()));
3593       Builder.CreateAlignedStore(Data, Ptr, Align(1));
3594       // Replace the original call result with the first result of the new call.
3595       Value *CF = Builder.CreateExtractValue(NewCall, 0);
3596 
3597       CI->replaceAllUsesWith(CF);
3598       Rep = nullptr;
3599     } else if (IsX86 && Name.startswith("avx512.mask.") &&
3600                upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3601       // Rep will be updated by the call in the condition.
3602     } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
3603       Value *Arg = CI->getArgOperand(0);
3604       Value *Neg = Builder.CreateNeg(Arg, "neg");
3605       Value *Cmp = Builder.CreateICmpSGE(
3606           Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
3607       Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
3608     } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
3609                           Name.startswith("atomic.load.add.f64.p"))) {
3610       Value *Ptr = CI->getArgOperand(0);
3611       Value *Val = CI->getArgOperand(1);
3612       Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
3613                                     AtomicOrdering::SequentiallyConsistent);
3614     } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
3615                           Name == "max.ui" || Name == "max.ull")) {
3616       Value *Arg0 = CI->getArgOperand(0);
3617       Value *Arg1 = CI->getArgOperand(1);
3618       Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3619                        ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
3620                        : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
3621       Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
3622     } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
3623                           Name == "min.ui" || Name == "min.ull")) {
3624       Value *Arg0 = CI->getArgOperand(0);
3625       Value *Arg1 = CI->getArgOperand(1);
3626       Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3627                        ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
3628                        : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
3629       Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
3630     } else if (IsNVVM && Name == "clz.ll") {
3631       // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
3632       Value *Arg = CI->getArgOperand(0);
3633       Value *Ctlz = Builder.CreateCall(
3634           Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
3635                                     {Arg->getType()}),
3636           {Arg, Builder.getFalse()}, "ctlz");
3637       Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3638     } else if (IsNVVM && Name == "popc.ll") {
3639       // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
3640       // i64.
3641       Value *Arg = CI->getArgOperand(0);
3642       Value *Popc = Builder.CreateCall(
3643           Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
3644                                     {Arg->getType()}),
3645           Arg, "ctpop");
3646       Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3647     } else if (IsNVVM && Name == "h2f") {
3648       Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3649                                    F->getParent(), Intrinsic::convert_from_fp16,
3650                                    {Builder.getFloatTy()}),
3651                                CI->getArgOperand(0), "h2f");
3652     } else {
3653       llvm_unreachable("Unknown function for CallInst upgrade.");
3654     }
3655 
3656     if (Rep)
3657       CI->replaceAllUsesWith(Rep);
3658     CI->eraseFromParent();
3659     return;
3660   }
3661 
3662   const auto &DefaultCase = [&NewFn, &CI]() -> void {
3663     // Handle generic mangling change, but nothing else
3664     assert(
3665         (CI->getCalledFunction()->getName() != NewFn->getName()) &&
3666         "Unknown function for CallInst upgrade and isn't just a name change");
3667     CI->setCalledFunction(NewFn);
3668   };
3669   CallInst *NewCall = nullptr;
3670   switch (NewFn->getIntrinsicID()) {
3671   default: {
3672     DefaultCase();
3673     return;
3674   }
3675   case Intrinsic::arm_neon_vld1:
3676   case Intrinsic::arm_neon_vld2:
3677   case Intrinsic::arm_neon_vld3:
3678   case Intrinsic::arm_neon_vld4:
3679   case Intrinsic::arm_neon_vld2lane:
3680   case Intrinsic::arm_neon_vld3lane:
3681   case Intrinsic::arm_neon_vld4lane:
3682   case Intrinsic::arm_neon_vst1:
3683   case Intrinsic::arm_neon_vst2:
3684   case Intrinsic::arm_neon_vst3:
3685   case Intrinsic::arm_neon_vst4:
3686   case Intrinsic::arm_neon_vst2lane:
3687   case Intrinsic::arm_neon_vst3lane:
3688   case Intrinsic::arm_neon_vst4lane: {
3689     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3690                                  CI->arg_operands().end());
3691     NewCall = Builder.CreateCall(NewFn, Args);
3692     break;
3693   }
3694 
3695   case Intrinsic::arm_neon_bfdot:
3696   case Intrinsic::arm_neon_bfmmla:
3697   case Intrinsic::arm_neon_bfmlalb:
3698   case Intrinsic::arm_neon_bfmlalt:
3699   case Intrinsic::aarch64_neon_bfdot:
3700   case Intrinsic::aarch64_neon_bfmmla:
3701   case Intrinsic::aarch64_neon_bfmlalb:
3702   case Intrinsic::aarch64_neon_bfmlalt: {
3703     SmallVector<Value *, 3> Args;
3704     assert(CI->getNumArgOperands() == 3 &&
3705            "Mismatch between function args and call args");
3706     size_t OperandWidth =
3707         CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
3708     assert((OperandWidth == 64 || OperandWidth == 128) &&
3709            "Unexpected operand width");
3710     Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
3711     auto Iter = CI->arg_operands().begin();
3712     Args.push_back(*Iter++);
3713     Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
3714     Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
3715     NewCall = Builder.CreateCall(NewFn, Args);
3716     break;
3717   }
3718 
3719   case Intrinsic::bitreverse:
3720     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3721     break;
3722 
3723   case Intrinsic::ctlz:
3724   case Intrinsic::cttz:
3725     assert(CI->getNumArgOperands() == 1 &&
3726            "Mismatch between function args and call args");
3727     NewCall =
3728         Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
3729     break;
3730 
3731   case Intrinsic::objectsize: {
3732     Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
3733                                    ? Builder.getFalse()
3734                                    : CI->getArgOperand(2);
3735     Value *Dynamic =
3736         CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
3737     NewCall = Builder.CreateCall(
3738         NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
3739     break;
3740   }
3741 
3742   case Intrinsic::ctpop:
3743     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3744     break;
3745 
3746   case Intrinsic::convert_from_fp16:
3747     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3748     break;
3749 
3750   case Intrinsic::dbg_value:
3751     // Upgrade from the old version that had an extra offset argument.
3752     assert(CI->getNumArgOperands() == 4);
3753     // Drop nonzero offsets instead of attempting to upgrade them.
3754     if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
3755       if (Offset->isZeroValue()) {
3756         NewCall = Builder.CreateCall(
3757             NewFn,
3758             {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
3759         break;
3760       }
3761     CI->eraseFromParent();
3762     return;
3763 
3764   case Intrinsic::ptr_annotation:
3765     // Upgrade from versions that lacked the annotation attribute argument.
3766     assert(CI->getNumArgOperands() == 4 &&
3767            "Before LLVM 12.0 this intrinsic took four arguments");
3768     // Create a new call with an added null annotation attribute argument.
3769     NewCall = Builder.CreateCall(
3770         NewFn,
3771         {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
3772          CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
3773     NewCall->takeName(CI);
3774     CI->replaceAllUsesWith(NewCall);
3775     CI->eraseFromParent();
3776     return;
3777 
3778   case Intrinsic::var_annotation:
3779     // Upgrade from versions that lacked the annotation attribute argument.
3780     assert(CI->getNumArgOperands() == 4 &&
3781            "Before LLVM 12.0 this intrinsic took four arguments");
3782     // Create a new call with an added null annotation attribute argument.
3783     NewCall = Builder.CreateCall(
3784         NewFn,
3785         {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
3786          CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
3787     CI->eraseFromParent();
3788     return;
3789 
3790   case Intrinsic::x86_xop_vfrcz_ss:
3791   case Intrinsic::x86_xop_vfrcz_sd:
3792     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
3793     break;
3794 
3795   case Intrinsic::x86_xop_vpermil2pd:
3796   case Intrinsic::x86_xop_vpermil2ps:
3797   case Intrinsic::x86_xop_vpermil2pd_256:
3798   case Intrinsic::x86_xop_vpermil2ps_256: {
3799     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3800                                  CI->arg_operands().end());
3801     VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
3802     VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
3803     Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
3804     NewCall = Builder.CreateCall(NewFn, Args);
3805     break;
3806   }
3807 
3808   case Intrinsic::x86_sse41_ptestc:
3809   case Intrinsic::x86_sse41_ptestz:
3810   case Intrinsic::x86_sse41_ptestnzc: {
3811     // The arguments for these intrinsics used to be v4f32, and changed
3812     // to v2i64. This is purely a nop, since those are bitwise intrinsics.
3813     // So, the only thing required is a bitcast for both arguments.
3814     // First, check the arguments have the old type.
3815     Value *Arg0 = CI->getArgOperand(0);
3816     if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
3817       return;
3818 
3819     // Old intrinsic, add bitcasts
3820     Value *Arg1 = CI->getArgOperand(1);
3821 
3822     auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3823 
3824     Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
3825     Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3826 
3827     NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
3828     break;
3829   }
3830 
3831   case Intrinsic::x86_rdtscp: {
3832     // This used to take 1 arguments. If we have no arguments, it is already
3833     // upgraded.
3834     if (CI->getNumOperands() == 0)
3835       return;
3836 
3837     NewCall = Builder.CreateCall(NewFn);
3838     // Extract the second result and store it.
3839     Value *Data = Builder.CreateExtractValue(NewCall, 1);
3840     // Cast the pointer to the right type.
3841     Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
3842                                  llvm::PointerType::getUnqual(Data->getType()));
3843     Builder.CreateAlignedStore(Data, Ptr, Align(1));
3844     // Replace the original call result with the first result of the new call.
3845     Value *TSC = Builder.CreateExtractValue(NewCall, 0);
3846 
3847     NewCall->takeName(CI);
3848     CI->replaceAllUsesWith(TSC);
3849     CI->eraseFromParent();
3850     return;
3851   }
3852 
3853   case Intrinsic::x86_sse41_insertps:
3854   case Intrinsic::x86_sse41_dppd:
3855   case Intrinsic::x86_sse41_dpps:
3856   case Intrinsic::x86_sse41_mpsadbw:
3857   case Intrinsic::x86_avx_dp_ps_256:
3858   case Intrinsic::x86_avx2_mpsadbw: {
3859     // Need to truncate the last argument from i32 to i8 -- this argument models
3860     // an inherently 8-bit immediate operand to these x86 instructions.
3861     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3862                                  CI->arg_operands().end());
3863 
3864     // Replace the last argument with a trunc.
3865     Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
3866     NewCall = Builder.CreateCall(NewFn, Args);
3867     break;
3868   }
3869 
3870   case Intrinsic::x86_avx512_mask_cmp_pd_128:
3871   case Intrinsic::x86_avx512_mask_cmp_pd_256:
3872   case Intrinsic::x86_avx512_mask_cmp_pd_512:
3873   case Intrinsic::x86_avx512_mask_cmp_ps_128:
3874   case Intrinsic::x86_avx512_mask_cmp_ps_256:
3875   case Intrinsic::x86_avx512_mask_cmp_ps_512: {
3876     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3877                                  CI->arg_operands().end());
3878     unsigned NumElts =
3879         cast<FixedVectorType>(Args[0]->getType())->getNumElements();
3880     Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
3881 
3882     NewCall = Builder.CreateCall(NewFn, Args);
3883     Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
3884 
3885     NewCall->takeName(CI);
3886     CI->replaceAllUsesWith(Res);
3887     CI->eraseFromParent();
3888     return;
3889   }
3890 
3891   case Intrinsic::thread_pointer: {
3892     NewCall = Builder.CreateCall(NewFn, {});
3893     break;
3894   }
3895 
3896   case Intrinsic::invariant_start:
3897   case Intrinsic::invariant_end: {
3898     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3899                                  CI->arg_operands().end());
3900     NewCall = Builder.CreateCall(NewFn, Args);
3901     break;
3902   }
3903   case Intrinsic::masked_load:
3904   case Intrinsic::masked_store:
3905   case Intrinsic::masked_gather:
3906   case Intrinsic::masked_scatter: {
3907     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3908                                  CI->arg_operands().end());
3909     NewCall = Builder.CreateCall(NewFn, Args);
3910     NewCall->copyMetadata(*CI);
3911     break;
3912   }
3913 
3914   case Intrinsic::memcpy:
3915   case Intrinsic::memmove:
3916   case Intrinsic::memset: {
3917     // We have to make sure that the call signature is what we're expecting.
3918     // We only want to change the old signatures by removing the alignment arg:
3919     //  @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
3920     //    -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
3921     //  @llvm.memset...(i8*, i8, i[32|64], i32, i1)
3922     //    -> @llvm.memset...(i8*, i8, i[32|64], i1)
3923     // Note: i8*'s in the above can be any pointer type
3924     if (CI->getNumArgOperands() != 5) {
3925       DefaultCase();
3926       return;
3927     }
3928     // Remove alignment argument (3), and add alignment attributes to the
3929     // dest/src pointers.
3930     Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
3931                       CI->getArgOperand(2), CI->getArgOperand(4)};
3932     NewCall = Builder.CreateCall(NewFn, Args);
3933     auto *MemCI = cast<MemIntrinsic>(NewCall);
3934     // All mem intrinsics support dest alignment.
3935     const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
3936     MemCI->setDestAlignment(Align->getMaybeAlignValue());
3937     // Memcpy/Memmove also support source alignment.
3938     if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
3939       MTI->setSourceAlignment(Align->getMaybeAlignValue());
3940     break;
3941   }
3942   }
3943   assert(NewCall && "Should have either set this variable or returned through "
3944                     "the default case");
3945   NewCall->takeName(CI);
3946   CI->replaceAllUsesWith(NewCall);
3947   CI->eraseFromParent();
3948 }
3949 
3950 void llvm::UpgradeCallsToIntrinsic(Function *F) {
3951   assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
3952 
3953   // Check if this function should be upgraded and get the replacement function
3954   // if there is one.
3955   Function *NewFn;
3956   if (UpgradeIntrinsicFunction(F, NewFn)) {
3957     // Replace all users of the old function with the new function or new
3958     // instructions. This is not a range loop because the call is deleted.
3959     for (User *U : make_early_inc_range(F->users()))
3960       if (CallInst *CI = dyn_cast<CallInst>(U))
3961         UpgradeIntrinsicCall(CI, NewFn);
3962 
3963     // Remove old function, no longer used, from the module.
3964     F->eraseFromParent();
3965   }
3966 }
3967 
3968 MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
3969   // Check if the tag uses struct-path aware TBAA format.
3970   if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)
3971     return &MD;
3972 
3973   auto &Context = MD.getContext();
3974   if (MD.getNumOperands() == 3) {
3975     Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
3976     MDNode *ScalarType = MDNode::get(Context, Elts);
3977     // Create a MDNode <ScalarType, ScalarType, offset 0, const>
3978     Metadata *Elts2[] = {ScalarType, ScalarType,
3979                          ConstantAsMetadata::get(
3980                              Constant::getNullValue(Type::getInt64Ty(Context))),
3981                          MD.getOperand(2)};
3982     return MDNode::get(Context, Elts2);
3983   }
3984   // Create a MDNode <MD, MD, offset 0>
3985   Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
3986                                     Type::getInt64Ty(Context)))};
3987   return MDNode::get(Context, Elts);
3988 }
3989 
3990 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
3991                                       Instruction *&Temp) {
3992   if (Opc != Instruction::BitCast)
3993     return nullptr;
3994 
3995   Temp = nullptr;
3996   Type *SrcTy = V->getType();
3997   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
3998       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
3999     LLVMContext &Context = V->getContext();
4000 
4001     // We have no information about target data layout, so we assume that
4002     // the maximum pointer size is 64bit.
4003     Type *MidTy = Type::getInt64Ty(Context);
4004     Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
4005 
4006     return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
4007   }
4008 
4009   return nullptr;
4010 }
4011 
4012 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
4013   if (Opc != Instruction::BitCast)
4014     return nullptr;
4015 
4016   Type *SrcTy = C->getType();
4017   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
4018       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
4019     LLVMContext &Context = C->getContext();
4020 
4021     // We have no information about target data layout, so we assume that
4022     // the maximum pointer size is 64bit.
4023     Type *MidTy = Type::getInt64Ty(Context);
4024 
4025     return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
4026                                      DestTy);
4027   }
4028 
4029   return nullptr;
4030 }
4031 
4032 /// Check the debug info version number, if it is out-dated, drop the debug
4033 /// info. Return true if module is modified.
4034 bool llvm::UpgradeDebugInfo(Module &M) {
4035   unsigned Version = getDebugMetadataVersionFromModule(M);
4036   if (Version == DEBUG_METADATA_VERSION) {
4037     bool BrokenDebugInfo = false;
4038     if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
4039       report_fatal_error("Broken module found, compilation aborted!");
4040     if (!BrokenDebugInfo)
4041       // Everything is ok.
4042       return false;
4043     else {
4044       // Diagnose malformed debug info.
4045       DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
4046       M.getContext().diagnose(Diag);
4047     }
4048   }
4049   bool Modified = StripDebugInfo(M);
4050   if (Modified && Version != DEBUG_METADATA_VERSION) {
4051     // Diagnose a version mismatch.
4052     DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
4053     M.getContext().diagnose(DiagVersion);
4054   }
4055   return Modified;
4056 }
4057 
4058 /// This checks for objc retain release marker which should be upgraded. It
4059 /// returns true if module is modified.
4060 static bool UpgradeRetainReleaseMarker(Module &M) {
4061   bool Changed = false;
4062   const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
4063   NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4064   if (ModRetainReleaseMarker) {
4065     MDNode *Op = ModRetainReleaseMarker->getOperand(0);
4066     if (Op) {
4067       MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
4068       if (ID) {
4069         SmallVector<StringRef, 4> ValueComp;
4070         ID->getString().split(ValueComp, "#");
4071         if (ValueComp.size() == 2) {
4072           std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
4073           ID = MDString::get(M.getContext(), NewValue);
4074         }
4075         M.addModuleFlag(Module::Error, MarkerKey, ID);
4076         M.eraseNamedMetadata(ModRetainReleaseMarker);
4077         Changed = true;
4078       }
4079     }
4080   }
4081   return Changed;
4082 }
4083 
4084 void llvm::UpgradeARCRuntime(Module &M) {
4085   // This lambda converts normal function calls to ARC runtime functions to
4086   // intrinsic calls.
4087   auto UpgradeToIntrinsic = [&](const char *OldFunc,
4088                                 llvm::Intrinsic::ID IntrinsicFunc) {
4089     Function *Fn = M.getFunction(OldFunc);
4090 
4091     if (!Fn)
4092       return;
4093 
4094     Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
4095 
4096     for (User *U : make_early_inc_range(Fn->users())) {
4097       CallInst *CI = dyn_cast<CallInst>(U);
4098       if (!CI || CI->getCalledFunction() != Fn)
4099         continue;
4100 
4101       IRBuilder<> Builder(CI->getParent(), CI->getIterator());
4102       FunctionType *NewFuncTy = NewFn->getFunctionType();
4103       SmallVector<Value *, 2> Args;
4104 
4105       // Don't upgrade the intrinsic if it's not valid to bitcast the return
4106       // value to the return type of the old function.
4107       if (NewFuncTy->getReturnType() != CI->getType() &&
4108           !CastInst::castIsValid(Instruction::BitCast, CI,
4109                                  NewFuncTy->getReturnType()))
4110         continue;
4111 
4112       bool InvalidCast = false;
4113 
4114       for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
4115         Value *Arg = CI->getArgOperand(I);
4116 
4117         // Bitcast argument to the parameter type of the new function if it's
4118         // not a variadic argument.
4119         if (I < NewFuncTy->getNumParams()) {
4120           // Don't upgrade the intrinsic if it's not valid to bitcast the argument
4121           // to the parameter type of the new function.
4122           if (!CastInst::castIsValid(Instruction::BitCast, Arg,
4123                                      NewFuncTy->getParamType(I))) {
4124             InvalidCast = true;
4125             break;
4126           }
4127           Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
4128         }
4129         Args.push_back(Arg);
4130       }
4131 
4132       if (InvalidCast)
4133         continue;
4134 
4135       // Create a call instruction that calls the new function.
4136       CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
4137       NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4138       NewCall->takeName(CI);
4139 
4140       // Bitcast the return value back to the type of the old call.
4141       Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
4142 
4143       if (!CI->use_empty())
4144         CI->replaceAllUsesWith(NewRetVal);
4145       CI->eraseFromParent();
4146     }
4147 
4148     if (Fn->use_empty())
4149       Fn->eraseFromParent();
4150   };
4151 
4152   // Unconditionally convert a call to "clang.arc.use" to a call to
4153   // "llvm.objc.clang.arc.use".
4154   UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
4155 
4156   // Upgrade the retain release marker. If there is no need to upgrade
4157   // the marker, that means either the module is already new enough to contain
4158   // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
4159   if (!UpgradeRetainReleaseMarker(M))
4160     return;
4161 
4162   std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
4163       {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
4164       {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
4165       {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
4166       {"objc_autoreleaseReturnValue",
4167        llvm::Intrinsic::objc_autoreleaseReturnValue},
4168       {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
4169       {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
4170       {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
4171       {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
4172       {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
4173       {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
4174       {"objc_release", llvm::Intrinsic::objc_release},
4175       {"objc_retain", llvm::Intrinsic::objc_retain},
4176       {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
4177       {"objc_retainAutoreleaseReturnValue",
4178        llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
4179       {"objc_retainAutoreleasedReturnValue",
4180        llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
4181       {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
4182       {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
4183       {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
4184       {"objc_unsafeClaimAutoreleasedReturnValue",
4185        llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
4186       {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
4187       {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
4188       {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
4189       {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
4190       {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
4191       {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
4192       {"objc_arc_annotation_topdown_bbstart",
4193        llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
4194       {"objc_arc_annotation_topdown_bbend",
4195        llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
4196       {"objc_arc_annotation_bottomup_bbstart",
4197        llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
4198       {"objc_arc_annotation_bottomup_bbend",
4199        llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
4200 
4201   for (auto &I : RuntimeFuncs)
4202     UpgradeToIntrinsic(I.first, I.second);
4203 }
4204 
4205 bool llvm::UpgradeModuleFlags(Module &M) {
4206   NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
4207   if (!ModFlags)
4208     return false;
4209 
4210   bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
4211   bool HasSwiftVersionFlag = false;
4212   uint8_t SwiftMajorVersion, SwiftMinorVersion;
4213   uint32_t SwiftABIVersion;
4214   auto Int8Ty = Type::getInt8Ty(M.getContext());
4215   auto Int32Ty = Type::getInt32Ty(M.getContext());
4216 
4217   for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
4218     MDNode *Op = ModFlags->getOperand(I);
4219     if (Op->getNumOperands() != 3)
4220       continue;
4221     MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
4222     if (!ID)
4223       continue;
4224     if (ID->getString() == "Objective-C Image Info Version")
4225       HasObjCFlag = true;
4226     if (ID->getString() == "Objective-C Class Properties")
4227       HasClassProperties = true;
4228     // Upgrade PIC/PIE Module Flags. The module flag behavior for these two
4229     // field was Error and now they are Max.
4230     if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {
4231       if (auto *Behavior =
4232               mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
4233         if (Behavior->getLimitedValue() == Module::Error) {
4234           Type *Int32Ty = Type::getInt32Ty(M.getContext());
4235           Metadata *Ops[3] = {
4236               ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),
4237               MDString::get(M.getContext(), ID->getString()),
4238               Op->getOperand(2)};
4239           ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4240           Changed = true;
4241         }
4242       }
4243     }
4244     // Upgrade Objective-C Image Info Section. Removed the whitespce in the
4245     // section name so that llvm-lto will not complain about mismatching
4246     // module flags that is functionally the same.
4247     if (ID->getString() == "Objective-C Image Info Section") {
4248       if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
4249         SmallVector<StringRef, 4> ValueComp;
4250         Value->getString().split(ValueComp, " ");
4251         if (ValueComp.size() != 1) {
4252           std::string NewValue;
4253           for (auto &S : ValueComp)
4254             NewValue += S.str();
4255           Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
4256                               MDString::get(M.getContext(), NewValue)};
4257           ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4258           Changed = true;
4259         }
4260       }
4261     }
4262 
4263     // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
4264     // If the higher bits are set, it adds new module flag for swift info.
4265     if (ID->getString() == "Objective-C Garbage Collection") {
4266       auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
4267       if (Md) {
4268         assert(Md->getValue() && "Expected non-empty metadata");
4269         auto Type = Md->getValue()->getType();
4270         if (Type == Int8Ty)
4271           continue;
4272         unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
4273         if ((Val & 0xff) != Val) {
4274           HasSwiftVersionFlag = true;
4275           SwiftABIVersion = (Val & 0xff00) >> 8;
4276           SwiftMajorVersion = (Val & 0xff000000) >> 24;
4277           SwiftMinorVersion = (Val & 0xff0000) >> 16;
4278         }
4279         Metadata *Ops[3] = {
4280           ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
4281           Op->getOperand(1),
4282           ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
4283         ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4284         Changed = true;
4285       }
4286     }
4287   }
4288 
4289   // "Objective-C Class Properties" is recently added for Objective-C. We
4290   // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
4291   // flag of value 0, so we can correclty downgrade this flag when trying to
4292   // link an ObjC bitcode without this module flag with an ObjC bitcode with
4293   // this module flag.
4294   if (HasObjCFlag && !HasClassProperties) {
4295     M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
4296                     (uint32_t)0);
4297     Changed = true;
4298   }
4299 
4300   if (HasSwiftVersionFlag) {
4301     M.addModuleFlag(Module::Error, "Swift ABI Version",
4302                     SwiftABIVersion);
4303     M.addModuleFlag(Module::Error, "Swift Major Version",
4304                     ConstantInt::get(Int8Ty, SwiftMajorVersion));
4305     M.addModuleFlag(Module::Error, "Swift Minor Version",
4306                     ConstantInt::get(Int8Ty, SwiftMinorVersion));
4307     Changed = true;
4308   }
4309 
4310   return Changed;
4311 }
4312 
4313 void llvm::UpgradeSectionAttributes(Module &M) {
4314   auto TrimSpaces = [](StringRef Section) -> std::string {
4315     SmallVector<StringRef, 5> Components;
4316     Section.split(Components, ',');
4317 
4318     SmallString<32> Buffer;
4319     raw_svector_ostream OS(Buffer);
4320 
4321     for (auto Component : Components)
4322       OS << ',' << Component.trim();
4323 
4324     return std::string(OS.str().substr(1));
4325   };
4326 
4327   for (auto &GV : M.globals()) {
4328     if (!GV.hasSection())
4329       continue;
4330 
4331     StringRef Section = GV.getSection();
4332 
4333     if (!Section.startswith("__DATA, __objc_catlist"))
4334       continue;
4335 
4336     // __DATA, __objc_catlist, regular, no_dead_strip
4337     // __DATA,__objc_catlist,regular,no_dead_strip
4338     GV.setSection(TrimSpaces(Section));
4339   }
4340 }
4341 
4342 namespace {
4343 // Prior to LLVM 10.0, the strictfp attribute could be used on individual
4344 // callsites within a function that did not also have the strictfp attribute.
4345 // Since 10.0, if strict FP semantics are needed within a function, the
4346 // function must have the strictfp attribute and all calls within the function
4347 // must also have the strictfp attribute. This latter restriction is
4348 // necessary to prevent unwanted libcall simplification when a function is
4349 // being cloned (such as for inlining).
4350 //
4351 // The "dangling" strictfp attribute usage was only used to prevent constant
4352 // folding and other libcall simplification. The nobuiltin attribute on the
4353 // callsite has the same effect.
4354 struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
4355   StrictFPUpgradeVisitor() {}
4356 
4357   void visitCallBase(CallBase &Call) {
4358     if (!Call.isStrictFP())
4359       return;
4360     if (isa<ConstrainedFPIntrinsic>(&Call))
4361       return;
4362     // If we get here, the caller doesn't have the strictfp attribute
4363     // but this callsite does. Replace the strictfp attribute with nobuiltin.
4364     Call.removeAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
4365     Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);
4366   }
4367 };
4368 } // namespace
4369 
4370 void llvm::UpgradeFunctionAttributes(Function &F) {
4371   // If a function definition doesn't have the strictfp attribute,
4372   // convert any callsite strictfp attributes to nobuiltin.
4373   if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
4374     StrictFPUpgradeVisitor SFPV;
4375     SFPV.visit(F);
4376   }
4377 
4378   if (F.getCallingConv() == CallingConv::X86_INTR &&
4379       !F.arg_empty() && !F.hasParamAttribute(0, Attribute::ByVal)) {
4380     Type *ByValTy = cast<PointerType>(F.getArg(0)->getType())->getElementType();
4381     Attribute NewAttr = Attribute::getWithByValType(F.getContext(), ByValTy);
4382     F.addParamAttr(0, NewAttr);
4383   }
4384 
4385   // Remove all incompatibile attributes from function.
4386   F.removeAttributes(AttributeList::ReturnIndex,
4387                      AttributeFuncs::typeIncompatible(F.getReturnType()));
4388   for (auto &Arg : F.args())
4389     Arg.removeAttrs(AttributeFuncs::typeIncompatible(Arg.getType()));
4390 }
4391 
4392 static bool isOldLoopArgument(Metadata *MD) {
4393   auto *T = dyn_cast_or_null<MDTuple>(MD);
4394   if (!T)
4395     return false;
4396   if (T->getNumOperands() < 1)
4397     return false;
4398   auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
4399   if (!S)
4400     return false;
4401   return S->getString().startswith("llvm.vectorizer.");
4402 }
4403 
4404 static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
4405   StringRef OldPrefix = "llvm.vectorizer.";
4406   assert(OldTag.startswith(OldPrefix) && "Expected old prefix");
4407 
4408   if (OldTag == "llvm.vectorizer.unroll")
4409     return MDString::get(C, "llvm.loop.interleave.count");
4410 
4411   return MDString::get(
4412       C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
4413              .str());
4414 }
4415 
4416 static Metadata *upgradeLoopArgument(Metadata *MD) {
4417   auto *T = dyn_cast_or_null<MDTuple>(MD);
4418   if (!T)
4419     return MD;
4420   if (T->getNumOperands() < 1)
4421     return MD;
4422   auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
4423   if (!OldTag)
4424     return MD;
4425   if (!OldTag->getString().startswith("llvm.vectorizer."))
4426     return MD;
4427 
4428   // This has an old tag.  Upgrade it.
4429   SmallVector<Metadata *, 8> Ops;
4430   Ops.reserve(T->getNumOperands());
4431   Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
4432   for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
4433     Ops.push_back(T->getOperand(I));
4434 
4435   return MDTuple::get(T->getContext(), Ops);
4436 }
4437 
4438 MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
4439   auto *T = dyn_cast<MDTuple>(&N);
4440   if (!T)
4441     return &N;
4442 
4443   if (none_of(T->operands(), isOldLoopArgument))
4444     return &N;
4445 
4446   SmallVector<Metadata *, 8> Ops;
4447   Ops.reserve(T->getNumOperands());
4448   for (Metadata *MD : T->operands())
4449     Ops.push_back(upgradeLoopArgument(MD));
4450 
4451   return MDTuple::get(T->getContext(), Ops);
4452 }
4453 
4454 std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
4455   Triple T(TT);
4456   // For AMDGPU we uprgrade older DataLayouts to include the default globals
4457   // address space of 1.
4458   if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) {
4459     return DL.empty() ? std::string("G1") : (DL + "-G1").str();
4460   }
4461 
4462   std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
4463   // If X86, and the datalayout matches the expected format, add pointer size
4464   // address spaces to the datalayout.
4465   if (!T.isX86() || DL.contains(AddrSpaces))
4466     return std::string(DL);
4467 
4468   SmallVector<StringRef, 4> Groups;
4469   Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
4470   if (!R.match(DL, &Groups))
4471     return std::string(DL);
4472 
4473   return (Groups[1] + AddrSpaces + Groups[3]).str();
4474 }
4475 
4476 void llvm::UpgradeAttributes(AttrBuilder &B) {
4477   StringRef FramePointer;
4478   if (B.contains("no-frame-pointer-elim")) {
4479     // The value can be "true" or "false".
4480     for (const auto &I : B.td_attrs())
4481       if (I.first == "no-frame-pointer-elim")
4482         FramePointer = I.second == "true" ? "all" : "none";
4483     B.removeAttribute("no-frame-pointer-elim");
4484   }
4485   if (B.contains("no-frame-pointer-elim-non-leaf")) {
4486     // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
4487     if (FramePointer != "all")
4488       FramePointer = "non-leaf";
4489     B.removeAttribute("no-frame-pointer-elim-non-leaf");
4490   }
4491   if (!FramePointer.empty())
4492     B.addAttribute("frame-pointer", FramePointer);
4493 
4494   if (B.contains("null-pointer-is-valid")) {
4495     // The value can be "true" or "false".
4496     bool NullPointerIsValid = false;
4497     for (const auto &I : B.td_attrs())
4498       if (I.first == "null-pointer-is-valid")
4499         NullPointerIsValid = I.second == "true";
4500     B.removeAttribute("null-pointer-is-valid");
4501     if (NullPointerIsValid)
4502       B.addAttribute(Attribute::NullPointerIsValid);
4503   }
4504 }
4505