xref: /freebsd/contrib/llvm-project/llvm/lib/IR/AutoUpgrade.cpp (revision ae7e8a02e6e93455e026036132c4d053b2c12ad9)
1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the auto-upgrade helper functions.
10 // This is where deprecated IR intrinsics and other IR features are updated to
11 // current specifications.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/IR/AutoUpgrade.h"
16 #include "llvm/ADT/StringSwitch.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instruction.h"
24 #include "llvm/IR/InstVisitor.h"
25 #include "llvm/IR/IntrinsicInst.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include "llvm/IR/IntrinsicsAArch64.h"
28 #include "llvm/IR/IntrinsicsARM.h"
29 #include "llvm/IR/IntrinsicsX86.h"
30 #include "llvm/IR/LLVMContext.h"
31 #include "llvm/IR/Module.h"
32 #include "llvm/IR/Verifier.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/Regex.h"
35 #include <cstring>
36 using namespace llvm;
37 
38 static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
39 
40 // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
41 // changed their type from v4f32 to v2i64.
42 static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
43                                   Function *&NewFn) {
44   // Check whether this is an old version of the function, which received
45   // v4f32 arguments.
46   Type *Arg0Type = F->getFunctionType()->getParamType(0);
47   if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
48     return false;
49 
50   // Yes, it's old, replace it with new version.
51   rename(F);
52   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
53   return true;
54 }
55 
56 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
57 // arguments have changed their type from i32 to i8.
58 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
59                                              Function *&NewFn) {
60   // Check that the last argument is an i32.
61   Type *LastArgType = F->getFunctionType()->getParamType(
62      F->getFunctionType()->getNumParams() - 1);
63   if (!LastArgType->isIntegerTy(32))
64     return false;
65 
66   // Move this function aside and map down.
67   rename(F);
68   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
69   return true;
70 }
71 
72 // Upgrade the declaration of fp compare intrinsics that change return type
73 // from scalar to vXi1 mask.
74 static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
75                                       Function *&NewFn) {
76   // Check if the return type is a vector.
77   if (F->getReturnType()->isVectorTy())
78     return false;
79 
80   rename(F);
81   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
82   return true;
83 }
84 
85 static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
86   // All of the intrinsics matches below should be marked with which llvm
87   // version started autoupgrading them. At some point in the future we would
88   // like to use this information to remove upgrade code for some older
89   // intrinsics. It is currently undecided how we will determine that future
90   // point.
91   if (Name == "addcarryx.u32" || // Added in 8.0
92       Name == "addcarryx.u64" || // Added in 8.0
93       Name == "addcarry.u32" || // Added in 8.0
94       Name == "addcarry.u64" || // Added in 8.0
95       Name == "subborrow.u32" || // Added in 8.0
96       Name == "subborrow.u64" || // Added in 8.0
97       Name.startswith("sse2.padds.") || // Added in 8.0
98       Name.startswith("sse2.psubs.") || // Added in 8.0
99       Name.startswith("sse2.paddus.") || // Added in 8.0
100       Name.startswith("sse2.psubus.") || // Added in 8.0
101       Name.startswith("avx2.padds.") || // Added in 8.0
102       Name.startswith("avx2.psubs.") || // Added in 8.0
103       Name.startswith("avx2.paddus.") || // Added in 8.0
104       Name.startswith("avx2.psubus.") || // Added in 8.0
105       Name.startswith("avx512.padds.") || // Added in 8.0
106       Name.startswith("avx512.psubs.") || // Added in 8.0
107       Name.startswith("avx512.mask.padds.") || // Added in 8.0
108       Name.startswith("avx512.mask.psubs.") || // Added in 8.0
109       Name.startswith("avx512.mask.paddus.") || // Added in 8.0
110       Name.startswith("avx512.mask.psubus.") || // Added in 8.0
111       Name=="ssse3.pabs.b.128" || // Added in 6.0
112       Name=="ssse3.pabs.w.128" || // Added in 6.0
113       Name=="ssse3.pabs.d.128" || // Added in 6.0
114       Name.startswith("fma4.vfmadd.s") || // Added in 7.0
115       Name.startswith("fma.vfmadd.") || // Added in 7.0
116       Name.startswith("fma.vfmsub.") || // Added in 7.0
117       Name.startswith("fma.vfmsubadd.") || // Added in 7.0
118       Name.startswith("fma.vfnmadd.") || // Added in 7.0
119       Name.startswith("fma.vfnmsub.") || // Added in 7.0
120       Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
121       Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
122       Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
123       Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
124       Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
125       Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
126       Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
127       Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
128       Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
129       Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
130       Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
131       Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
132       Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
133       Name.startswith("avx512.kunpck") || //added in 6.0
134       Name.startswith("avx2.pabs.") || // Added in 6.0
135       Name.startswith("avx512.mask.pabs.") || // Added in 6.0
136       Name.startswith("avx512.broadcastm") || // Added in 6.0
137       Name == "sse.sqrt.ss" || // Added in 7.0
138       Name == "sse2.sqrt.sd" || // Added in 7.0
139       Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
140       Name.startswith("avx.sqrt.p") || // Added in 7.0
141       Name.startswith("sse2.sqrt.p") || // Added in 7.0
142       Name.startswith("sse.sqrt.p") || // Added in 7.0
143       Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
144       Name.startswith("sse2.pcmpeq.") || // Added in 3.1
145       Name.startswith("sse2.pcmpgt.") || // Added in 3.1
146       Name.startswith("avx2.pcmpeq.") || // Added in 3.1
147       Name.startswith("avx2.pcmpgt.") || // Added in 3.1
148       Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
149       Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
150       Name.startswith("avx.vperm2f128.") || // Added in 6.0
151       Name == "avx2.vperm2i128" || // Added in 6.0
152       Name == "sse.add.ss" || // Added in 4.0
153       Name == "sse2.add.sd" || // Added in 4.0
154       Name == "sse.sub.ss" || // Added in 4.0
155       Name == "sse2.sub.sd" || // Added in 4.0
156       Name == "sse.mul.ss" || // Added in 4.0
157       Name == "sse2.mul.sd" || // Added in 4.0
158       Name == "sse.div.ss" || // Added in 4.0
159       Name == "sse2.div.sd" || // Added in 4.0
160       Name == "sse41.pmaxsb" || // Added in 3.9
161       Name == "sse2.pmaxs.w" || // Added in 3.9
162       Name == "sse41.pmaxsd" || // Added in 3.9
163       Name == "sse2.pmaxu.b" || // Added in 3.9
164       Name == "sse41.pmaxuw" || // Added in 3.9
165       Name == "sse41.pmaxud" || // Added in 3.9
166       Name == "sse41.pminsb" || // Added in 3.9
167       Name == "sse2.pmins.w" || // Added in 3.9
168       Name == "sse41.pminsd" || // Added in 3.9
169       Name == "sse2.pminu.b" || // Added in 3.9
170       Name == "sse41.pminuw" || // Added in 3.9
171       Name == "sse41.pminud" || // Added in 3.9
172       Name == "avx512.kand.w" || // Added in 7.0
173       Name == "avx512.kandn.w" || // Added in 7.0
174       Name == "avx512.knot.w" || // Added in 7.0
175       Name == "avx512.kor.w" || // Added in 7.0
176       Name == "avx512.kxor.w" || // Added in 7.0
177       Name == "avx512.kxnor.w" || // Added in 7.0
178       Name == "avx512.kortestc.w" || // Added in 7.0
179       Name == "avx512.kortestz.w" || // Added in 7.0
180       Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
181       Name.startswith("avx2.pmax") || // Added in 3.9
182       Name.startswith("avx2.pmin") || // Added in 3.9
183       Name.startswith("avx512.mask.pmax") || // Added in 4.0
184       Name.startswith("avx512.mask.pmin") || // Added in 4.0
185       Name.startswith("avx2.vbroadcast") || // Added in 3.8
186       Name.startswith("avx2.pbroadcast") || // Added in 3.8
187       Name.startswith("avx.vpermil.") || // Added in 3.1
188       Name.startswith("sse2.pshuf") || // Added in 3.9
189       Name.startswith("avx512.pbroadcast") || // Added in 3.9
190       Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
191       Name.startswith("avx512.mask.movddup") || // Added in 3.9
192       Name.startswith("avx512.mask.movshdup") || // Added in 3.9
193       Name.startswith("avx512.mask.movsldup") || // Added in 3.9
194       Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
195       Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
196       Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
197       Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
198       Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
199       Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
200       Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
201       Name.startswith("avx512.mask.punpckl") || // Added in 3.9
202       Name.startswith("avx512.mask.punpckh") || // Added in 3.9
203       Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
204       Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
205       Name.startswith("avx512.mask.pand.") || // Added in 3.9
206       Name.startswith("avx512.mask.pandn.") || // Added in 3.9
207       Name.startswith("avx512.mask.por.") || // Added in 3.9
208       Name.startswith("avx512.mask.pxor.") || // Added in 3.9
209       Name.startswith("avx512.mask.and.") || // Added in 3.9
210       Name.startswith("avx512.mask.andn.") || // Added in 3.9
211       Name.startswith("avx512.mask.or.") || // Added in 3.9
212       Name.startswith("avx512.mask.xor.") || // Added in 3.9
213       Name.startswith("avx512.mask.padd.") || // Added in 4.0
214       Name.startswith("avx512.mask.psub.") || // Added in 4.0
215       Name.startswith("avx512.mask.pmull.") || // Added in 4.0
216       Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
217       Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
218       Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
219       Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
220       Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
221       Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
222       Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0
223       Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0
224       Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
225       Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
226       Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
227       Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
228       Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
229       Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
230       Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
231       Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
232       Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
233       Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
234       Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
235       Name == "avx512.cvtusi2sd" || // Added in 7.0
236       Name.startswith("avx512.mask.permvar.") || // Added in 7.0
237       Name == "sse2.pmulu.dq" || // Added in 7.0
238       Name == "sse41.pmuldq" || // Added in 7.0
239       Name == "avx2.pmulu.dq" || // Added in 7.0
240       Name == "avx2.pmul.dq" || // Added in 7.0
241       Name == "avx512.pmulu.dq.512" || // Added in 7.0
242       Name == "avx512.pmul.dq.512" || // Added in 7.0
243       Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
244       Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
245       Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
246       Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
247       Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
248       Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
249       Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
250       Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
251       Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
252       Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
253       Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
254       Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
255       Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
256       Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
257       Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
258       Name.startswith("avx512.cmp.p") || // Added in 12.0
259       Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
260       Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
261       Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
262       Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
263       Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
264       Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
265       Name.startswith("avx512.mask.psll.d") || // Added in 4.0
266       Name.startswith("avx512.mask.psll.q") || // Added in 4.0
267       Name.startswith("avx512.mask.psll.w") || // Added in 4.0
268       Name.startswith("avx512.mask.psra.d") || // Added in 4.0
269       Name.startswith("avx512.mask.psra.q") || // Added in 4.0
270       Name.startswith("avx512.mask.psra.w") || // Added in 4.0
271       Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
272       Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
273       Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
274       Name.startswith("avx512.mask.pslli") || // Added in 4.0
275       Name.startswith("avx512.mask.psrai") || // Added in 4.0
276       Name.startswith("avx512.mask.psrli") || // Added in 4.0
277       Name.startswith("avx512.mask.psllv") || // Added in 4.0
278       Name.startswith("avx512.mask.psrav") || // Added in 4.0
279       Name.startswith("avx512.mask.psrlv") || // Added in 4.0
280       Name.startswith("sse41.pmovsx") || // Added in 3.8
281       Name.startswith("sse41.pmovzx") || // Added in 3.9
282       Name.startswith("avx2.pmovsx") || // Added in 3.9
283       Name.startswith("avx2.pmovzx") || // Added in 3.9
284       Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
285       Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
286       Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
287       Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
288       Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
289       Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
290       Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
291       Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
292       Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
293       Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
294       Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
295       Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
296       Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
297       Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
298       Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
299       Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
300       Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
301       Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
302       Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
303       Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
304       Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
305       Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
306       Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
307       Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
308       Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
309       Name.startswith("avx512.vpshld.") || // Added in 8.0
310       Name.startswith("avx512.vpshrd.") || // Added in 8.0
311       Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
312       Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
313       Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
314       Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
315       Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
316       Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
317       Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
318       Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
319       Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
320       Name.startswith("avx512.mask.conflict.") || // Added in 9.0
321       Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
322       Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
323       Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
324       Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
325       Name == "sse.cvtsi2ss" || // Added in 7.0
326       Name == "sse.cvtsi642ss" || // Added in 7.0
327       Name == "sse2.cvtsi2sd" || // Added in 7.0
328       Name == "sse2.cvtsi642sd" || // Added in 7.0
329       Name == "sse2.cvtss2sd" || // Added in 7.0
330       Name == "sse2.cvtdq2pd" || // Added in 3.9
331       Name == "sse2.cvtdq2ps" || // Added in 7.0
332       Name == "sse2.cvtps2pd" || // Added in 3.9
333       Name == "avx.cvtdq2.pd.256" || // Added in 3.9
334       Name == "avx.cvtdq2.ps.256" || // Added in 7.0
335       Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
336       Name.startswith("vcvtph2ps.") || // Added in 11.0
337       Name.startswith("avx.vinsertf128.") || // Added in 3.7
338       Name == "avx2.vinserti128" || // Added in 3.7
339       Name.startswith("avx512.mask.insert") || // Added in 4.0
340       Name.startswith("avx.vextractf128.") || // Added in 3.7
341       Name == "avx2.vextracti128" || // Added in 3.7
342       Name.startswith("avx512.mask.vextract") || // Added in 4.0
343       Name.startswith("sse4a.movnt.") || // Added in 3.9
344       Name.startswith("avx.movnt.") || // Added in 3.2
345       Name.startswith("avx512.storent.") || // Added in 3.9
346       Name == "sse41.movntdqa" || // Added in 5.0
347       Name == "avx2.movntdqa" || // Added in 5.0
348       Name == "avx512.movntdqa" || // Added in 5.0
349       Name == "sse2.storel.dq" || // Added in 3.9
350       Name.startswith("sse.storeu.") || // Added in 3.9
351       Name.startswith("sse2.storeu.") || // Added in 3.9
352       Name.startswith("avx.storeu.") || // Added in 3.9
353       Name.startswith("avx512.mask.storeu.") || // Added in 3.9
354       Name.startswith("avx512.mask.store.p") || // Added in 3.9
355       Name.startswith("avx512.mask.store.b.") || // Added in 3.9
356       Name.startswith("avx512.mask.store.w.") || // Added in 3.9
357       Name.startswith("avx512.mask.store.d.") || // Added in 3.9
358       Name.startswith("avx512.mask.store.q.") || // Added in 3.9
359       Name == "avx512.mask.store.ss" || // Added in 7.0
360       Name.startswith("avx512.mask.loadu.") || // Added in 3.9
361       Name.startswith("avx512.mask.load.") || // Added in 3.9
362       Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
363       Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
364       Name.startswith("avx512.mask.expand.b") || // Added in 9.0
365       Name.startswith("avx512.mask.expand.w") || // Added in 9.0
366       Name.startswith("avx512.mask.expand.d") || // Added in 9.0
367       Name.startswith("avx512.mask.expand.q") || // Added in 9.0
368       Name.startswith("avx512.mask.expand.p") || // Added in 9.0
369       Name.startswith("avx512.mask.compress.b") || // Added in 9.0
370       Name.startswith("avx512.mask.compress.w") || // Added in 9.0
371       Name.startswith("avx512.mask.compress.d") || // Added in 9.0
372       Name.startswith("avx512.mask.compress.q") || // Added in 9.0
373       Name.startswith("avx512.mask.compress.p") || // Added in 9.0
374       Name == "sse42.crc32.64.8" || // Added in 3.4
375       Name.startswith("avx.vbroadcast.s") || // Added in 3.5
376       Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
377       Name.startswith("avx512.mask.palignr.") || // Added in 3.9
378       Name.startswith("avx512.mask.valign.") || // Added in 4.0
379       Name.startswith("sse2.psll.dq") || // Added in 3.7
380       Name.startswith("sse2.psrl.dq") || // Added in 3.7
381       Name.startswith("avx2.psll.dq") || // Added in 3.7
382       Name.startswith("avx2.psrl.dq") || // Added in 3.7
383       Name.startswith("avx512.psll.dq") || // Added in 3.9
384       Name.startswith("avx512.psrl.dq") || // Added in 3.9
385       Name == "sse41.pblendw" || // Added in 3.7
386       Name.startswith("sse41.blendp") || // Added in 3.7
387       Name.startswith("avx.blend.p") || // Added in 3.7
388       Name == "avx2.pblendw" || // Added in 3.7
389       Name.startswith("avx2.pblendd.") || // Added in 3.7
390       Name.startswith("avx.vbroadcastf128") || // Added in 4.0
391       Name == "avx2.vbroadcasti128" || // Added in 3.7
392       Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0
393       Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0
394       Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0
395       Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0
396       Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0
397       Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0
398       Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0
399       Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0
400       Name == "xop.vpcmov" || // Added in 3.8
401       Name == "xop.vpcmov.256" || // Added in 5.0
402       Name.startswith("avx512.mask.move.s") || // Added in 4.0
403       Name.startswith("avx512.cvtmask2") || // Added in 5.0
404       Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
405       Name.startswith("xop.vprot") || // Added in 8.0
406       Name.startswith("avx512.prol") || // Added in 8.0
407       Name.startswith("avx512.pror") || // Added in 8.0
408       Name.startswith("avx512.mask.prorv.") || // Added in 8.0
409       Name.startswith("avx512.mask.pror.") ||  // Added in 8.0
410       Name.startswith("avx512.mask.prolv.") || // Added in 8.0
411       Name.startswith("avx512.mask.prol.") ||  // Added in 8.0
412       Name.startswith("avx512.ptestm") || //Added in 6.0
413       Name.startswith("avx512.ptestnm") || //Added in 6.0
414       Name.startswith("avx512.mask.pavg")) // Added in 6.0
415     return true;
416 
417   return false;
418 }
419 
420 static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
421                                         Function *&NewFn) {
422   // Only handle intrinsics that start with "x86.".
423   if (!Name.startswith("x86."))
424     return false;
425   // Remove "x86." prefix.
426   Name = Name.substr(4);
427 
428   if (ShouldUpgradeX86Intrinsic(F, Name)) {
429     NewFn = nullptr;
430     return true;
431   }
432 
433   if (Name == "rdtscp") { // Added in 8.0
434     // If this intrinsic has 0 operands, it's the new version.
435     if (F->getFunctionType()->getNumParams() == 0)
436       return false;
437 
438     rename(F);
439     NewFn = Intrinsic::getDeclaration(F->getParent(),
440                                       Intrinsic::x86_rdtscp);
441     return true;
442   }
443 
444   // SSE4.1 ptest functions may have an old signature.
445   if (Name.startswith("sse41.ptest")) { // Added in 3.2
446     if (Name.substr(11) == "c")
447       return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
448     if (Name.substr(11) == "z")
449       return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
450     if (Name.substr(11) == "nzc")
451       return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
452   }
453   // Several blend and other instructions with masks used the wrong number of
454   // bits.
455   if (Name == "sse41.insertps") // Added in 3.6
456     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
457                                             NewFn);
458   if (Name == "sse41.dppd") // Added in 3.6
459     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
460                                             NewFn);
461   if (Name == "sse41.dpps") // Added in 3.6
462     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
463                                             NewFn);
464   if (Name == "sse41.mpsadbw") // Added in 3.6
465     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
466                                             NewFn);
467   if (Name == "avx.dp.ps.256") // Added in 3.6
468     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
469                                             NewFn);
470   if (Name == "avx2.mpsadbw") // Added in 3.6
471     return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
472                                             NewFn);
473   if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
474     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
475                                      NewFn);
476   if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
477     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
478                                      NewFn);
479   if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
480     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
481                                      NewFn);
482   if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
483     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
484                                      NewFn);
485   if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
486     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
487                                      NewFn);
488   if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
489     return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
490                                      NewFn);
491 
492   // frcz.ss/sd may need to have an argument dropped. Added in 3.2
493   if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
494     rename(F);
495     NewFn = Intrinsic::getDeclaration(F->getParent(),
496                                       Intrinsic::x86_xop_vfrcz_ss);
497     return true;
498   }
499   if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
500     rename(F);
501     NewFn = Intrinsic::getDeclaration(F->getParent(),
502                                       Intrinsic::x86_xop_vfrcz_sd);
503     return true;
504   }
505   // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
506   if (Name.startswith("xop.vpermil2")) { // Added in 3.9
507     auto Idx = F->getFunctionType()->getParamType(2);
508     if (Idx->isFPOrFPVectorTy()) {
509       rename(F);
510       unsigned IdxSize = Idx->getPrimitiveSizeInBits();
511       unsigned EltSize = Idx->getScalarSizeInBits();
512       Intrinsic::ID Permil2ID;
513       if (EltSize == 64 && IdxSize == 128)
514         Permil2ID = Intrinsic::x86_xop_vpermil2pd;
515       else if (EltSize == 32 && IdxSize == 128)
516         Permil2ID = Intrinsic::x86_xop_vpermil2ps;
517       else if (EltSize == 64 && IdxSize == 256)
518         Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
519       else
520         Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
521       NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
522       return true;
523     }
524   }
525 
526   if (Name == "seh.recoverfp") {
527     NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
528     return true;
529   }
530 
531   return false;
532 }
533 
534 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
535   assert(F && "Illegal to upgrade a non-existent Function.");
536 
537   // Quickly eliminate it, if it's not a candidate.
538   StringRef Name = F->getName();
539   if (Name.size() <= 8 || !Name.startswith("llvm."))
540     return false;
541   Name = Name.substr(5); // Strip off "llvm."
542 
543   switch (Name[0]) {
544   default: break;
545   case 'a': {
546     if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
547       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
548                                         F->arg_begin()->getType());
549       return true;
550     }
551     if (Name.startswith("arm.neon.vclz")) {
552       Type* args[2] = {
553         F->arg_begin()->getType(),
554         Type::getInt1Ty(F->getContext())
555       };
556       // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
557       // the end of the name. Change name from llvm.arm.neon.vclz.* to
558       //  llvm.ctlz.*
559       FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
560       NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
561                                "llvm.ctlz." + Name.substr(14), F->getParent());
562       return true;
563     }
564     if (Name.startswith("arm.neon.vcnt")) {
565       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
566                                         F->arg_begin()->getType());
567       return true;
568     }
569     static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
570     if (vldRegex.match(Name)) {
571       auto fArgs = F->getFunctionType()->params();
572       SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
573       // Can't use Intrinsic::getDeclaration here as the return types might
574       // then only be structurally equal.
575       FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);
576       NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
577                                "llvm." + Name + ".p0i8", F->getParent());
578       return true;
579     }
580     static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
581     if (vstRegex.match(Name)) {
582       static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
583                                                 Intrinsic::arm_neon_vst2,
584                                                 Intrinsic::arm_neon_vst3,
585                                                 Intrinsic::arm_neon_vst4};
586 
587       static const Intrinsic::ID StoreLaneInts[] = {
588         Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
589         Intrinsic::arm_neon_vst4lane
590       };
591 
592       auto fArgs = F->getFunctionType()->params();
593       Type *Tys[] = {fArgs[0], fArgs[1]};
594       if (Name.find("lane") == StringRef::npos)
595         NewFn = Intrinsic::getDeclaration(F->getParent(),
596                                           StoreInts[fArgs.size() - 3], Tys);
597       else
598         NewFn = Intrinsic::getDeclaration(F->getParent(),
599                                           StoreLaneInts[fArgs.size() - 5], Tys);
600       return true;
601     }
602     if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
603       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
604       return true;
605     }
606     if (Name.startswith("arm.neon.vqadds.")) {
607       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
608                                         F->arg_begin()->getType());
609       return true;
610     }
611     if (Name.startswith("arm.neon.vqaddu.")) {
612       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
613                                         F->arg_begin()->getType());
614       return true;
615     }
616     if (Name.startswith("arm.neon.vqsubs.")) {
617       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
618                                         F->arg_begin()->getType());
619       return true;
620     }
621     if (Name.startswith("arm.neon.vqsubu.")) {
622       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
623                                         F->arg_begin()->getType());
624       return true;
625     }
626     if (Name.startswith("aarch64.neon.addp")) {
627       if (F->arg_size() != 2)
628         break; // Invalid IR.
629       VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
630       if (Ty && Ty->getElementType()->isFloatingPointTy()) {
631         NewFn = Intrinsic::getDeclaration(F->getParent(),
632                                           Intrinsic::aarch64_neon_faddp, Ty);
633         return true;
634       }
635     }
636 
637     // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
638     // respectively
639     if ((Name.startswith("arm.neon.bfdot.") ||
640          Name.startswith("aarch64.neon.bfdot.")) &&
641         Name.endswith("i8")) {
642       Intrinsic::ID IID =
643           StringSwitch<Intrinsic::ID>(Name)
644               .Cases("arm.neon.bfdot.v2f32.v8i8",
645                      "arm.neon.bfdot.v4f32.v16i8",
646                      Intrinsic::arm_neon_bfdot)
647               .Cases("aarch64.neon.bfdot.v2f32.v8i8",
648                      "aarch64.neon.bfdot.v4f32.v16i8",
649                      Intrinsic::aarch64_neon_bfdot)
650               .Default(Intrinsic::not_intrinsic);
651       if (IID == Intrinsic::not_intrinsic)
652         break;
653 
654       size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
655       assert((OperandWidth == 64 || OperandWidth == 128) &&
656              "Unexpected operand width");
657       LLVMContext &Ctx = F->getParent()->getContext();
658       std::array<Type *, 2> Tys {{
659         F->getReturnType(),
660         FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
661       }};
662       NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
663       return true;
664     }
665 
666     // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
667     // and accept v8bf16 instead of v16i8
668     if ((Name.startswith("arm.neon.bfm") ||
669          Name.startswith("aarch64.neon.bfm")) &&
670         Name.endswith(".v4f32.v16i8")) {
671       Intrinsic::ID IID =
672           StringSwitch<Intrinsic::ID>(Name)
673               .Case("arm.neon.bfmmla.v4f32.v16i8",
674                     Intrinsic::arm_neon_bfmmla)
675               .Case("arm.neon.bfmlalb.v4f32.v16i8",
676                     Intrinsic::arm_neon_bfmlalb)
677               .Case("arm.neon.bfmlalt.v4f32.v16i8",
678                     Intrinsic::arm_neon_bfmlalt)
679               .Case("aarch64.neon.bfmmla.v4f32.v16i8",
680                     Intrinsic::aarch64_neon_bfmmla)
681               .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
682                     Intrinsic::aarch64_neon_bfmlalb)
683               .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
684                     Intrinsic::aarch64_neon_bfmlalt)
685               .Default(Intrinsic::not_intrinsic);
686       if (IID == Intrinsic::not_intrinsic)
687         break;
688 
689       std::array<Type *, 0> Tys;
690       NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
691       return true;
692     }
693     break;
694   }
695 
696   case 'c': {
697     if (Name.startswith("ctlz.") && F->arg_size() == 1) {
698       rename(F);
699       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
700                                         F->arg_begin()->getType());
701       return true;
702     }
703     if (Name.startswith("cttz.") && F->arg_size() == 1) {
704       rename(F);
705       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
706                                         F->arg_begin()->getType());
707       return true;
708     }
709     break;
710   }
711   case 'd': {
712     if (Name == "dbg.value" && F->arg_size() == 4) {
713       rename(F);
714       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
715       return true;
716     }
717     break;
718   }
719   case 'e': {
720     SmallVector<StringRef, 2> Groups;
721     static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
722     if (R.match(Name, &Groups)) {
723       Intrinsic::ID ID;
724       ID = StringSwitch<Intrinsic::ID>(Groups[1])
725                .Case("add", Intrinsic::vector_reduce_add)
726                .Case("mul", Intrinsic::vector_reduce_mul)
727                .Case("and", Intrinsic::vector_reduce_and)
728                .Case("or", Intrinsic::vector_reduce_or)
729                .Case("xor", Intrinsic::vector_reduce_xor)
730                .Case("smax", Intrinsic::vector_reduce_smax)
731                .Case("smin", Intrinsic::vector_reduce_smin)
732                .Case("umax", Intrinsic::vector_reduce_umax)
733                .Case("umin", Intrinsic::vector_reduce_umin)
734                .Case("fmax", Intrinsic::vector_reduce_fmax)
735                .Case("fmin", Intrinsic::vector_reduce_fmin)
736                .Default(Intrinsic::not_intrinsic);
737       if (ID != Intrinsic::not_intrinsic) {
738         rename(F);
739         auto Args = F->getFunctionType()->params();
740         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});
741         return true;
742       }
743     }
744     static const Regex R2(
745         "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
746     Groups.clear();
747     if (R2.match(Name, &Groups)) {
748       Intrinsic::ID ID = Intrinsic::not_intrinsic;
749       if (Groups[1] == "fadd")
750         ID = Intrinsic::vector_reduce_fadd;
751       if (Groups[1] == "fmul")
752         ID = Intrinsic::vector_reduce_fmul;
753       if (ID != Intrinsic::not_intrinsic) {
754         rename(F);
755         auto Args = F->getFunctionType()->params();
756         Type *Tys[] = {Args[1]};
757         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
758         return true;
759       }
760     }
761     break;
762   }
763   case 'i':
764   case 'l': {
765     bool IsLifetimeStart = Name.startswith("lifetime.start");
766     if (IsLifetimeStart || Name.startswith("invariant.start")) {
767       Intrinsic::ID ID = IsLifetimeStart ?
768         Intrinsic::lifetime_start : Intrinsic::invariant_start;
769       auto Args = F->getFunctionType()->params();
770       Type* ObjectPtr[1] = {Args[1]};
771       if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
772         rename(F);
773         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
774         return true;
775       }
776     }
777 
778     bool IsLifetimeEnd = Name.startswith("lifetime.end");
779     if (IsLifetimeEnd || Name.startswith("invariant.end")) {
780       Intrinsic::ID ID = IsLifetimeEnd ?
781         Intrinsic::lifetime_end : Intrinsic::invariant_end;
782 
783       auto Args = F->getFunctionType()->params();
784       Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
785       if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
786         rename(F);
787         NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
788         return true;
789       }
790     }
791     if (Name.startswith("invariant.group.barrier")) {
792       // Rename invariant.group.barrier to launder.invariant.group
793       auto Args = F->getFunctionType()->params();
794       Type* ObjectPtr[1] = {Args[0]};
795       rename(F);
796       NewFn = Intrinsic::getDeclaration(F->getParent(),
797           Intrinsic::launder_invariant_group, ObjectPtr);
798       return true;
799 
800     }
801 
802     break;
803   }
804   case 'm': {
805     if (Name.startswith("masked.load.")) {
806       Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
807       if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {
808         rename(F);
809         NewFn = Intrinsic::getDeclaration(F->getParent(),
810                                           Intrinsic::masked_load,
811                                           Tys);
812         return true;
813       }
814     }
815     if (Name.startswith("masked.store.")) {
816       auto Args = F->getFunctionType()->params();
817       Type *Tys[] = { Args[0], Args[1] };
818       if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {
819         rename(F);
820         NewFn = Intrinsic::getDeclaration(F->getParent(),
821                                           Intrinsic::masked_store,
822                                           Tys);
823         return true;
824       }
825     }
826     // Renaming gather/scatter intrinsics with no address space overloading
827     // to the new overload which includes an address space
828     if (Name.startswith("masked.gather.")) {
829       Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
830       if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) {
831         rename(F);
832         NewFn = Intrinsic::getDeclaration(F->getParent(),
833                                           Intrinsic::masked_gather, Tys);
834         return true;
835       }
836     }
837     if (Name.startswith("masked.scatter.")) {
838       auto Args = F->getFunctionType()->params();
839       Type *Tys[] = {Args[0], Args[1]};
840       if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) {
841         rename(F);
842         NewFn = Intrinsic::getDeclaration(F->getParent(),
843                                           Intrinsic::masked_scatter, Tys);
844         return true;
845       }
846     }
847     // Updating the memory intrinsics (memcpy/memmove/memset) that have an
848     // alignment parameter to embedding the alignment as an attribute of
849     // the pointer args.
850     if (Name.startswith("memcpy.") && F->arg_size() == 5) {
851       rename(F);
852       // Get the types of dest, src, and len
853       ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
854       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
855                                         ParamTypes);
856       return true;
857     }
858     if (Name.startswith("memmove.") && F->arg_size() == 5) {
859       rename(F);
860       // Get the types of dest, src, and len
861       ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
862       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
863                                         ParamTypes);
864       return true;
865     }
866     if (Name.startswith("memset.") && F->arg_size() == 5) {
867       rename(F);
868       // Get the types of dest, and len
869       const auto *FT = F->getFunctionType();
870       Type *ParamTypes[2] = {
871           FT->getParamType(0), // Dest
872           FT->getParamType(2)  // len
873       };
874       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
875                                         ParamTypes);
876       return true;
877     }
878     break;
879   }
880   case 'n': {
881     if (Name.startswith("nvvm.")) {
882       Name = Name.substr(5);
883 
884       // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
885       Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
886                               .Cases("brev32", "brev64", Intrinsic::bitreverse)
887                               .Case("clz.i", Intrinsic::ctlz)
888                               .Case("popc.i", Intrinsic::ctpop)
889                               .Default(Intrinsic::not_intrinsic);
890       if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
891         NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
892                                           {F->getReturnType()});
893         return true;
894       }
895 
896       // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
897       // not to an intrinsic alone.  We expand them in UpgradeIntrinsicCall.
898       //
899       // TODO: We could add lohi.i2d.
900       bool Expand = StringSwitch<bool>(Name)
901                         .Cases("abs.i", "abs.ll", true)
902                         .Cases("clz.ll", "popc.ll", "h2f", true)
903                         .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
904                         .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
905                         .StartsWith("atomic.load.add.f32.p", true)
906                         .StartsWith("atomic.load.add.f64.p", true)
907                         .Default(false);
908       if (Expand) {
909         NewFn = nullptr;
910         return true;
911       }
912     }
913     break;
914   }
915   case 'o':
916     // We only need to change the name to match the mangling including the
917     // address space.
918     if (Name.startswith("objectsize.")) {
919       Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
920       if (F->arg_size() == 2 || F->arg_size() == 3 ||
921           F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
922         rename(F);
923         NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
924                                           Tys);
925         return true;
926       }
927     }
928     break;
929 
930   case 'p':
931     if (Name == "prefetch") {
932       // Handle address space overloading.
933       Type *Tys[] = {F->arg_begin()->getType()};
934       if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) {
935         rename(F);
936         NewFn =
937             Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
938         return true;
939       }
940     } else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {
941       rename(F);
942       NewFn = Intrinsic::getDeclaration(F->getParent(),
943                                         Intrinsic::ptr_annotation,
944                                         F->arg_begin()->getType());
945       return true;
946     }
947     break;
948 
949   case 's':
950     if (Name == "stackprotectorcheck") {
951       NewFn = nullptr;
952       return true;
953     }
954     break;
955 
956   case 'v': {
957     if (Name == "var.annotation" && F->arg_size() == 4) {
958       rename(F);
959       NewFn = Intrinsic::getDeclaration(F->getParent(),
960                                         Intrinsic::var_annotation);
961       return true;
962     }
963     break;
964   }
965 
966   case 'x':
967     if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
968       return true;
969   }
970   // Remangle our intrinsic since we upgrade the mangling
971   auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
972   if (Result != None) {
973     NewFn = Result.getValue();
974     return true;
975   }
976 
977   //  This may not belong here. This function is effectively being overloaded
978   //  to both detect an intrinsic which needs upgrading, and to provide the
979   //  upgraded form of the intrinsic. We should perhaps have two separate
980   //  functions for this.
981   return false;
982 }
983 
984 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
985   NewFn = nullptr;
986   bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
987   assert(F != NewFn && "Intrinsic function upgraded to the same function");
988 
989   // Upgrade intrinsic attributes.  This does not change the function.
990   if (NewFn)
991     F = NewFn;
992   if (Intrinsic::ID id = F->getIntrinsicID())
993     F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
994   return Upgraded;
995 }
996 
997 GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
998   if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
999                           GV->getName() == "llvm.global_dtors")) ||
1000       !GV->hasInitializer())
1001     return nullptr;
1002   ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
1003   if (!ATy)
1004     return nullptr;
1005   StructType *STy = dyn_cast<StructType>(ATy->getElementType());
1006   if (!STy || STy->getNumElements() != 2)
1007     return nullptr;
1008 
1009   LLVMContext &C = GV->getContext();
1010   IRBuilder<> IRB(C);
1011   auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
1012                                IRB.getInt8PtrTy());
1013   Constant *Init = GV->getInitializer();
1014   unsigned N = Init->getNumOperands();
1015   std::vector<Constant *> NewCtors(N);
1016   for (unsigned i = 0; i != N; ++i) {
1017     auto Ctor = cast<Constant>(Init->getOperand(i));
1018     NewCtors[i] = ConstantStruct::get(
1019         EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1020         Constant::getNullValue(IRB.getInt8PtrTy()));
1021   }
1022   Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
1023 
1024   return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
1025                             NewInit, GV->getName());
1026 }
1027 
1028 // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
1029 // to byte shuffles.
1030 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
1031                                          Value *Op, unsigned Shift) {
1032   auto *ResultTy = cast<FixedVectorType>(Op->getType());
1033   unsigned NumElts = ResultTy->getNumElements() * 8;
1034 
1035   // Bitcast from a 64-bit element type to a byte element type.
1036   Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1037   Op = Builder.CreateBitCast(Op, VecTy, "cast");
1038 
1039   // We'll be shuffling in zeroes.
1040   Value *Res = Constant::getNullValue(VecTy);
1041 
1042   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1043   // we'll just return the zero vector.
1044   if (Shift < 16) {
1045     int Idxs[64];
1046     // 256/512-bit version is split into 2/4 16-byte lanes.
1047     for (unsigned l = 0; l != NumElts; l += 16)
1048       for (unsigned i = 0; i != 16; ++i) {
1049         unsigned Idx = NumElts + i - Shift;
1050         if (Idx < NumElts)
1051           Idx -= NumElts - 16; // end of lane, switch operand.
1052         Idxs[l + i] = Idx + l;
1053       }
1054 
1055     Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
1056   }
1057 
1058   // Bitcast back to a 64-bit element type.
1059   return Builder.CreateBitCast(Res, ResultTy, "cast");
1060 }
1061 
1062 // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
1063 // to byte shuffles.
1064 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1065                                          unsigned Shift) {
1066   auto *ResultTy = cast<FixedVectorType>(Op->getType());
1067   unsigned NumElts = ResultTy->getNumElements() * 8;
1068 
1069   // Bitcast from a 64-bit element type to a byte element type.
1070   Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1071   Op = Builder.CreateBitCast(Op, VecTy, "cast");
1072 
1073   // We'll be shuffling in zeroes.
1074   Value *Res = Constant::getNullValue(VecTy);
1075 
1076   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1077   // we'll just return the zero vector.
1078   if (Shift < 16) {
1079     int Idxs[64];
1080     // 256/512-bit version is split into 2/4 16-byte lanes.
1081     for (unsigned l = 0; l != NumElts; l += 16)
1082       for (unsigned i = 0; i != 16; ++i) {
1083         unsigned Idx = i + Shift;
1084         if (Idx >= 16)
1085           Idx += NumElts - 16; // end of lane, switch operand.
1086         Idxs[l + i] = Idx + l;
1087       }
1088 
1089     Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
1090   }
1091 
1092   // Bitcast back to a 64-bit element type.
1093   return Builder.CreateBitCast(Res, ResultTy, "cast");
1094 }
1095 
1096 static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
1097                             unsigned NumElts) {
1098   assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
1099   llvm::VectorType *MaskTy = FixedVectorType::get(
1100       Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
1101   Mask = Builder.CreateBitCast(Mask, MaskTy);
1102 
1103   // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
1104   // i8 and we need to extract down to the right number of elements.
1105   if (NumElts <= 4) {
1106     int Indices[4];
1107     for (unsigned i = 0; i != NumElts; ++i)
1108       Indices[i] = i;
1109     Mask = Builder.CreateShuffleVector(
1110         Mask, Mask, makeArrayRef(Indices, NumElts), "extract");
1111   }
1112 
1113   return Mask;
1114 }
1115 
1116 static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
1117                             Value *Op0, Value *Op1) {
1118   // If the mask is all ones just emit the first operation.
1119   if (const auto *C = dyn_cast<Constant>(Mask))
1120     if (C->isAllOnesValue())
1121       return Op0;
1122 
1123   Mask = getX86MaskVec(Builder, Mask,
1124                        cast<FixedVectorType>(Op0->getType())->getNumElements());
1125   return Builder.CreateSelect(Mask, Op0, Op1);
1126 }
1127 
1128 static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
1129                                   Value *Op0, Value *Op1) {
1130   // If the mask is all ones just emit the first operation.
1131   if (const auto *C = dyn_cast<Constant>(Mask))
1132     if (C->isAllOnesValue())
1133       return Op0;
1134 
1135   auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
1136                                       Mask->getType()->getIntegerBitWidth());
1137   Mask = Builder.CreateBitCast(Mask, MaskTy);
1138   Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
1139   return Builder.CreateSelect(Mask, Op0, Op1);
1140 }
1141 
1142 // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
1143 // PALIGNR handles large immediates by shifting while VALIGN masks the immediate
1144 // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
1145 static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
1146                                         Value *Op1, Value *Shift,
1147                                         Value *Passthru, Value *Mask,
1148                                         bool IsVALIGN) {
1149   unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1150 
1151   unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1152   assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
1153   assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
1154   assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
1155 
1156   // Mask the immediate for VALIGN.
1157   if (IsVALIGN)
1158     ShiftVal &= (NumElts - 1);
1159 
1160   // If palignr is shifting the pair of vectors more than the size of two
1161   // lanes, emit zero.
1162   if (ShiftVal >= 32)
1163     return llvm::Constant::getNullValue(Op0->getType());
1164 
1165   // If palignr is shifting the pair of input vectors more than one lane,
1166   // but less than two lanes, convert to shifting in zeroes.
1167   if (ShiftVal > 16) {
1168     ShiftVal -= 16;
1169     Op1 = Op0;
1170     Op0 = llvm::Constant::getNullValue(Op0->getType());
1171   }
1172 
1173   int Indices[64];
1174   // 256-bit palignr operates on 128-bit lanes so we need to handle that
1175   for (unsigned l = 0; l < NumElts; l += 16) {
1176     for (unsigned i = 0; i != 16; ++i) {
1177       unsigned Idx = ShiftVal + i;
1178       if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
1179         Idx += NumElts - 16; // End of lane, switch operand.
1180       Indices[l + i] = Idx + l;
1181     }
1182   }
1183 
1184   Value *Align = Builder.CreateShuffleVector(Op1, Op0,
1185                                              makeArrayRef(Indices, NumElts),
1186                                              "palignr");
1187 
1188   return EmitX86Select(Builder, Mask, Align, Passthru);
1189 }
1190 
1191 static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
1192                                           bool ZeroMask, bool IndexForm) {
1193   Type *Ty = CI.getType();
1194   unsigned VecWidth = Ty->getPrimitiveSizeInBits();
1195   unsigned EltWidth = Ty->getScalarSizeInBits();
1196   bool IsFloat = Ty->isFPOrFPVectorTy();
1197   Intrinsic::ID IID;
1198   if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1199     IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1200   else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1201     IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1202   else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1203     IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1204   else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1205     IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1206   else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1207     IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1208   else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1209     IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1210   else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1211     IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1212   else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1213     IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1214   else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1215     IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1216   else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1217     IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1218   else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1219     IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1220   else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1221     IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1222   else if (VecWidth == 128 && EltWidth == 16)
1223     IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1224   else if (VecWidth == 256 && EltWidth == 16)
1225     IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1226   else if (VecWidth == 512 && EltWidth == 16)
1227     IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1228   else if (VecWidth == 128 && EltWidth == 8)
1229     IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1230   else if (VecWidth == 256 && EltWidth == 8)
1231     IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1232   else if (VecWidth == 512 && EltWidth == 8)
1233     IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1234   else
1235     llvm_unreachable("Unexpected intrinsic");
1236 
1237   Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
1238                     CI.getArgOperand(2) };
1239 
1240   // If this isn't index form we need to swap operand 0 and 1.
1241   if (!IndexForm)
1242     std::swap(Args[0], Args[1]);
1243 
1244   Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1245                                 Args);
1246   Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
1247                              : Builder.CreateBitCast(CI.getArgOperand(1),
1248                                                      Ty);
1249   return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1250 }
1251 
1252 static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI,
1253                                          Intrinsic::ID IID) {
1254   Type *Ty = CI.getType();
1255   Value *Op0 = CI.getOperand(0);
1256   Value *Op1 = CI.getOperand(1);
1257   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1258   Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
1259 
1260   if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
1261     Value *VecSrc = CI.getOperand(2);
1262     Value *Mask = CI.getOperand(3);
1263     Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1264   }
1265   return Res;
1266 }
1267 
1268 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
1269                                bool IsRotateRight) {
1270   Type *Ty = CI.getType();
1271   Value *Src = CI.getArgOperand(0);
1272   Value *Amt = CI.getArgOperand(1);
1273 
1274   // Amount may be scalar immediate, in which case create a splat vector.
1275   // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1276   // we only care about the lowest log2 bits anyway.
1277   if (Amt->getType() != Ty) {
1278     unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1279     Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1280     Amt = Builder.CreateVectorSplat(NumElts, Amt);
1281   }
1282 
1283   Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1284   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1285   Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
1286 
1287   if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
1288     Value *VecSrc = CI.getOperand(2);
1289     Value *Mask = CI.getOperand(3);
1290     Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1291   }
1292   return Res;
1293 }
1294 
1295 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
1296                               bool IsSigned) {
1297   Type *Ty = CI.getType();
1298   Value *LHS = CI.getArgOperand(0);
1299   Value *RHS = CI.getArgOperand(1);
1300 
1301   CmpInst::Predicate Pred;
1302   switch (Imm) {
1303   case 0x0:
1304     Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1305     break;
1306   case 0x1:
1307     Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1308     break;
1309   case 0x2:
1310     Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1311     break;
1312   case 0x3:
1313     Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1314     break;
1315   case 0x4:
1316     Pred = ICmpInst::ICMP_EQ;
1317     break;
1318   case 0x5:
1319     Pred = ICmpInst::ICMP_NE;
1320     break;
1321   case 0x6:
1322     return Constant::getNullValue(Ty); // FALSE
1323   case 0x7:
1324     return Constant::getAllOnesValue(Ty); // TRUE
1325   default:
1326     llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
1327   }
1328 
1329   Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
1330   Value *Ext = Builder.CreateSExt(Cmp, Ty);
1331   return Ext;
1332 }
1333 
1334 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
1335                                     bool IsShiftRight, bool ZeroMask) {
1336   Type *Ty = CI.getType();
1337   Value *Op0 = CI.getArgOperand(0);
1338   Value *Op1 = CI.getArgOperand(1);
1339   Value *Amt = CI.getArgOperand(2);
1340 
1341   if (IsShiftRight)
1342     std::swap(Op0, Op1);
1343 
1344   // Amount may be scalar immediate, in which case create a splat vector.
1345   // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1346   // we only care about the lowest log2 bits anyway.
1347   if (Amt->getType() != Ty) {
1348     unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1349     Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1350     Amt = Builder.CreateVectorSplat(NumElts, Amt);
1351   }
1352 
1353   Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1354   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1355   Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1356 
1357   unsigned NumArgs = CI.getNumArgOperands();
1358   if (NumArgs >= 4) { // For masked intrinsics.
1359     Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
1360                     ZeroMask     ? ConstantAggregateZero::get(CI.getType()) :
1361                                    CI.getArgOperand(0);
1362     Value *Mask = CI.getOperand(NumArgs - 1);
1363     Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1364   }
1365   return Res;
1366 }
1367 
1368 static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
1369                                  Value *Ptr, Value *Data, Value *Mask,
1370                                  bool Aligned) {
1371   // Cast the pointer to the right type.
1372   Ptr = Builder.CreateBitCast(Ptr,
1373                               llvm::PointerType::getUnqual(Data->getType()));
1374   const Align Alignment =
1375       Aligned
1376           ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)
1377           : Align(1);
1378 
1379   // If the mask is all ones just emit a regular store.
1380   if (const auto *C = dyn_cast<Constant>(Mask))
1381     if (C->isAllOnesValue())
1382       return Builder.CreateAlignedStore(Data, Ptr, Alignment);
1383 
1384   // Convert the mask from an integer type to a vector of i1.
1385   unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1386   Mask = getX86MaskVec(Builder, Mask, NumElts);
1387   return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
1388 }
1389 
1390 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
1391                                 Value *Ptr, Value *Passthru, Value *Mask,
1392                                 bool Aligned) {
1393   Type *ValTy = Passthru->getType();
1394   // Cast the pointer to the right type.
1395   Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
1396   const Align Alignment =
1397       Aligned
1398           ? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() /
1399                   8)
1400           : Align(1);
1401 
1402   // If the mask is all ones just emit a regular store.
1403   if (const auto *C = dyn_cast<Constant>(Mask))
1404     if (C->isAllOnesValue())
1405       return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1406 
1407   // Convert the mask from an integer type to a vector of i1.
1408   unsigned NumElts =
1409       cast<FixedVectorType>(Passthru->getType())->getNumElements();
1410   Mask = getX86MaskVec(Builder, Mask, NumElts);
1411   return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);
1412 }
1413 
1414 static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
1415   Type *Ty = CI.getType();
1416   Value *Op0 = CI.getArgOperand(0);
1417   Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
1418   Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
1419   if (CI.getNumArgOperands() == 3)
1420     Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
1421   return Res;
1422 }
1423 
1424 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
1425   Type *Ty = CI.getType();
1426 
1427   // Arguments have a vXi32 type so cast to vXi64.
1428   Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1429   Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1430 
1431   if (IsSigned) {
1432     // Shift left then arithmetic shift right.
1433     Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1434     LHS = Builder.CreateShl(LHS, ShiftAmt);
1435     LHS = Builder.CreateAShr(LHS, ShiftAmt);
1436     RHS = Builder.CreateShl(RHS, ShiftAmt);
1437     RHS = Builder.CreateAShr(RHS, ShiftAmt);
1438   } else {
1439     // Clear the upper bits.
1440     Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1441     LHS = Builder.CreateAnd(LHS, Mask);
1442     RHS = Builder.CreateAnd(RHS, Mask);
1443   }
1444 
1445   Value *Res = Builder.CreateMul(LHS, RHS);
1446 
1447   if (CI.getNumArgOperands() == 4)
1448     Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1449 
1450   return Res;
1451 }
1452 
1453 // Applying mask on vector of i1's and make sure result is at least 8 bits wide.
1454 static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
1455                                      Value *Mask) {
1456   unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
1457   if (Mask) {
1458     const auto *C = dyn_cast<Constant>(Mask);
1459     if (!C || !C->isAllOnesValue())
1460       Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
1461   }
1462 
1463   if (NumElts < 8) {
1464     int Indices[8];
1465     for (unsigned i = 0; i != NumElts; ++i)
1466       Indices[i] = i;
1467     for (unsigned i = NumElts; i != 8; ++i)
1468       Indices[i] = NumElts + i % NumElts;
1469     Vec = Builder.CreateShuffleVector(Vec,
1470                                       Constant::getNullValue(Vec->getType()),
1471                                       Indices);
1472   }
1473   return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
1474 }
1475 
1476 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
1477                                    unsigned CC, bool Signed) {
1478   Value *Op0 = CI.getArgOperand(0);
1479   unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1480 
1481   Value *Cmp;
1482   if (CC == 3) {
1483     Cmp = Constant::getNullValue(
1484         FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1485   } else if (CC == 7) {
1486     Cmp = Constant::getAllOnesValue(
1487         FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1488   } else {
1489     ICmpInst::Predicate Pred;
1490     switch (CC) {
1491     default: llvm_unreachable("Unknown condition code");
1492     case 0: Pred = ICmpInst::ICMP_EQ;  break;
1493     case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
1494     case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
1495     case 4: Pred = ICmpInst::ICMP_NE;  break;
1496     case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
1497     case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
1498     }
1499     Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1500   }
1501 
1502   Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
1503 
1504   return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
1505 }
1506 
1507 // Replace a masked intrinsic with an older unmasked intrinsic.
1508 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
1509                                     Intrinsic::ID IID) {
1510   Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1511   Value *Rep = Builder.CreateCall(Intrin,
1512                                  { CI.getArgOperand(0), CI.getArgOperand(1) });
1513   return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1514 }
1515 
1516 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
1517   Value* A = CI.getArgOperand(0);
1518   Value* B = CI.getArgOperand(1);
1519   Value* Src = CI.getArgOperand(2);
1520   Value* Mask = CI.getArgOperand(3);
1521 
1522   Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
1523   Value* Cmp = Builder.CreateIsNotNull(AndNode);
1524   Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
1525   Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
1526   Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
1527   return Builder.CreateInsertElement(A, Select, (uint64_t)0);
1528 }
1529 
1530 
1531 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
1532   Value* Op = CI.getArgOperand(0);
1533   Type* ReturnOp = CI.getType();
1534   unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
1535   Value *Mask = getX86MaskVec(Builder, Op, NumElts);
1536   return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
1537 }
1538 
1539 // Replace intrinsic with unmasked version and a select.
1540 static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
1541                                       CallInst &CI, Value *&Rep) {
1542   Name = Name.substr(12); // Remove avx512.mask.
1543 
1544   unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
1545   unsigned EltWidth = CI.getType()->getScalarSizeInBits();
1546   Intrinsic::ID IID;
1547   if (Name.startswith("max.p")) {
1548     if (VecWidth == 128 && EltWidth == 32)
1549       IID = Intrinsic::x86_sse_max_ps;
1550     else if (VecWidth == 128 && EltWidth == 64)
1551       IID = Intrinsic::x86_sse2_max_pd;
1552     else if (VecWidth == 256 && EltWidth == 32)
1553       IID = Intrinsic::x86_avx_max_ps_256;
1554     else if (VecWidth == 256 && EltWidth == 64)
1555       IID = Intrinsic::x86_avx_max_pd_256;
1556     else
1557       llvm_unreachable("Unexpected intrinsic");
1558   } else if (Name.startswith("min.p")) {
1559     if (VecWidth == 128 && EltWidth == 32)
1560       IID = Intrinsic::x86_sse_min_ps;
1561     else if (VecWidth == 128 && EltWidth == 64)
1562       IID = Intrinsic::x86_sse2_min_pd;
1563     else if (VecWidth == 256 && EltWidth == 32)
1564       IID = Intrinsic::x86_avx_min_ps_256;
1565     else if (VecWidth == 256 && EltWidth == 64)
1566       IID = Intrinsic::x86_avx_min_pd_256;
1567     else
1568       llvm_unreachable("Unexpected intrinsic");
1569   } else if (Name.startswith("pshuf.b.")) {
1570     if (VecWidth == 128)
1571       IID = Intrinsic::x86_ssse3_pshuf_b_128;
1572     else if (VecWidth == 256)
1573       IID = Intrinsic::x86_avx2_pshuf_b;
1574     else if (VecWidth == 512)
1575       IID = Intrinsic::x86_avx512_pshuf_b_512;
1576     else
1577       llvm_unreachable("Unexpected intrinsic");
1578   } else if (Name.startswith("pmul.hr.sw.")) {
1579     if (VecWidth == 128)
1580       IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1581     else if (VecWidth == 256)
1582       IID = Intrinsic::x86_avx2_pmul_hr_sw;
1583     else if (VecWidth == 512)
1584       IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1585     else
1586       llvm_unreachable("Unexpected intrinsic");
1587   } else if (Name.startswith("pmulh.w.")) {
1588     if (VecWidth == 128)
1589       IID = Intrinsic::x86_sse2_pmulh_w;
1590     else if (VecWidth == 256)
1591       IID = Intrinsic::x86_avx2_pmulh_w;
1592     else if (VecWidth == 512)
1593       IID = Intrinsic::x86_avx512_pmulh_w_512;
1594     else
1595       llvm_unreachable("Unexpected intrinsic");
1596   } else if (Name.startswith("pmulhu.w.")) {
1597     if (VecWidth == 128)
1598       IID = Intrinsic::x86_sse2_pmulhu_w;
1599     else if (VecWidth == 256)
1600       IID = Intrinsic::x86_avx2_pmulhu_w;
1601     else if (VecWidth == 512)
1602       IID = Intrinsic::x86_avx512_pmulhu_w_512;
1603     else
1604       llvm_unreachable("Unexpected intrinsic");
1605   } else if (Name.startswith("pmaddw.d.")) {
1606     if (VecWidth == 128)
1607       IID = Intrinsic::x86_sse2_pmadd_wd;
1608     else if (VecWidth == 256)
1609       IID = Intrinsic::x86_avx2_pmadd_wd;
1610     else if (VecWidth == 512)
1611       IID = Intrinsic::x86_avx512_pmaddw_d_512;
1612     else
1613       llvm_unreachable("Unexpected intrinsic");
1614   } else if (Name.startswith("pmaddubs.w.")) {
1615     if (VecWidth == 128)
1616       IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1617     else if (VecWidth == 256)
1618       IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1619     else if (VecWidth == 512)
1620       IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1621     else
1622       llvm_unreachable("Unexpected intrinsic");
1623   } else if (Name.startswith("packsswb.")) {
1624     if (VecWidth == 128)
1625       IID = Intrinsic::x86_sse2_packsswb_128;
1626     else if (VecWidth == 256)
1627       IID = Intrinsic::x86_avx2_packsswb;
1628     else if (VecWidth == 512)
1629       IID = Intrinsic::x86_avx512_packsswb_512;
1630     else
1631       llvm_unreachable("Unexpected intrinsic");
1632   } else if (Name.startswith("packssdw.")) {
1633     if (VecWidth == 128)
1634       IID = Intrinsic::x86_sse2_packssdw_128;
1635     else if (VecWidth == 256)
1636       IID = Intrinsic::x86_avx2_packssdw;
1637     else if (VecWidth == 512)
1638       IID = Intrinsic::x86_avx512_packssdw_512;
1639     else
1640       llvm_unreachable("Unexpected intrinsic");
1641   } else if (Name.startswith("packuswb.")) {
1642     if (VecWidth == 128)
1643       IID = Intrinsic::x86_sse2_packuswb_128;
1644     else if (VecWidth == 256)
1645       IID = Intrinsic::x86_avx2_packuswb;
1646     else if (VecWidth == 512)
1647       IID = Intrinsic::x86_avx512_packuswb_512;
1648     else
1649       llvm_unreachable("Unexpected intrinsic");
1650   } else if (Name.startswith("packusdw.")) {
1651     if (VecWidth == 128)
1652       IID = Intrinsic::x86_sse41_packusdw;
1653     else if (VecWidth == 256)
1654       IID = Intrinsic::x86_avx2_packusdw;
1655     else if (VecWidth == 512)
1656       IID = Intrinsic::x86_avx512_packusdw_512;
1657     else
1658       llvm_unreachable("Unexpected intrinsic");
1659   } else if (Name.startswith("vpermilvar.")) {
1660     if (VecWidth == 128 && EltWidth == 32)
1661       IID = Intrinsic::x86_avx_vpermilvar_ps;
1662     else if (VecWidth == 128 && EltWidth == 64)
1663       IID = Intrinsic::x86_avx_vpermilvar_pd;
1664     else if (VecWidth == 256 && EltWidth == 32)
1665       IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1666     else if (VecWidth == 256 && EltWidth == 64)
1667       IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1668     else if (VecWidth == 512 && EltWidth == 32)
1669       IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1670     else if (VecWidth == 512 && EltWidth == 64)
1671       IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1672     else
1673       llvm_unreachable("Unexpected intrinsic");
1674   } else if (Name == "cvtpd2dq.256") {
1675     IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1676   } else if (Name == "cvtpd2ps.256") {
1677     IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1678   } else if (Name == "cvttpd2dq.256") {
1679     IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1680   } else if (Name == "cvttps2dq.128") {
1681     IID = Intrinsic::x86_sse2_cvttps2dq;
1682   } else if (Name == "cvttps2dq.256") {
1683     IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1684   } else if (Name.startswith("permvar.")) {
1685     bool IsFloat = CI.getType()->isFPOrFPVectorTy();
1686     if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1687       IID = Intrinsic::x86_avx2_permps;
1688     else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1689       IID = Intrinsic::x86_avx2_permd;
1690     else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1691       IID = Intrinsic::x86_avx512_permvar_df_256;
1692     else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1693       IID = Intrinsic::x86_avx512_permvar_di_256;
1694     else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1695       IID = Intrinsic::x86_avx512_permvar_sf_512;
1696     else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1697       IID = Intrinsic::x86_avx512_permvar_si_512;
1698     else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1699       IID = Intrinsic::x86_avx512_permvar_df_512;
1700     else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1701       IID = Intrinsic::x86_avx512_permvar_di_512;
1702     else if (VecWidth == 128 && EltWidth == 16)
1703       IID = Intrinsic::x86_avx512_permvar_hi_128;
1704     else if (VecWidth == 256 && EltWidth == 16)
1705       IID = Intrinsic::x86_avx512_permvar_hi_256;
1706     else if (VecWidth == 512 && EltWidth == 16)
1707       IID = Intrinsic::x86_avx512_permvar_hi_512;
1708     else if (VecWidth == 128 && EltWidth == 8)
1709       IID = Intrinsic::x86_avx512_permvar_qi_128;
1710     else if (VecWidth == 256 && EltWidth == 8)
1711       IID = Intrinsic::x86_avx512_permvar_qi_256;
1712     else if (VecWidth == 512 && EltWidth == 8)
1713       IID = Intrinsic::x86_avx512_permvar_qi_512;
1714     else
1715       llvm_unreachable("Unexpected intrinsic");
1716   } else if (Name.startswith("dbpsadbw.")) {
1717     if (VecWidth == 128)
1718       IID = Intrinsic::x86_avx512_dbpsadbw_128;
1719     else if (VecWidth == 256)
1720       IID = Intrinsic::x86_avx512_dbpsadbw_256;
1721     else if (VecWidth == 512)
1722       IID = Intrinsic::x86_avx512_dbpsadbw_512;
1723     else
1724       llvm_unreachable("Unexpected intrinsic");
1725   } else if (Name.startswith("pmultishift.qb.")) {
1726     if (VecWidth == 128)
1727       IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1728     else if (VecWidth == 256)
1729       IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1730     else if (VecWidth == 512)
1731       IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1732     else
1733       llvm_unreachable("Unexpected intrinsic");
1734   } else if (Name.startswith("conflict.")) {
1735     if (Name[9] == 'd' && VecWidth == 128)
1736       IID = Intrinsic::x86_avx512_conflict_d_128;
1737     else if (Name[9] == 'd' && VecWidth == 256)
1738       IID = Intrinsic::x86_avx512_conflict_d_256;
1739     else if (Name[9] == 'd' && VecWidth == 512)
1740       IID = Intrinsic::x86_avx512_conflict_d_512;
1741     else if (Name[9] == 'q' && VecWidth == 128)
1742       IID = Intrinsic::x86_avx512_conflict_q_128;
1743     else if (Name[9] == 'q' && VecWidth == 256)
1744       IID = Intrinsic::x86_avx512_conflict_q_256;
1745     else if (Name[9] == 'q' && VecWidth == 512)
1746       IID = Intrinsic::x86_avx512_conflict_q_512;
1747     else
1748       llvm_unreachable("Unexpected intrinsic");
1749   } else if (Name.startswith("pavg.")) {
1750     if (Name[5] == 'b' && VecWidth == 128)
1751       IID = Intrinsic::x86_sse2_pavg_b;
1752     else if (Name[5] == 'b' && VecWidth == 256)
1753       IID = Intrinsic::x86_avx2_pavg_b;
1754     else if (Name[5] == 'b' && VecWidth == 512)
1755       IID = Intrinsic::x86_avx512_pavg_b_512;
1756     else if (Name[5] == 'w' && VecWidth == 128)
1757       IID = Intrinsic::x86_sse2_pavg_w;
1758     else if (Name[5] == 'w' && VecWidth == 256)
1759       IID = Intrinsic::x86_avx2_pavg_w;
1760     else if (Name[5] == 'w' && VecWidth == 512)
1761       IID = Intrinsic::x86_avx512_pavg_w_512;
1762     else
1763       llvm_unreachable("Unexpected intrinsic");
1764   } else
1765     return false;
1766 
1767   SmallVector<Value *, 4> Args(CI.arg_operands().begin(),
1768                                CI.arg_operands().end());
1769   Args.pop_back();
1770   Args.pop_back();
1771   Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1772                            Args);
1773   unsigned NumArgs = CI.getNumArgOperands();
1774   Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
1775                       CI.getArgOperand(NumArgs - 2));
1776   return true;
1777 }
1778 
1779 /// Upgrade comment in call to inline asm that represents an objc retain release
1780 /// marker.
1781 void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
1782   size_t Pos;
1783   if (AsmStr->find("mov\tfp") == 0 &&
1784       AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
1785       (Pos = AsmStr->find("# marker")) != std::string::npos) {
1786     AsmStr->replace(Pos, 1, ";");
1787   }
1788 }
1789 
1790 /// Upgrade a call to an old intrinsic. All argument and return casting must be
1791 /// provided to seamlessly integrate with existing context.
1792 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
1793   Function *F = CI->getCalledFunction();
1794   LLVMContext &C = CI->getContext();
1795   IRBuilder<> Builder(C);
1796   Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
1797 
1798   assert(F && "Intrinsic call is not direct?");
1799 
1800   if (!NewFn) {
1801     // Get the Function's name.
1802     StringRef Name = F->getName();
1803 
1804     assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
1805     Name = Name.substr(5);
1806 
1807     bool IsX86 = Name.startswith("x86.");
1808     if (IsX86)
1809       Name = Name.substr(4);
1810     bool IsNVVM = Name.startswith("nvvm.");
1811     if (IsNVVM)
1812       Name = Name.substr(5);
1813 
1814     if (IsX86 && Name.startswith("sse4a.movnt.")) {
1815       Module *M = F->getParent();
1816       SmallVector<Metadata *, 1> Elts;
1817       Elts.push_back(
1818           ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
1819       MDNode *Node = MDNode::get(C, Elts);
1820 
1821       Value *Arg0 = CI->getArgOperand(0);
1822       Value *Arg1 = CI->getArgOperand(1);
1823 
1824       // Nontemporal (unaligned) store of the 0'th element of the float/double
1825       // vector.
1826       Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
1827       PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
1828       Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
1829       Value *Extract =
1830           Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
1831 
1832       StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
1833       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1834 
1835       // Remove intrinsic.
1836       CI->eraseFromParent();
1837       return;
1838     }
1839 
1840     if (IsX86 && (Name.startswith("avx.movnt.") ||
1841                   Name.startswith("avx512.storent."))) {
1842       Module *M = F->getParent();
1843       SmallVector<Metadata *, 1> Elts;
1844       Elts.push_back(
1845           ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
1846       MDNode *Node = MDNode::get(C, Elts);
1847 
1848       Value *Arg0 = CI->getArgOperand(0);
1849       Value *Arg1 = CI->getArgOperand(1);
1850 
1851       // Convert the type of the pointer to a pointer to the stored type.
1852       Value *BC = Builder.CreateBitCast(Arg0,
1853                                         PointerType::getUnqual(Arg1->getType()),
1854                                         "cast");
1855       StoreInst *SI = Builder.CreateAlignedStore(
1856           Arg1, BC,
1857           Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
1858       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1859 
1860       // Remove intrinsic.
1861       CI->eraseFromParent();
1862       return;
1863     }
1864 
1865     if (IsX86 && Name == "sse2.storel.dq") {
1866       Value *Arg0 = CI->getArgOperand(0);
1867       Value *Arg1 = CI->getArgOperand(1);
1868 
1869       auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
1870       Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
1871       Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
1872       Value *BC = Builder.CreateBitCast(Arg0,
1873                                         PointerType::getUnqual(Elt->getType()),
1874                                         "cast");
1875       Builder.CreateAlignedStore(Elt, BC, Align(1));
1876 
1877       // Remove intrinsic.
1878       CI->eraseFromParent();
1879       return;
1880     }
1881 
1882     if (IsX86 && (Name.startswith("sse.storeu.") ||
1883                   Name.startswith("sse2.storeu.") ||
1884                   Name.startswith("avx.storeu."))) {
1885       Value *Arg0 = CI->getArgOperand(0);
1886       Value *Arg1 = CI->getArgOperand(1);
1887 
1888       Arg0 = Builder.CreateBitCast(Arg0,
1889                                    PointerType::getUnqual(Arg1->getType()),
1890                                    "cast");
1891       Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
1892 
1893       // Remove intrinsic.
1894       CI->eraseFromParent();
1895       return;
1896     }
1897 
1898     if (IsX86 && Name == "avx512.mask.store.ss") {
1899       Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
1900       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1901                          Mask, false);
1902 
1903       // Remove intrinsic.
1904       CI->eraseFromParent();
1905       return;
1906     }
1907 
1908     if (IsX86 && (Name.startswith("avx512.mask.store"))) {
1909       // "avx512.mask.storeu." or "avx512.mask.store."
1910       bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
1911       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1912                          CI->getArgOperand(2), Aligned);
1913 
1914       // Remove intrinsic.
1915       CI->eraseFromParent();
1916       return;
1917     }
1918 
1919     Value *Rep;
1920     // Upgrade packed integer vector compare intrinsics to compare instructions.
1921     if (IsX86 && (Name.startswith("sse2.pcmp") ||
1922                   Name.startswith("avx2.pcmp"))) {
1923       // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
1924       bool CmpEq = Name[9] == 'e';
1925       Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
1926                                CI->getArgOperand(0), CI->getArgOperand(1));
1927       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
1928     } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
1929       Type *ExtTy = Type::getInt32Ty(C);
1930       if (CI->getOperand(0)->getType()->isIntegerTy(8))
1931         ExtTy = Type::getInt64Ty(C);
1932       unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
1933                          ExtTy->getPrimitiveSizeInBits();
1934       Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
1935       Rep = Builder.CreateVectorSplat(NumElts, Rep);
1936     } else if (IsX86 && (Name == "sse.sqrt.ss" ||
1937                          Name == "sse2.sqrt.sd")) {
1938       Value *Vec = CI->getArgOperand(0);
1939       Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
1940       Function *Intr = Intrinsic::getDeclaration(F->getParent(),
1941                                                  Intrinsic::sqrt, Elt0->getType());
1942       Elt0 = Builder.CreateCall(Intr, Elt0);
1943       Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
1944     } else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
1945                          Name.startswith("sse2.sqrt.p") ||
1946                          Name.startswith("sse.sqrt.p"))) {
1947       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1948                                                          Intrinsic::sqrt,
1949                                                          CI->getType()),
1950                                {CI->getArgOperand(0)});
1951     } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
1952       if (CI->getNumArgOperands() == 4 &&
1953           (!isa<ConstantInt>(CI->getArgOperand(3)) ||
1954            cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
1955         Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
1956                                             : Intrinsic::x86_avx512_sqrt_pd_512;
1957 
1958         Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
1959         Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
1960                                                            IID), Args);
1961       } else {
1962         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1963                                                            Intrinsic::sqrt,
1964                                                            CI->getType()),
1965                                  {CI->getArgOperand(0)});
1966       }
1967       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1968                           CI->getArgOperand(1));
1969     } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
1970                          Name.startswith("avx512.ptestnm"))) {
1971       Value *Op0 = CI->getArgOperand(0);
1972       Value *Op1 = CI->getArgOperand(1);
1973       Value *Mask = CI->getArgOperand(2);
1974       Rep = Builder.CreateAnd(Op0, Op1);
1975       llvm::Type *Ty = Op0->getType();
1976       Value *Zero = llvm::Constant::getNullValue(Ty);
1977       ICmpInst::Predicate Pred =
1978         Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
1979       Rep = Builder.CreateICmp(Pred, Rep, Zero);
1980       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
1981     } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
1982       unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
1983                              ->getNumElements();
1984       Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
1985       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1986                           CI->getArgOperand(1));
1987     } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
1988       unsigned NumElts = CI->getType()->getScalarSizeInBits();
1989       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
1990       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
1991       int Indices[64];
1992       for (unsigned i = 0; i != NumElts; ++i)
1993         Indices[i] = i;
1994 
1995       // First extract half of each vector. This gives better codegen than
1996       // doing it in a single shuffle.
1997       LHS = Builder.CreateShuffleVector(LHS, LHS,
1998                                         makeArrayRef(Indices, NumElts / 2));
1999       RHS = Builder.CreateShuffleVector(RHS, RHS,
2000                                         makeArrayRef(Indices, NumElts / 2));
2001       // Concat the vectors.
2002       // NOTE: Operands have to be swapped to match intrinsic definition.
2003       Rep = Builder.CreateShuffleVector(RHS, LHS,
2004                                         makeArrayRef(Indices, NumElts));
2005       Rep = Builder.CreateBitCast(Rep, CI->getType());
2006     } else if (IsX86 && Name == "avx512.kand.w") {
2007       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2008       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2009       Rep = Builder.CreateAnd(LHS, RHS);
2010       Rep = Builder.CreateBitCast(Rep, CI->getType());
2011     } else if (IsX86 && Name == "avx512.kandn.w") {
2012       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2013       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2014       LHS = Builder.CreateNot(LHS);
2015       Rep = Builder.CreateAnd(LHS, RHS);
2016       Rep = Builder.CreateBitCast(Rep, CI->getType());
2017     } else if (IsX86 && Name == "avx512.kor.w") {
2018       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2019       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2020       Rep = Builder.CreateOr(LHS, RHS);
2021       Rep = Builder.CreateBitCast(Rep, CI->getType());
2022     } else if (IsX86 && Name == "avx512.kxor.w") {
2023       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2024       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2025       Rep = Builder.CreateXor(LHS, RHS);
2026       Rep = Builder.CreateBitCast(Rep, CI->getType());
2027     } else if (IsX86 && Name == "avx512.kxnor.w") {
2028       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2029       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2030       LHS = Builder.CreateNot(LHS);
2031       Rep = Builder.CreateXor(LHS, RHS);
2032       Rep = Builder.CreateBitCast(Rep, CI->getType());
2033     } else if (IsX86 && Name == "avx512.knot.w") {
2034       Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2035       Rep = Builder.CreateNot(Rep);
2036       Rep = Builder.CreateBitCast(Rep, CI->getType());
2037     } else if (IsX86 &&
2038                (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {
2039       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2040       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2041       Rep = Builder.CreateOr(LHS, RHS);
2042       Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
2043       Value *C;
2044       if (Name[14] == 'c')
2045         C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
2046       else
2047         C = ConstantInt::getNullValue(Builder.getInt16Ty());
2048       Rep = Builder.CreateICmpEQ(Rep, C);
2049       Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
2050     } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
2051                          Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
2052                          Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
2053                          Name == "sse.div.ss" || Name == "sse2.div.sd")) {
2054       Type *I32Ty = Type::getInt32Ty(C);
2055       Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
2056                                                  ConstantInt::get(I32Ty, 0));
2057       Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
2058                                                  ConstantInt::get(I32Ty, 0));
2059       Value *EltOp;
2060       if (Name.contains(".add."))
2061         EltOp = Builder.CreateFAdd(Elt0, Elt1);
2062       else if (Name.contains(".sub."))
2063         EltOp = Builder.CreateFSub(Elt0, Elt1);
2064       else if (Name.contains(".mul."))
2065         EltOp = Builder.CreateFMul(Elt0, Elt1);
2066       else
2067         EltOp = Builder.CreateFDiv(Elt0, Elt1);
2068       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
2069                                         ConstantInt::get(I32Ty, 0));
2070     } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
2071       // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
2072       bool CmpEq = Name[16] == 'e';
2073       Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
2074     } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
2075       Type *OpTy = CI->getArgOperand(0)->getType();
2076       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2077       Intrinsic::ID IID;
2078       switch (VecWidth) {
2079       default: llvm_unreachable("Unexpected intrinsic");
2080       case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
2081       case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
2082       case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
2083       }
2084 
2085       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2086                                { CI->getOperand(0), CI->getArgOperand(1) });
2087       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2088     } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
2089       Type *OpTy = CI->getArgOperand(0)->getType();
2090       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2091       unsigned EltWidth = OpTy->getScalarSizeInBits();
2092       Intrinsic::ID IID;
2093       if (VecWidth == 128 && EltWidth == 32)
2094         IID = Intrinsic::x86_avx512_fpclass_ps_128;
2095       else if (VecWidth == 256 && EltWidth == 32)
2096         IID = Intrinsic::x86_avx512_fpclass_ps_256;
2097       else if (VecWidth == 512 && EltWidth == 32)
2098         IID = Intrinsic::x86_avx512_fpclass_ps_512;
2099       else if (VecWidth == 128 && EltWidth == 64)
2100         IID = Intrinsic::x86_avx512_fpclass_pd_128;
2101       else if (VecWidth == 256 && EltWidth == 64)
2102         IID = Intrinsic::x86_avx512_fpclass_pd_256;
2103       else if (VecWidth == 512 && EltWidth == 64)
2104         IID = Intrinsic::x86_avx512_fpclass_pd_512;
2105       else
2106         llvm_unreachable("Unexpected intrinsic");
2107 
2108       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2109                                { CI->getOperand(0), CI->getArgOperand(1) });
2110       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2111     } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
2112       SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2113                                    CI->arg_operands().end());
2114       Type *OpTy = Args[0]->getType();
2115       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2116       unsigned EltWidth = OpTy->getScalarSizeInBits();
2117       Intrinsic::ID IID;
2118       if (VecWidth == 128 && EltWidth == 32)
2119         IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2120       else if (VecWidth == 256 && EltWidth == 32)
2121         IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2122       else if (VecWidth == 512 && EltWidth == 32)
2123         IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2124       else if (VecWidth == 128 && EltWidth == 64)
2125         IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2126       else if (VecWidth == 256 && EltWidth == 64)
2127         IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2128       else if (VecWidth == 512 && EltWidth == 64)
2129         IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2130       else
2131         llvm_unreachable("Unexpected intrinsic");
2132 
2133       Value *Mask = Constant::getAllOnesValue(CI->getType());
2134       if (VecWidth == 512)
2135         std::swap(Mask, Args.back());
2136       Args.push_back(Mask);
2137 
2138       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2139                                Args);
2140     } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
2141       // Integer compare intrinsics.
2142       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2143       Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2144     } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
2145       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2146       Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2147     } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
2148                          Name.startswith("avx512.cvtw2mask.") ||
2149                          Name.startswith("avx512.cvtd2mask.") ||
2150                          Name.startswith("avx512.cvtq2mask."))) {
2151       Value *Op = CI->getArgOperand(0);
2152       Value *Zero = llvm::Constant::getNullValue(Op->getType());
2153       Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2154       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
2155     } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
2156                         Name == "ssse3.pabs.w.128" ||
2157                         Name == "ssse3.pabs.d.128" ||
2158                         Name.startswith("avx2.pabs") ||
2159                         Name.startswith("avx512.mask.pabs"))) {
2160       Rep = upgradeAbs(Builder, *CI);
2161     } else if (IsX86 && (Name == "sse41.pmaxsb" ||
2162                          Name == "sse2.pmaxs.w" ||
2163                          Name == "sse41.pmaxsd" ||
2164                          Name.startswith("avx2.pmaxs") ||
2165                          Name.startswith("avx512.mask.pmaxs"))) {
2166       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
2167     } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
2168                          Name == "sse41.pmaxuw" ||
2169                          Name == "sse41.pmaxud" ||
2170                          Name.startswith("avx2.pmaxu") ||
2171                          Name.startswith("avx512.mask.pmaxu"))) {
2172       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
2173     } else if (IsX86 && (Name == "sse41.pminsb" ||
2174                          Name == "sse2.pmins.w" ||
2175                          Name == "sse41.pminsd" ||
2176                          Name.startswith("avx2.pmins") ||
2177                          Name.startswith("avx512.mask.pmins"))) {
2178       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
2179     } else if (IsX86 && (Name == "sse2.pminu.b" ||
2180                          Name == "sse41.pminuw" ||
2181                          Name == "sse41.pminud" ||
2182                          Name.startswith("avx2.pminu") ||
2183                          Name.startswith("avx512.mask.pminu"))) {
2184       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
2185     } else if (IsX86 && (Name == "sse2.pmulu.dq" ||
2186                          Name == "avx2.pmulu.dq" ||
2187                          Name == "avx512.pmulu.dq.512" ||
2188                          Name.startswith("avx512.mask.pmulu.dq."))) {
2189       Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2190     } else if (IsX86 && (Name == "sse41.pmuldq" ||
2191                          Name == "avx2.pmul.dq" ||
2192                          Name == "avx512.pmul.dq.512" ||
2193                          Name.startswith("avx512.mask.pmul.dq."))) {
2194       Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2195     } else if (IsX86 && (Name == "sse.cvtsi2ss" ||
2196                          Name == "sse2.cvtsi2sd" ||
2197                          Name == "sse.cvtsi642ss" ||
2198                          Name == "sse2.cvtsi642sd")) {
2199       Rep = Builder.CreateSIToFP(
2200           CI->getArgOperand(1),
2201           cast<VectorType>(CI->getType())->getElementType());
2202       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2203     } else if (IsX86 && Name == "avx512.cvtusi2sd") {
2204       Rep = Builder.CreateUIToFP(
2205           CI->getArgOperand(1),
2206           cast<VectorType>(CI->getType())->getElementType());
2207       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2208     } else if (IsX86 && Name == "sse2.cvtss2sd") {
2209       Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2210       Rep = Builder.CreateFPExt(
2211           Rep, cast<VectorType>(CI->getType())->getElementType());
2212       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2213     } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
2214                          Name == "sse2.cvtdq2ps" ||
2215                          Name == "avx.cvtdq2.pd.256" ||
2216                          Name == "avx.cvtdq2.ps.256" ||
2217                          Name.startswith("avx512.mask.cvtdq2pd.") ||
2218                          Name.startswith("avx512.mask.cvtudq2pd.") ||
2219                          Name.startswith("avx512.mask.cvtdq2ps.") ||
2220                          Name.startswith("avx512.mask.cvtudq2ps.") ||
2221                          Name.startswith("avx512.mask.cvtqq2pd.") ||
2222                          Name.startswith("avx512.mask.cvtuqq2pd.") ||
2223                          Name == "avx512.mask.cvtqq2ps.256" ||
2224                          Name == "avx512.mask.cvtqq2ps.512" ||
2225                          Name == "avx512.mask.cvtuqq2ps.256" ||
2226                          Name == "avx512.mask.cvtuqq2ps.512" ||
2227                          Name == "sse2.cvtps2pd" ||
2228                          Name == "avx.cvt.ps2.pd.256" ||
2229                          Name == "avx512.mask.cvtps2pd.128" ||
2230                          Name == "avx512.mask.cvtps2pd.256")) {
2231       auto *DstTy = cast<FixedVectorType>(CI->getType());
2232       Rep = CI->getArgOperand(0);
2233       auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2234 
2235       unsigned NumDstElts = DstTy->getNumElements();
2236       if (NumDstElts < SrcTy->getNumElements()) {
2237         assert(NumDstElts == 2 && "Unexpected vector size");
2238         Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
2239       }
2240 
2241       bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2242       bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
2243       if (IsPS2PD)
2244         Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2245       else if (CI->getNumArgOperands() == 4 &&
2246                (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2247                 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2248         Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2249                                        : Intrinsic::x86_avx512_sitofp_round;
2250         Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2251                                                 { DstTy, SrcTy });
2252         Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2253       } else {
2254         Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2255                          : Builder.CreateSIToFP(Rep, DstTy, "cvt");
2256       }
2257 
2258       if (CI->getNumArgOperands() >= 3)
2259         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2260                             CI->getArgOperand(1));
2261     } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
2262                          Name.startswith("vcvtph2ps."))) {
2263       auto *DstTy = cast<FixedVectorType>(CI->getType());
2264       Rep = CI->getArgOperand(0);
2265       auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2266       unsigned NumDstElts = DstTy->getNumElements();
2267       if (NumDstElts != SrcTy->getNumElements()) {
2268         assert(NumDstElts == 4 && "Unexpected vector size");
2269         Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
2270       }
2271       Rep = Builder.CreateBitCast(
2272           Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
2273       Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2274       if (CI->getNumArgOperands() >= 3)
2275         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2276                             CI->getArgOperand(1));
2277     } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
2278       Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2279                               CI->getArgOperand(1), CI->getArgOperand(2),
2280                               /*Aligned*/false);
2281     } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
2282       Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2283                               CI->getArgOperand(1),CI->getArgOperand(2),
2284                               /*Aligned*/true);
2285     } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
2286       auto *ResultTy = cast<FixedVectorType>(CI->getType());
2287       Type *PtrTy = ResultTy->getElementType();
2288 
2289       // Cast the pointer to element type.
2290       Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2291                                          llvm::PointerType::getUnqual(PtrTy));
2292 
2293       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2294                                      ResultTy->getNumElements());
2295 
2296       Function *ELd = Intrinsic::getDeclaration(F->getParent(),
2297                                                 Intrinsic::masked_expandload,
2298                                                 ResultTy);
2299       Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2300     } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
2301       auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
2302       Type *PtrTy = ResultTy->getElementType();
2303 
2304       // Cast the pointer to element type.
2305       Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2306                                          llvm::PointerType::getUnqual(PtrTy));
2307 
2308       Value *MaskVec =
2309           getX86MaskVec(Builder, CI->getArgOperand(2),
2310                         cast<FixedVectorType>(ResultTy)->getNumElements());
2311 
2312       Function *CSt = Intrinsic::getDeclaration(F->getParent(),
2313                                                 Intrinsic::masked_compressstore,
2314                                                 ResultTy);
2315       Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2316     } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
2317                          Name.startswith("avx512.mask.expand."))) {
2318       auto *ResultTy = cast<FixedVectorType>(CI->getType());
2319 
2320       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2321                                      ResultTy->getNumElements());
2322 
2323       bool IsCompress = Name[12] == 'c';
2324       Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2325                                      : Intrinsic::x86_avx512_mask_expand;
2326       Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
2327       Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2328                                        MaskVec });
2329     } else if (IsX86 && Name.startswith("xop.vpcom")) {
2330       bool IsSigned;
2331       if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
2332           Name.endswith("uq"))
2333         IsSigned = false;
2334       else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
2335                Name.endswith("q"))
2336         IsSigned = true;
2337       else
2338         llvm_unreachable("Unknown suffix");
2339 
2340       unsigned Imm;
2341       if (CI->getNumArgOperands() == 3) {
2342         Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2343       } else {
2344         Name = Name.substr(9); // strip off "xop.vpcom"
2345         if (Name.startswith("lt"))
2346           Imm = 0;
2347         else if (Name.startswith("le"))
2348           Imm = 1;
2349         else if (Name.startswith("gt"))
2350           Imm = 2;
2351         else if (Name.startswith("ge"))
2352           Imm = 3;
2353         else if (Name.startswith("eq"))
2354           Imm = 4;
2355         else if (Name.startswith("ne"))
2356           Imm = 5;
2357         else if (Name.startswith("false"))
2358           Imm = 6;
2359         else if (Name.startswith("true"))
2360           Imm = 7;
2361         else
2362           llvm_unreachable("Unknown condition");
2363       }
2364 
2365       Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2366     } else if (IsX86 && Name.startswith("xop.vpcmov")) {
2367       Value *Sel = CI->getArgOperand(2);
2368       Value *NotSel = Builder.CreateNot(Sel);
2369       Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2370       Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2371       Rep = Builder.CreateOr(Sel0, Sel1);
2372     } else if (IsX86 && (Name.startswith("xop.vprot") ||
2373                          Name.startswith("avx512.prol") ||
2374                          Name.startswith("avx512.mask.prol"))) {
2375       Rep = upgradeX86Rotate(Builder, *CI, false);
2376     } else if (IsX86 && (Name.startswith("avx512.pror") ||
2377                          Name.startswith("avx512.mask.pror"))) {
2378       Rep = upgradeX86Rotate(Builder, *CI, true);
2379     } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
2380                          Name.startswith("avx512.mask.vpshld") ||
2381                          Name.startswith("avx512.maskz.vpshld"))) {
2382       bool ZeroMask = Name[11] == 'z';
2383       Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2384     } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
2385                          Name.startswith("avx512.mask.vpshrd") ||
2386                          Name.startswith("avx512.maskz.vpshrd"))) {
2387       bool ZeroMask = Name[11] == 'z';
2388       Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2389     } else if (IsX86 && Name == "sse42.crc32.64.8") {
2390       Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
2391                                                Intrinsic::x86_sse42_crc32_32_8);
2392       Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2393       Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2394       Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2395     } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
2396                          Name.startswith("avx512.vbroadcast.s"))) {
2397       // Replace broadcasts with a series of insertelements.
2398       auto *VecTy = cast<FixedVectorType>(CI->getType());
2399       Type *EltTy = VecTy->getElementType();
2400       unsigned EltNum = VecTy->getNumElements();
2401       Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2402                                           EltTy->getPointerTo());
2403       Value *Load = Builder.CreateLoad(EltTy, Cast);
2404       Type *I32Ty = Type::getInt32Ty(C);
2405       Rep = UndefValue::get(VecTy);
2406       for (unsigned I = 0; I < EltNum; ++I)
2407         Rep = Builder.CreateInsertElement(Rep, Load,
2408                                           ConstantInt::get(I32Ty, I));
2409     } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
2410                          Name.startswith("sse41.pmovzx") ||
2411                          Name.startswith("avx2.pmovsx") ||
2412                          Name.startswith("avx2.pmovzx") ||
2413                          Name.startswith("avx512.mask.pmovsx") ||
2414                          Name.startswith("avx512.mask.pmovzx"))) {
2415       auto *DstTy = cast<FixedVectorType>(CI->getType());
2416       unsigned NumDstElts = DstTy->getNumElements();
2417 
2418       // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
2419       SmallVector<int, 8> ShuffleMask(NumDstElts);
2420       for (unsigned i = 0; i != NumDstElts; ++i)
2421         ShuffleMask[i] = i;
2422 
2423       Value *SV =
2424           Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
2425 
2426       bool DoSext = (StringRef::npos != Name.find("pmovsx"));
2427       Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
2428                    : Builder.CreateZExt(SV, DstTy);
2429       // If there are 3 arguments, it's a masked intrinsic so we need a select.
2430       if (CI->getNumArgOperands() == 3)
2431         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2432                             CI->getArgOperand(1));
2433     } else if (Name == "avx512.mask.pmov.qd.256" ||
2434                Name == "avx512.mask.pmov.qd.512" ||
2435                Name == "avx512.mask.pmov.wb.256" ||
2436                Name == "avx512.mask.pmov.wb.512") {
2437       Type *Ty = CI->getArgOperand(1)->getType();
2438       Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2439       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2440                           CI->getArgOperand(1));
2441     } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
2442                          Name == "avx2.vbroadcasti128")) {
2443       // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
2444       Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
2445       unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
2446       auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
2447       Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2448                                             PointerType::getUnqual(VT));
2449       Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
2450       if (NumSrcElts == 2)
2451         Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
2452       else
2453         Rep = Builder.CreateShuffleVector(
2454             Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
2455     } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
2456                          Name.startswith("avx512.mask.shuf.f"))) {
2457       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2458       Type *VT = CI->getType();
2459       unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
2460       unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
2461       unsigned ControlBitsMask = NumLanes - 1;
2462       unsigned NumControlBits = NumLanes / 2;
2463       SmallVector<int, 8> ShuffleMask(0);
2464 
2465       for (unsigned l = 0; l != NumLanes; ++l) {
2466         unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2467         // We actually need the other source.
2468         if (l >= NumLanes / 2)
2469           LaneMask += NumLanes;
2470         for (unsigned i = 0; i != NumElementsInLane; ++i)
2471           ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
2472       }
2473       Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2474                                         CI->getArgOperand(1), ShuffleMask);
2475       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2476                           CI->getArgOperand(3));
2477     }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
2478                          Name.startswith("avx512.mask.broadcasti"))) {
2479       unsigned NumSrcElts =
2480           cast<FixedVectorType>(CI->getArgOperand(0)->getType())
2481               ->getNumElements();
2482       unsigned NumDstElts =
2483           cast<FixedVectorType>(CI->getType())->getNumElements();
2484 
2485       SmallVector<int, 8> ShuffleMask(NumDstElts);
2486       for (unsigned i = 0; i != NumDstElts; ++i)
2487         ShuffleMask[i] = i % NumSrcElts;
2488 
2489       Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2490                                         CI->getArgOperand(0),
2491                                         ShuffleMask);
2492       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2493                           CI->getArgOperand(1));
2494     } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
2495                          Name.startswith("avx2.vbroadcast") ||
2496                          Name.startswith("avx512.pbroadcast") ||
2497                          Name.startswith("avx512.mask.broadcast.s"))) {
2498       // Replace vp?broadcasts with a vector shuffle.
2499       Value *Op = CI->getArgOperand(0);
2500       ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
2501       Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
2502       SmallVector<int, 8> M;
2503       ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
2504       Rep = Builder.CreateShuffleVector(Op, M);
2505 
2506       if (CI->getNumArgOperands() == 3)
2507         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2508                             CI->getArgOperand(1));
2509     } else if (IsX86 && (Name.startswith("sse2.padds.") ||
2510                          Name.startswith("avx2.padds.") ||
2511                          Name.startswith("avx512.padds.") ||
2512                          Name.startswith("avx512.mask.padds."))) {
2513       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
2514     } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
2515                          Name.startswith("avx2.psubs.") ||
2516                          Name.startswith("avx512.psubs.") ||
2517                          Name.startswith("avx512.mask.psubs."))) {
2518       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
2519     } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
2520                          Name.startswith("avx2.paddus.") ||
2521                          Name.startswith("avx512.mask.paddus."))) {
2522       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
2523     } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
2524                          Name.startswith("avx2.psubus.") ||
2525                          Name.startswith("avx512.mask.psubus."))) {
2526       Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
2527     } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
2528       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2529                                       CI->getArgOperand(1),
2530                                       CI->getArgOperand(2),
2531                                       CI->getArgOperand(3),
2532                                       CI->getArgOperand(4),
2533                                       false);
2534     } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
2535       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2536                                       CI->getArgOperand(1),
2537                                       CI->getArgOperand(2),
2538                                       CI->getArgOperand(3),
2539                                       CI->getArgOperand(4),
2540                                       true);
2541     } else if (IsX86 && (Name == "sse2.psll.dq" ||
2542                          Name == "avx2.psll.dq")) {
2543       // 128/256-bit shift left specified in bits.
2544       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2545       Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2546                                        Shift / 8); // Shift is in bits.
2547     } else if (IsX86 && (Name == "sse2.psrl.dq" ||
2548                          Name == "avx2.psrl.dq")) {
2549       // 128/256-bit shift right specified in bits.
2550       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2551       Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2552                                        Shift / 8); // Shift is in bits.
2553     } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
2554                          Name == "avx2.psll.dq.bs" ||
2555                          Name == "avx512.psll.dq.512")) {
2556       // 128/256/512-bit shift left specified in bytes.
2557       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2558       Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2559     } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
2560                          Name == "avx2.psrl.dq.bs" ||
2561                          Name == "avx512.psrl.dq.512")) {
2562       // 128/256/512-bit shift right specified in bytes.
2563       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2564       Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2565     } else if (IsX86 && (Name == "sse41.pblendw" ||
2566                          Name.startswith("sse41.blendp") ||
2567                          Name.startswith("avx.blend.p") ||
2568                          Name == "avx2.pblendw" ||
2569                          Name.startswith("avx2.pblendd."))) {
2570       Value *Op0 = CI->getArgOperand(0);
2571       Value *Op1 = CI->getArgOperand(1);
2572       unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2573       auto *VecTy = cast<FixedVectorType>(CI->getType());
2574       unsigned NumElts = VecTy->getNumElements();
2575 
2576       SmallVector<int, 16> Idxs(NumElts);
2577       for (unsigned i = 0; i != NumElts; ++i)
2578         Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
2579 
2580       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2581     } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
2582                          Name == "avx2.vinserti128" ||
2583                          Name.startswith("avx512.mask.insert"))) {
2584       Value *Op0 = CI->getArgOperand(0);
2585       Value *Op1 = CI->getArgOperand(1);
2586       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2587       unsigned DstNumElts =
2588           cast<FixedVectorType>(CI->getType())->getNumElements();
2589       unsigned SrcNumElts =
2590           cast<FixedVectorType>(Op1->getType())->getNumElements();
2591       unsigned Scale = DstNumElts / SrcNumElts;
2592 
2593       // Mask off the high bits of the immediate value; hardware ignores those.
2594       Imm = Imm % Scale;
2595 
2596       // Extend the second operand into a vector the size of the destination.
2597       SmallVector<int, 8> Idxs(DstNumElts);
2598       for (unsigned i = 0; i != SrcNumElts; ++i)
2599         Idxs[i] = i;
2600       for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
2601         Idxs[i] = SrcNumElts;
2602       Rep = Builder.CreateShuffleVector(Op1, Idxs);
2603 
2604       // Insert the second operand into the first operand.
2605 
2606       // Note that there is no guarantee that instruction lowering will actually
2607       // produce a vinsertf128 instruction for the created shuffles. In
2608       // particular, the 0 immediate case involves no lane changes, so it can
2609       // be handled as a blend.
2610 
2611       // Example of shuffle mask for 32-bit elements:
2612       // Imm = 1  <i32 0, i32 1, i32 2,  i32 3,  i32 8, i32 9, i32 10, i32 11>
2613       // Imm = 0  <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6,  i32 7 >
2614 
2615       // First fill with identify mask.
2616       for (unsigned i = 0; i != DstNumElts; ++i)
2617         Idxs[i] = i;
2618       // Then replace the elements where we need to insert.
2619       for (unsigned i = 0; i != SrcNumElts; ++i)
2620         Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
2621       Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
2622 
2623       // If the intrinsic has a mask operand, handle that.
2624       if (CI->getNumArgOperands() == 5)
2625         Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2626                             CI->getArgOperand(3));
2627     } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
2628                          Name == "avx2.vextracti128" ||
2629                          Name.startswith("avx512.mask.vextract"))) {
2630       Value *Op0 = CI->getArgOperand(0);
2631       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2632       unsigned DstNumElts =
2633           cast<FixedVectorType>(CI->getType())->getNumElements();
2634       unsigned SrcNumElts =
2635           cast<FixedVectorType>(Op0->getType())->getNumElements();
2636       unsigned Scale = SrcNumElts / DstNumElts;
2637 
2638       // Mask off the high bits of the immediate value; hardware ignores those.
2639       Imm = Imm % Scale;
2640 
2641       // Get indexes for the subvector of the input vector.
2642       SmallVector<int, 8> Idxs(DstNumElts);
2643       for (unsigned i = 0; i != DstNumElts; ++i) {
2644         Idxs[i] = i + (Imm * DstNumElts);
2645       }
2646       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2647 
2648       // If the intrinsic has a mask operand, handle that.
2649       if (CI->getNumArgOperands() == 4)
2650         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2651                             CI->getArgOperand(2));
2652     } else if (!IsX86 && Name == "stackprotectorcheck") {
2653       Rep = nullptr;
2654     } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
2655                          Name.startswith("avx512.mask.perm.di."))) {
2656       Value *Op0 = CI->getArgOperand(0);
2657       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2658       auto *VecTy = cast<FixedVectorType>(CI->getType());
2659       unsigned NumElts = VecTy->getNumElements();
2660 
2661       SmallVector<int, 8> Idxs(NumElts);
2662       for (unsigned i = 0; i != NumElts; ++i)
2663         Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
2664 
2665       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2666 
2667       if (CI->getNumArgOperands() == 4)
2668         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2669                             CI->getArgOperand(2));
2670     } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
2671                          Name == "avx2.vperm2i128")) {
2672       // The immediate permute control byte looks like this:
2673       //    [1:0] - select 128 bits from sources for low half of destination
2674       //    [2]   - ignore
2675       //    [3]   - zero low half of destination
2676       //    [5:4] - select 128 bits from sources for high half of destination
2677       //    [6]   - ignore
2678       //    [7]   - zero high half of destination
2679 
2680       uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2681 
2682       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2683       unsigned HalfSize = NumElts / 2;
2684       SmallVector<int, 8> ShuffleMask(NumElts);
2685 
2686       // Determine which operand(s) are actually in use for this instruction.
2687       Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2688       Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2689 
2690       // If needed, replace operands based on zero mask.
2691       V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
2692       V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
2693 
2694       // Permute low half of result.
2695       unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
2696       for (unsigned i = 0; i < HalfSize; ++i)
2697         ShuffleMask[i] = StartIndex + i;
2698 
2699       // Permute high half of result.
2700       StartIndex = (Imm & 0x10) ? HalfSize : 0;
2701       for (unsigned i = 0; i < HalfSize; ++i)
2702         ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
2703 
2704       Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2705 
2706     } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
2707                          Name == "sse2.pshuf.d" ||
2708                          Name.startswith("avx512.mask.vpermil.p") ||
2709                          Name.startswith("avx512.mask.pshuf.d."))) {
2710       Value *Op0 = CI->getArgOperand(0);
2711       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2712       auto *VecTy = cast<FixedVectorType>(CI->getType());
2713       unsigned NumElts = VecTy->getNumElements();
2714       // Calculate the size of each index in the immediate.
2715       unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
2716       unsigned IdxMask = ((1 << IdxSize) - 1);
2717 
2718       SmallVector<int, 8> Idxs(NumElts);
2719       // Lookup the bits for this element, wrapping around the immediate every
2720       // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
2721       // to offset by the first index of each group.
2722       for (unsigned i = 0; i != NumElts; ++i)
2723         Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
2724 
2725       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2726 
2727       if (CI->getNumArgOperands() == 4)
2728         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2729                             CI->getArgOperand(2));
2730     } else if (IsX86 && (Name == "sse2.pshufl.w" ||
2731                          Name.startswith("avx512.mask.pshufl.w."))) {
2732       Value *Op0 = CI->getArgOperand(0);
2733       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2734       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2735 
2736       SmallVector<int, 16> Idxs(NumElts);
2737       for (unsigned l = 0; l != NumElts; l += 8) {
2738         for (unsigned i = 0; i != 4; ++i)
2739           Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
2740         for (unsigned i = 4; i != 8; ++i)
2741           Idxs[i + l] = i + l;
2742       }
2743 
2744       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2745 
2746       if (CI->getNumArgOperands() == 4)
2747         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2748                             CI->getArgOperand(2));
2749     } else if (IsX86 && (Name == "sse2.pshufh.w" ||
2750                          Name.startswith("avx512.mask.pshufh.w."))) {
2751       Value *Op0 = CI->getArgOperand(0);
2752       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2753       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2754 
2755       SmallVector<int, 16> Idxs(NumElts);
2756       for (unsigned l = 0; l != NumElts; l += 8) {
2757         for (unsigned i = 0; i != 4; ++i)
2758           Idxs[i + l] = i + l;
2759         for (unsigned i = 0; i != 4; ++i)
2760           Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
2761       }
2762 
2763       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2764 
2765       if (CI->getNumArgOperands() == 4)
2766         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2767                             CI->getArgOperand(2));
2768     } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
2769       Value *Op0 = CI->getArgOperand(0);
2770       Value *Op1 = CI->getArgOperand(1);
2771       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2772       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2773 
2774       unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2775       unsigned HalfLaneElts = NumLaneElts / 2;
2776 
2777       SmallVector<int, 16> Idxs(NumElts);
2778       for (unsigned i = 0; i != NumElts; ++i) {
2779         // Base index is the starting element of the lane.
2780         Idxs[i] = i - (i % NumLaneElts);
2781         // If we are half way through the lane switch to the other source.
2782         if ((i % NumLaneElts) >= HalfLaneElts)
2783           Idxs[i] += NumElts;
2784         // Now select the specific element. By adding HalfLaneElts bits from
2785         // the immediate. Wrapping around the immediate every 8-bits.
2786         Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
2787       }
2788 
2789       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2790 
2791       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2792                           CI->getArgOperand(3));
2793     } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
2794                          Name.startswith("avx512.mask.movshdup") ||
2795                          Name.startswith("avx512.mask.movsldup"))) {
2796       Value *Op0 = CI->getArgOperand(0);
2797       unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2798       unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2799 
2800       unsigned Offset = 0;
2801       if (Name.startswith("avx512.mask.movshdup."))
2802         Offset = 1;
2803 
2804       SmallVector<int, 16> Idxs(NumElts);
2805       for (unsigned l = 0; l != NumElts; l += NumLaneElts)
2806         for (unsigned i = 0; i != NumLaneElts; i += 2) {
2807           Idxs[i + l + 0] = i + l + Offset;
2808           Idxs[i + l + 1] = i + l + Offset;
2809         }
2810 
2811       Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2812 
2813       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2814                           CI->getArgOperand(1));
2815     } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
2816                          Name.startswith("avx512.mask.unpckl."))) {
2817       Value *Op0 = CI->getArgOperand(0);
2818       Value *Op1 = CI->getArgOperand(1);
2819       int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2820       int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2821 
2822       SmallVector<int, 64> Idxs(NumElts);
2823       for (int l = 0; l != NumElts; l += NumLaneElts)
2824         for (int i = 0; i != NumLaneElts; ++i)
2825           Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
2826 
2827       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2828 
2829       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2830                           CI->getArgOperand(2));
2831     } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
2832                          Name.startswith("avx512.mask.unpckh."))) {
2833       Value *Op0 = CI->getArgOperand(0);
2834       Value *Op1 = CI->getArgOperand(1);
2835       int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2836       int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2837 
2838       SmallVector<int, 64> Idxs(NumElts);
2839       for (int l = 0; l != NumElts; l += NumLaneElts)
2840         for (int i = 0; i != NumLaneElts; ++i)
2841           Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
2842 
2843       Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2844 
2845       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2846                           CI->getArgOperand(2));
2847     } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
2848                          Name.startswith("avx512.mask.pand."))) {
2849       VectorType *FTy = cast<VectorType>(CI->getType());
2850       VectorType *ITy = VectorType::getInteger(FTy);
2851       Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2852                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2853       Rep = Builder.CreateBitCast(Rep, FTy);
2854       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2855                           CI->getArgOperand(2));
2856     } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
2857                          Name.startswith("avx512.mask.pandn."))) {
2858       VectorType *FTy = cast<VectorType>(CI->getType());
2859       VectorType *ITy = VectorType::getInteger(FTy);
2860       Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
2861       Rep = Builder.CreateAnd(Rep,
2862                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2863       Rep = Builder.CreateBitCast(Rep, FTy);
2864       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2865                           CI->getArgOperand(2));
2866     } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
2867                          Name.startswith("avx512.mask.por."))) {
2868       VectorType *FTy = cast<VectorType>(CI->getType());
2869       VectorType *ITy = VectorType::getInteger(FTy);
2870       Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2871                              Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2872       Rep = Builder.CreateBitCast(Rep, FTy);
2873       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2874                           CI->getArgOperand(2));
2875     } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
2876                          Name.startswith("avx512.mask.pxor."))) {
2877       VectorType *FTy = cast<VectorType>(CI->getType());
2878       VectorType *ITy = VectorType::getInteger(FTy);
2879       Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2880                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2881       Rep = Builder.CreateBitCast(Rep, FTy);
2882       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2883                           CI->getArgOperand(2));
2884     } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
2885       Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2886       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2887                           CI->getArgOperand(2));
2888     } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
2889       Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
2890       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2891                           CI->getArgOperand(2));
2892     } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
2893       Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
2894       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2895                           CI->getArgOperand(2));
2896     } else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
2897       if (Name.endswith(".512")) {
2898         Intrinsic::ID IID;
2899         if (Name[17] == 's')
2900           IID = Intrinsic::x86_avx512_add_ps_512;
2901         else
2902           IID = Intrinsic::x86_avx512_add_pd_512;
2903 
2904         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2905                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2906                                    CI->getArgOperand(4) });
2907       } else {
2908         Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2909       }
2910       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2911                           CI->getArgOperand(2));
2912     } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
2913       if (Name.endswith(".512")) {
2914         Intrinsic::ID IID;
2915         if (Name[17] == 's')
2916           IID = Intrinsic::x86_avx512_div_ps_512;
2917         else
2918           IID = Intrinsic::x86_avx512_div_pd_512;
2919 
2920         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2921                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2922                                    CI->getArgOperand(4) });
2923       } else {
2924         Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
2925       }
2926       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2927                           CI->getArgOperand(2));
2928     } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
2929       if (Name.endswith(".512")) {
2930         Intrinsic::ID IID;
2931         if (Name[17] == 's')
2932           IID = Intrinsic::x86_avx512_mul_ps_512;
2933         else
2934           IID = Intrinsic::x86_avx512_mul_pd_512;
2935 
2936         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2937                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2938                                    CI->getArgOperand(4) });
2939       } else {
2940         Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
2941       }
2942       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2943                           CI->getArgOperand(2));
2944     } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
2945       if (Name.endswith(".512")) {
2946         Intrinsic::ID IID;
2947         if (Name[17] == 's')
2948           IID = Intrinsic::x86_avx512_sub_ps_512;
2949         else
2950           IID = Intrinsic::x86_avx512_sub_pd_512;
2951 
2952         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2953                                  { CI->getArgOperand(0), CI->getArgOperand(1),
2954                                    CI->getArgOperand(4) });
2955       } else {
2956         Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
2957       }
2958       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2959                           CI->getArgOperand(2));
2960     } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
2961                          Name.startswith("avx512.mask.min.p")) &&
2962                Name.drop_front(18) == ".512") {
2963       bool IsDouble = Name[17] == 'd';
2964       bool IsMin = Name[13] == 'i';
2965       static const Intrinsic::ID MinMaxTbl[2][2] = {
2966         { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
2967         { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
2968       };
2969       Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
2970 
2971       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2972                                { CI->getArgOperand(0), CI->getArgOperand(1),
2973                                  CI->getArgOperand(4) });
2974       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2975                           CI->getArgOperand(2));
2976     } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
2977       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2978                                                          Intrinsic::ctlz,
2979                                                          CI->getType()),
2980                                { CI->getArgOperand(0), Builder.getInt1(false) });
2981       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2982                           CI->getArgOperand(1));
2983     } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
2984       bool IsImmediate = Name[16] == 'i' ||
2985                          (Name.size() > 18 && Name[18] == 'i');
2986       bool IsVariable = Name[16] == 'v';
2987       char Size = Name[16] == '.' ? Name[17] :
2988                   Name[17] == '.' ? Name[18] :
2989                   Name[18] == '.' ? Name[19] :
2990                                     Name[20];
2991 
2992       Intrinsic::ID IID;
2993       if (IsVariable && Name[17] != '.') {
2994         if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
2995           IID = Intrinsic::x86_avx2_psllv_q;
2996         else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
2997           IID = Intrinsic::x86_avx2_psllv_q_256;
2998         else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
2999           IID = Intrinsic::x86_avx2_psllv_d;
3000         else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
3001           IID = Intrinsic::x86_avx2_psllv_d_256;
3002         else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
3003           IID = Intrinsic::x86_avx512_psllv_w_128;
3004         else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
3005           IID = Intrinsic::x86_avx512_psllv_w_256;
3006         else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
3007           IID = Intrinsic::x86_avx512_psllv_w_512;
3008         else
3009           llvm_unreachable("Unexpected size");
3010       } else if (Name.endswith(".128")) {
3011         if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
3012           IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3013                             : Intrinsic::x86_sse2_psll_d;
3014         else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
3015           IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3016                             : Intrinsic::x86_sse2_psll_q;
3017         else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
3018           IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3019                             : Intrinsic::x86_sse2_psll_w;
3020         else
3021           llvm_unreachable("Unexpected size");
3022       } else if (Name.endswith(".256")) {
3023         if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
3024           IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3025                             : Intrinsic::x86_avx2_psll_d;
3026         else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
3027           IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3028                             : Intrinsic::x86_avx2_psll_q;
3029         else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
3030           IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3031                             : Intrinsic::x86_avx2_psll_w;
3032         else
3033           llvm_unreachable("Unexpected size");
3034       } else {
3035         if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
3036           IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3037                 IsVariable  ? Intrinsic::x86_avx512_psllv_d_512 :
3038                               Intrinsic::x86_avx512_psll_d_512;
3039         else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
3040           IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3041                 IsVariable  ? Intrinsic::x86_avx512_psllv_q_512 :
3042                               Intrinsic::x86_avx512_psll_q_512;
3043         else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
3044           IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3045                             : Intrinsic::x86_avx512_psll_w_512;
3046         else
3047           llvm_unreachable("Unexpected size");
3048       }
3049 
3050       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3051     } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
3052       bool IsImmediate = Name[16] == 'i' ||
3053                          (Name.size() > 18 && Name[18] == 'i');
3054       bool IsVariable = Name[16] == 'v';
3055       char Size = Name[16] == '.' ? Name[17] :
3056                   Name[17] == '.' ? Name[18] :
3057                   Name[18] == '.' ? Name[19] :
3058                                     Name[20];
3059 
3060       Intrinsic::ID IID;
3061       if (IsVariable && Name[17] != '.') {
3062         if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
3063           IID = Intrinsic::x86_avx2_psrlv_q;
3064         else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
3065           IID = Intrinsic::x86_avx2_psrlv_q_256;
3066         else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
3067           IID = Intrinsic::x86_avx2_psrlv_d;
3068         else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
3069           IID = Intrinsic::x86_avx2_psrlv_d_256;
3070         else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
3071           IID = Intrinsic::x86_avx512_psrlv_w_128;
3072         else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
3073           IID = Intrinsic::x86_avx512_psrlv_w_256;
3074         else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
3075           IID = Intrinsic::x86_avx512_psrlv_w_512;
3076         else
3077           llvm_unreachable("Unexpected size");
3078       } else if (Name.endswith(".128")) {
3079         if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
3080           IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3081                             : Intrinsic::x86_sse2_psrl_d;
3082         else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
3083           IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3084                             : Intrinsic::x86_sse2_psrl_q;
3085         else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
3086           IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3087                             : Intrinsic::x86_sse2_psrl_w;
3088         else
3089           llvm_unreachable("Unexpected size");
3090       } else if (Name.endswith(".256")) {
3091         if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
3092           IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3093                             : Intrinsic::x86_avx2_psrl_d;
3094         else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
3095           IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3096                             : Intrinsic::x86_avx2_psrl_q;
3097         else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
3098           IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3099                             : Intrinsic::x86_avx2_psrl_w;
3100         else
3101           llvm_unreachable("Unexpected size");
3102       } else {
3103         if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
3104           IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3105                 IsVariable  ? Intrinsic::x86_avx512_psrlv_d_512 :
3106                               Intrinsic::x86_avx512_psrl_d_512;
3107         else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
3108           IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3109                 IsVariable  ? Intrinsic::x86_avx512_psrlv_q_512 :
3110                               Intrinsic::x86_avx512_psrl_q_512;
3111         else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
3112           IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3113                             : Intrinsic::x86_avx512_psrl_w_512;
3114         else
3115           llvm_unreachable("Unexpected size");
3116       }
3117 
3118       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3119     } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
3120       bool IsImmediate = Name[16] == 'i' ||
3121                          (Name.size() > 18 && Name[18] == 'i');
3122       bool IsVariable = Name[16] == 'v';
3123       char Size = Name[16] == '.' ? Name[17] :
3124                   Name[17] == '.' ? Name[18] :
3125                   Name[18] == '.' ? Name[19] :
3126                                     Name[20];
3127 
3128       Intrinsic::ID IID;
3129       if (IsVariable && Name[17] != '.') {
3130         if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
3131           IID = Intrinsic::x86_avx2_psrav_d;
3132         else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
3133           IID = Intrinsic::x86_avx2_psrav_d_256;
3134         else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
3135           IID = Intrinsic::x86_avx512_psrav_w_128;
3136         else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
3137           IID = Intrinsic::x86_avx512_psrav_w_256;
3138         else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
3139           IID = Intrinsic::x86_avx512_psrav_w_512;
3140         else
3141           llvm_unreachable("Unexpected size");
3142       } else if (Name.endswith(".128")) {
3143         if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
3144           IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3145                             : Intrinsic::x86_sse2_psra_d;
3146         else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
3147           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3148                 IsVariable  ? Intrinsic::x86_avx512_psrav_q_128 :
3149                               Intrinsic::x86_avx512_psra_q_128;
3150         else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
3151           IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3152                             : Intrinsic::x86_sse2_psra_w;
3153         else
3154           llvm_unreachable("Unexpected size");
3155       } else if (Name.endswith(".256")) {
3156         if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
3157           IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3158                             : Intrinsic::x86_avx2_psra_d;
3159         else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
3160           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3161                 IsVariable  ? Intrinsic::x86_avx512_psrav_q_256 :
3162                               Intrinsic::x86_avx512_psra_q_256;
3163         else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
3164           IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3165                             : Intrinsic::x86_avx2_psra_w;
3166         else
3167           llvm_unreachable("Unexpected size");
3168       } else {
3169         if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
3170           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3171                 IsVariable  ? Intrinsic::x86_avx512_psrav_d_512 :
3172                               Intrinsic::x86_avx512_psra_d_512;
3173         else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
3174           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3175                 IsVariable  ? Intrinsic::x86_avx512_psrav_q_512 :
3176                               Intrinsic::x86_avx512_psra_q_512;
3177         else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
3178           IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3179                             : Intrinsic::x86_avx512_psra_w_512;
3180         else
3181           llvm_unreachable("Unexpected size");
3182       }
3183 
3184       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3185     } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
3186       Rep = upgradeMaskedMove(Builder, *CI);
3187     } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
3188       Rep = UpgradeMaskToInt(Builder, *CI);
3189     } else if (IsX86 && Name.endswith(".movntdqa")) {
3190       Module *M = F->getParent();
3191       MDNode *Node = MDNode::get(
3192           C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3193 
3194       Value *Ptr = CI->getArgOperand(0);
3195 
3196       // Convert the type of the pointer to a pointer to the stored type.
3197       Value *BC = Builder.CreateBitCast(
3198           Ptr, PointerType::getUnqual(CI->getType()), "cast");
3199       LoadInst *LI = Builder.CreateAlignedLoad(
3200           CI->getType(), BC,
3201           Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
3202       LI->setMetadata(M->getMDKindID("nontemporal"), Node);
3203       Rep = LI;
3204     } else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
3205                          Name.startswith("fma.vfmsub.") ||
3206                          Name.startswith("fma.vfnmadd.") ||
3207                          Name.startswith("fma.vfnmsub."))) {
3208       bool NegMul = Name[6] == 'n';
3209       bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
3210       bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
3211 
3212       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3213                        CI->getArgOperand(2) };
3214 
3215       if (IsScalar) {
3216         Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3217         Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3218         Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3219       }
3220 
3221       if (NegMul && !IsScalar)
3222         Ops[0] = Builder.CreateFNeg(Ops[0]);
3223       if (NegMul && IsScalar)
3224         Ops[1] = Builder.CreateFNeg(Ops[1]);
3225       if (NegAcc)
3226         Ops[2] = Builder.CreateFNeg(Ops[2]);
3227 
3228       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3229                                                          Intrinsic::fma,
3230                                                          Ops[0]->getType()),
3231                                Ops);
3232 
3233       if (IsScalar)
3234         Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3235                                           (uint64_t)0);
3236     } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
3237       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3238                        CI->getArgOperand(2) };
3239 
3240       Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3241       Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3242       Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3243 
3244       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3245                                                          Intrinsic::fma,
3246                                                          Ops[0]->getType()),
3247                                Ops);
3248 
3249       Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3250                                         Rep, (uint64_t)0);
3251     } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
3252                          Name.startswith("avx512.maskz.vfmadd.s") ||
3253                          Name.startswith("avx512.mask3.vfmadd.s") ||
3254                          Name.startswith("avx512.mask3.vfmsub.s") ||
3255                          Name.startswith("avx512.mask3.vfnmsub.s"))) {
3256       bool IsMask3 = Name[11] == '3';
3257       bool IsMaskZ = Name[11] == 'z';
3258       // Drop the "avx512.mask." to make it easier.
3259       Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3260       bool NegMul = Name[2] == 'n';
3261       bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3262 
3263       Value *A = CI->getArgOperand(0);
3264       Value *B = CI->getArgOperand(1);
3265       Value *C = CI->getArgOperand(2);
3266 
3267       if (NegMul && (IsMask3 || IsMaskZ))
3268         A = Builder.CreateFNeg(A);
3269       if (NegMul && !(IsMask3 || IsMaskZ))
3270         B = Builder.CreateFNeg(B);
3271       if (NegAcc)
3272         C = Builder.CreateFNeg(C);
3273 
3274       A = Builder.CreateExtractElement(A, (uint64_t)0);
3275       B = Builder.CreateExtractElement(B, (uint64_t)0);
3276       C = Builder.CreateExtractElement(C, (uint64_t)0);
3277 
3278       if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3279           cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3280         Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
3281 
3282         Intrinsic::ID IID;
3283         if (Name.back() == 'd')
3284           IID = Intrinsic::x86_avx512_vfmadd_f64;
3285         else
3286           IID = Intrinsic::x86_avx512_vfmadd_f32;
3287         Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3288         Rep = Builder.CreateCall(FMA, Ops);
3289       } else {
3290         Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3291                                                   Intrinsic::fma,
3292                                                   A->getType());
3293         Rep = Builder.CreateCall(FMA, { A, B, C });
3294       }
3295 
3296       Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
3297                         IsMask3 ? C : A;
3298 
3299       // For Mask3 with NegAcc, we need to create a new extractelement that
3300       // avoids the negation above.
3301       if (NegAcc && IsMask3)
3302         PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3303                                                 (uint64_t)0);
3304 
3305       Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3306                                 Rep, PassThru);
3307       Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3308                                         Rep, (uint64_t)0);
3309     } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
3310                          Name.startswith("avx512.mask.vfnmadd.p") ||
3311                          Name.startswith("avx512.mask.vfnmsub.p") ||
3312                          Name.startswith("avx512.mask3.vfmadd.p") ||
3313                          Name.startswith("avx512.mask3.vfmsub.p") ||
3314                          Name.startswith("avx512.mask3.vfnmsub.p") ||
3315                          Name.startswith("avx512.maskz.vfmadd.p"))) {
3316       bool IsMask3 = Name[11] == '3';
3317       bool IsMaskZ = Name[11] == 'z';
3318       // Drop the "avx512.mask." to make it easier.
3319       Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3320       bool NegMul = Name[2] == 'n';
3321       bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3322 
3323       Value *A = CI->getArgOperand(0);
3324       Value *B = CI->getArgOperand(1);
3325       Value *C = CI->getArgOperand(2);
3326 
3327       if (NegMul && (IsMask3 || IsMaskZ))
3328         A = Builder.CreateFNeg(A);
3329       if (NegMul && !(IsMask3 || IsMaskZ))
3330         B = Builder.CreateFNeg(B);
3331       if (NegAcc)
3332         C = Builder.CreateFNeg(C);
3333 
3334       if (CI->getNumArgOperands() == 5 &&
3335           (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3336            cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3337         Intrinsic::ID IID;
3338         // Check the character before ".512" in string.
3339         if (Name[Name.size()-5] == 's')
3340           IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3341         else
3342           IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3343 
3344         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3345                                  { A, B, C, CI->getArgOperand(4) });
3346       } else {
3347         Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3348                                                   Intrinsic::fma,
3349                                                   A->getType());
3350         Rep = Builder.CreateCall(FMA, { A, B, C });
3351       }
3352 
3353       Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3354                         IsMask3 ? CI->getArgOperand(2) :
3355                                   CI->getArgOperand(0);
3356 
3357       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3358     } else if (IsX86 &&  Name.startswith("fma.vfmsubadd.p")) {
3359       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3360       unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3361       Intrinsic::ID IID;
3362       if (VecWidth == 128 && EltWidth == 32)
3363         IID = Intrinsic::x86_fma_vfmaddsub_ps;
3364       else if (VecWidth == 256 && EltWidth == 32)
3365         IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3366       else if (VecWidth == 128 && EltWidth == 64)
3367         IID = Intrinsic::x86_fma_vfmaddsub_pd;
3368       else if (VecWidth == 256 && EltWidth == 64)
3369         IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3370       else
3371         llvm_unreachable("Unexpected intrinsic");
3372 
3373       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3374                        CI->getArgOperand(2) };
3375       Ops[2] = Builder.CreateFNeg(Ops[2]);
3376       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3377                                Ops);
3378     } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
3379                          Name.startswith("avx512.mask3.vfmaddsub.p") ||
3380                          Name.startswith("avx512.maskz.vfmaddsub.p") ||
3381                          Name.startswith("avx512.mask3.vfmsubadd.p"))) {
3382       bool IsMask3 = Name[11] == '3';
3383       bool IsMaskZ = Name[11] == 'z';
3384       // Drop the "avx512.mask." to make it easier.
3385       Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3386       bool IsSubAdd = Name[3] == 's';
3387       if (CI->getNumArgOperands() == 5) {
3388         Intrinsic::ID IID;
3389         // Check the character before ".512" in string.
3390         if (Name[Name.size()-5] == 's')
3391           IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3392         else
3393           IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3394 
3395         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3396                          CI->getArgOperand(2), CI->getArgOperand(4) };
3397         if (IsSubAdd)
3398           Ops[2] = Builder.CreateFNeg(Ops[2]);
3399 
3400         Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3401                                  Ops);
3402       } else {
3403         int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3404 
3405         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3406                          CI->getArgOperand(2) };
3407 
3408         Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3409                                                   Ops[0]->getType());
3410         Value *Odd = Builder.CreateCall(FMA, Ops);
3411         Ops[2] = Builder.CreateFNeg(Ops[2]);
3412         Value *Even = Builder.CreateCall(FMA, Ops);
3413 
3414         if (IsSubAdd)
3415           std::swap(Even, Odd);
3416 
3417         SmallVector<int, 32> Idxs(NumElts);
3418         for (int i = 0; i != NumElts; ++i)
3419           Idxs[i] = i + (i % 2) * NumElts;
3420 
3421         Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
3422       }
3423 
3424       Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3425                         IsMask3 ? CI->getArgOperand(2) :
3426                                   CI->getArgOperand(0);
3427 
3428       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3429     } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
3430                          Name.startswith("avx512.maskz.pternlog."))) {
3431       bool ZeroMask = Name[11] == 'z';
3432       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3433       unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3434       Intrinsic::ID IID;
3435       if (VecWidth == 128 && EltWidth == 32)
3436         IID = Intrinsic::x86_avx512_pternlog_d_128;
3437       else if (VecWidth == 256 && EltWidth == 32)
3438         IID = Intrinsic::x86_avx512_pternlog_d_256;
3439       else if (VecWidth == 512 && EltWidth == 32)
3440         IID = Intrinsic::x86_avx512_pternlog_d_512;
3441       else if (VecWidth == 128 && EltWidth == 64)
3442         IID = Intrinsic::x86_avx512_pternlog_q_128;
3443       else if (VecWidth == 256 && EltWidth == 64)
3444         IID = Intrinsic::x86_avx512_pternlog_q_256;
3445       else if (VecWidth == 512 && EltWidth == 64)
3446         IID = Intrinsic::x86_avx512_pternlog_q_512;
3447       else
3448         llvm_unreachable("Unexpected intrinsic");
3449 
3450       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3451                         CI->getArgOperand(2), CI->getArgOperand(3) };
3452       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3453                                Args);
3454       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3455                                  : CI->getArgOperand(0);
3456       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3457     } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
3458                          Name.startswith("avx512.maskz.vpmadd52"))) {
3459       bool ZeroMask = Name[11] == 'z';
3460       bool High = Name[20] == 'h' || Name[21] == 'h';
3461       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3462       Intrinsic::ID IID;
3463       if (VecWidth == 128 && !High)
3464         IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3465       else if (VecWidth == 256 && !High)
3466         IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3467       else if (VecWidth == 512 && !High)
3468         IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3469       else if (VecWidth == 128 && High)
3470         IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3471       else if (VecWidth == 256 && High)
3472         IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3473       else if (VecWidth == 512 && High)
3474         IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3475       else
3476         llvm_unreachable("Unexpected intrinsic");
3477 
3478       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3479                         CI->getArgOperand(2) };
3480       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3481                                Args);
3482       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3483                                  : CI->getArgOperand(0);
3484       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3485     } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
3486                          Name.startswith("avx512.mask.vpermt2var.") ||
3487                          Name.startswith("avx512.maskz.vpermt2var."))) {
3488       bool ZeroMask = Name[11] == 'z';
3489       bool IndexForm = Name[17] == 'i';
3490       Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3491     } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
3492                          Name.startswith("avx512.maskz.vpdpbusd.") ||
3493                          Name.startswith("avx512.mask.vpdpbusds.") ||
3494                          Name.startswith("avx512.maskz.vpdpbusds."))) {
3495       bool ZeroMask = Name[11] == 'z';
3496       bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3497       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3498       Intrinsic::ID IID;
3499       if (VecWidth == 128 && !IsSaturating)
3500         IID = Intrinsic::x86_avx512_vpdpbusd_128;
3501       else if (VecWidth == 256 && !IsSaturating)
3502         IID = Intrinsic::x86_avx512_vpdpbusd_256;
3503       else if (VecWidth == 512 && !IsSaturating)
3504         IID = Intrinsic::x86_avx512_vpdpbusd_512;
3505       else if (VecWidth == 128 && IsSaturating)
3506         IID = Intrinsic::x86_avx512_vpdpbusds_128;
3507       else if (VecWidth == 256 && IsSaturating)
3508         IID = Intrinsic::x86_avx512_vpdpbusds_256;
3509       else if (VecWidth == 512 && IsSaturating)
3510         IID = Intrinsic::x86_avx512_vpdpbusds_512;
3511       else
3512         llvm_unreachable("Unexpected intrinsic");
3513 
3514       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3515                         CI->getArgOperand(2)  };
3516       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3517                                Args);
3518       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3519                                  : CI->getArgOperand(0);
3520       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3521     } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
3522                          Name.startswith("avx512.maskz.vpdpwssd.") ||
3523                          Name.startswith("avx512.mask.vpdpwssds.") ||
3524                          Name.startswith("avx512.maskz.vpdpwssds."))) {
3525       bool ZeroMask = Name[11] == 'z';
3526       bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3527       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3528       Intrinsic::ID IID;
3529       if (VecWidth == 128 && !IsSaturating)
3530         IID = Intrinsic::x86_avx512_vpdpwssd_128;
3531       else if (VecWidth == 256 && !IsSaturating)
3532         IID = Intrinsic::x86_avx512_vpdpwssd_256;
3533       else if (VecWidth == 512 && !IsSaturating)
3534         IID = Intrinsic::x86_avx512_vpdpwssd_512;
3535       else if (VecWidth == 128 && IsSaturating)
3536         IID = Intrinsic::x86_avx512_vpdpwssds_128;
3537       else if (VecWidth == 256 && IsSaturating)
3538         IID = Intrinsic::x86_avx512_vpdpwssds_256;
3539       else if (VecWidth == 512 && IsSaturating)
3540         IID = Intrinsic::x86_avx512_vpdpwssds_512;
3541       else
3542         llvm_unreachable("Unexpected intrinsic");
3543 
3544       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3545                         CI->getArgOperand(2)  };
3546       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3547                                Args);
3548       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3549                                  : CI->getArgOperand(0);
3550       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3551     } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
3552                          Name == "addcarry.u32" || Name == "addcarry.u64" ||
3553                          Name == "subborrow.u32" || Name == "subborrow.u64")) {
3554       Intrinsic::ID IID;
3555       if (Name[0] == 'a' && Name.back() == '2')
3556         IID = Intrinsic::x86_addcarry_32;
3557       else if (Name[0] == 'a' && Name.back() == '4')
3558         IID = Intrinsic::x86_addcarry_64;
3559       else if (Name[0] == 's' && Name.back() == '2')
3560         IID = Intrinsic::x86_subborrow_32;
3561       else if (Name[0] == 's' && Name.back() == '4')
3562         IID = Intrinsic::x86_subborrow_64;
3563       else
3564         llvm_unreachable("Unexpected intrinsic");
3565 
3566       // Make a call with 3 operands.
3567       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3568                         CI->getArgOperand(2)};
3569       Value *NewCall = Builder.CreateCall(
3570                                 Intrinsic::getDeclaration(CI->getModule(), IID),
3571                                 Args);
3572 
3573       // Extract the second result and store it.
3574       Value *Data = Builder.CreateExtractValue(NewCall, 1);
3575       // Cast the pointer to the right type.
3576       Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3577                                  llvm::PointerType::getUnqual(Data->getType()));
3578       Builder.CreateAlignedStore(Data, Ptr, Align(1));
3579       // Replace the original call result with the first result of the new call.
3580       Value *CF = Builder.CreateExtractValue(NewCall, 0);
3581 
3582       CI->replaceAllUsesWith(CF);
3583       Rep = nullptr;
3584     } else if (IsX86 && Name.startswith("avx512.mask.") &&
3585                upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3586       // Rep will be updated by the call in the condition.
3587     } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
3588       Value *Arg = CI->getArgOperand(0);
3589       Value *Neg = Builder.CreateNeg(Arg, "neg");
3590       Value *Cmp = Builder.CreateICmpSGE(
3591           Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
3592       Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
3593     } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
3594                           Name.startswith("atomic.load.add.f64.p"))) {
3595       Value *Ptr = CI->getArgOperand(0);
3596       Value *Val = CI->getArgOperand(1);
3597       Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
3598                                     AtomicOrdering::SequentiallyConsistent);
3599     } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
3600                           Name == "max.ui" || Name == "max.ull")) {
3601       Value *Arg0 = CI->getArgOperand(0);
3602       Value *Arg1 = CI->getArgOperand(1);
3603       Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3604                        ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
3605                        : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
3606       Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
3607     } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
3608                           Name == "min.ui" || Name == "min.ull")) {
3609       Value *Arg0 = CI->getArgOperand(0);
3610       Value *Arg1 = CI->getArgOperand(1);
3611       Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3612                        ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
3613                        : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
3614       Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
3615     } else if (IsNVVM && Name == "clz.ll") {
3616       // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
3617       Value *Arg = CI->getArgOperand(0);
3618       Value *Ctlz = Builder.CreateCall(
3619           Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
3620                                     {Arg->getType()}),
3621           {Arg, Builder.getFalse()}, "ctlz");
3622       Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3623     } else if (IsNVVM && Name == "popc.ll") {
3624       // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
3625       // i64.
3626       Value *Arg = CI->getArgOperand(0);
3627       Value *Popc = Builder.CreateCall(
3628           Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
3629                                     {Arg->getType()}),
3630           Arg, "ctpop");
3631       Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3632     } else if (IsNVVM && Name == "h2f") {
3633       Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3634                                    F->getParent(), Intrinsic::convert_from_fp16,
3635                                    {Builder.getFloatTy()}),
3636                                CI->getArgOperand(0), "h2f");
3637     } else {
3638       llvm_unreachable("Unknown function for CallInst upgrade.");
3639     }
3640 
3641     if (Rep)
3642       CI->replaceAllUsesWith(Rep);
3643     CI->eraseFromParent();
3644     return;
3645   }
3646 
3647   const auto &DefaultCase = [&NewFn, &CI]() -> void {
3648     // Handle generic mangling change, but nothing else
3649     assert(
3650         (CI->getCalledFunction()->getName() != NewFn->getName()) &&
3651         "Unknown function for CallInst upgrade and isn't just a name change");
3652     CI->setCalledFunction(NewFn);
3653   };
3654   CallInst *NewCall = nullptr;
3655   switch (NewFn->getIntrinsicID()) {
3656   default: {
3657     DefaultCase();
3658     return;
3659   }
3660   case Intrinsic::arm_neon_vld1:
3661   case Intrinsic::arm_neon_vld2:
3662   case Intrinsic::arm_neon_vld3:
3663   case Intrinsic::arm_neon_vld4:
3664   case Intrinsic::arm_neon_vld2lane:
3665   case Intrinsic::arm_neon_vld3lane:
3666   case Intrinsic::arm_neon_vld4lane:
3667   case Intrinsic::arm_neon_vst1:
3668   case Intrinsic::arm_neon_vst2:
3669   case Intrinsic::arm_neon_vst3:
3670   case Intrinsic::arm_neon_vst4:
3671   case Intrinsic::arm_neon_vst2lane:
3672   case Intrinsic::arm_neon_vst3lane:
3673   case Intrinsic::arm_neon_vst4lane: {
3674     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3675                                  CI->arg_operands().end());
3676     NewCall = Builder.CreateCall(NewFn, Args);
3677     break;
3678   }
3679 
3680   case Intrinsic::arm_neon_bfdot:
3681   case Intrinsic::arm_neon_bfmmla:
3682   case Intrinsic::arm_neon_bfmlalb:
3683   case Intrinsic::arm_neon_bfmlalt:
3684   case Intrinsic::aarch64_neon_bfdot:
3685   case Intrinsic::aarch64_neon_bfmmla:
3686   case Intrinsic::aarch64_neon_bfmlalb:
3687   case Intrinsic::aarch64_neon_bfmlalt: {
3688     SmallVector<Value *, 3> Args;
3689     assert(CI->getNumArgOperands() == 3 &&
3690            "Mismatch between function args and call args");
3691     size_t OperandWidth =
3692         CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
3693     assert((OperandWidth == 64 || OperandWidth == 128) &&
3694            "Unexpected operand width");
3695     Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
3696     auto Iter = CI->arg_operands().begin();
3697     Args.push_back(*Iter++);
3698     Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
3699     Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
3700     NewCall = Builder.CreateCall(NewFn, Args);
3701     break;
3702   }
3703 
3704   case Intrinsic::bitreverse:
3705     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3706     break;
3707 
3708   case Intrinsic::ctlz:
3709   case Intrinsic::cttz:
3710     assert(CI->getNumArgOperands() == 1 &&
3711            "Mismatch between function args and call args");
3712     NewCall =
3713         Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
3714     break;
3715 
3716   case Intrinsic::objectsize: {
3717     Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
3718                                    ? Builder.getFalse()
3719                                    : CI->getArgOperand(2);
3720     Value *Dynamic =
3721         CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
3722     NewCall = Builder.CreateCall(
3723         NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
3724     break;
3725   }
3726 
3727   case Intrinsic::ctpop:
3728     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3729     break;
3730 
3731   case Intrinsic::convert_from_fp16:
3732     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3733     break;
3734 
3735   case Intrinsic::dbg_value:
3736     // Upgrade from the old version that had an extra offset argument.
3737     assert(CI->getNumArgOperands() == 4);
3738     // Drop nonzero offsets instead of attempting to upgrade them.
3739     if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
3740       if (Offset->isZeroValue()) {
3741         NewCall = Builder.CreateCall(
3742             NewFn,
3743             {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
3744         break;
3745       }
3746     CI->eraseFromParent();
3747     return;
3748 
3749   case Intrinsic::ptr_annotation:
3750     // Upgrade from versions that lacked the annotation attribute argument.
3751     assert(CI->getNumArgOperands() == 4 &&
3752            "Before LLVM 12.0 this intrinsic took four arguments");
3753     // Create a new call with an added null annotation attribute argument.
3754     NewCall = Builder.CreateCall(
3755         NewFn,
3756         {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
3757          CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
3758     NewCall->takeName(CI);
3759     CI->replaceAllUsesWith(NewCall);
3760     CI->eraseFromParent();
3761     return;
3762 
3763   case Intrinsic::var_annotation:
3764     // Upgrade from versions that lacked the annotation attribute argument.
3765     assert(CI->getNumArgOperands() == 4 &&
3766            "Before LLVM 12.0 this intrinsic took four arguments");
3767     // Create a new call with an added null annotation attribute argument.
3768     NewCall = Builder.CreateCall(
3769         NewFn,
3770         {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
3771          CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
3772     CI->eraseFromParent();
3773     return;
3774 
3775   case Intrinsic::x86_xop_vfrcz_ss:
3776   case Intrinsic::x86_xop_vfrcz_sd:
3777     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
3778     break;
3779 
3780   case Intrinsic::x86_xop_vpermil2pd:
3781   case Intrinsic::x86_xop_vpermil2ps:
3782   case Intrinsic::x86_xop_vpermil2pd_256:
3783   case Intrinsic::x86_xop_vpermil2ps_256: {
3784     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3785                                  CI->arg_operands().end());
3786     VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
3787     VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
3788     Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
3789     NewCall = Builder.CreateCall(NewFn, Args);
3790     break;
3791   }
3792 
3793   case Intrinsic::x86_sse41_ptestc:
3794   case Intrinsic::x86_sse41_ptestz:
3795   case Intrinsic::x86_sse41_ptestnzc: {
3796     // The arguments for these intrinsics used to be v4f32, and changed
3797     // to v2i64. This is purely a nop, since those are bitwise intrinsics.
3798     // So, the only thing required is a bitcast for both arguments.
3799     // First, check the arguments have the old type.
3800     Value *Arg0 = CI->getArgOperand(0);
3801     if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
3802       return;
3803 
3804     // Old intrinsic, add bitcasts
3805     Value *Arg1 = CI->getArgOperand(1);
3806 
3807     auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3808 
3809     Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
3810     Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3811 
3812     NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
3813     break;
3814   }
3815 
3816   case Intrinsic::x86_rdtscp: {
3817     // This used to take 1 arguments. If we have no arguments, it is already
3818     // upgraded.
3819     if (CI->getNumOperands() == 0)
3820       return;
3821 
3822     NewCall = Builder.CreateCall(NewFn);
3823     // Extract the second result and store it.
3824     Value *Data = Builder.CreateExtractValue(NewCall, 1);
3825     // Cast the pointer to the right type.
3826     Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
3827                                  llvm::PointerType::getUnqual(Data->getType()));
3828     Builder.CreateAlignedStore(Data, Ptr, Align(1));
3829     // Replace the original call result with the first result of the new call.
3830     Value *TSC = Builder.CreateExtractValue(NewCall, 0);
3831 
3832     NewCall->takeName(CI);
3833     CI->replaceAllUsesWith(TSC);
3834     CI->eraseFromParent();
3835     return;
3836   }
3837 
3838   case Intrinsic::x86_sse41_insertps:
3839   case Intrinsic::x86_sse41_dppd:
3840   case Intrinsic::x86_sse41_dpps:
3841   case Intrinsic::x86_sse41_mpsadbw:
3842   case Intrinsic::x86_avx_dp_ps_256:
3843   case Intrinsic::x86_avx2_mpsadbw: {
3844     // Need to truncate the last argument from i32 to i8 -- this argument models
3845     // an inherently 8-bit immediate operand to these x86 instructions.
3846     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3847                                  CI->arg_operands().end());
3848 
3849     // Replace the last argument with a trunc.
3850     Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
3851     NewCall = Builder.CreateCall(NewFn, Args);
3852     break;
3853   }
3854 
3855   case Intrinsic::x86_avx512_mask_cmp_pd_128:
3856   case Intrinsic::x86_avx512_mask_cmp_pd_256:
3857   case Intrinsic::x86_avx512_mask_cmp_pd_512:
3858   case Intrinsic::x86_avx512_mask_cmp_ps_128:
3859   case Intrinsic::x86_avx512_mask_cmp_ps_256:
3860   case Intrinsic::x86_avx512_mask_cmp_ps_512: {
3861     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3862                                  CI->arg_operands().end());
3863     unsigned NumElts =
3864         cast<FixedVectorType>(Args[0]->getType())->getNumElements();
3865     Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
3866 
3867     NewCall = Builder.CreateCall(NewFn, Args);
3868     Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
3869 
3870     NewCall->takeName(CI);
3871     CI->replaceAllUsesWith(Res);
3872     CI->eraseFromParent();
3873     return;
3874   }
3875 
3876   case Intrinsic::thread_pointer: {
3877     NewCall = Builder.CreateCall(NewFn, {});
3878     break;
3879   }
3880 
3881   case Intrinsic::invariant_start:
3882   case Intrinsic::invariant_end:
3883   case Intrinsic::masked_load:
3884   case Intrinsic::masked_store:
3885   case Intrinsic::masked_gather:
3886   case Intrinsic::masked_scatter: {
3887     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3888                                  CI->arg_operands().end());
3889     NewCall = Builder.CreateCall(NewFn, Args);
3890     break;
3891   }
3892 
3893   case Intrinsic::memcpy:
3894   case Intrinsic::memmove:
3895   case Intrinsic::memset: {
3896     // We have to make sure that the call signature is what we're expecting.
3897     // We only want to change the old signatures by removing the alignment arg:
3898     //  @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
3899     //    -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
3900     //  @llvm.memset...(i8*, i8, i[32|64], i32, i1)
3901     //    -> @llvm.memset...(i8*, i8, i[32|64], i1)
3902     // Note: i8*'s in the above can be any pointer type
3903     if (CI->getNumArgOperands() != 5) {
3904       DefaultCase();
3905       return;
3906     }
3907     // Remove alignment argument (3), and add alignment attributes to the
3908     // dest/src pointers.
3909     Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
3910                       CI->getArgOperand(2), CI->getArgOperand(4)};
3911     NewCall = Builder.CreateCall(NewFn, Args);
3912     auto *MemCI = cast<MemIntrinsic>(NewCall);
3913     // All mem intrinsics support dest alignment.
3914     const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
3915     MemCI->setDestAlignment(Align->getMaybeAlignValue());
3916     // Memcpy/Memmove also support source alignment.
3917     if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
3918       MTI->setSourceAlignment(Align->getMaybeAlignValue());
3919     break;
3920   }
3921   }
3922   assert(NewCall && "Should have either set this variable or returned through "
3923                     "the default case");
3924   NewCall->takeName(CI);
3925   CI->replaceAllUsesWith(NewCall);
3926   CI->eraseFromParent();
3927 }
3928 
3929 void llvm::UpgradeCallsToIntrinsic(Function *F) {
3930   assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
3931 
3932   // Check if this function should be upgraded and get the replacement function
3933   // if there is one.
3934   Function *NewFn;
3935   if (UpgradeIntrinsicFunction(F, NewFn)) {
3936     // Replace all users of the old function with the new function or new
3937     // instructions. This is not a range loop because the call is deleted.
3938     for (User *U : make_early_inc_range(F->users()))
3939       if (CallInst *CI = dyn_cast<CallInst>(U))
3940         UpgradeIntrinsicCall(CI, NewFn);
3941 
3942     // Remove old function, no longer used, from the module.
3943     F->eraseFromParent();
3944   }
3945 }
3946 
3947 MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
3948   // Check if the tag uses struct-path aware TBAA format.
3949   if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)
3950     return &MD;
3951 
3952   auto &Context = MD.getContext();
3953   if (MD.getNumOperands() == 3) {
3954     Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
3955     MDNode *ScalarType = MDNode::get(Context, Elts);
3956     // Create a MDNode <ScalarType, ScalarType, offset 0, const>
3957     Metadata *Elts2[] = {ScalarType, ScalarType,
3958                          ConstantAsMetadata::get(
3959                              Constant::getNullValue(Type::getInt64Ty(Context))),
3960                          MD.getOperand(2)};
3961     return MDNode::get(Context, Elts2);
3962   }
3963   // Create a MDNode <MD, MD, offset 0>
3964   Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
3965                                     Type::getInt64Ty(Context)))};
3966   return MDNode::get(Context, Elts);
3967 }
3968 
3969 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
3970                                       Instruction *&Temp) {
3971   if (Opc != Instruction::BitCast)
3972     return nullptr;
3973 
3974   Temp = nullptr;
3975   Type *SrcTy = V->getType();
3976   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
3977       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
3978     LLVMContext &Context = V->getContext();
3979 
3980     // We have no information about target data layout, so we assume that
3981     // the maximum pointer size is 64bit.
3982     Type *MidTy = Type::getInt64Ty(Context);
3983     Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
3984 
3985     return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
3986   }
3987 
3988   return nullptr;
3989 }
3990 
3991 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
3992   if (Opc != Instruction::BitCast)
3993     return nullptr;
3994 
3995   Type *SrcTy = C->getType();
3996   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
3997       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
3998     LLVMContext &Context = C->getContext();
3999 
4000     // We have no information about target data layout, so we assume that
4001     // the maximum pointer size is 64bit.
4002     Type *MidTy = Type::getInt64Ty(Context);
4003 
4004     return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
4005                                      DestTy);
4006   }
4007 
4008   return nullptr;
4009 }
4010 
4011 /// Check the debug info version number, if it is out-dated, drop the debug
4012 /// info. Return true if module is modified.
4013 bool llvm::UpgradeDebugInfo(Module &M) {
4014   unsigned Version = getDebugMetadataVersionFromModule(M);
4015   if (Version == DEBUG_METADATA_VERSION) {
4016     bool BrokenDebugInfo = false;
4017     if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
4018       report_fatal_error("Broken module found, compilation aborted!");
4019     if (!BrokenDebugInfo)
4020       // Everything is ok.
4021       return false;
4022     else {
4023       // Diagnose malformed debug info.
4024       DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
4025       M.getContext().diagnose(Diag);
4026     }
4027   }
4028   bool Modified = StripDebugInfo(M);
4029   if (Modified && Version != DEBUG_METADATA_VERSION) {
4030     // Diagnose a version mismatch.
4031     DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
4032     M.getContext().diagnose(DiagVersion);
4033   }
4034   return Modified;
4035 }
4036 
4037 /// This checks for objc retain release marker which should be upgraded. It
4038 /// returns true if module is modified.
4039 static bool UpgradeRetainReleaseMarker(Module &M) {
4040   bool Changed = false;
4041   const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
4042   NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4043   if (ModRetainReleaseMarker) {
4044     MDNode *Op = ModRetainReleaseMarker->getOperand(0);
4045     if (Op) {
4046       MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
4047       if (ID) {
4048         SmallVector<StringRef, 4> ValueComp;
4049         ID->getString().split(ValueComp, "#");
4050         if (ValueComp.size() == 2) {
4051           std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
4052           ID = MDString::get(M.getContext(), NewValue);
4053         }
4054         M.addModuleFlag(Module::Error, MarkerKey, ID);
4055         M.eraseNamedMetadata(ModRetainReleaseMarker);
4056         Changed = true;
4057       }
4058     }
4059   }
4060   return Changed;
4061 }
4062 
4063 void llvm::UpgradeARCRuntime(Module &M) {
4064   // This lambda converts normal function calls to ARC runtime functions to
4065   // intrinsic calls.
4066   auto UpgradeToIntrinsic = [&](const char *OldFunc,
4067                                 llvm::Intrinsic::ID IntrinsicFunc) {
4068     Function *Fn = M.getFunction(OldFunc);
4069 
4070     if (!Fn)
4071       return;
4072 
4073     Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
4074 
4075     for (User *U : make_early_inc_range(Fn->users())) {
4076       CallInst *CI = dyn_cast<CallInst>(U);
4077       if (!CI || CI->getCalledFunction() != Fn)
4078         continue;
4079 
4080       IRBuilder<> Builder(CI->getParent(), CI->getIterator());
4081       FunctionType *NewFuncTy = NewFn->getFunctionType();
4082       SmallVector<Value *, 2> Args;
4083 
4084       // Don't upgrade the intrinsic if it's not valid to bitcast the return
4085       // value to the return type of the old function.
4086       if (NewFuncTy->getReturnType() != CI->getType() &&
4087           !CastInst::castIsValid(Instruction::BitCast, CI,
4088                                  NewFuncTy->getReturnType()))
4089         continue;
4090 
4091       bool InvalidCast = false;
4092 
4093       for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
4094         Value *Arg = CI->getArgOperand(I);
4095 
4096         // Bitcast argument to the parameter type of the new function if it's
4097         // not a variadic argument.
4098         if (I < NewFuncTy->getNumParams()) {
4099           // Don't upgrade the intrinsic if it's not valid to bitcast the argument
4100           // to the parameter type of the new function.
4101           if (!CastInst::castIsValid(Instruction::BitCast, Arg,
4102                                      NewFuncTy->getParamType(I))) {
4103             InvalidCast = true;
4104             break;
4105           }
4106           Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
4107         }
4108         Args.push_back(Arg);
4109       }
4110 
4111       if (InvalidCast)
4112         continue;
4113 
4114       // Create a call instruction that calls the new function.
4115       CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
4116       NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4117       NewCall->takeName(CI);
4118 
4119       // Bitcast the return value back to the type of the old call.
4120       Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
4121 
4122       if (!CI->use_empty())
4123         CI->replaceAllUsesWith(NewRetVal);
4124       CI->eraseFromParent();
4125     }
4126 
4127     if (Fn->use_empty())
4128       Fn->eraseFromParent();
4129   };
4130 
4131   // Unconditionally convert a call to "clang.arc.use" to a call to
4132   // "llvm.objc.clang.arc.use".
4133   UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
4134 
4135   // Upgrade the retain release marker. If there is no need to upgrade
4136   // the marker, that means either the module is already new enough to contain
4137   // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
4138   if (!UpgradeRetainReleaseMarker(M))
4139     return;
4140 
4141   std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
4142       {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
4143       {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
4144       {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
4145       {"objc_autoreleaseReturnValue",
4146        llvm::Intrinsic::objc_autoreleaseReturnValue},
4147       {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
4148       {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
4149       {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
4150       {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
4151       {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
4152       {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
4153       {"objc_release", llvm::Intrinsic::objc_release},
4154       {"objc_retain", llvm::Intrinsic::objc_retain},
4155       {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
4156       {"objc_retainAutoreleaseReturnValue",
4157        llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
4158       {"objc_retainAutoreleasedReturnValue",
4159        llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
4160       {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
4161       {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
4162       {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
4163       {"objc_unsafeClaimAutoreleasedReturnValue",
4164        llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
4165       {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
4166       {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
4167       {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
4168       {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
4169       {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
4170       {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
4171       {"objc_arc_annotation_topdown_bbstart",
4172        llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
4173       {"objc_arc_annotation_topdown_bbend",
4174        llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
4175       {"objc_arc_annotation_bottomup_bbstart",
4176        llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
4177       {"objc_arc_annotation_bottomup_bbend",
4178        llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
4179 
4180   for (auto &I : RuntimeFuncs)
4181     UpgradeToIntrinsic(I.first, I.second);
4182 }
4183 
4184 bool llvm::UpgradeModuleFlags(Module &M) {
4185   NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
4186   if (!ModFlags)
4187     return false;
4188 
4189   bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
4190   bool HasSwiftVersionFlag = false;
4191   uint8_t SwiftMajorVersion, SwiftMinorVersion;
4192   uint32_t SwiftABIVersion;
4193   auto Int8Ty = Type::getInt8Ty(M.getContext());
4194   auto Int32Ty = Type::getInt32Ty(M.getContext());
4195 
4196   for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
4197     MDNode *Op = ModFlags->getOperand(I);
4198     if (Op->getNumOperands() != 3)
4199       continue;
4200     MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
4201     if (!ID)
4202       continue;
4203     if (ID->getString() == "Objective-C Image Info Version")
4204       HasObjCFlag = true;
4205     if (ID->getString() == "Objective-C Class Properties")
4206       HasClassProperties = true;
4207     // Upgrade PIC/PIE Module Flags. The module flag behavior for these two
4208     // field was Error and now they are Max.
4209     if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {
4210       if (auto *Behavior =
4211               mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
4212         if (Behavior->getLimitedValue() == Module::Error) {
4213           Type *Int32Ty = Type::getInt32Ty(M.getContext());
4214           Metadata *Ops[3] = {
4215               ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),
4216               MDString::get(M.getContext(), ID->getString()),
4217               Op->getOperand(2)};
4218           ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4219           Changed = true;
4220         }
4221       }
4222     }
4223     // Upgrade Objective-C Image Info Section. Removed the whitespce in the
4224     // section name so that llvm-lto will not complain about mismatching
4225     // module flags that is functionally the same.
4226     if (ID->getString() == "Objective-C Image Info Section") {
4227       if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
4228         SmallVector<StringRef, 4> ValueComp;
4229         Value->getString().split(ValueComp, " ");
4230         if (ValueComp.size() != 1) {
4231           std::string NewValue;
4232           for (auto &S : ValueComp)
4233             NewValue += S.str();
4234           Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
4235                               MDString::get(M.getContext(), NewValue)};
4236           ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4237           Changed = true;
4238         }
4239       }
4240     }
4241 
4242     // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
4243     // If the higher bits are set, it adds new module flag for swift info.
4244     if (ID->getString() == "Objective-C Garbage Collection") {
4245       auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
4246       if (Md) {
4247         assert(Md->getValue() && "Expected non-empty metadata");
4248         auto Type = Md->getValue()->getType();
4249         if (Type == Int8Ty)
4250           continue;
4251         unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
4252         if ((Val & 0xff) != Val) {
4253           HasSwiftVersionFlag = true;
4254           SwiftABIVersion = (Val & 0xff00) >> 8;
4255           SwiftMajorVersion = (Val & 0xff000000) >> 24;
4256           SwiftMinorVersion = (Val & 0xff0000) >> 16;
4257         }
4258         Metadata *Ops[3] = {
4259           ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
4260           Op->getOperand(1),
4261           ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
4262         ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4263         Changed = true;
4264       }
4265     }
4266   }
4267 
4268   // "Objective-C Class Properties" is recently added for Objective-C. We
4269   // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
4270   // flag of value 0, so we can correclty downgrade this flag when trying to
4271   // link an ObjC bitcode without this module flag with an ObjC bitcode with
4272   // this module flag.
4273   if (HasObjCFlag && !HasClassProperties) {
4274     M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
4275                     (uint32_t)0);
4276     Changed = true;
4277   }
4278 
4279   if (HasSwiftVersionFlag) {
4280     M.addModuleFlag(Module::Error, "Swift ABI Version",
4281                     SwiftABIVersion);
4282     M.addModuleFlag(Module::Error, "Swift Major Version",
4283                     ConstantInt::get(Int8Ty, SwiftMajorVersion));
4284     M.addModuleFlag(Module::Error, "Swift Minor Version",
4285                     ConstantInt::get(Int8Ty, SwiftMinorVersion));
4286     Changed = true;
4287   }
4288 
4289   return Changed;
4290 }
4291 
4292 void llvm::UpgradeSectionAttributes(Module &M) {
4293   auto TrimSpaces = [](StringRef Section) -> std::string {
4294     SmallVector<StringRef, 5> Components;
4295     Section.split(Components, ',');
4296 
4297     SmallString<32> Buffer;
4298     raw_svector_ostream OS(Buffer);
4299 
4300     for (auto Component : Components)
4301       OS << ',' << Component.trim();
4302 
4303     return std::string(OS.str().substr(1));
4304   };
4305 
4306   for (auto &GV : M.globals()) {
4307     if (!GV.hasSection())
4308       continue;
4309 
4310     StringRef Section = GV.getSection();
4311 
4312     if (!Section.startswith("__DATA, __objc_catlist"))
4313       continue;
4314 
4315     // __DATA, __objc_catlist, regular, no_dead_strip
4316     // __DATA,__objc_catlist,regular,no_dead_strip
4317     GV.setSection(TrimSpaces(Section));
4318   }
4319 }
4320 
4321 namespace {
4322 // Prior to LLVM 10.0, the strictfp attribute could be used on individual
4323 // callsites within a function that did not also have the strictfp attribute.
4324 // Since 10.0, if strict FP semantics are needed within a function, the
4325 // function must have the strictfp attribute and all calls within the function
4326 // must also have the strictfp attribute. This latter restriction is
4327 // necessary to prevent unwanted libcall simplification when a function is
4328 // being cloned (such as for inlining).
4329 //
4330 // The "dangling" strictfp attribute usage was only used to prevent constant
4331 // folding and other libcall simplification. The nobuiltin attribute on the
4332 // callsite has the same effect.
4333 struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
4334   StrictFPUpgradeVisitor() {}
4335 
4336   void visitCallBase(CallBase &Call) {
4337     if (!Call.isStrictFP())
4338       return;
4339     if (isa<ConstrainedFPIntrinsic>(&Call))
4340       return;
4341     // If we get here, the caller doesn't have the strictfp attribute
4342     // but this callsite does. Replace the strictfp attribute with nobuiltin.
4343     Call.removeAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
4344     Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);
4345   }
4346 };
4347 } // namespace
4348 
4349 void llvm::UpgradeFunctionAttributes(Function &F) {
4350   // If a function definition doesn't have the strictfp attribute,
4351   // convert any callsite strictfp attributes to nobuiltin.
4352   if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
4353     StrictFPUpgradeVisitor SFPV;
4354     SFPV.visit(F);
4355   }
4356 
4357   if (F.getCallingConv() == CallingConv::X86_INTR &&
4358       !F.arg_empty() && !F.hasParamAttribute(0, Attribute::ByVal)) {
4359     Type *ByValTy = cast<PointerType>(F.getArg(0)->getType())->getElementType();
4360     Attribute NewAttr = Attribute::getWithByValType(F.getContext(), ByValTy);
4361     F.addParamAttr(0, NewAttr);
4362   }
4363 }
4364 
4365 static bool isOldLoopArgument(Metadata *MD) {
4366   auto *T = dyn_cast_or_null<MDTuple>(MD);
4367   if (!T)
4368     return false;
4369   if (T->getNumOperands() < 1)
4370     return false;
4371   auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
4372   if (!S)
4373     return false;
4374   return S->getString().startswith("llvm.vectorizer.");
4375 }
4376 
4377 static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
4378   StringRef OldPrefix = "llvm.vectorizer.";
4379   assert(OldTag.startswith(OldPrefix) && "Expected old prefix");
4380 
4381   if (OldTag == "llvm.vectorizer.unroll")
4382     return MDString::get(C, "llvm.loop.interleave.count");
4383 
4384   return MDString::get(
4385       C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
4386              .str());
4387 }
4388 
4389 static Metadata *upgradeLoopArgument(Metadata *MD) {
4390   auto *T = dyn_cast_or_null<MDTuple>(MD);
4391   if (!T)
4392     return MD;
4393   if (T->getNumOperands() < 1)
4394     return MD;
4395   auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
4396   if (!OldTag)
4397     return MD;
4398   if (!OldTag->getString().startswith("llvm.vectorizer."))
4399     return MD;
4400 
4401   // This has an old tag.  Upgrade it.
4402   SmallVector<Metadata *, 8> Ops;
4403   Ops.reserve(T->getNumOperands());
4404   Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
4405   for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
4406     Ops.push_back(T->getOperand(I));
4407 
4408   return MDTuple::get(T->getContext(), Ops);
4409 }
4410 
4411 MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
4412   auto *T = dyn_cast<MDTuple>(&N);
4413   if (!T)
4414     return &N;
4415 
4416   if (none_of(T->operands(), isOldLoopArgument))
4417     return &N;
4418 
4419   SmallVector<Metadata *, 8> Ops;
4420   Ops.reserve(T->getNumOperands());
4421   for (Metadata *MD : T->operands())
4422     Ops.push_back(upgradeLoopArgument(MD));
4423 
4424   return MDTuple::get(T->getContext(), Ops);
4425 }
4426 
4427 std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
4428   Triple T(TT);
4429   // For AMDGPU we uprgrade older DataLayouts to include the default globals
4430   // address space of 1.
4431   if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) {
4432     return DL.empty() ? std::string("G1") : (DL + "-G1").str();
4433   }
4434 
4435   std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
4436   // If X86, and the datalayout matches the expected format, add pointer size
4437   // address spaces to the datalayout.
4438   if (!T.isX86() || DL.contains(AddrSpaces))
4439     return std::string(DL);
4440 
4441   SmallVector<StringRef, 4> Groups;
4442   Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
4443   if (!R.match(DL, &Groups))
4444     return std::string(DL);
4445 
4446   return (Groups[1] + AddrSpaces + Groups[3]).str();
4447 }
4448 
4449 void llvm::UpgradeAttributes(AttrBuilder &B) {
4450   StringRef FramePointer;
4451   if (B.contains("no-frame-pointer-elim")) {
4452     // The value can be "true" or "false".
4453     for (const auto &I : B.td_attrs())
4454       if (I.first == "no-frame-pointer-elim")
4455         FramePointer = I.second == "true" ? "all" : "none";
4456     B.removeAttribute("no-frame-pointer-elim");
4457   }
4458   if (B.contains("no-frame-pointer-elim-non-leaf")) {
4459     // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
4460     if (FramePointer != "all")
4461       FramePointer = "non-leaf";
4462     B.removeAttribute("no-frame-pointer-elim-non-leaf");
4463   }
4464   if (!FramePointer.empty())
4465     B.addAttribute("frame-pointer", FramePointer);
4466 
4467   if (B.contains("null-pointer-is-valid")) {
4468     // The value can be "true" or "false".
4469     bool NullPointerIsValid = false;
4470     for (const auto &I : B.td_attrs())
4471       if (I.first == "null-pointer-is-valid")
4472         NullPointerIsValid = I.second == "true";
4473     B.removeAttribute("null-pointer-is-valid");
4474     if (NullPointerIsValid)
4475       B.addAttribute(Attribute::NullPointerIsValid);
4476   }
4477 }
4478