1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Define several functions to decode x86 specific shuffle semantics using 10 // constants from the constant pool. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "Utils/X86ShuffleDecode.h" 15 #include "llvm/ADT/APInt.h" 16 #include "llvm/IR/Constants.h" 17 18 //===----------------------------------------------------------------------===// 19 // Vector Mask Decoding 20 //===----------------------------------------------------------------------===// 21 22 namespace llvm { 23 24 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, 25 APInt &UndefElts, 26 SmallVectorImpl<uint64_t> &RawMask) { 27 // It is not an error for shuffle masks to not be a vector of 28 // MaskEltSizeInBits because the constant pool uniques constants by their 29 // bit representation. 30 // e.g. the following take up the same space in the constant pool: 31 // i128 -170141183420855150465331762880109871104 32 // 33 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160> 34 // 35 // <4 x i32> <i32 -2147483648, i32 -2147483648, 36 // i32 -2147483648, i32 -2147483648> 37 Type *CstTy = C->getType(); 38 if (!CstTy->isVectorTy()) 39 return false; 40 41 Type *CstEltTy = CstTy->getVectorElementType(); 42 if (!CstEltTy->isIntegerTy()) 43 return false; 44 45 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); 46 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits(); 47 unsigned NumCstElts = CstTy->getVectorNumElements(); 48 49 assert((CstSizeInBits % MaskEltSizeInBits) == 0 && 50 "Unaligned shuffle mask size"); 51 52 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits; 53 UndefElts = APInt(NumMaskElts, 0); 54 RawMask.resize(NumMaskElts, 0); 55 56 // Fast path - if the constants match the mask size then copy direct. 57 if (MaskEltSizeInBits == CstEltSizeInBits) { 58 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size"); 59 for (unsigned i = 0; i != NumMaskElts; ++i) { 60 Constant *COp = C->getAggregateElement(i); 61 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) 62 return false; 63 64 if (isa<UndefValue>(COp)) { 65 UndefElts.setBit(i); 66 RawMask[i] = 0; 67 continue; 68 } 69 70 auto *Elt = cast<ConstantInt>(COp); 71 RawMask[i] = Elt->getValue().getZExtValue(); 72 } 73 return true; 74 } 75 76 // Extract all the undef/constant element data and pack into single bitsets. 77 APInt UndefBits(CstSizeInBits, 0); 78 APInt MaskBits(CstSizeInBits, 0); 79 for (unsigned i = 0; i != NumCstElts; ++i) { 80 Constant *COp = C->getAggregateElement(i); 81 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) 82 return false; 83 84 unsigned BitOffset = i * CstEltSizeInBits; 85 86 if (isa<UndefValue>(COp)) { 87 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits); 88 continue; 89 } 90 91 MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset); 92 } 93 94 // Now extract the undef/constant bit data into the raw shuffle masks. 95 for (unsigned i = 0; i != NumMaskElts; ++i) { 96 unsigned BitOffset = i * MaskEltSizeInBits; 97 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset); 98 99 // Only treat the element as UNDEF if all bits are UNDEF, otherwise 100 // treat it as zero. 101 if (EltUndef.isAllOnesValue()) { 102 UndefElts.setBit(i); 103 RawMask[i] = 0; 104 continue; 105 } 106 107 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset); 108 RawMask[i] = EltBits.getZExtValue(); 109 } 110 111 return true; 112 } 113 114 void DecodePSHUFBMask(const Constant *C, unsigned Width, 115 SmallVectorImpl<int> &ShuffleMask) { 116 assert((Width == 128 || Width == 256 || Width == 512) && 117 C->getType()->getPrimitiveSizeInBits() >= Width && 118 "Unexpected vector size."); 119 120 // The shuffle mask requires a byte vector. 121 APInt UndefElts; 122 SmallVector<uint64_t, 64> RawMask; 123 if (!extractConstantMask(C, 8, UndefElts, RawMask)) 124 return; 125 126 unsigned NumElts = Width / 8; 127 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) && 128 "Unexpected number of vector elements."); 129 130 for (unsigned i = 0; i != NumElts; ++i) { 131 if (UndefElts[i]) { 132 ShuffleMask.push_back(SM_SentinelUndef); 133 continue; 134 } 135 136 uint64_t Element = RawMask[i]; 137 // If the high bit (7) of the byte is set, the element is zeroed. 138 if (Element & (1 << 7)) 139 ShuffleMask.push_back(SM_SentinelZero); 140 else { 141 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte 142 // lane of the vector we're inside. 143 unsigned Base = i & ~0xf; 144 145 // Only the least significant 4 bits of the byte are used. 146 int Index = Base + (Element & 0xf); 147 ShuffleMask.push_back(Index); 148 } 149 } 150 } 151 152 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width, 153 SmallVectorImpl<int> &ShuffleMask) { 154 assert((Width == 128 || Width == 256 || Width == 512) && 155 C->getType()->getPrimitiveSizeInBits() >= Width && 156 "Unexpected vector size."); 157 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size."); 158 159 // The shuffle mask requires elements the same size as the target. 160 APInt UndefElts; 161 SmallVector<uint64_t, 16> RawMask; 162 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 163 return; 164 165 unsigned NumElts = Width / ElSize; 166 unsigned NumEltsPerLane = 128 / ElSize; 167 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) && 168 "Unexpected number of vector elements."); 169 170 for (unsigned i = 0; i != NumElts; ++i) { 171 if (UndefElts[i]) { 172 ShuffleMask.push_back(SM_SentinelUndef); 173 continue; 174 } 175 176 int Index = i & ~(NumEltsPerLane - 1); 177 uint64_t Element = RawMask[i]; 178 if (ElSize == 64) 179 Index += (Element >> 1) & 0x1; 180 else 181 Index += Element & 0x3; 182 183 ShuffleMask.push_back(Index); 184 } 185 } 186 187 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize, 188 unsigned Width, 189 SmallVectorImpl<int> &ShuffleMask) { 190 Type *MaskTy = C->getType(); 191 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 192 (void)MaskTySize; 193 assert((MaskTySize == 128 || MaskTySize == 256) && 194 Width >= MaskTySize && "Unexpected vector size."); 195 196 // The shuffle mask requires elements the same size as the target. 197 APInt UndefElts; 198 SmallVector<uint64_t, 8> RawMask; 199 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 200 return; 201 202 unsigned NumElts = Width / ElSize; 203 unsigned NumEltsPerLane = 128 / ElSize; 204 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) && 205 "Unexpected number of vector elements."); 206 207 for (unsigned i = 0; i != NumElts; ++i) { 208 if (UndefElts[i]) { 209 ShuffleMask.push_back(SM_SentinelUndef); 210 continue; 211 } 212 213 // VPERMIL2 Operation. 214 // Bits[3] - Match Bit. 215 // Bits[2:1] - (Per Lane) PD Shuffle Mask. 216 // Bits[2:0] - (Per Lane) PS Shuffle Mask. 217 uint64_t Selector = RawMask[i]; 218 unsigned MatchBit = (Selector >> 3) & 0x1; 219 220 // M2Z[0:1] MatchBit 221 // 0Xb X Source selected by Selector index. 222 // 10b 0 Source selected by Selector index. 223 // 10b 1 Zero. 224 // 11b 0 Zero. 225 // 11b 1 Source selected by Selector index. 226 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) { 227 ShuffleMask.push_back(SM_SentinelZero); 228 continue; 229 } 230 231 int Index = i & ~(NumEltsPerLane - 1); 232 if (ElSize == 64) 233 Index += (Selector >> 1) & 0x1; 234 else 235 Index += Selector & 0x3; 236 237 int Src = (Selector >> 2) & 0x1; 238 Index += Src * NumElts; 239 ShuffleMask.push_back(Index); 240 } 241 } 242 243 void DecodeVPPERMMask(const Constant *C, unsigned Width, 244 SmallVectorImpl<int> &ShuffleMask) { 245 Type *MaskTy = C->getType(); 246 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 247 (void)MaskTySize; 248 assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size."); 249 250 // The shuffle mask requires a byte vector. 251 APInt UndefElts; 252 SmallVector<uint64_t, 16> RawMask; 253 if (!extractConstantMask(C, 8, UndefElts, RawMask)) 254 return; 255 256 unsigned NumElts = Width / 8; 257 assert(NumElts == 16 && "Unexpected number of vector elements."); 258 259 for (unsigned i = 0; i != NumElts; ++i) { 260 if (UndefElts[i]) { 261 ShuffleMask.push_back(SM_SentinelUndef); 262 continue; 263 } 264 265 // VPPERM Operation 266 // Bits[4:0] - Byte Index (0 - 31) 267 // Bits[7:5] - Permute Operation 268 // 269 // Permute Operation: 270 // 0 - Source byte (no logical operation). 271 // 1 - Invert source byte. 272 // 2 - Bit reverse of source byte. 273 // 3 - Bit reverse of inverted source byte. 274 // 4 - 00h (zero - fill). 275 // 5 - FFh (ones - fill). 276 // 6 - Most significant bit of source byte replicated in all bit positions. 277 // 7 - Invert most significant bit of source byte and replicate in all bit 278 // positions. 279 uint64_t Element = RawMask[i]; 280 uint64_t Index = Element & 0x1F; 281 uint64_t PermuteOp = (Element >> 5) & 0x7; 282 283 if (PermuteOp == 4) { 284 ShuffleMask.push_back(SM_SentinelZero); 285 continue; 286 } 287 if (PermuteOp != 0) { 288 ShuffleMask.clear(); 289 return; 290 } 291 ShuffleMask.push_back((int)Index); 292 } 293 } 294 295 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width, 296 SmallVectorImpl<int> &ShuffleMask) { 297 assert((Width == 128 || Width == 256 || Width == 512) && 298 C->getType()->getPrimitiveSizeInBits() >= Width && 299 "Unexpected vector size."); 300 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && 301 "Unexpected vector element size."); 302 303 // The shuffle mask requires elements the same size as the target. 304 APInt UndefElts; 305 SmallVector<uint64_t, 64> RawMask; 306 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 307 return; 308 309 unsigned NumElts = Width / ElSize; 310 311 for (unsigned i = 0; i != NumElts; ++i) { 312 if (UndefElts[i]) { 313 ShuffleMask.push_back(SM_SentinelUndef); 314 continue; 315 } 316 int Index = RawMask[i] & (NumElts - 1); 317 ShuffleMask.push_back(Index); 318 } 319 } 320 321 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width, 322 SmallVectorImpl<int> &ShuffleMask) { 323 assert((Width == 128 || Width == 256 || Width == 512) && 324 C->getType()->getPrimitiveSizeInBits() >= Width && 325 "Unexpected vector size."); 326 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && 327 "Unexpected vector element size."); 328 329 // The shuffle mask requires elements the same size as the target. 330 APInt UndefElts; 331 SmallVector<uint64_t, 64> RawMask; 332 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 333 return; 334 335 unsigned NumElts = Width / ElSize; 336 337 for (unsigned i = 0; i != NumElts; ++i) { 338 if (UndefElts[i]) { 339 ShuffleMask.push_back(SM_SentinelUndef); 340 continue; 341 } 342 int Index = RawMask[i] & (NumElts*2 - 1); 343 ShuffleMask.push_back(Index); 344 } 345 } 346 } // llvm namespace 347