//===- AArch64ExpandImm.h - AArch64 Immediate Expansion -------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the AArch64ExpandImm stuff. // //===----------------------------------------------------------------------===// #include "AArch64.h" #include "AArch64ExpandImm.h" #include "MCTargetDesc/AArch64AddressingModes.h" using namespace llvm; using namespace llvm::AArch64_IMM; /// Helper function which extracts the specified 16-bit chunk from a /// 64-bit value. static uint64_t getChunk(uint64_t Imm, unsigned ChunkIdx) { assert(ChunkIdx < 4 && "Out of range chunk index specified!"); return (Imm >> (ChunkIdx * 16)) & 0xFFFF; } /// Check whether the given 16-bit chunk replicated to full 64-bit width /// can be materialized with an ORR instruction. static bool canUseOrr(uint64_t Chunk, uint64_t &Encoding) { Chunk = (Chunk << 48) | (Chunk << 32) | (Chunk << 16) | Chunk; return AArch64_AM::processLogicalImmediate(Chunk, 64, Encoding); } /// Check for identical 16-bit chunks within the constant and if so /// materialize them with a single ORR instruction. The remaining one or two /// 16-bit chunks will be materialized with MOVK instructions. /// /// This allows us to materialize constants like |A|B|A|A| or |A|B|C|A| (order /// of the chunks doesn't matter), assuming |A|A|A|A| can be materialized with /// an ORR instruction. static bool tryToreplicateChunks(uint64_t UImm, SmallVectorImpl &Insn) { using CountMap = DenseMap; CountMap Counts; // Scan the constant and count how often every chunk occurs. for (unsigned Idx = 0; Idx < 4; ++Idx) ++Counts[getChunk(UImm, Idx)]; // Traverse the chunks to find one which occurs more than once. for (const auto &Chunk : Counts) { const uint64_t ChunkVal = Chunk.first; const unsigned Count = Chunk.second; uint64_t Encoding = 0; // We are looking for chunks which have two or three instances and can be // materialized with an ORR instruction. if ((Count != 2 && Count != 3) || !canUseOrr(ChunkVal, Encoding)) continue; const bool CountThree = Count == 3; Insn.push_back({ AArch64::ORRXri, 0, Encoding }); unsigned ShiftAmt = 0; uint64_t Imm16 = 0; // Find the first chunk not materialized with the ORR instruction. for (; ShiftAmt < 64; ShiftAmt += 16) { Imm16 = (UImm >> ShiftAmt) & 0xFFFF; if (Imm16 != ChunkVal) break; } // Create the first MOVK instruction. Insn.push_back({ AArch64::MOVKXi, Imm16, AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt) }); // In case we have three instances the whole constant is now materialized // and we can exit. if (CountThree) return true; // Find the remaining chunk which needs to be materialized. for (ShiftAmt += 16; ShiftAmt < 64; ShiftAmt += 16) { Imm16 = (UImm >> ShiftAmt) & 0xFFFF; if (Imm16 != ChunkVal) break; } Insn.push_back({ AArch64::MOVKXi, Imm16, AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt) }); return true; } return false; } /// Check whether this chunk matches the pattern '1...0...'. This pattern /// starts a contiguous sequence of ones if we look at the bits from the LSB /// towards the MSB. static bool isStartChunk(uint64_t Chunk) { if (Chunk == 0 || Chunk == std::numeric_limits::max()) return false; return isMask_64(~Chunk); } /// Check whether this chunk matches the pattern '0...1...' This pattern /// ends a contiguous sequence of ones if we look at the bits from the LSB /// towards the MSB. static bool isEndChunk(uint64_t Chunk) { if (Chunk == 0 || Chunk == std::numeric_limits::max()) return false; return isMask_64(Chunk); } /// Clear or set all bits in the chunk at the given index. static uint64_t updateImm(uint64_t Imm, unsigned Idx, bool Clear) { const uint64_t Mask = 0xFFFF; if (Clear) // Clear chunk in the immediate. Imm &= ~(Mask << (Idx * 16)); else // Set all bits in the immediate for the particular chunk. Imm |= Mask << (Idx * 16); return Imm; } /// Check whether the constant contains a sequence of contiguous ones, /// which might be interrupted by one or two chunks. If so, materialize the /// sequence of contiguous ones with an ORR instruction. /// Materialize the chunks which are either interrupting the sequence or outside /// of the sequence with a MOVK instruction. /// /// Assuming S is a chunk which starts the sequence (1...0...), E is a chunk /// which ends the sequence (0...1...). Then we are looking for constants which /// contain at least one S and E chunk. /// E.g. |E|A|B|S|, |A|E|B|S| or |A|B|E|S|. /// /// We are also looking for constants like |S|A|B|E| where the contiguous /// sequence of ones wraps around the MSB into the LSB. static bool trySequenceOfOnes(uint64_t UImm, SmallVectorImpl &Insn) { const int NotSet = -1; const uint64_t Mask = 0xFFFF; int StartIdx = NotSet; int EndIdx = NotSet; // Try to find the chunks which start/end a contiguous sequence of ones. for (int Idx = 0; Idx < 4; ++Idx) { int64_t Chunk = getChunk(UImm, Idx); // Sign extend the 16-bit chunk to 64-bit. Chunk = (Chunk << 48) >> 48; if (isStartChunk(Chunk)) StartIdx = Idx; else if (isEndChunk(Chunk)) EndIdx = Idx; } // Early exit in case we can't find a start/end chunk. if (StartIdx == NotSet || EndIdx == NotSet) return false; // Outside of the contiguous sequence of ones everything needs to be zero. uint64_t Outside = 0; // Chunks between the start and end chunk need to have all their bits set. uint64_t Inside = Mask; // If our contiguous sequence of ones wraps around from the MSB into the LSB, // just swap indices and pretend we are materializing a contiguous sequence // of zeros surrounded by a contiguous sequence of ones. if (StartIdx > EndIdx) { std::swap(StartIdx, EndIdx); std::swap(Outside, Inside); } uint64_t OrrImm = UImm; int FirstMovkIdx = NotSet; int SecondMovkIdx = NotSet; // Find out which chunks we need to patch up to obtain a contiguous sequence // of ones. for (int Idx = 0; Idx < 4; ++Idx) { const uint64_t Chunk = getChunk(UImm, Idx); // Check whether we are looking at a chunk which is not part of the // contiguous sequence of ones. if ((Idx < StartIdx || EndIdx < Idx) && Chunk != Outside) { OrrImm = updateImm(OrrImm, Idx, Outside == 0); // Remember the index we need to patch. if (FirstMovkIdx == NotSet) FirstMovkIdx = Idx; else SecondMovkIdx = Idx; // Check whether we are looking a chunk which is part of the contiguous // sequence of ones. } else if (Idx > StartIdx && Idx < EndIdx && Chunk != Inside) { OrrImm = updateImm(OrrImm, Idx, Inside != Mask); // Remember the index we need to patch. if (FirstMovkIdx == NotSet) FirstMovkIdx = Idx; else SecondMovkIdx = Idx; } } assert(FirstMovkIdx != NotSet && "Constant materializable with single ORR!"); // Create the ORR-immediate instruction. uint64_t Encoding = 0; AArch64_AM::processLogicalImmediate(OrrImm, 64, Encoding); Insn.push_back({ AArch64::ORRXri, 0, Encoding }); const bool SingleMovk = SecondMovkIdx == NotSet; Insn.push_back({ AArch64::MOVKXi, getChunk(UImm, FirstMovkIdx), AArch64_AM::getShifterImm(AArch64_AM::LSL, FirstMovkIdx * 16) }); // Early exit in case we only need to emit a single MOVK instruction. if (SingleMovk) return true; // Create the second MOVK instruction. Insn.push_back({ AArch64::MOVKXi, getChunk(UImm, SecondMovkIdx), AArch64_AM::getShifterImm(AArch64_AM::LSL, SecondMovkIdx * 16) }); return true; } static uint64_t GetRunOfOnesStartingAt(uint64_t V, uint64_t StartPosition) { uint64_t NumOnes = llvm::countr_one(V >> StartPosition); uint64_t UnshiftedOnes; if (NumOnes == 64) { UnshiftedOnes = ~0ULL; } else { UnshiftedOnes = (1ULL << NumOnes) - 1; } return UnshiftedOnes << StartPosition; } static uint64_t MaximallyReplicateSubImmediate(uint64_t V, uint64_t Subset) { uint64_t Result = Subset; // 64, 32, 16, 8, 4, 2 for (uint64_t i = 0; i < 6; ++i) { uint64_t Rotation = 1ULL << (6 - i); uint64_t Closure = Result | llvm::rotl(Result, Rotation); if (Closure != (Closure & V)) { break; } Result = Closure; } return Result; } // Find the logical immediate that covers the most bits in RemainingBits, // allowing for additional bits to be set that were set in OriginalBits. static uint64_t maximalLogicalImmWithin(uint64_t RemainingBits, uint64_t OriginalBits) { // Find the first set bit. uint32_t Position = llvm::countr_zero(RemainingBits); // Get the first run of set bits. uint64_t FirstRun = GetRunOfOnesStartingAt(OriginalBits, Position); // Replicate the run as many times as possible, as long as the bits are set in // RemainingBits. uint64_t MaximalImm = MaximallyReplicateSubImmediate(OriginalBits, FirstRun); return MaximalImm; } static std::optional> decomposeIntoOrrOfLogicalImmediates(uint64_t UImm) { if (UImm == 0 || ~UImm == 0) return std::nullopt; // Make sure we don't have a run of ones split around the rotation boundary. uint32_t InitialTrailingOnes = llvm::countr_one(UImm); uint64_t RotatedBits = llvm::rotr(UImm, InitialTrailingOnes); // Find the largest logical immediate that fits within the full immediate. uint64_t MaximalImm1 = maximalLogicalImmWithin(RotatedBits, RotatedBits); // Remove all bits that are set by this mask. uint64_t RemainingBits = RotatedBits & ~MaximalImm1; // Find the largest logical immediate covering the remaining bits, allowing // for additional bits to be set that were also set in the original immediate. uint64_t MaximalImm2 = maximalLogicalImmWithin(RemainingBits, RotatedBits); // If any bits still haven't been covered, then give up. if (RemainingBits & ~MaximalImm2) return std::nullopt; // Make sure to un-rotate the immediates. return std::make_pair(rotl(MaximalImm1, InitialTrailingOnes), rotl(MaximalImm2, InitialTrailingOnes)); } // Attempt to expand an immediate as the ORR of a pair of logical immediates. static bool tryOrrOfLogicalImmediates(uint64_t UImm, SmallVectorImpl &Insn) { auto MaybeDecomposition = decomposeIntoOrrOfLogicalImmediates(UImm); if (MaybeDecomposition == std::nullopt) return false; uint64_t Imm1 = MaybeDecomposition->first; uint64_t Imm2 = MaybeDecomposition->second; uint64_t Encoding1, Encoding2; bool Imm1Success = AArch64_AM::processLogicalImmediate(Imm1, 64, Encoding1); bool Imm2Success = AArch64_AM::processLogicalImmediate(Imm2, 64, Encoding2); if (Imm1Success && Imm2Success) { // Create the ORR-immediate instructions. Insn.push_back({AArch64::ORRXri, 0, Encoding1}); Insn.push_back({AArch64::ORRXri, 1, Encoding2}); return true; } return false; } // Attempt to expand an immediate as the AND of a pair of logical immediates. // This is done by applying DeMorgan's law, under which logical immediates // are closed. static bool tryAndOfLogicalImmediates(uint64_t UImm, SmallVectorImpl &Insn) { // Apply DeMorgan's law to turn this into an ORR problem. auto MaybeDecomposition = decomposeIntoOrrOfLogicalImmediates(~UImm); if (MaybeDecomposition == std::nullopt) return false; uint64_t Imm1 = MaybeDecomposition->first; uint64_t Imm2 = MaybeDecomposition->second; uint64_t Encoding1, Encoding2; bool Imm1Success = AArch64_AM::processLogicalImmediate(~Imm1, 64, Encoding1); bool Imm2Success = AArch64_AM::processLogicalImmediate(~Imm2, 64, Encoding2); if (Imm1Success && Imm2Success) { // Materialize Imm1, the LHS of the AND Insn.push_back({AArch64::ORRXri, 0, Encoding1}); // AND Imm1 with Imm2 Insn.push_back({AArch64::ANDXri, 1, Encoding2}); return true; } return false; } // Check whether the constant can be represented by exclusive-or of two 64-bit // logical immediates. If so, materialize it with an ORR instruction followed // by an EOR instruction. // // This encoding allows all remaining repeated byte patterns, and many repeated // 16-bit values, to be encoded without needing four instructions. It can also // represent some irregular bitmasks (although those would mostly only need // three instructions otherwise). static bool tryEorOfLogicalImmediates(uint64_t Imm, SmallVectorImpl &Insn) { // Determine the larger repetition size of the two possible logical // immediates, by finding the repetition size of Imm. unsigned BigSize = 64; do { BigSize /= 2; uint64_t Mask = (1ULL << BigSize) - 1; if ((Imm & Mask) != ((Imm >> BigSize) & Mask)) { BigSize *= 2; break; } } while (BigSize > 2); uint64_t BigMask = ((uint64_t)-1LL) >> (64 - BigSize); // Find the last bit of each run of ones, circularly. For runs which wrap // around from bit 0 to bit 63, this is the bit before the most-significant // zero, otherwise it is the least-significant bit in the run of ones. uint64_t RunStarts = Imm & ~rotl(Imm, 1); // Find the smaller repetition size of the two possible logical immediates by // counting the number of runs of one-bits within the BigSize-bit value. Both // sizes may be the same. The EOR may add one or subtract one from the // power-of-two count that can be represented by a logical immediate, or it // may be left unchanged. int RunsPerBigChunk = popcount(RunStarts & BigMask); static const int8_t BigToSmallSizeTable[32] = { -1, -1, 0, 1, 2, 2, -1, 3, 3, 3, -1, -1, -1, -1, -1, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 5, }; int BigToSmallShift = BigToSmallSizeTable[RunsPerBigChunk]; // Early-exit if the big chunk couldn't be a power-of-two number of runs // EORed with another single run. if (BigToSmallShift == -1) return false; unsigned SmallSize = BigSize >> BigToSmallShift; // 64-bit values with a bit set every (1 << index) bits. static const uint64_t RepeatedOnesTable[] = { 0xffffffffffffffff, 0x5555555555555555, 0x1111111111111111, 0x0101010101010101, 0x0001000100010001, 0x0000000100000001, 0x0000000000000001, }; // This RepeatedOnesTable lookup is a faster implementation of the division // 0xffffffffffffffff / ((1 << SmallSize) - 1), and can be thought of as // dividing the 64-bit value into fields of width SmallSize, and placing a // one in the least significant bit of each field. uint64_t SmallOnes = RepeatedOnesTable[countr_zero(SmallSize)]; // Now we try to find the number of ones in each of the smaller repetitions, // by looking at runs of ones in Imm. This can take three attempts, as the // EOR may have changed the length of the first two runs we find. // Rotate a run of ones so we can count the number of trailing set bits. int Rotation = countr_zero(RunStarts); uint64_t RotatedImm = rotr(Imm, Rotation); for (int Attempt = 0; Attempt < 3; ++Attempt) { unsigned RunLength = countr_one(RotatedImm); // Construct candidate values BigImm and SmallImm, such that if these two // values are encodable, we have a solution. (SmallImm is constructed to be // encodable, but this isn't guaranteed when RunLength >= SmallSize) uint64_t SmallImm = rotl((SmallOnes << RunLength) - SmallOnes, Rotation); uint64_t BigImm = Imm ^ SmallImm; uint64_t BigEncoding = 0; uint64_t SmallEncoding = 0; if (AArch64_AM::processLogicalImmediate(BigImm, 64, BigEncoding) && AArch64_AM::processLogicalImmediate(SmallImm, 64, SmallEncoding)) { Insn.push_back({AArch64::ORRXri, 0, SmallEncoding}); Insn.push_back({AArch64::EORXri, 1, BigEncoding}); return true; } // Rotate to the next run of ones Rotation += countr_zero(rotr(RunStarts, Rotation) & ~1); RotatedImm = rotr(Imm, Rotation); } return false; } /// \brief Expand a MOVi32imm or MOVi64imm pseudo instruction to a /// MOVZ or MOVN of width BitSize followed by up to 3 MOVK instructions. static inline void expandMOVImmSimple(uint64_t Imm, unsigned BitSize, unsigned OneChunks, unsigned ZeroChunks, SmallVectorImpl &Insn) { const unsigned Mask = 0xFFFF; // Use a MOVZ or MOVN instruction to set the high bits, followed by one or // more MOVK instructions to insert additional 16-bit portions into the // lower bits. bool isNeg = false; // Use MOVN to materialize the high bits if we have more all one chunks // than all zero chunks. if (OneChunks > ZeroChunks) { isNeg = true; Imm = ~Imm; } unsigned FirstOpc; if (BitSize == 32) { Imm &= (1LL << 32) - 1; FirstOpc = (isNeg ? AArch64::MOVNWi : AArch64::MOVZWi); } else { FirstOpc = (isNeg ? AArch64::MOVNXi : AArch64::MOVZXi); } unsigned Shift = 0; // LSL amount for high bits with MOVZ/MOVN unsigned LastShift = 0; // LSL amount for last MOVK if (Imm != 0) { unsigned LZ = llvm::countl_zero(Imm); unsigned TZ = llvm::countr_zero(Imm); Shift = (TZ / 16) * 16; LastShift = ((63 - LZ) / 16) * 16; } unsigned Imm16 = (Imm >> Shift) & Mask; Insn.push_back({ FirstOpc, Imm16, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift) }); if (Shift == LastShift) return; // If a MOVN was used for the high bits of a negative value, flip the rest // of the bits back for use with MOVK. if (isNeg) Imm = ~Imm; unsigned Opc = (BitSize == 32 ? AArch64::MOVKWi : AArch64::MOVKXi); while (Shift < LastShift) { Shift += 16; Imm16 = (Imm >> Shift) & Mask; if (Imm16 == (isNeg ? Mask : 0)) continue; // This 16-bit portion is already set correctly. Insn.push_back({ Opc, Imm16, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift) }); } } /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more /// real move-immediate instructions to synthesize the immediate. void AArch64_IMM::expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl &Insn) { const unsigned Mask = 0xFFFF; // Scan the immediate and count the number of 16-bit chunks which are either // all ones or all zeros. unsigned OneChunks = 0; unsigned ZeroChunks = 0; for (unsigned Shift = 0; Shift < BitSize; Shift += 16) { const unsigned Chunk = (Imm >> Shift) & Mask; if (Chunk == Mask) OneChunks++; else if (Chunk == 0) ZeroChunks++; } // Prefer MOVZ/MOVN over ORR because of the rules for the "mov" alias. if ((BitSize / 16) - OneChunks <= 1 || (BitSize / 16) - ZeroChunks <= 1) { expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); return; } // Try a single ORR. uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize); uint64_t Encoding; if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) { unsigned Opc = (BitSize == 32 ? AArch64::ORRWri : AArch64::ORRXri); Insn.push_back({ Opc, 0, Encoding }); return; } // One to up three instruction sequences. // // Prefer MOVZ/MOVN followed by MOVK; it's more readable, and possibly the // fastest sequence with fast literal generation. if (OneChunks >= (BitSize / 16) - 2 || ZeroChunks >= (BitSize / 16) - 2) { expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); return; } assert(BitSize == 64 && "All 32-bit immediates can be expanded with a" "MOVZ/MOVK pair"); // Try other two-instruction sequences. // 64-bit ORR followed by MOVK. // We try to construct the ORR immediate in three different ways: either we // zero out the chunk which will be replaced, we fill the chunk which will // be replaced with ones, or we take the bit pattern from the other half of // the 64-bit immediate. This is comprehensive because of the way ORR // immediates are constructed. for (unsigned Shift = 0; Shift < BitSize; Shift += 16) { uint64_t ShiftedMask = (0xFFFFULL << Shift); uint64_t ZeroChunk = UImm & ~ShiftedMask; uint64_t OneChunk = UImm | ShiftedMask; uint64_t RotatedImm = (UImm << 32) | (UImm >> 32); uint64_t ReplicateChunk = ZeroChunk | (RotatedImm & ShiftedMask); if (AArch64_AM::processLogicalImmediate(ZeroChunk, BitSize, Encoding) || AArch64_AM::processLogicalImmediate(OneChunk, BitSize, Encoding) || AArch64_AM::processLogicalImmediate(ReplicateChunk, BitSize, Encoding)) { // Create the ORR-immediate instruction. Insn.push_back({ AArch64::ORRXri, 0, Encoding }); // Create the MOVK instruction. const unsigned Imm16 = getChunk(UImm, Shift / 16); Insn.push_back({ AArch64::MOVKXi, Imm16, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift) }); return; } } // Attempt to use a sequence of two ORR-immediate instructions. if (tryOrrOfLogicalImmediates(Imm, Insn)) return; // Attempt to use a sequence of ORR-immediate followed by AND-immediate. if (tryAndOfLogicalImmediates(Imm, Insn)) return; // Attempt to use a sequence of ORR-immediate followed by EOR-immediate. if (tryEorOfLogicalImmediates(UImm, Insn)) return; // FIXME: Add more two-instruction sequences. // Three instruction sequences. // // Prefer MOVZ/MOVN followed by two MOVK; it's more readable, and possibly // the fastest sequence with fast literal generation. (If neither MOVK is // part of a fast literal generation pair, it could be slower than the // four-instruction sequence, but we won't worry about that for now.) if (OneChunks || ZeroChunks) { expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); return; } // Check for identical 16-bit chunks within the constant and if so materialize // them with a single ORR instruction. The remaining one or two 16-bit chunks // will be materialized with MOVK instructions. if (BitSize == 64 && tryToreplicateChunks(UImm, Insn)) return; // Check whether the constant contains a sequence of contiguous ones, which // might be interrupted by one or two chunks. If so, materialize the sequence // of contiguous ones with an ORR instruction. Materialize the chunks which // are either interrupting the sequence or outside of the sequence with a // MOVK instruction. if (BitSize == 64 && trySequenceOfOnes(UImm, Insn)) return; // We found no possible two or three instruction sequence; use the general // four-instruction sequence. expandMOVImmSimple(Imm, BitSize, OneChunks, ZeroChunks, Insn); }