1 //===- VPlanTransforms.h - Utility VPlan to VPlan transforms --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file provides utility VPlan to VPlan transformations. 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H 14 #define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H 15 16 #include "VPlan.h" 17 #include "llvm/ADT/STLFunctionalExtras.h" 18 19 namespace llvm { 20 21 class InductionDescriptor; 22 class Instruction; 23 class PHINode; 24 class ScalarEvolution; 25 class PredicatedScalarEvolution; 26 class TargetLibraryInfo; 27 class VPBuilder; 28 29 struct VPlanTransforms { 30 /// Replaces the VPInstructions in \p Plan with corresponding 31 /// widen recipes. 32 static void 33 VPInstructionsToVPRecipes(VPlanPtr &Plan, 34 function_ref<const InductionDescriptor *(PHINode *)> 35 GetIntOrFpInductionDescriptor, 36 ScalarEvolution &SE, const TargetLibraryInfo &TLI); 37 38 /// Sink users of fixed-order recurrences after the recipe defining their 39 /// previous value. Then introduce FirstOrderRecurrenceSplice VPInstructions 40 /// to combine the value from the recurrence phis and previous values. The 41 /// current implementation assumes all users can be sunk after the previous 42 /// value, which is enforced by earlier legality checks. 43 /// \returns true if all users of fixed-order recurrences could be re-arranged 44 /// as needed or false if it is not possible. In the latter case, \p Plan is 45 /// not valid. 46 static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder); 47 48 /// Clear NSW/NUW flags from reduction instructions if necessary. 49 static void clearReductionWrapFlags(VPlan &Plan); 50 51 /// Optimize \p Plan based on \p BestVF and \p BestUF. This may restrict the 52 /// resulting plan to \p BestVF and \p BestUF. 53 static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, 54 unsigned BestUF, 55 PredicatedScalarEvolution &PSE); 56 57 /// Apply VPlan-to-VPlan optimizations to \p Plan, including induction recipe 58 /// optimizations, dead recipe removal, replicate region optimizations and 59 /// block merging. 60 static void optimize(VPlan &Plan, ScalarEvolution &SE); 61 62 /// Wrap predicated VPReplicateRecipes with a mask operand in an if-then 63 /// region block and remove the mask operand. Optimize the created regions by 64 /// iteratively sinking scalar operands into the region, followed by merging 65 /// regions until no improvements are remaining. 66 static void createAndOptimizeReplicateRegions(VPlan &Plan); 67 68 /// Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an 69 /// (active-lane-mask recipe, wide canonical IV, trip-count). If \p 70 /// UseActiveLaneMaskForControlFlow is true, introduce an 71 /// VPActiveLaneMaskPHIRecipe. If \p DataAndControlFlowWithoutRuntimeCheck is 72 /// true, no minimum-iteration runtime check will be created (during skeleton 73 /// creation) and instead it is handled using active-lane-mask. \p 74 /// DataAndControlFlowWithoutRuntimeCheck implies \p 75 /// UseActiveLaneMaskForControlFlow. 76 static void addActiveLaneMask(VPlan &Plan, 77 bool UseActiveLaneMaskForControlFlow, 78 bool DataAndControlFlowWithoutRuntimeCheck); 79 80 /// Insert truncates and extends for any truncated recipe. Redundant casts 81 /// will be folded later. 82 static void 83 truncateToMinimalBitwidths(VPlan &Plan, 84 const MapVector<Instruction *, uint64_t> &MinBWs, 85 LLVMContext &Ctx); 86 87 /// Drop poison flags from recipes that may generate a poison value that is 88 /// used after vectorization, even when their operands are not poison. Those 89 /// recipes meet the following conditions: 90 /// * Contribute to the address computation of a recipe generating a widen 91 /// memory load/store (VPWidenMemoryInstructionRecipe or 92 /// VPInterleaveRecipe). 93 /// * Such a widen memory load/store has at least one underlying Instruction 94 /// that is in a basic block that needs predication and after vectorization 95 /// the generated instruction won't be predicated. 96 /// Uses \p BlockNeedsPredication to check if a block needs predicating. 97 /// TODO: Replace BlockNeedsPredication callback with retrieving info from 98 /// VPlan directly. 99 static void dropPoisonGeneratingRecipes( 100 VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication); 101 102 /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and 103 /// replaces all uses except the canonical IV increment of 104 /// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe. 105 /// VPCanonicalIVPHIRecipe is only used to control the loop after 106 /// this transformation. 107 /// \returns true if the transformation succeeds, or false if it doesn't. 108 static bool tryAddExplicitVectorLength(VPlan &Plan); 109 }; 110 111 } // namespace llvm 112 113 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H 114