LoopVectorize.cpp File Reference

#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <memory>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes
class	llvm::InnerLoopVectorizer
	InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF). More...
struct	llvm::EpilogueLoopVectorizationInfo
	Encapsulate information regarding vectorization of a loop and its epilogue. More...
class	llvm::InnerLoopAndEpilogueVectorizer
	An extension of the inner loop vectorizer that creates a skeleton for a vectorized loop that has its epilogue (residual) also vectorized. More...
class	llvm::EpilogueVectorizerMainLoop
	A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the process of vectorizing loops and their epilogues. More...
class	llvm::EpilogueVectorizerEpilogueLoop
class	llvm::LoopVectorizationCostModel
	LoopVectorizationCostModel - estimates the expected speedups due to vectorization. More...
struct	llvm::LoopVectorizationCostModel::CallWideningDecision

Namespaces
namespace	PreferPredicateTy
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	LV_NAME   "loop-vectorize"
#define	DEBUG_TYPE   LV_NAME

Enumerations
enum	PreferPredicateTy::Option { PreferPredicateTy::ScalarEpilogue = 0 , PreferPredicateTy::PredicateElseScalarEpilogue , PreferPredicateTy::PredicateOrDontVectorize }
enum	llvm::ScalarEpilogueLowering {   llvm::CM_ScalarEpilogueAllowed , llvm::CM_ScalarEpilogueNotAllowedOptSize , llvm::CM_ScalarEpilogueNotAllowedLowTripLoop , llvm::CM_ScalarEpilogueNotNeededUsePredicate ,   llvm::CM_ScalarEpilogueNotAllowedUsePredicate }

Functions
	STATISTIC (LoopsVectorized, "Number of loops vectorized")
	STATISTIC (LoopsAnalyzed, "Number of loops analyzed for vectorization")
	STATISTIC (LoopsEpilogueVectorized, "Number of epilogues vectorized")
	STATISTIC (LoopsEarlyExitVectorized, "Number of early exit loops vectorized")
static bool	hasIrregularType (Type *Ty, const DataLayout &DL)
	A helper function that returns true if the given type is irregular.
static ElementCount	getSmallConstantTripCount (ScalarEvolution *SE, const Loop *L)
	A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops whose trip count is a function of vscale.
static std::optional< ElementCount >	getSmallBestKnownTC (PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
	Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an estimate cannot be made (including when the trip count would overflow), for the specified loop L as defined by the following procedure: 1) Returns exact trip count if it is known.
static DebugLoc	getDebugLocFromInstOrOperands (Instruction *I)
	Look for a meaningful debug location on the instruction or its operands.
static void	debugVectorizationMessage (const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
	Write a DebugMsg about vectorization to the debug output stream.
static OptimizationRemarkAnalysis	createLVAnalysis (const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
	Create an analysis remark that explains why vectorization failed.
Value *	llvm::createStepForVF (IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
	Return a value for Step multiplied by VF.
Value *	llvm::getRuntimeVF (IRBuilderBase &B, Type *Ty, ElementCount VF)
	Return the runtime value for VF.
LLVM_ABI void	llvm::reportVectorizationFailure (const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
	Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding optimization remark RemarkName.
static void	llvm::reportVectorizationInfo (const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
	Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
static void	llvm::reportVectorization (OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
	Report successful vectorization of the loop.
static bool	useActiveLaneMask (TailFoldingStyle Style)
static bool	useActiveLaneMaskForControlFlow (TailFoldingStyle Style)
static bool	isExplicitVecOuterLoop (Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static void	collectSupportedLoops (Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static Value *	emitTransformedIndex (IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
	Compute the transformed value of Index at offset StartValue using step StepValue.
static std::optional< unsigned >	getMaxVScale (const Function &F, const TargetTransformInfo &TTI)
static bool	isIndvarOverflowCheckKnownFalse (const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
	For the given VF and UF and maximum trip count computed for the loop, return whether the induction variable might overflow in the vectorized loop.
static bool	useMaskedInterleavedAccesses (const TargetTransformInfo &TTI)
static VPIRBasicBlock *	replaceVPBBWithIRVPBB (VPBasicBlock *VPBB, BasicBlock *IRBB)
	Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static Value *	getExpandedStep (const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
	Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static void	addFullyUnrolledInstructionsToIgnore (Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
	Knowing that loop L executes a single vector iteration, add instructions that will get simplified and thus should not have any cost to InstsToIgnore.
static void	cse (BasicBlock *BB)
	Perform cse of induction variable instructions.
static unsigned	estimateElementCount (ElementCount VF, std::optional< unsigned > VScale)
	This function attempts to return a value that represents the ElementCount at runtime.
static Type *	maybeVectorizeType (Type *Ty, ElementCount VF)
static bool	willGenerateVectors (VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
	Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static bool	hasReplicatorRegion (VPlan &Plan)
static const SCEV *	getAddressAccessSCEV (Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
	Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.
static ElementCount	determineVPlanVF (const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static bool	planContainsAdditionalSimplifications (VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop, ElementCount VF)
	Return true if the original loop \ TheLoop contains any instructions that do not have corresponding recipes in Plan and are not marked to be ignored in CostCtx.
static Value *	getStartValueFromReductionResult (VPInstruction *RdxResult)
static void	fixReductionScalarResumeWhenVectorizingEpilog (VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock)
static VPWidenIntOrFpInductionRecipe *	createWidenInductionRecipes (PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
	Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static VPInstruction *	addResumePhiRecipeForInduction (VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC)
	Create and return a ResumePhi for WideIV, unless it is truncated.
static void	addScalarResumePhis (VPRecipeBuilder &Builder, VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)
	Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions, and update the VPIRInstructions wrapping the original phis in the scalar header.
static void	addExitUsersForFirstOrderRecurrences (VPlan &Plan, VFRange &Range)
	Handle users in the exit block for first order reductions in the original exit block.
static ScalarEpilogueLowering	getScalarEpilogueLowering (Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static bool	processLoopInVPlanNativePath (Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static void	checkMixedPrecision (Loop *L, OptimizationRemarkEmitter *ORE)
static InstructionCost	calculateEarlyExitCost (VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
	For loops with uncountable early exits, find the cost of doing work when exiting the loop early, such as calculating the final exit values of variables used outside the loop.
static bool	isOutsideLoopWorkProfitable (GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale)
	This function determines whether or not it's still profitable to vectorize the loop given the extra work we have to do outside of the loop:
static void	preparePlanForMainVectorLoop (VPlan &MainPlan, VPlan &EpiPlan)
	Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static void	preparePlanForEpilogueVectorLoop (VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, EpilogueLoopVectorizationInfo &EPI)
	Prepare Plan for vectorizing the epilogue loop.
static Value *	createInductionAdditionalBypassValues (PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, Instruction *OldInduction)
static void	fixScalarResumeValuesFromBypass (BasicBlock *BypassBlock, Loop *L, VPlan &BestEpiPlan, LoopVectorizationLegality &LVL, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)

Variables
const char	VerboseDebug [] = DEBUG_TYPE "-verbose"
static cl::opt< bool >	EnableEpilogueVectorization ("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static cl::opt< unsigned >	EpilogueVectorizationForceVF ("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static cl::opt< unsigned >	EpilogueVectorizationMinVF ("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned >	TinyTripCountVectorThreshold ("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
	Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.
static cl::opt< unsigned >	VectorizeMemoryCheckThreshold ("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static cl::opt< PreferPredicateTy::Option >	PreferPredicateOverEpilogue ("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< TailFoldingStyle >	ForceTailFoldingStyle ("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool >	MaximizeBandwidth ("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static cl::opt< bool >	EnableInterleavedMemAccesses ("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool >	EnableMaskedInterleavedMemAccesses ("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
	An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.
static cl::opt< unsigned >	ForceTargetNumScalarRegs ("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< unsigned >	ForceTargetNumVectorRegs ("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static cl::opt< unsigned >	ForceTargetMaxScalarInterleaveFactor ("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static cl::opt< unsigned >	ForceTargetMaxVectorInterleaveFactor ("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static cl::opt< bool >	ForceTargetSupportsScalableVectors ("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static cl::opt< unsigned >	SmallLoopCost ("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static cl::opt< bool >	LoopVectorizeWithBlockFrequency ("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static cl::opt< bool >	EnableLoadStoreRuntimeInterleave ("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< unsigned >	NumberOfStoresToPredicate ("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
	The number of stores in a loop that are allowed to need predication.
static cl::opt< bool >	EnableIndVarRegisterHeur ("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< bool >	EnableCondStoresVectorization ("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static cl::opt< unsigned >	MaxNestedScalarReductionIC ("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< bool >	PreferInLoopReductions ("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static cl::opt< bool >	ForceOrderedReductions ("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static cl::opt< bool >	PreferPredicatedReductionSelect ("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static cl::opt< bool >	VPlanBuildStressTest ("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static cl::opt< cl::boolOrDefault >	ForceSafeDivisor ("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static cl::opt< bool >	UseWiderVFIfCallVariantsPresent ("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static cl::opt< bool >	EnableEarlyExitVectorization ("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool >	ConsiderRegPressure ("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high."))
static constexpr uint32_t	MinItersBypassWeights [] = {1, 127}

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   LV_NAME

Definition at line 162 of file LoopVectorize.cpp.

LV_NAME

#define LV_NAME   "loop-vectorize"

Definition at line 161 of file LoopVectorize.cpp.

Function Documentation

addExitUsersForFirstOrderRecurrences()

void addExitUsersForFirstOrderRecurrences ( VPlan & Plan, VFRange & Range )

static

Handle users in the exit block for first order reductions in the original exit block.

The penultimate value of recurrences is fed to their LCSSA phi users in the original exit block using the VPIRInstruction wrapping to the LCSSA phi.

Definition at line 8346 of file LoopVectorize.cpp.

References assert(), llvm::cast(), llvm::VPBuilder::createNaryOp(), llvm::dyn_cast(), llvm::VPInstruction::ExtractPenultimateElement, llvm::LoopVectorizationPlanner::getDecisionAndClampRange(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getMiddleBlock(), llvm::ElementCount::getScalable(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlanPatternMatch::m_ExtractLastElement(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), Range, and llvm::VPValue::replaceAllUsesWith().

addFullyUnrolledInstructionsToIgnore()

void addFullyUnrolledInstructionsToIgnore ( Loop * L, const LoopVectorizationLegality::InductionList & IL, SmallPtrSetImpl< Instruction * > & InstsToIgnore )

static

Knowing that loop L executes a single vector iteration, add instructions that will get simplified and thus should not have any cost to InstsToIgnore.

Definition at line 2401 of file LoopVectorize.cpp.

References llvm::all_of(), llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::insert(), IV, and llvm::Value::users().

Referenced by llvm::LoopVectorizationCostModel::expectedCost().

addResumePhiRecipeForInduction()

VPInstruction * addResumePhiRecipeForInduction ( VPWidenInductionRecipe * WideIV, VPBuilder & VectorPHBuilder, VPBuilder & ScalarPHBuilder, VPTypeAnalysis & TypeInfo, VPValue * VectorTC )

static

Create and return a ResumePhi for WideIV, unless it is truncated.

If the induction recipe is not canonical, creates a VPDerivedIVRecipe to compute the end value of the induction.

Definition at line 8253 of file LoopVectorize.cpp.

References llvm::VPBuilder::createDerivedIV(), llvm::VPBuilder::createScalarCast(), llvm::VPBuilder::createScalarPhi(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPWidenInductionRecipe::getInductionDescriptor(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPWidenInductionRecipe::getStepValue(), and llvm::VPTypeAnalysis::inferScalarType().

Referenced by addScalarResumePhis().

addScalarResumePhis()

void addScalarResumePhis ( VPRecipeBuilder & Builder, VPlan & Plan, DenseMap< VPValue *, VPValue * > & IVEndValues )

static

Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions, and update the VPIRInstructions wrapping the original phis in the scalar header.

End values for inductions are added to IVEndValues.

Definition at line 8290 of file LoopVectorize.cpp.

References llvm::VPUser::addOperand(), addResumePhiRecipeForInduction(), assert(), llvm::cast(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createScalarPhi(), llvm::dyn_cast(), llvm::VPInstruction::ExtractLastElement, llvm::VPlan::getMiddleBlock(), llvm::VPlan::getScalarHeader(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::isa(), and llvm::VPBasicBlock::phis().

calculateEarlyExitCost()

InstructionCost calculateEarlyExitCost ( VPCostContext & CostCtx, VPlan & Plan, ElementCount VF )

static

For loops with uncountable early exits, find the cost of doing work when exiting the loop early, such as calculating the final exit values of variables used outside the loop.

TODO: This is currently overly pessimistic because the loop may not take the early exit, but better to keep this conservative for now. In future, it might be possible to relax this by using branch probabilities.

Definition at line 9358 of file LoopVectorize.cpp.

References llvm::dbgs(), llvm::VPlan::getExitBlocks(), llvm::VPlan::getMiddleBlock(), and LLVM_DEBUG.

Referenced by isOutsideLoopWorkProfitable().

checkMixedPrecision()

void checkMixedPrecision ( Loop * L, OptimizationRemarkEmitter * ORE )

static

Definition at line 9311 of file LoopVectorize.cpp.

References llvm::dyn_cast(), llvm::OptimizationRemarkEmitter::emit(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), LV_NAME, llvm::SmallVectorImpl< T >::pop_back_val(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::LoopVectorizePass::processLoop().

collectSupportedLoops()

void collectSupportedLoops ( Loop & L, LoopInfo * LI, OptimizationRemarkEmitter * ORE, SmallVectorImpl< Loop * > & V )

static

Definition at line 2086 of file LoopVectorize.cpp.

References collectSupportedLoops(), llvm::containsIrreducibleCFG(), llvm::EnableVPlanNativePath, isExplicitVecOuterLoop(), llvm::LoopBlocksRPO::perform(), and VPlanBuildStressTest.

Referenced by collectSupportedLoops(), and llvm::LoopVectorizePass::runImpl().

createInductionAdditionalBypassValues()

Value * createInductionAdditionalBypassValues ( PHINode * OrigPhi, const InductionDescriptor & II, IRBuilder<> & BypassBuilder, const SCEV2ValueTy & ExpandedSCEVs, Value * MainVectorTripCount, Instruction * OldInduction )

static

Definition at line 9733 of file LoopVectorize.cpp.

References emitTransformedIndex(), getExpandedStep(), II, llvm::isa_and_nonnull(), llvm::IRBuilderBase::setFastMathFlags(), and llvm::Value::setName().

Referenced by fixScalarResumeValuesFromBypass().

createLVAnalysis()

OptimizationRemarkAnalysis createLVAnalysis ( const char * PassName, StringRef RemarkName, Loop * TheLoop, Instruction * I, DebugLoc DL = {} )

static

Create an analysis remark that explains why vectorization failed.

PassName is the name of the pass (e.g. can be AlwaysPrint). RemarkName is the identifier for the remark. If I is passed it is an instruction that prevents vectorization. Otherwise TheLoop is used for the location of the remark. If DL is passed, use it as debug location for the remark.

Returns

the remark object that can be streamed to.

Definition at line 777 of file LoopVectorize.cpp.

References DL, I, and PassName.

Referenced by llvm::reportVectorizationFailure().

createWidenInductionRecipes()

VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes ( PHINode * Phi, Instruction * PhiOrTrunc, VPValue * Start, const InductionDescriptor & IndDesc, VPlan & Plan, ScalarEvolution & SE, Loop & OrigLoop )

static

Creates a VPWidenIntOrFpInductionRecpipe for Phi.

If needed, it will also insert a recipe to expand the step for the induction recipe.

Definition at line 7591 of file LoopVectorize.cpp.

References assert(), llvm::dyn_cast(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::InductionDescriptor::getStartValue(), llvm::InductionDescriptor::getStep(), llvm::VPlan::getVF(), llvm::isa(), and llvm::ScalarEvolution::isLoopInvariant().

cse()

void cse ( BasicBlock * BB )

static

Perform cse of induction variable instructions.

Definition at line 2460 of file LoopVectorize.cpp.

References llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), and llvm::make_early_inc_range().

Referenced by llvm::InnerLoopVectorizer::fixVectorizedLoop().

debugVectorizationMessage()

void debugVectorizationMessage ( const StringRef Prefix, const StringRef DebugMsg, Instruction * I )

static

Write a DebugMsg about vectorization to the debug output stream.

If I is passed, the message relates to that particular instruction.

Definition at line 757 of file LoopVectorize.cpp.

References llvm::dbgs(), and I.

Referenced by llvm::reportVectorization(), and llvm::reportVectorizationFailure().

determineVPlanVF()

ElementCount determineVPlanVF ( const TargetTransformInfo & TTI, LoopVectorizationCostModel & CM )

static

Definition at line 6553 of file LoopVectorize.cpp.

References llvm::ElementCount::get(), llvm::LoopVectorizationCostModel::getSmallestAndWidestTypes(), N, RegSize, llvm::TargetTransformInfo::RGK_FixedWidthVector, and llvm::TargetTransformInfo::RGK_ScalableVector.

Referenced by llvm::LoopVectorizationPlanner::planInVPlanNativePath().

emitTransformedIndex()

Value * emitTransformedIndex ( IRBuilderBase & B, Value * Index, Value * StartValue, Value * Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator * InductionBinOp )

static

Compute the transformed value of Index at offset StartValue using step StepValue.

For integer induction, returns StartValue + Index * StepValue. For pointer induction, returns StartValue[Index * StepValue]. FIXME: The newly created binary instructions should contain nsw/nuw flags, which can be found from the original scalar operations.

Definition at line 2123 of file LoopVectorize.cpp.

References assert(), B(), llvm::cast(), CreateAdd(), CreateMul(), llvm::dyn_cast(), llvm::Value::getName(), llvm::BinaryOperator::getOpcode(), llvm::Value::getType(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_NoInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::isa(), llvm::Type::isFloatingPointTy(), llvm::Type::isIntegerTy(), llvm_unreachable, llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), llvm::Offset, llvm::Value::setName(), X, and Y.

Referenced by createInductionAdditionalBypassValues(), and llvm::VPDerivedIVRecipe::execute().

estimateElementCount()

unsigned estimateElementCount ( ElementCount VF, std::optional< unsigned > VScale )

static

This function attempts to return a value that represents the ElementCount at runtime.

For fixed-width VFs we know this precisely at compile time, but for scalable VFs we calculate it based on an estimate of the vscale value.

Definition at line 2483 of file LoopVectorize.cpp.

References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced by llvm::EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(), llvm::LoopVectorizationPlanner::executePlan(), llvm::LoopVectorizationCostModel::isEpilogueVectorizationProfitable(), isOutsideLoopWorkProfitable(), llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor(), and llvm::LoopVectorizationPlanner::selectInterleaveCount().

fixReductionScalarResumeWhenVectorizingEpilog()

void fixReductionScalarResumeWhenVectorizingEpilog ( VPPhi * EpiResumePhiR, PHINode & EpiResumePhi, BasicBlock * BypassBlock )

static

Definition at line 7101 of file LoopVectorize.cpp.

References assert(), llvm::VPInstruction::Broadcast, llvm::cast(), llvm::VPInstruction::ComputeAnyOfResult, llvm::VPInstruction::ComputeFindIVResult, llvm::VPInstruction::ComputeReductionResult, llvm::dyn_cast(), llvm::PHINode::getIncomingValueForBlock(), llvm::VPUser::getOperand(), getStartValueFromReductionResult(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::RecurrenceDescriptor::isAnyOfRecurrenceKind(), llvm::RecurrenceDescriptor::isFindIVRecurrenceKind(), llvm::isGuaranteedNotToBeUndefOrPoison(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::PatternMatch::m_Value(), llvm::VPlanPatternMatch::m_VPValue(), llvm::VPlanPatternMatch::m_ZExtOrSExt(), llvm::PatternMatch::match(), llvm::VPInstruction::ReductionStartVector, and llvm::PHINode::setIncomingValueForBlock().

Referenced by fixScalarResumeValuesFromBypass().

fixScalarResumeValuesFromBypass()

void fixScalarResumeValuesFromBypass ( BasicBlock * BypassBlock, Loop * L, VPlan & BestEpiPlan, LoopVectorizationLegality & LVL, const SCEV2ValueTy & ExpandedSCEVs, Value * MainVectorTripCount )

static

Definition at line 9756 of file LoopVectorize.cpp.

References llvm::cast(), createInductionAdditionalBypassValues(), fixReductionScalarResumeWhenVectorizingEpilog(), llvm::BasicBlock::getFirstInsertionPt(), llvm::LoopVectorizationLegality::getInductionVars(), llvm::LoopVectorizationLegality::getPrimaryInduction(), llvm::VPlan::getScalarPreheader(), II, llvm::BasicBlock::phis(), llvm::predecessors(), and llvm::zip().

Referenced by llvm::LoopVectorizePass::processLoop().

getAddressAccessSCEV()

const SCEV * getAddressAccessSCEV ( Value * Ptr, LoopVectorizationLegality * Legal, PredicatedScalarEvolution & PSE, const Loop * TheLoop )

static

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.

This SCEV can be sent to the Target in order to estimate the address calculation cost.

Definition at line 5100 of file LoopVectorize.cpp.

References llvm::dyn_cast(), llvm::PredicatedScalarEvolution::getSCEV(), llvm::ScalarEvolution::getSCEV(), llvm::PredicatedScalarEvolution::getSE(), llvm::ScalarEvolution::isLoopInvariant(), llvm::IRSimilarity::Legal, and Ptr.

getDebugLocFromInstOrOperands()

DebugLoc getDebugLocFromInstOrOperands ( Instruction * I )

static

Look for a meaningful debug location on the instruction or its operands.

Definition at line 737 of file LoopVectorize.cpp.

References DenseMapInfo< LocallyHashedType >::Empty, llvm::dyn_cast(), llvm::DebugLoc::getUnknown(), and I.

getExpandedStep()

Value * getExpandedStep ( const InductionDescriptor & ID, const SCEV2ValueTy & ExpandedSCEVs )

static

Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.

Definition at line 2386 of file LoopVectorize.cpp.

References assert(), llvm::CallingConv::C, and llvm::dyn_cast().

Referenced by createInductionAdditionalBypassValues().

getMaxVScale()

std::optional< unsigned > getMaxVScale ( const Function & F, const TargetTransformInfo & TTI )

static

Definition at line 2199 of file LoopVectorize.cpp.

References F.

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), and isIndvarOverflowCheckKnownFalse().

getScalarEpilogueLowering()

ScalarEpilogueLowering getScalarEpilogueLowering ( Function * F, Loop * L, LoopVectorizeHints & Hints, ProfileSummaryInfo * PSI, BlockFrequencyInfo * BFI, TargetTransformInfo * TTI, TargetLibraryInfo * TLI, LoopVectorizationLegality & LVL, InterleavedAccessInfo * IAI )

static

Definition at line 9197 of file LoopVectorize.cpp.

References llvm::CM_ScalarEpilogueAllowed, llvm::CM_ScalarEpilogueNotAllowedOptSize, llvm::CM_ScalarEpilogueNotAllowedUsePredicate, llvm::CM_ScalarEpilogueNotNeededUsePredicate, F, llvm::LoopVectorizeHints::FK_Disabled, llvm::LoopVectorizeHints::FK_Enabled, llvm::LoopVectorizeHints::getForce(), llvm::LoopVectorizeHints::getPredicate(), llvm::IRPass, PreferPredicateTy::PredicateElseScalarEpilogue, PreferPredicateTy::PredicateOrDontVectorize, PreferPredicateOverEpilogue, PreferPredicateTy::ScalarEpilogue, and llvm::shouldOptimizeForSize().

Referenced by llvm::LoopVectorizePass::processLoop(), and processLoopInVPlanNativePath().

getSmallBestKnownTC()

std::optional< ElementCount > getSmallBestKnownTC ( PredicatedScalarEvolution & PSE, Loop * L, bool CanUseConstantMax = true )

static

Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an estimate cannot be made (including when the trip count would overflow), for the specified loop L as defined by the following procedure: 1) Returns exact trip count if it is known.

2) Returns expected trip count according to profile data if any. 3) Returns upper bound estimate if known, and if CanUseConstantMax. 4) Returns std::nullopt if all of the above failed.

Definition at line 447 of file LoopVectorize.cpp.

References llvm::ElementCount::getFixed(), llvm::getLoopEstimatedTripCount(), llvm::PredicatedScalarEvolution::getSE(), llvm::PredicatedScalarEvolution::getSmallConstantMaxTripCount(), getSmallConstantTripCount(), and LoopVectorizeWithBlockFrequency.

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), isOutsideLoopWorkProfitable(), llvm::LoopVectorizePass::processLoop(), and llvm::LoopVectorizationPlanner::selectInterleaveCount().

getSmallConstantTripCount()

ElementCount getSmallConstantTripCount ( ScalarEvolution * SE, const Loop * L )

static

A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops whose trip count is a function of vscale.

Definition at line 416 of file LoopVectorize.cpp.

References llvm::cast(), llvm::APInt::getActiveBits(), llvm::ScalarEvolution::getBackedgeTakenCount(), llvm::ElementCount::getFixed(), llvm::ElementCount::getScalable(), llvm::ScalarEvolution::getSmallConstantTripCount(), llvm::ScalarEvolution::getTripCountFromExitCount(), llvm::SCEV::getType(), llvm::APInt::getZExtValue(), hasNoUnsignedWrap(), llvm::isa(), llvm::SCEVPatternMatch::m_scev_APInt(), llvm::SCEVPatternMatch::m_scev_Mul(), llvm::SCEVPatternMatch::m_SCEVVScale(), and llvm::SCEVPatternMatch::match().

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), llvm::LoopVectorizationCostModel::expectedCost(), getSmallBestKnownTC(), and llvm::LoopVectorizationPlanner::selectInterleaveCount().

getStartValueFromReductionResult()

Value * getStartValueFromReductionResult ( VPInstruction * RdxResult )

static

Definition at line 7089 of file LoopVectorize.cpp.

References assert(), llvm::VPInstruction::ComputeFindIVResult, llvm::VPValue::getLiveInIRValue(), llvm::VPInstruction::getOpcode(), llvm::VPUser::getOperand(), llvm::PatternMatch::m_Freeze(), and llvm::PatternMatch::match().

Referenced by fixReductionScalarResumeWhenVectorizingEpilog(), and preparePlanForEpilogueVectorLoop().

hasIrregularType()

bool hasIrregularType ( Type * Ty, const DataLayout & DL )

static

A helper function that returns true if the given type is irregular.

The type is irregular if its allocated size doesn't equal the store size of an element of the corresponding vector type.

Definition at line 407 of file LoopVectorize.cpp.

References DL.

Referenced by llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), and llvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened().

hasReplicatorRegion()

bool hasReplicatorRegion ( VPlan & Plan )

static

Definition at line 4119 of file LoopVectorize.cpp.

References llvm::any_of(), llvm::VPBlockUtils::blocksOnly(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::computeBestVF().

isExplicitVecOuterLoop()

bool isExplicitVecOuterLoop ( Loop * OuterLp, OptimizationRemarkEmitter * ORE )

static

Definition at line 2058 of file LoopVectorize.cpp.

References llvm::LoopVectorizeHints::allowVectorization(), assert(), llvm::dbgs(), llvm::LoopVectorizeHints::emitRemarkWithHints(), llvm::LoopVectorizeHints::FK_Undefined, llvm::LoopVectorizeHints::getForce(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizeHints::getInterleave(), llvm::BasicBlock::getParent(), llvm::LoopBase< BlockT, LoopT >::isInnermost(), and LLVM_DEBUG.

Referenced by collectSupportedLoops().

isIndvarOverflowCheckKnownFalse()

bool isIndvarOverflowCheckKnownFalse ( const LoopVectorizationCostModel * Cost, ElementCount VF, std::optional< unsigned > UF = std::nullopt )

static

For the given VF and UF and maximum trip count computed for the loop, return whether the induction variable might overflow in the vectorized loop.

If not, then we know a runtime overflow check always evaluates to false and can be removed.

Definition at line 2214 of file LoopVectorize.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::IntegerType::getMask(), getMaxVScale(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced by llvm::LoopVectorizationPlanner::addMinimumIterationCheck(), and llvm::EpilogueVectorizerMainLoop::createIterationCountCheck().

isOutsideLoopWorkProfitable()

bool isOutsideLoopWorkProfitable ( GeneratedRTChecks & Checks, VectorizationFactor & VF, Loop * L, PredicatedScalarEvolution & PSE, VPCostContext & CostCtx, VPlan & Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale )

static

This function determines whether or not it's still profitable to vectorize the loop given the extra work we have to do outside of the loop:

1. Perform the runtime checks before entering the loop to ensure it's safe to vectorize.
2. In the case of loops with uncountable early exits, we may have to do extra work when exiting the loop early, such as calculating the final exit values of variables used outside the loop.

Definition at line 9383 of file LoopVectorize.cpp.

References llvm::alignTo(), calculateEarlyExitCost(), llvm::CM_ScalarEpilogueAllowed, llvm::VectorizationFactor::Cost, llvm::dbgs(), llvm::divideCeil(), estimateElementCount(), llvm::ElementCount::getFixed(), getSmallBestKnownTC(), llvm::InstructionCost::getValue(), llvm::details::FixedOrScalableQuantity< ElementCount, unsigned >::isKnownLT(), llvm::ElementCount::isScalar(), llvm::InstructionCost::isValid(), LLVM_DEBUG, llvm::VectorizationFactor::MinProfitableTripCount, llvm::VectorizationFactor::ScalarCost, VectorizeMemoryCheckThreshold, and llvm::VectorizationFactor::Width.

Referenced by llvm::LoopVectorizePass::processLoop().

maybeVectorizeType()

Type * maybeVectorizeType ( Type * Ty, ElementCount VF )

static

Definition at line 2521 of file LoopVectorize.cpp.

References llvm::canVectorizeTy(), llvm::ElementCount::isScalar(), and llvm::toVectorizedTy().

Referenced by llvm::LoopVectorizationCostModel::getVectorIntrinsicCost().

planContainsAdditionalSimplifications()

bool planContainsAdditionalSimplifications ( VPlan & Plan, VPCostContext & CostCtx, Loop * TheLoop, ElementCount VF )

static

Return true if the original loop \ TheLoop contains any instructions that do not have corresponding recipes in Plan and are not marked to be ignored in CostCtx.

This means the VPlan contains simplification that the legacy cost-model did not account for.

If a VPlan transform folded a recipe to one producing a single-scalar, but the original instruction wasn't uniform-after-vectorization in the legacy cost model, the legacy cost overestimates the actual cost.

Definition at line 6881 of file LoopVectorize.cpp.

References llvm::any_of(), llvm::LoopBase< BlockT, LoopT >::blocks(), llvm::VPBlockUtils::blocksOnly(), llvm::cast(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::VPRegionBlock::getEntry(), llvm::getLoadStorePointerOperand(), llvm::VPlan::getVectorLoopRegion(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), IR, llvm::isa(), llvm::VPCostContext::isLegacyUniformAfterVectorization(), llvm::vputils::isSingleScalar(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::match(), llvm::none_of(), and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::computeBestVF().

preparePlanForEpilogueVectorLoop()

void preparePlanForEpilogueVectorLoop ( VPlan & Plan, Loop * L, const SCEV2ValueTy & ExpandedSCEVs, EpilogueLoopVectorizationInfo & EPI )

static

Prepare Plan for vectorizing the epilogue loop.

That is, re-use expanded SCEVs from ExpandedSCEVs and set resume values for header recipes.

Definition at line 9583 of file LoopVectorize.cpp.

References llvm::all_of(), assert(), llvm::cast(), llvm::dyn_cast(), llvm::find_if(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::BasicBlock::getFirstNonPHIIt(), llvm::PHINode::getIncomingValueForBlock(), llvm::VPValue::getLiveInIRValue(), llvm::PHINode::getNumIncomingValues(), llvm::User::getOperand(), llvm::VPlan::getOrAddLiveIn(), llvm::BasicBlock::getParent(), llvm::VPHeaderPHIRecipe::getStartValue(), getStartValueFromReductionResult(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorLoopRegion(), llvm::PHINode::incoming_values(), llvm::RecurrenceDescriptor::isAnyOfRecurrenceKind(), llvm::RecurrenceDescriptor::isFindIVRecurrenceKind(), IV, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::PatternMatch::m_SpecificInt(), llvm::EpilogueLoopVectorizationInfo::MainLoopIterationCountCheck, llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPInstruction::ReductionStartVector, llvm::VPValue::replaceAllUsesWith(), llvm::VPlan::resetTripCount(), llvm::Sentinel, llvm::VPBlockBase::setName(), and llvm::EpilogueLoopVectorizationInfo::VectorTripCount.

Referenced by llvm::LoopVectorizePass::processLoop().

preparePlanForMainVectorLoop()

void preparePlanForMainVectorLoop ( VPlan & MainPlan, VPlan & EpiPlan )

static

Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.

Remove ResumePhis from MainPlan for inductions that don't have a corresponding wide induction in EpiPlan.

Definition at line 9496 of file LoopVectorize.cpp.

References llvm::VPBasicBlock::begin(), llvm::cast(), llvm::VPInstruction::ComputeFindIVResult, llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createScalarPhi(), llvm::dyn_cast(), llvm::iterator_range< IteratorT >::end(), llvm::VPBasicBlock::end(), llvm::VPRecipeBase::eraseFromParent(), llvm::find_if(), llvm::VPlan::getCanonicalIV(), llvm::VPValue::getDefiningRecipe(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPValue::getLiveInIRValue(), llvm::VPlan::getMiddleBlock(), llvm::VPUser::getOperand(), llvm::VPlan::getScalarHeader(), llvm::VPlan::getScalarPreheader(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), llvm::isGuaranteedNotToBeUndefOrPoison(), llvm::make_early_inc_range(), llvm::VPRecipeBase::moveBefore(), llvm::VPBasicBlock::phis(), llvm::VPlanTransforms::removeDeadRecipes(), llvm::VPValue::replaceUsesWithIf(), llvm::VPInstruction::ResumeForEpilogue, llvm::VPlanTransforms::runPass(), and llvm::VPUser::setOperand().

Referenced by llvm::LoopVectorizePass::processLoop().

processLoopInVPlanNativePath()

bool processLoopInVPlanNativePath ( Loop * L, PredicatedScalarEvolution & PSE, LoopInfo * LI, DominatorTree * DT, LoopVectorizationLegality * LVL, TargetTransformInfo * TTI, TargetLibraryInfo * TLI, DemandedBits * DB, AssumptionCache * AC, OptimizationRemarkEmitter * ORE, BlockFrequencyInfo * BFI, ProfileSummaryInfo * PSI, LoopVectorizeHints & Hints, LoopVectorizationRequirements & Requirements )

static

Definition at line 9246 of file LoopVectorize.cpp.

References llvm::LoopVectorizationPlanner::addMinimumIterationCheck(), assert(), llvm::LoopVectorizationCostModel::collectElementTypesForWidening(), llvm::LoopVectorizationCostModel::CostKind, llvm::dbgs(), llvm::VectorizationFactor::Disabled(), llvm::EnableVPlanNativePath, llvm::LoopVectorizationPlanner::executePlan(), F, llvm::PredicatedScalarEvolution::getBackedgeTakenCount(), llvm::LoopVectorizationLegality::getLAI(), llvm::LoopVectorizationPlanner::getPlanFor(), getScalarEpilogueLowering(), llvm::LoopVectorizeHints::getWidth(), llvm::isa(), LLVM_DEBUG, llvm::VectorizationFactor::MinProfitableTripCount, llvm::LoopVectorizationPlanner::planInVPlanNativePath(), llvm::reportVectorization(), llvm::verifyFunction(), VPlanBuildStressTest, and llvm::VectorizationFactor::Width.

Referenced by llvm::LoopVectorizePass::processLoop().

replaceVPBBWithIRVPBB()

VPIRBasicBlock * replaceVPBBWithIRVPBB ( VPBasicBlock * VPBB, BasicBlock * IRBB )

static

Replace VPBB with a VPIRBasicBlock wrapping IRBB.

All recipes from VPBB are moved to the end of the newly created VPIRBasicBlock. VPBB must have a single predecessor, which is rewired to the new VPIRBasicBlock. All successors of VPBB, if any, are rewired to the new VPIRBasicBlock.

Definition at line 2353 of file LoopVectorize.cpp.

References llvm::VPBasicBlock::begin(), llvm::VPlan::createVPIRBasicBlock(), llvm::VPBasicBlock::end(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::VPBlockBase::getPlan(), llvm::make_early_inc_range(), llvm::make_range(), llvm::VPBasicBlock::phis(), and llvm::VPBlockUtils::reassociateBlocks().

Referenced by llvm::EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton(), llvm::EpilogueVectorizerMainLoop::emitIterationCountCheck(), and llvm::LoopVectorizationPlanner::executePlan().

STATISTIC() [1/4]

STATISTIC	(	LoopsAnalyzed	,
		"Number of loops analyzed for vectorization"	)

STATISTIC() [2/4]

STATISTIC	(	LoopsEarlyExitVectorized	,
		"Number of early exit loops vectorized"	)

STATISTIC() [3/4]

STATISTIC	(	LoopsEpilogueVectorized	,
		"Number of epilogues vectorized"	)

STATISTIC() [4/4]

STATISTIC	(	LoopsVectorized	,
		"Number of loops vectorized"	)

useActiveLaneMask()

bool useActiveLaneMask ( TailFoldingStyle Style )

static

Definition at line 2033 of file LoopVectorize.cpp.

References llvm::Data, llvm::DataAndControlFlow, and llvm::DataAndControlFlowWithoutRuntimeCheck.

useActiveLaneMaskForControlFlow()

bool useActiveLaneMaskForControlFlow ( TailFoldingStyle Style )

static

Definition at line 2039 of file LoopVectorize.cpp.

References llvm::DataAndControlFlow, and llvm::DataAndControlFlowWithoutRuntimeCheck.

useMaskedInterleavedAccesses()

bool useMaskedInterleavedAccesses ( const TargetTransformInfo & TTI )

static

Definition at line 2245 of file LoopVectorize.cpp.

References EnableMaskedInterleavedMemAccesses.

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), llvm::LoopVectorizationPlanner::plan(), and llvm::LoopVectorizePass::processLoop().

willGenerateVectors()

bool willGenerateVectors ( VPlan & Plan, ElementCount VF, const TargetTransformInfo & TTI )

static

Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.

Definition at line 4023 of file LoopVectorize.cpp.

References llvm::any_of(), assert(), llvm::VPBlockUtils::blocksOnly(), llvm::collectEphemeralRecipesForVPlan(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::contains(), llvm::getContainedTypes(), llvm::VPRegionBlock::getEntry(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPTypeAnalysis::inferScalarType(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::isa(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), llvm::ElementCount::isVector(), llvm_unreachable, llvm::toVectorizedTy(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::computeBestVF().

Variable Documentation

ConsiderRegPressure

cl::opt< bool > ConsiderRegPressure("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high.")) ( "vectorizer-consider-reg-pressure" , cl::init(false) , cl::Hidden , cl::desc("Discard VFs if their register pressure is too high.")  )

static

Referenced by llvm::LoopVectorizationCostModel::shouldConsiderRegPressureForVF().

EnableCondStoresVectorization

cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization.")) ( "enable-cond-stores-vec" , cl::init(true) , cl::Hidden , cl::desc("Enable if predication of stores during vectorization.")  )

static

EnableEarlyExitVectorization

cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc( "Enable vectorization of early exit loops with uncountable exits.")) ( "enable-early-exit-vectorization" , cl::init(true) , cl::Hidden , cl::desc( "Enable vectorization of early exit loops with uncountable exits.")  )

static

Referenced by llvm::LoopVectorizePass::processLoop(), and llvm::LoopVectorizationCostModel::requiresScalarEpilogue().

EnableEpilogueVectorization

cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops.")) ( "enable-epilogue-vectorization" , cl::init(true) , cl::Hidden , cl::desc("Enable vectorization of epilogue loops.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

EnableIndVarRegisterHeur

cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving")) ( "enable-ind-var-reg-heur" , cl::init(true) , cl::Hidden , cl::desc("Count the induction variable only once when interleaving")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

EnableInterleavedMemAccesses

cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop")) ( "enable-interleaved-mem-accesses" , cl::init(false) , cl::Hidden , cl::desc("Enable vectorization on interleaved memory accesses in a loop")  )

static

Referenced by llvm::LoopVectorizePass::processLoop().

EnableLoadStoreRuntimeInterleave

cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc( "Enable runtime interleaving until load/store ports are saturated")) ( "enable-loadstore-runtime-interleave" , cl::init(true) , cl::Hidden , cl::desc( "Enable runtime interleaving until load/store ports are saturated")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

EnableMaskedInterleavedMemAccesses

cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop")) ( "enable-masked-interleaved-mem-accesses" , cl::init(false) , cl::Hidden , cl::desc("Enable vectorization on masked interleaved memory accesses in a loop")  )

static

An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.

Referenced by useMaskedInterleavedAccesses().

EpilogueVectorizationForceVF

cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops.")) ( "epilogue-vectorization-force-VF" , cl::init(1) , cl::Hidden , cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

EpilogueVectorizationMinVF

cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization.")) ( "epilogue-vectorization-minimum-VF" , cl::Hidden , cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization.")  )

static

Referenced by llvm::LoopVectorizationCostModel::isEpilogueVectorizationProfitable().

ForceOrderedReductions

cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions")) ( "force-ordered-reductions" , cl::init(false) , cl::Hidden , cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions")  )

static

Referenced by llvm::LoopVectorizePass::processLoop().

ForceSafeDivisor

cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc( "Override cost based safe divisor widening for div/rem instructions")) ( "force-widen-divrem-via-safe-divisor" , cl::Hidden , cl::desc( "Override cost based safe divisor widening for div/rem instructions")  )

static

Referenced by llvm::LoopVectorizationCostModel::isDivRemScalarWithPredication().

ForceTailFoldingStyle

cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values( clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN( TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask."))) ( "force-tail-folding-style" , cl::desc("Force the tail folding style") , cl::init(TailFoldingStyle::None) , cl::values( clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN( TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask."))  )

static

Referenced by llvm::LoopVectorizationCostModel::setTailFoldingStyles().

ForceTargetMaxScalarInterleaveFactor

cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops.")) ( "force-target-max-scalar-interleave" , cl::init(0) , cl::Hidden , cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

ForceTargetMaxVectorInterleaveFactor

cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops.")) ( "force-target-max-vector-interleave" , cl::init(0) , cl::Hidden , cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

ForceTargetNumScalarRegs

cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers.")) ( "force-target-num-scalar-regs" , cl::init(0) , cl::Hidden , cl::desc("A flag that overrides the target's number of scalar registers.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

ForceTargetNumVectorRegs

cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers.")) ( "force-target-num-vector-regs" , cl::init(0) , cl::Hidden , cl::desc("A flag that overrides the target's number of vector registers.")  )

static

Referenced by llvm::LoopVectorizationPlanner::computeBestVF(), and llvm::LoopVectorizationPlanner::selectInterleaveCount().

ForceTargetSupportsScalableVectors

cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc( "Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing.")) ( "force-target-supports-scalable-vectors" , cl::init(false) , cl::Hidden , cl::desc( "Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing.")  )

static

Referenced by llvm::LoopVectorizationCostModel::LoopVectorizationCostModel(), and llvm::LoopVectorizationPlanner::planInVPlanNativePath().

LoopVectorizeWithBlockFrequency

cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions.")) ( "loop-vectorize-with-block-frequency" , cl::init(true) , cl::Hidden , cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions.")  )

static

Referenced by getSmallBestKnownTC().

MaximizeBandwidth

cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop.")) ( "vectorizer-maximize-bandwidth" , cl::init(false) , cl::Hidden , cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop.")  )

static

Referenced by llvm::LoopVectorizationCostModel::useMaxBandwidth().

MaxNestedScalarReductionIC

cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop.")) ( "max-nested-scalar-reduction-interleave" , cl::init(2) , cl::Hidden , cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

MinItersBypassWeights

uint32_t MinItersBypassWeights[] = {1, 127}

staticconstexpr

Definition at line 402 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationPlanner::addMinimumIterationCheck(), llvm::VPlanTransforms::addMinimumIterationCheck(), and llvm::EpilogueVectorizerMainLoop::emitIterationCountCheck().

NumberOfStoresToPredicate

cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if.")) ( "vectorize-num-stores-pred" , cl::init(1) , cl::Hidden , cl::desc("Max number of stores to be predicated behind an if.")  )

static

The number of stores in a loop that are allowed to need predication.

PreferInLoopReductions

cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference.")) ( "prefer-inloop-reductions" , cl::init(false) , cl::Hidden , cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference.")  )

static

Referenced by llvm::LoopVectorizationCostModel::collectElementTypesForWidening(), and llvm::LoopVectorizationCostModel::collectInLoopReductions().

PreferPredicatedReductionSelect

cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc( "Prefer predicating a reduction operation over an after loop select.")) ( "prefer-predicated-reduction-select" , cl::init(false) , cl::Hidden , cl::desc( "Prefer predicating a reduction operation over an after loop select.")  )

static

Referenced by llvm::LoopVectorizationCostModel::usePredicatedReductionSelect().

PreferPredicateOverEpilogue

cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails."))) ( "prefer-predicate-over-epilogue" , cl::init(PreferPredicateTy::ScalarEpilogue) , cl::Hidden , cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop.") , cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails."))  )

static

Referenced by getScalarEpilogueLowering().

SmallLoopCost

cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc( "The cost of a loop that is considered 'small' by the interleaver.")) ( "small-loop-cost" , cl::init(20) , cl::Hidden , cl::desc( "The cost of a loop that is considered 'small' by the interleaver.")  )

static

Referenced by llvm::LoopVectorizationPlanner::selectInterleaveCount().

TinyTripCountVectorThreshold

cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred.")) ( "vectorizer-min-trip-count" , cl::init(16) , cl::Hidden , cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred.")  )

static

Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.

Referenced by llvm::LoopVectorizePass::processLoop().

UseWiderVFIfCallVariantsPresent

cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants")) ( "vectorizer-maximize-bandwidth-for-vector-calls" , cl::init(true) , cl::Hidden , cl::desc("Try wider VFs if they enable the use of vector variants")  )

static

Referenced by llvm::LoopVectorizationCostModel::useMaxBandwidth().

VectorizeMemoryCheckThreshold

cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks")) ( "vectorize-memory-check-threshold" , cl::init(128) , cl::Hidden , cl::desc("The maximum allowed number of runtime memory checks")  )

static

Referenced by isOutsideLoopWorkProfitable().

VerboseDebug

const char VerboseDebug[] = DEBUG_TYPE "-verbose"

Definition at line 165 of file LoopVectorize.cpp.

VPlanBuildStressTest

cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc( "Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path).")) ( "vplan-build-stress-test" , cl::init(false) , cl::Hidden , cl::desc( "Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path).")  )

static

Referenced by collectSupportedLoops(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), and processLoopInVPlanNativePath().