74#define DEBUG_TYPE "hash-recognize"
85 while (!Worklist.
empty()) {
92 for (
const Use &U :
I->operands()) {
100 return std::distance(Latch->
begin(), Latch->
end()) != Visited.
size();
118 OS.
indent(Indent) <<
"Phi: ";
121 OS.
indent(Indent) <<
"BinaryOperator: ";
124 OS.
indent(Indent) <<
"Start: ";
127 OS.
indent(Indent) <<
"Step: ";
131 OS.
indent(Indent) <<
"ExtraConst: ";
137#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
167 bool ByteOrderSwapped) {
183 bool LWellFormed = ByteOrderSwapped ?
match(L, MatchPred)
197 if (AllowedByR == CheckAllowedByR)
198 return TV == BitShift &&
201 if (AllowedByR.inverse() == CheckAllowedByR)
202 return FV == BitShift &&
238 while (!Worklist.
empty()) {
251 if (
I->getOpcode() == BOWithConstOpToMatch) {
260 for (Use &U :
I->operands())
286 if (
Phi->getNumIncomingValues() != 2)
289 for (
unsigned Idx = 0; Idx != 2; ++Idx) {
290 Value *FoundStep =
Phi->getIncomingValue(Idx);
291 Value *FoundStart =
Phi->getIncomingValue(!Idx);
294 if (!
match(FoundStep,
300 BinaryOperator *FoundBO = digRecurrence(TV, BOWithConstOpToMatch);
302 if (!FoundBO || FoundBO != AltBO)
305 if (BOWithConstOpToMatch != Instruction::BinaryOpsEnd && !
ExtraConst) {
306 LLVM_DEBUG(
dbgs() <<
"HashRecognize: Unable to match single BinaryOp "
307 "with constant in conditional recurrence\n");
321static std::optional<std::pair<RecurrenceInfo, RecurrenceInfo>>
323 auto Phis = LoopLatch->
phis();
324 unsigned NumPhis = std::distance(Phis.begin(), Phis.end());
325 if (NumPhis != 2 && NumPhis != 3)
333 if (!SimpleRecurrence)
335 if (!ConditionalRecurrence)
337 &
P, Instruction::BinaryOps::Xor);
339 if (NumPhis == 3 && (!SimpleRecurrence || !ConditionalRecurrence))
341 return std::make_pair(SimpleRecurrence, ConditionalRecurrence);
354 bool ByteOrderSwapped) {
359 if (ByteOrderSwapped) {
361 for (
unsigned I = 1;
I < 256;
I <<= 1) {
362 CRCInit = CRCInit.
shl(1) ^
364 for (
unsigned J = 0; J <
I; ++J)
365 Table[
I + J] = CRCInit ^ Table[J];
370 APInt CRCInit(BW, 1);
371 for (
unsigned I = 128;
I;
I >>= 1) {
373 for (
unsigned J = 0; J < 256; J += (
I << 1))
374 Table[
I + J] = CRCInit ^ Table[J];
401 while (!Worklist.
empty()) {
414 for (
const Use &U :
I->operands())
427 if (!V->getType()->isIntegerTy())
441 if (!L.isInnermost())
442 return "Loop is not innermost";
445 const PHINode *IndVar = L.getCanonicalInductionVariable();
446 if (!Latch || !Exit || !IndVar || L.getNumBlocks() != 1)
447 return "Loop not in canonical form";
448 unsigned TC = SE.getSmallConstantTripCount(&L);
450 return "Unable to find a small constant byte-multiple trip count";
454 return "Found stray PHI";
455 auto [SimpleRecurrence, ConditionalRecurrence] = *R;
456 if (!ConditionalRecurrence)
457 return "Unable to find conditional recurrence";
461 std::optional<bool> ByteOrderSwapped =
463 if (!ByteOrderSwapped)
464 return "Loop with non-unit bitshifts";
465 if (SimpleRecurrence) {
467 return "Loop with non-unit bitshifts";
471 if (!ConditionalRecurrence.Phi->hasNUses(2) ||
472 !SimpleRecurrence.Phi->hasNUses(2))
473 return "Recurrences have stray uses";
478 SimpleRecurrence.Phi,
479 ConditionalRecurrence.Phi, L))
480 return "Recurrences not intertwined with XOR";
484 Value *LHS = ConditionalRecurrence.Start;
485 Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start :
nullptr;
487 : LHS->getType()->getIntegerBitWidth()))
488 return "Loop iterations exceed bitwidth of data";
494 if (
none_of(ComputedValue->users(), [Exit](
User *U) {
495 auto *UI = dyn_cast<Instruction>(U);
496 return UI && UI->getParent() == Exit;
498 return "Unable to find use of computed value in loop exit block";
500 assert(ConditionalRecurrence.ExtraConst &&
501 "Expected ExtraConst in conditional recurrence");
502 const APInt &GenPoly = *ConditionalRecurrence.ExtraConst;
506 return "Malformed significant-bit check";
512 if (SimpleRecurrence)
515 return "Found stray unvisited instructions";
517 return PolynomialInfo(TC, LHS, GenPoly, ComputedValue, *ByteOrderSwapped,
522 for (
unsigned I = 0;
I < 256;
I++) {
523 (*this)[
I].print(OS,
false);
524 OS << (
I % 16 == 15 ?
'\n' :
' ');
528#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
533 if (!L.isInnermost())
535 OS <<
"HashRecognize: Checking a loop in '"
536 << L.getHeader()->getParent()->getName() <<
"' from " << L.getLocStr()
539 if (!std::holds_alternative<PolynomialInfo>(Ret)) {
540 OS <<
"Did not find a hash algorithm\n";
541 if (std::holds_alternative<StringRef>(Ret))
542 OS <<
"Reason: " << std::get<StringRef>(Ret) <<
"\n";
546 auto Info = std::get<PolynomialInfo>(Ret);
547 OS <<
"Found" << (Info.ByteOrderSwapped ?
" big-endian " :
" little-endian ")
548 <<
"CRC-" << Info.RHS.getBitWidth() <<
" loop with trip count "
549 << Info.TripCount <<
"\n";
550 OS.
indent(2) <<
"Initial CRC: ";
553 OS.
indent(2) <<
"Generating polynomial: ";
554 Info.RHS.print(OS,
false);
556 OS.
indent(2) <<
"Computed CRC: ";
557 Info.ComputedValue->print(OS);
560 OS.
indent(2) <<
"Auxiliary data: ";
561 Info.LHSAux->print(OS);
564 OS.
indent(2) <<
"Computed CRC lookup table:\n";
568#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
574 if (std::holds_alternative<PolynomialInfo>(Res))
575 return std::get<PolynomialInfo>(Res);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file implements a class to represent arbitrary precision integral constant values and operations...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static bool containsUnreachable(const Loop &L, ArrayRef< const Instruction * > Roots)
Checks if there's a stray instruction in the loop L outside of the use-def chains from Roots,...
static bool isSignificantBitCheckWellFormed(const RecurrenceInfo &ConditionalRecurrence, const RecurrenceInfo &SimpleRecurrence, bool ByteOrderSwapped)
Check the well-formedness of the (most|least) significant bit check given ConditionalRecurrence,...
static bool isConditionalOnXorOfPHIs(const SelectInst *SI, const PHINode *P1, const PHINode *P2, const Loop &L)
Checks that P1 and P2 are used together in an XOR in the use-def chain of SI's condition,...
static std::optional< std::pair< RecurrenceInfo, RecurrenceInfo > > getRecurrences(BasicBlock *LoopLatch, const PHINode *IndVar, const Loop &L)
Iterates over all the phis in LoopLatch, and attempts to extract a Conditional Recurrence and an opti...
static std::optional< bool > isBigEndianBitShift(Value *V, ScalarEvolution &SE)
This header provides classes for managing per-loop analyses.
Class for arbitrary precision integers.
unsigned getBitWidth() const
Return the number of bits in the APInt.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
This class represents a range of values.
static LLVM_ABI ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
static LLVM_ABI ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &)
static CRCTable genSarwateTable(const APInt &GenPoly, bool ByteOrderSwapped)
Generate a lookup table of 256 entries by interleaving the generating polynomial.
std::optional< PolynomialInfo > getResult() const
LLVM_DUMP_METHOD void dump() const
HashRecognize(const Loop &L, ScalarEvolution &SE)
void print(raw_ostream &OS) const
std::variant< PolynomialInfo, StringRef > recognizeCRC() const
The main entry point for analyzing a loop and recognizing the CRC algorithm.
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
Represents a single loop in the control flow graph.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
LLVM_ABI unsigned getIntegerBitWidth() const
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
@ C
The default llvm calling convention, compatible with C.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
AnyBinaryOp_match< LHS, RHS, true > m_c_BinOp(const LHS &L, const RHS &R)
Matches a BinaryOperator with LHS and RHS in either order.
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
match_combine_or< match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, TruncInst > >, OpTy > m_ZExtOrTruncOrSelf(const OpTy &Op)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t > m_scev_Mul(const Op0_t &Op0, const Op1_t &Op1)
bool match(const SCEV *S, const Pattern &P)
SCEVBinaryExpr_match< SCEVUDivExpr, Op0_t, Op1_t > m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1)
cst_pred_ty< is_specific_cst > m_scev_SpecificInt(uint64_t V)
Match an SCEV constant with a plain unsigned integer.
class_match< const SCEV > m_SCEV()
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
AnalysisManager< Loop, LoopStandardAnalysisResults & > LoopAnalysisManager
The loop analysis manager.
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
A structure that can hold either a Simple Recurrence or a Conditional Recurrence.
LLVM_DUMP_METHOD void dump() const
bool matchConditionalRecurrence(const PHINode *P, Instruction::BinaryOps BOWithConstOpToMatch=Instruction::BinaryOpsEnd)
A Conditional Recurrence is a recurrence of the form:
void print(raw_ostream &OS, unsigned Indent=0) const
std::optional< APInt > ExtraConst
bool matchSimpleRecurrence(const PHINode *P)
Wraps llvm::matchSimpleRecurrence.
RecurrenceInfo(const Loop &L)
A custom std::array with 256 entries, that also has a print function.
LLVM_DUMP_METHOD void dump() const
void print(raw_ostream &OS) const
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
unsigned getBitWidth() const
Get the bit width of this value.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
The structure that is returned when a polynomial algorithm was recognized by the analysis.
PolynomialInfo(unsigned TripCount, Value *LHS, const APInt &RHS, Value *ComputedValue, bool ByteOrderSwapped, Value *LHSAux=nullptr)