diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h index f98bd684149f9..16795969d4cd1 100644 --- a/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -47,994 +47,908 @@ #include "llvm/Support/Compiler.h" namespace llvm { - class AAResults; - template class ArrayRef; - class Loop; - class LoopInfo; - class SCEVConstant; - class raw_ostream; - - /// Dependence - This class represents a dependence between two memory - /// memory references in a function. It contains minimal information and - /// is used in the very common situation where the compiler is unable to - /// determine anything beyond the existence of a dependence; that is, it - /// represents a confused dependence (see also FullDependence). In most - /// cases (for output, flow, and anti dependences), the dependence implies - /// an ordering, where the source must precede the destination; in contrast, - /// input dependences are unordered. - /// - /// When a dependence graph is built, each Dependence will be a member of - /// the set of predecessor edges for its destination instruction and a set - /// if successor edges for its source instruction. These sets are represented - /// as singly-linked lists, with the "next" fields stored in the dependence - /// itelf. - class LLVM_ABI Dependence { - protected: - Dependence(Dependence &&) = default; - Dependence &operator=(Dependence &&) = default; - - public: - Dependence(Instruction *Source, Instruction *Destination, - const SCEVUnionPredicate &A) - : Src(Source), Dst(Destination), Assumptions(A) {} - virtual ~Dependence() = default; - - /// Dependence::DVEntry - Each level in the distance/direction vector - /// has a direction (or perhaps a union of several directions), and - /// perhaps a distance. - struct DVEntry { - enum : unsigned char { - NONE = 0, - LT = 1, - EQ = 2, - LE = 3, - GT = 4, - NE = 5, - GE = 6, - ALL = 7 - }; - unsigned char Direction : 3; // Init to ALL, then refine. - bool Scalar : 1; // Init to true. - bool PeelFirst : 1; // Peeling the first iteration will break dependence. - bool PeelLast : 1; // Peeling the last iteration will break the dependence. - bool Splitable : 1; // Splitting the loop will break dependence. - const SCEV *Distance = nullptr; // NULL implies no distance available. - DVEntry() - : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false), - Splitable(false) {} +class AAResults; +template class ArrayRef; +class Loop; +class LoopInfo; +class SCEVConstant; +class raw_ostream; + +/// Dependence - This class represents a dependence between two memory +/// memory references in a function. It contains minimal information and +/// is used in the very common situation where the compiler is unable to +/// determine anything beyond the existence of a dependence; that is, it +/// represents a confused dependence (see also FullDependence). In most +/// cases (for output, flow, and anti dependences), the dependence implies +/// an ordering, where the source must precede the destination; in contrast, +/// input dependences are unordered. +/// +/// When a dependence graph is built, each Dependence will be a member of +/// the set of predecessor edges for its destination instruction and a set +/// if successor edges for its source instruction. These sets are represented +/// as singly-linked lists, with the "next" fields stored in the dependence +/// itelf. +class LLVM_ABI Dependence { +protected: + Dependence(Dependence &&) = default; + Dependence &operator=(Dependence &&) = default; + +public: + Dependence(Instruction *Source, Instruction *Destination, + const SCEVUnionPredicate &A) + : Src(Source), Dst(Destination), Assumptions(A) {} + virtual ~Dependence() = default; + + /// Dependence::DVEntry - Each level in the distance/direction vector + /// has a direction (or perhaps a union of several directions), and + /// perhaps a distance. + struct DVEntry { + enum : unsigned char { + NONE = 0, + LT = 1, + EQ = 2, + LE = 3, + GT = 4, + NE = 5, + GE = 6, + ALL = 7 }; + unsigned char Direction : 3; // Init to ALL, then refine. + bool Scalar : 1; // Init to true. + bool PeelFirst : 1; // Peeling the first iteration will break dependence. + bool PeelLast : 1; // Peeling the last iteration will break the dependence. + bool Splitable : 1; // Splitting the loop will break dependence. + const SCEV *Distance = nullptr; // NULL implies no distance available. + DVEntry() + : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false), + Splitable(false) {} + }; - /// getSrc - Returns the source instruction for this dependence. - /// - Instruction *getSrc() const { return Src; } - - /// getDst - Returns the destination instruction for this dependence. - /// - Instruction *getDst() const { return Dst; } - - /// isInput - Returns true if this is an input dependence. - /// - bool isInput() const; - - /// isOutput - Returns true if this is an output dependence. - /// - bool isOutput() const; - - /// isFlow - Returns true if this is a flow (aka true) dependence. - /// - bool isFlow() const; - - /// isAnti - Returns true if this is an anti dependence. - /// - bool isAnti() const; - - /// isOrdered - Returns true if dependence is Output, Flow, or Anti - /// - bool isOrdered() const { return isOutput() || isFlow() || isAnti(); } + /// getSrc - Returns the source instruction for this dependence. + Instruction *getSrc() const { return Src; } - /// isUnordered - Returns true if dependence is Input - /// - bool isUnordered() const { return isInput(); } + /// getDst - Returns the destination instruction for this dependence. + Instruction *getDst() const { return Dst; } - /// isLoopIndependent - Returns true if this is a loop-independent - /// dependence. - virtual bool isLoopIndependent() const { return true; } + /// isInput - Returns true if this is an input dependence. + bool isInput() const; - /// isConfused - Returns true if this dependence is confused - /// (the compiler understands nothing and makes worst-case - /// assumptions). - virtual bool isConfused() const { return true; } + /// isOutput - Returns true if this is an output dependence. + bool isOutput() const; - /// isConsistent - Returns true if this dependence is consistent - /// (occurs every time the source and destination are executed). - virtual bool isConsistent() const { return false; } + /// isFlow - Returns true if this is a flow (aka true) dependence. + bool isFlow() const; - /// getLevels - Returns the number of common loops surrounding the - /// source and destination of the dependence. - virtual unsigned getLevels() const { return 0; } + /// isAnti - Returns true if this is an anti dependence. + bool isAnti() const; - /// getDirection - Returns the direction associated with a particular - /// level. - virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; } + /// isOrdered - Returns true if dependence is Output, Flow, or Anti + bool isOrdered() const { return isOutput() || isFlow() || isAnti(); } - /// getDistance - Returns the distance (or NULL) associated with a - /// particular level. - virtual const SCEV *getDistance(unsigned Level) const { return nullptr; } + /// isUnordered - Returns true if dependence is Input + bool isUnordered() const { return isInput(); } - /// Check if the direction vector is negative. A negative direction - /// vector means Src and Dst are reversed in the actual program. - virtual bool isDirectionNegative() const { return false; } + /// isLoopIndependent - Returns true if this is a loop-independent + /// dependence. + virtual bool isLoopIndependent() const { return true; } - /// If the direction vector is negative, normalize the direction - /// vector to make it non-negative. Normalization is done by reversing - /// Src and Dst, plus reversing the dependence directions and distances - /// in the vector. - virtual bool normalize(ScalarEvolution *SE) { return false; } + /// isConfused - Returns true if this dependence is confused + /// (the compiler understands nothing and makes worst-case assumptions). + virtual bool isConfused() const { return true; } - /// isPeelFirst - Returns true if peeling the first iteration from - /// this loop will break this dependence. - virtual bool isPeelFirst(unsigned Level) const { return false; } + /// isConsistent - Returns true if this dependence is consistent + /// (occurs every time the source and destination are executed). + virtual bool isConsistent() const { return false; } - /// isPeelLast - Returns true if peeling the last iteration from - /// this loop will break this dependence. - virtual bool isPeelLast(unsigned Level) const { return false; } + /// getLevels - Returns the number of common loops surrounding the + /// source and destination of the dependence. + virtual unsigned getLevels() const { return 0; } - /// isSplitable - Returns true if splitting this loop will break - /// the dependence. - virtual bool isSplitable(unsigned Level) const { return false; } + /// getDirection - Returns the direction associated with a particular level. + virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; } - /// isScalar - Returns true if a particular level is scalar; that is, - /// if no subscript in the source or destination mention the induction - /// variable associated with the loop at this level. - virtual bool isScalar(unsigned Level) const; + /// getDistance - Returns the distance (or NULL) associated with a particular + /// level. + virtual const SCEV *getDistance(unsigned Level) const { return nullptr; } - /// getNextPredecessor - Returns the value of the NextPredecessor - /// field. - const Dependence *getNextPredecessor() const { return NextPredecessor; } + /// Check if the direction vector is negative. A negative direction + /// vector means Src and Dst are reversed in the actual program. + virtual bool isDirectionNegative() const { return false; } + + /// If the direction vector is negative, normalize the direction + /// vector to make it non-negative. Normalization is done by reversing + /// Src and Dst, plus reversing the dependence directions and distances + /// in the vector. + virtual bool normalize(ScalarEvolution *SE) { return false; } - /// getNextSuccessor - Returns the value of the NextSuccessor - /// field. - const Dependence *getNextSuccessor() const { return NextSuccessor; } + /// isPeelFirst - Returns true if peeling the first iteration from + /// this loop will break this dependence. + virtual bool isPeelFirst(unsigned Level) const { return false; } - /// setNextPredecessor - Sets the value of the NextPredecessor - /// field. - void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; } + /// isPeelLast - Returns true if peeling the last iteration from + /// this loop will break this dependence. + virtual bool isPeelLast(unsigned Level) const { return false; } - /// setNextSuccessor - Sets the value of the NextSuccessor - /// field. - void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; } + /// isSplitable - Returns true if splitting this loop will break the + /// dependence. + virtual bool isSplitable(unsigned Level) const { return false; } - /// getRuntimeAssumptions - Returns the runtime assumptions under which this - /// Dependence relation is valid. - SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; } + /// isScalar - Returns true if a particular level is scalar; that is, + /// if no subscript in the source or destination mention the induction + /// variable associated with the loop at this level. + virtual bool isScalar(unsigned Level) const; + + /// getNextPredecessor - Returns the value of the NextPredecessor field. + const Dependence *getNextPredecessor() const { return NextPredecessor; } + + /// getNextSuccessor - Returns the value of the NextSuccessor field. + const Dependence *getNextSuccessor() const { return NextSuccessor; } + + /// setNextPredecessor - Sets the value of the NextPredecessor + /// field. + void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; } + + /// setNextSuccessor - Sets the value of the NextSuccessor field. + void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; } + + /// getRuntimeAssumptions - Returns the runtime assumptions under which this + /// Dependence relation is valid. + SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; } + + /// dump - For debugging purposes, dumps a dependence to OS. + void dump(raw_ostream &OS) const; + +protected: + Instruction *Src, *Dst; + +private: + SCEVUnionPredicate Assumptions; + const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr; + friend class DependenceInfo; +}; + +/// FullDependence - This class represents a dependence between two memory +/// references in a function. It contains detailed information about the +/// dependence (direction vectors, etc.) and is used when the compiler is +/// able to accurately analyze the interaction of the references; that is, +/// it is not a confused dependence (see Dependence). In most cases +/// (for output, flow, and anti dependences), the dependence implies an +/// ordering, where the source must precede the destination; in contrast, +/// input dependences are unordered. +class LLVM_ABI FullDependence final : public Dependence { +public: + FullDependence(Instruction *Source, Instruction *Destination, + const SCEVUnionPredicate &Assumes, + bool PossiblyLoopIndependent, unsigned Levels); + + /// isLoopIndependent - Returns true if this is a loop-independent + /// dependence. + bool isLoopIndependent() const override { return LoopIndependent; } + + /// isConfused - Returns true if this dependence is confused + /// (the compiler understands nothing and makes worst-case + /// assumptions). + bool isConfused() const override { return false; } + + /// isConsistent - Returns true if this dependence is consistent + /// (occurs every time the source and destination are executed). + bool isConsistent() const override { return Consistent; } + + /// getLevels - Returns the number of common loops surrounding the + /// source and destination of the dependence. + unsigned getLevels() const override { return Levels; } + + /// getDirection - Returns the direction associated with a particular + /// level. + unsigned getDirection(unsigned Level) const override; + + /// getDistance - Returns the distance (or NULL) associated with a + /// particular level. + const SCEV *getDistance(unsigned Level) const override; + + /// Check if the direction vector is negative. A negative direction + /// vector means Src and Dst are reversed in the actual program. + bool isDirectionNegative() const override; + + /// If the direction vector is negative, normalize the direction + /// vector to make it non-negative. Normalization is done by reversing + /// Src and Dst, plus reversing the dependence directions and distances + /// in the vector. + bool normalize(ScalarEvolution *SE) override; + + /// isPeelFirst - Returns true if peeling the first iteration from + /// this loop will break this dependence. + bool isPeelFirst(unsigned Level) const override; + + /// isPeelLast - Returns true if peeling the last iteration from + /// this loop will break this dependence. + bool isPeelLast(unsigned Level) const override; + + /// isSplitable - Returns true if splitting the loop will break + /// the dependence. + bool isSplitable(unsigned Level) const override; + + /// isScalar - Returns true if a particular level is scalar; that is, + /// if no subscript in the source or destination mention the induction + /// variable associated with the loop at this level. + bool isScalar(unsigned Level) const override; + +private: + unsigned short Levels; + bool LoopIndependent; + bool Consistent; // Init to true, then refine. + std::unique_ptr DV; + friend class DependenceInfo; +}; + +/// DependenceInfo - This class is the main dependence-analysis driver. +class DependenceInfo { +public: + DependenceInfo(Function *F, AAResults *AA, ScalarEvolution *SE, LoopInfo *LI) + : AA(AA), SE(SE), LI(LI), F(F) {} + + /// Handle transitive invalidation when the cached analysis results go away. + LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); + + /// depends - Tests for a dependence between the Src and Dst instructions. + /// Returns NULL if no dependence; otherwise, returns a Dependence (or a + /// FullDependence) with as much information as can be gleaned. By default, + /// the dependence test collects a set of runtime assumptions that cannot be + /// solved at compilation time. By default UnderRuntimeAssumptions is false + /// for a safe approximation of the dependence relation that does not + /// require runtime checks. + LLVM_ABI std::unique_ptr + depends(Instruction *Src, Instruction *Dst, + bool UnderRuntimeAssumptions = false); + + /// getSplitIteration - Give a dependence that's splittable at some + /// particular level, return the iteration that should be used to split + /// the loop. + /// + /// Generally, the dependence analyzer will be used to build + /// a dependence graph for a function (basically a map from instructions + /// to dependences). Looking for cycles in the graph shows us loops + /// that cannot be trivially vectorized/parallelized. + /// + /// We can try to improve the situation by examining all the dependences + /// that make up the cycle, looking for ones we can break. + /// Sometimes, peeling the first or last iteration of a loop will break + /// dependences, and there are flags for those possibilities. + /// Sometimes, splitting a loop at some other iteration will do the trick, + /// and we've got a flag for that case. Rather than waste the space to + /// record the exact iteration (since we rarely know), we provide + /// a method that calculates the iteration. It's a drag that it must work + /// from scratch, but wonderful in that it's possible. + /// + /// Here's an example: + /// + /// for (i = 0; i < 10; i++) + /// A[i] = ... + /// ... = A[11 - i] + /// + /// There's a loop-carried flow dependence from the store to the load, + /// found by the weak-crossing SIV test. The dependence will have a flag, + /// indicating that the dependence can be broken by splitting the loop. + /// Calling getSplitIteration will return 5. + /// Splitting the loop breaks the dependence, like so: + /// + /// for (i = 0; i <= 5; i++) + /// A[i] = ... + /// ... = A[11 - i] + /// for (i = 6; i < 10; i++) + /// A[i] = ... + /// ... = A[11 - i] + /// + /// breaks the dependence and allows us to vectorize/parallelize + /// both loops. + LLVM_ABI const SCEV *getSplitIteration(const Dependence &Dep, unsigned Level); + + Function *getFunction() const { return F; } + + /// getRuntimeAssumptions - Returns all the runtime assumptions under which + /// the dependence test is valid. + LLVM_ABI SCEVUnionPredicate getRuntimeAssumptions() const; + +private: + AAResults *AA; + ScalarEvolution *SE; + LoopInfo *LI; + Function *F; + SmallVector Assumptions; + + /// Subscript - This private struct represents a pair of subscripts from + /// a pair of potentially multi-dimensional array references. We use a + /// vector of them to guide subscript partitioning. + struct Subscript { + const SCEV *Src; + const SCEV *Dst; + enum ClassificationKind { ZIV, SIV, RDIV, MIV, NonLinear } Classification; + SmallBitVector Loops; + SmallBitVector GroupLoops; + SmallBitVector Group; + }; - /// dump - For debugging purposes, dumps a dependence to OS. - /// - void dump(raw_ostream &OS) const; + struct CoefficientInfo { + const SCEV *Coeff; + const SCEV *PosPart; + const SCEV *NegPart; + const SCEV *Iterations; + }; - protected: - Instruction *Src, *Dst; + struct BoundInfo { + const SCEV *Iterations; + const SCEV *Upper[8]; + const SCEV *Lower[8]; + unsigned char Direction; + unsigned char DirSet; + }; + /// Constraint - This private class represents a constraint, as defined + /// in the paper + /// + /// Practical Dependence Testing + /// Goff, Kennedy, Tseng + /// PLDI 1991 + /// + /// There are 5 kinds of constraint, in a hierarchy. + /// 1) Any - indicates no constraint, any dependence is possible. + /// 2) Line - A line ax + by = c, where a, b, and c are parameters, + /// representing the dependence equation. + /// 3) Distance - The value d of the dependence distance; + /// 4) Point - A point representing the dependence from + /// iteration x to iteration y. + /// 5) Empty - No dependence is possible. + class Constraint { private: - SCEVUnionPredicate Assumptions; - const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr; - friend class DependenceInfo; - }; + enum ConstraintKind { Empty, Point, Distance, Line, Any } Kind; + ScalarEvolution *SE; + const SCEV *A; + const SCEV *B; + const SCEV *C; + const Loop *AssociatedLoop; - /// FullDependence - This class represents a dependence between two memory - /// references in a function. It contains detailed information about the - /// dependence (direction vectors, etc.) and is used when the compiler is - /// able to accurately analyze the interaction of the references; that is, - /// it is not a confused dependence (see Dependence). In most cases - /// (for output, flow, and anti dependences), the dependence implies an - /// ordering, where the source must precede the destination; in contrast, - /// input dependences are unordered. - class LLVM_ABI FullDependence final : public Dependence { public: - FullDependence(Instruction *Source, Instruction *Destination, - const SCEVUnionPredicate &Assumes, - bool PossiblyLoopIndependent, unsigned Levels); - - /// isLoopIndependent - Returns true if this is a loop-independent - /// dependence. - bool isLoopIndependent() const override { return LoopIndependent; } - - /// isConfused - Returns true if this dependence is confused - /// (the compiler understands nothing and makes worst-case - /// assumptions). - bool isConfused() const override { return false; } - - /// isConsistent - Returns true if this dependence is consistent - /// (occurs every time the source and destination are executed). - bool isConsistent() const override { return Consistent; } - - /// getLevels - Returns the number of common loops surrounding the - /// source and destination of the dependence. - unsigned getLevels() const override { return Levels; } - - /// getDirection - Returns the direction associated with a particular - /// level. - unsigned getDirection(unsigned Level) const override; - - /// getDistance - Returns the distance (or NULL) associated with a - /// particular level. - const SCEV *getDistance(unsigned Level) const override; - - /// Check if the direction vector is negative. A negative direction - /// vector means Src and Dst are reversed in the actual program. - bool isDirectionNegative() const override; - - /// If the direction vector is negative, normalize the direction - /// vector to make it non-negative. Normalization is done by reversing - /// Src and Dst, plus reversing the dependence directions and distances - /// in the vector. - bool normalize(ScalarEvolution *SE) override; - - /// isPeelFirst - Returns true if peeling the first iteration from - /// this loop will break this dependence. - bool isPeelFirst(unsigned Level) const override; - - /// isPeelLast - Returns true if peeling the last iteration from - /// this loop will break this dependence. - bool isPeelLast(unsigned Level) const override; - - /// isSplitable - Returns true if splitting the loop will break - /// the dependence. - bool isSplitable(unsigned Level) const override; - - /// isScalar - Returns true if a particular level is scalar; that is, - /// if no subscript in the source or destination mention the induction - /// variable associated with the loop at this level. - bool isScalar(unsigned Level) const override; + /// isEmpty - Return true if the constraint is of kind Empty. + bool isEmpty() const { return Kind == Empty; } - private: - unsigned short Levels; - bool LoopIndependent; - bool Consistent; // Init to true, then refine. - std::unique_ptr DV; - friend class DependenceInfo; - }; + /// isPoint - Return true if the constraint is of kind Point. + bool isPoint() const { return Kind == Point; } - /// DependenceInfo - This class is the main dependence-analysis driver. - /// - class DependenceInfo { - public: - DependenceInfo(Function *F, AAResults *AA, ScalarEvolution *SE, - LoopInfo *LI) - : AA(AA), SE(SE), LI(LI), F(F) {} - - /// Handle transitive invalidation when the cached analysis results go away. - LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA, - FunctionAnalysisManager::Invalidator &Inv); - - /// depends - Tests for a dependence between the Src and Dst instructions. - /// Returns NULL if no dependence; otherwise, returns a Dependence (or a - /// FullDependence) with as much information as can be gleaned. By default, - /// the dependence test collects a set of runtime assumptions that cannot be - /// solved at compilation time. By default UnderRuntimeAssumptions is false - /// for a safe approximation of the dependence relation that does not - /// require runtime checks. - LLVM_ABI std::unique_ptr - depends(Instruction *Src, Instruction *Dst, - bool UnderRuntimeAssumptions = false); - - /// getSplitIteration - Give a dependence that's splittable at some - /// particular level, return the iteration that should be used to split - /// the loop. - /// - /// Generally, the dependence analyzer will be used to build - /// a dependence graph for a function (basically a map from instructions - /// to dependences). Looking for cycles in the graph shows us loops - /// that cannot be trivially vectorized/parallelized. - /// - /// We can try to improve the situation by examining all the dependences - /// that make up the cycle, looking for ones we can break. - /// Sometimes, peeling the first or last iteration of a loop will break - /// dependences, and there are flags for those possibilities. - /// Sometimes, splitting a loop at some other iteration will do the trick, - /// and we've got a flag for that case. Rather than waste the space to - /// record the exact iteration (since we rarely know), we provide - /// a method that calculates the iteration. It's a drag that it must work - /// from scratch, but wonderful in that it's possible. - /// - /// Here's an example: - /// - /// for (i = 0; i < 10; i++) - /// A[i] = ... - /// ... = A[11 - i] - /// - /// There's a loop-carried flow dependence from the store to the load, - /// found by the weak-crossing SIV test. The dependence will have a flag, - /// indicating that the dependence can be broken by splitting the loop. - /// Calling getSplitIteration will return 5. - /// Splitting the loop breaks the dependence, like so: - /// - /// for (i = 0; i <= 5; i++) - /// A[i] = ... - /// ... = A[11 - i] - /// for (i = 6; i < 10; i++) - /// A[i] = ... - /// ... = A[11 - i] - /// - /// breaks the dependence and allows us to vectorize/parallelize - /// both loops. - LLVM_ABI const SCEV *getSplitIteration(const Dependence &Dep, - unsigned Level); - - Function *getFunction() const { return F; } - - /// getRuntimeAssumptions - Returns all the runtime assumptions under which - /// the dependence test is valid. - LLVM_ABI SCEVUnionPredicate getRuntimeAssumptions() const; + /// isDistance - Return true if the constraint is of kind Distance. + bool isDistance() const { return Kind == Distance; } - private: - AAResults *AA; - ScalarEvolution *SE; - LoopInfo *LI; - Function *F; - SmallVector Assumptions; - - /// Subscript - This private struct represents a pair of subscripts from - /// a pair of potentially multi-dimensional array references. We use a - /// vector of them to guide subscript partitioning. - struct Subscript { - const SCEV *Src; - const SCEV *Dst; - enum ClassificationKind { ZIV, SIV, RDIV, MIV, NonLinear } Classification; - SmallBitVector Loops; - SmallBitVector GroupLoops; - SmallBitVector Group; - }; + /// isLine - Return true if the constraint is of kind Line. + /// Since Distance's can also be represented as Lines, we also return + /// true if the constraint is of kind Distance. + bool isLine() const { return Kind == Line || Kind == Distance; } - struct CoefficientInfo { - const SCEV *Coeff; - const SCEV *PosPart; - const SCEV *NegPart; - const SCEV *Iterations; - }; + /// isAny - Return true if the constraint is of kind Any; + bool isAny() const { return Kind == Any; } - struct BoundInfo { - const SCEV *Iterations; - const SCEV *Upper[8]; - const SCEV *Lower[8]; - unsigned char Direction; - unsigned char DirSet; - }; + /// getX - If constraint is a point , returns X. + /// Otherwise assert. + LLVM_ABI const SCEV *getX() const; - /// Constraint - This private class represents a constraint, as defined - /// in the paper - /// - /// Practical Dependence Testing - /// Goff, Kennedy, Tseng - /// PLDI 1991 - /// - /// There are 5 kinds of constraint, in a hierarchy. - /// 1) Any - indicates no constraint, any dependence is possible. - /// 2) Line - A line ax + by = c, where a, b, and c are parameters, - /// representing the dependence equation. - /// 3) Distance - The value d of the dependence distance; - /// 4) Point - A point representing the dependence from - /// iteration x to iteration y. - /// 5) Empty - No dependence is possible. - class Constraint { - private: - enum ConstraintKind { Empty, Point, Distance, Line, Any } Kind; - ScalarEvolution *SE; - const SCEV *A; - const SCEV *B; - const SCEV *C; - const Loop *AssociatedLoop; - - public: - /// isEmpty - Return true if the constraint is of kind Empty. - bool isEmpty() const { return Kind == Empty; } - - /// isPoint - Return true if the constraint is of kind Point. - bool isPoint() const { return Kind == Point; } - - /// isDistance - Return true if the constraint is of kind Distance. - bool isDistance() const { return Kind == Distance; } - - /// isLine - Return true if the constraint is of kind Line. - /// Since Distance's can also be represented as Lines, we also return - /// true if the constraint is of kind Distance. - bool isLine() const { return Kind == Line || Kind == Distance; } - - /// isAny - Return true if the constraint is of kind Any; - bool isAny() const { return Kind == Any; } - - /// getX - If constraint is a point , returns X. - /// Otherwise assert. - LLVM_ABI const SCEV *getX() const; - - /// getY - If constraint is a point , returns Y. - /// Otherwise assert. - LLVM_ABI const SCEV *getY() const; - - /// getA - If constraint is a line AX + BY = C, returns A. - /// Otherwise assert. - LLVM_ABI const SCEV *getA() const; - - /// getB - If constraint is a line AX + BY = C, returns B. - /// Otherwise assert. - LLVM_ABI const SCEV *getB() const; - - /// getC - If constraint is a line AX + BY = C, returns C. - /// Otherwise assert. - LLVM_ABI const SCEV *getC() const; - - /// getD - If constraint is a distance, returns D. - /// Otherwise assert. - LLVM_ABI const SCEV *getD() const; - - /// getAssociatedLoop - Returns the loop associated with this constraint. - LLVM_ABI const Loop *getAssociatedLoop() const; - - /// setPoint - Change a constraint to Point. - LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y, - const Loop *CurrentLoop); - - /// setLine - Change a constraint to Line. - LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C, - const Loop *CurrentLoop); - - /// setDistance - Change a constraint to Distance. - LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop); - - /// setEmpty - Change a constraint to Empty. - LLVM_ABI void setEmpty(); - - /// setAny - Change a constraint to Any. - LLVM_ABI void setAny(ScalarEvolution *SE); - - /// dump - For debugging purposes. Dumps the constraint - /// out to OS. - LLVM_ABI void dump(raw_ostream &OS) const; - }; + /// getY - If constraint is a point , returns Y. + /// Otherwise assert. + LLVM_ABI const SCEV *getY() const; - /// establishNestingLevels - Examines the loop nesting of the Src and Dst - /// instructions and establishes their shared loops. Sets the variables - /// CommonLevels, SrcLevels, and MaxLevels. - /// The source and destination instructions needn't be contained in the same - /// loop. The routine establishNestingLevels finds the level of most deeply - /// nested loop that contains them both, CommonLevels. An instruction that's - /// not contained in a loop is at level = 0. MaxLevels is equal to the level - /// of the source plus the level of the destination, minus CommonLevels. - /// This lets us allocate vectors MaxLevels in length, with room for every - /// distinct loop referenced in both the source and destination subscripts. - /// The variable SrcLevels is the nesting depth of the source instruction. - /// It's used to help calculate distinct loops referenced by the destination. - /// Here's the map from loops to levels: - /// 0 - unused - /// 1 - outermost common loop - /// ... - other common loops - /// CommonLevels - innermost common loop - /// ... - loops containing Src but not Dst - /// SrcLevels - innermost loop containing Src but not Dst - /// ... - loops containing Dst but not Src - /// MaxLevels - innermost loop containing Dst but not Src - /// Consider the follow code fragment: - /// for (a = ...) { - /// for (b = ...) { - /// for (c = ...) { - /// for (d = ...) { - /// A[] = ...; - /// } - /// } - /// for (e = ...) { - /// for (f = ...) { - /// for (g = ...) { - /// ... = A[]; - /// } - /// } - /// } - /// } - /// } - /// If we're looking at the possibility of a dependence between the store - /// to A (the Src) and the load from A (the Dst), we'll note that they - /// have 2 loops in common, so CommonLevels will equal 2 and the direction - /// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7. - /// A map from loop names to level indices would look like - /// a - 1 - /// b - 2 = CommonLevels - /// c - 3 - /// d - 4 = SrcLevels - /// e - 5 - /// f - 6 - /// g - 7 = MaxLevels - void establishNestingLevels(const Instruction *Src, - const Instruction *Dst); - - unsigned CommonLevels, SrcLevels, MaxLevels; - - /// mapSrcLoop - Given one of the loops containing the source, return - /// its level index in our numbering scheme. - unsigned mapSrcLoop(const Loop *SrcLoop) const; - - /// mapDstLoop - Given one of the loops containing the destination, - /// return its level index in our numbering scheme. - unsigned mapDstLoop(const Loop *DstLoop) const; - - /// isLoopInvariant - Returns true if Expression is loop invariant - /// in LoopNest. - bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const; - - /// Makes sure all subscript pairs share the same integer type by - /// sign-extending as necessary. - /// Sign-extending a subscript is safe because getelementptr assumes the - /// array subscripts are signed. - void unifySubscriptType(ArrayRef Pairs); - - /// removeMatchingExtensions - Examines a subscript pair. - /// If the source and destination are identically sign (or zero) - /// extended, it strips off the extension in an effort to - /// simplify the actual analysis. - void removeMatchingExtensions(Subscript *Pair); - - /// collectCommonLoops - Finds the set of loops from the LoopNest that - /// have a level <= CommonLevels and are referred to by the SCEV Expression. - void collectCommonLoops(const SCEV *Expression, - const Loop *LoopNest, - SmallBitVector &Loops) const; - - /// checkSrcSubscript - Examines the SCEV Src, returning true iff it's - /// linear. Collect the set of loops mentioned by Src. - bool checkSrcSubscript(const SCEV *Src, - const Loop *LoopNest, - SmallBitVector &Loops); - - /// checkDstSubscript - Examines the SCEV Dst, returning true iff it's - /// linear. Collect the set of loops mentioned by Dst. - bool checkDstSubscript(const SCEV *Dst, - const Loop *LoopNest, - SmallBitVector &Loops); - - /// isKnownPredicate - Compare X and Y using the predicate Pred. - /// Basically a wrapper for SCEV::isKnownPredicate, - /// but tries harder, especially in the presence of sign and zero - /// extensions and symbolics. - bool isKnownPredicate(ICmpInst::Predicate Pred, - const SCEV *X, - const SCEV *Y) const; - - /// isKnownLessThan - Compare to see if S is less than Size - /// Another wrapper for isKnownNegative(S - max(Size, 1)) with some extra - /// checking if S is an AddRec and we can prove lessthan using the loop - /// bounds. - bool isKnownLessThan(const SCEV *S, const SCEV *Size) const; - - /// isKnownNonNegative - Compare to see if S is known not to be negative - /// Uses the fact that S comes from Ptr, which may be an inbound GEP, - /// Proving there is no wrapping going on. - bool isKnownNonNegative(const SCEV *S, const Value *Ptr) const; - - /// collectUpperBound - All subscripts are the same type (on my machine, - /// an i64). The loop bound may be a smaller type. collectUpperBound - /// find the bound, if available, and zero extends it to the Type T. - /// (I zero extend since the bound should always be >= 0.) - /// If no upper bound is available, return NULL. - const SCEV *collectUpperBound(const Loop *l, Type *T) const; - - /// collectConstantUpperBound - Calls collectUpperBound(), then - /// attempts to cast it to SCEVConstant. If the cast fails, - /// returns NULL. - const SCEVConstant *collectConstantUpperBound(const Loop *l, Type *T) const; - - /// classifyPair - Examines the subscript pair (the Src and Dst SCEVs) - /// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear. - /// Collects the associated loops in a set. - Subscript::ClassificationKind classifyPair(const SCEV *Src, - const Loop *SrcLoopNest, - const SCEV *Dst, - const Loop *DstLoopNest, - SmallBitVector &Loops); - - /// testZIV - Tests the ZIV subscript pair (Src and Dst) for dependence. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// If the dependence isn't proven to exist, - /// marks the Result as inconsistent. - bool testZIV(const SCEV *Src, - const SCEV *Dst, - FullDependence &Result) const; - - /// testSIV - Tests the SIV subscript pair (Src and Dst) for dependence. - /// Things of the form [c1 + a1*i] and [c2 + a2*j], where - /// i and j are induction variables, c1 and c2 are loop invariant, - /// and a1 and a2 are constant. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction vector entry and, when possible, - /// the distance vector entry. - /// If the dependence isn't proven to exist, - /// marks the Result as inconsistent. - bool testSIV(const SCEV *Src, - const SCEV *Dst, - unsigned &Level, - FullDependence &Result, - Constraint &NewConstraint, - const SCEV *&SplitIter) const; - - /// testRDIV - Tests the RDIV subscript pair (Src and Dst) for dependence. - /// Things of the form [c1 + a1*i] and [c2 + a2*j] - /// where i and j are induction variables, c1 and c2 are loop invariant, - /// and a1 and a2 are constant. - /// With minor algebra, this test can also be used for things like - /// [c1 + a1*i + a2*j][c2]. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Marks the Result as inconsistent. - bool testRDIV(const SCEV *Src, - const SCEV *Dst, - FullDependence &Result) const; + /// getA - If constraint is a line AX + BY = C, returns A. + /// Otherwise assert. + LLVM_ABI const SCEV *getA() const; - /// testMIV - Tests the MIV subscript pair (Src and Dst) for dependence. - /// Returns true if dependence disproved. - /// Can sometimes refine direction vectors. - bool testMIV(const SCEV *Src, - const SCEV *Dst, - const SmallBitVector &Loops, - FullDependence &Result) const; - - /// strongSIVtest - Tests the strong SIV subscript pair (Src and Dst) - /// for dependence. - /// Things of the form [c1 + a*i] and [c2 + a*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction and distance. - bool strongSIVtest(const SCEV *Coeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1 + a*i] and [c2 - a*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - /// Marks the dependence as splitable. - bool weakCrossingSIVtest(const SCEV *SrcCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint, - const SCEV *&SplitIter) const; - - /// ExactSIVtest - Tests the SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1 + a1*i] and [c2 + a2*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a1 and a2 are constant. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - bool exactSIVtest(const SCEV *SrcCoeff, - const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1] and [c2 + a*i], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant. See also weakZeroDstSIVtest. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - /// If loop peeling will break the dependence, mark appropriately. - bool weakZeroSrcSIVtest(const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair - /// (Src and Dst) for dependence. - /// Things of the form [c1 + a*i] and [c2], - /// where i is an induction variable, c1 and c2 are loop invariant, - /// and a is a constant. See also weakZeroSrcSIVtest. - /// Returns true if any possible dependence is disproved. - /// If there might be a dependence, returns false. - /// Sets appropriate direction entry. - /// Set consistent to false. - /// If loop peeling will break the dependence, mark appropriately. - bool weakZeroDstSIVtest(const SCEV *SrcCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *CurrentLoop, - unsigned Level, - FullDependence &Result, - Constraint &NewConstraint) const; - - /// exactRDIVtest - Tests the RDIV subscript pair for dependence. - /// Things of the form [c1 + a*i] and [c2 + b*j], - /// where i and j are induction variable, c1 and c2 are loop invariant, - /// and a and b are constants. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Works in some cases that symbolicRDIVtest doesn't, - /// and vice versa. - bool exactRDIVtest(const SCEV *SrcCoeff, - const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *SrcLoop, - const Loop *DstLoop, - FullDependence &Result) const; + /// getB - If constraint is a line AX + BY = C, returns B. + /// Otherwise assert. + LLVM_ABI const SCEV *getB() const; - /// symbolicRDIVtest - Tests the RDIV subscript pair for dependence. - /// Things of the form [c1 + a*i] and [c2 + b*j], - /// where i and j are induction variable, c1 and c2 are loop invariant, - /// and a and b are constants. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Works in some cases that exactRDIVtest doesn't, - /// and vice versa. Can also be used as a backup for - /// ordinary SIV tests. - bool symbolicRDIVtest(const SCEV *SrcCoeff, - const SCEV *DstCoeff, - const SCEV *SrcConst, - const SCEV *DstConst, - const Loop *SrcLoop, - const Loop *DstLoop) const; - - /// gcdMIVtest - Tests an MIV subscript pair for dependence. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Can sometimes disprove the equal direction for 1 or more loops. - // Can handle some symbolics that even the SIV tests don't get, - /// so we use it as a backup for everything. - bool gcdMIVtest(const SCEV *Src, - const SCEV *Dst, - FullDependence &Result) const; - - /// banerjeeMIVtest - Tests an MIV subscript pair for dependence. - /// Returns true if any possible dependence is disproved. - /// Marks the result as inconsistent. - /// Computes directions. - bool banerjeeMIVtest(const SCEV *Src, - const SCEV *Dst, - const SmallBitVector &Loops, - FullDependence &Result) const; - - /// collectCoefficientInfo - Walks through the subscript, - /// collecting each coefficient, the associated loop bounds, - /// and recording its positive and negative parts for later use. - CoefficientInfo *collectCoeffInfo(const SCEV *Subscript, - bool SrcFlag, - const SCEV *&Constant) const; - - /// getPositivePart - X^+ = max(X, 0). - /// - const SCEV *getPositivePart(const SCEV *X) const; - - /// getNegativePart - X^- = min(X, 0). - /// - const SCEV *getNegativePart(const SCEV *X) const; - - /// getLowerBound - Looks through all the bounds info and - /// computes the lower bound given the current direction settings - /// at each level. - const SCEV *getLowerBound(BoundInfo *Bound) const; - - /// getUpperBound - Looks through all the bounds info and - /// computes the upper bound given the current direction settings - /// at each level. - const SCEV *getUpperBound(BoundInfo *Bound) const; - - /// exploreDirections - Hierarchically expands the direction vector - /// search space, combining the directions of discovered dependences - /// in the DirSet field of Bound. Returns the number of distinct - /// dependences discovered. If the dependence is disproved, - /// it will return 0. - unsigned exploreDirections(unsigned Level, - CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - const SmallBitVector &Loops, - unsigned &DepthExpanded, - const SCEV *Delta) const; - - /// testBounds - Returns true iff the current bounds are plausible. - bool testBounds(unsigned char DirKind, - unsigned Level, - BoundInfo *Bound, - const SCEV *Delta) const; - - /// findBoundsALL - Computes the upper and lower bounds for level K - /// using the * direction. Records them in Bound. - void findBoundsALL(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// findBoundsLT - Computes the upper and lower bounds for level K - /// using the < direction. Records them in Bound. - void findBoundsLT(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// findBoundsGT - Computes the upper and lower bounds for level K - /// using the > direction. Records them in Bound. - void findBoundsGT(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// findBoundsEQ - Computes the upper and lower bounds for level K - /// using the = direction. Records them in Bound. - void findBoundsEQ(CoefficientInfo *A, - CoefficientInfo *B, - BoundInfo *Bound, - unsigned K) const; - - /// intersectConstraints - Updates X with the intersection - /// of the Constraints X and Y. Returns true if X has changed. - bool intersectConstraints(Constraint *X, - const Constraint *Y); - - /// propagate - Review the constraints, looking for opportunities - /// to simplify a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - /// If the simplification isn't exact (that is, if it is conservative - /// in terms of dependence), set consistent to false. - bool propagate(const SCEV *&Src, - const SCEV *&Dst, - SmallBitVector &Loops, - SmallVectorImpl &Constraints, - bool &Consistent); - - /// propagateDistance - Attempt to propagate a distance - /// constraint into a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - /// If the simplification isn't exact (that is, if it is conservative - /// in terms of dependence), set consistent to false. - bool propagateDistance(const SCEV *&Src, - const SCEV *&Dst, - Constraint &CurConstraint, - bool &Consistent); - - /// propagatePoint - Attempt to propagate a point - /// constraint into a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - bool propagatePoint(const SCEV *&Src, - const SCEV *&Dst, - Constraint &CurConstraint); - - /// propagateLine - Attempt to propagate a line - /// constraint into a subscript pair (Src and Dst). - /// Return true if some simplification occurs. - /// If the simplification isn't exact (that is, if it is conservative - /// in terms of dependence), set consistent to false. - bool propagateLine(const SCEV *&Src, - const SCEV *&Dst, - Constraint &CurConstraint, - bool &Consistent); - - /// findCoefficient - Given a linear SCEV, - /// return the coefficient corresponding to specified loop. - /// If there isn't one, return the SCEV constant 0. - /// For example, given a*i + b*j + c*k, returning the coefficient - /// corresponding to the j loop would yield b. - const SCEV *findCoefficient(const SCEV *Expr, - const Loop *TargetLoop) const; - - /// zeroCoefficient - Given a linear SCEV, - /// return the SCEV given by zeroing out the coefficient - /// corresponding to the specified loop. - /// For example, given a*i + b*j + c*k, zeroing the coefficient - /// corresponding to the j loop would yield a*i + c*k. - const SCEV *zeroCoefficient(const SCEV *Expr, - const Loop *TargetLoop) const; - - /// addToCoefficient - Given a linear SCEV Expr, - /// return the SCEV given by adding some Value to the - /// coefficient corresponding to the specified TargetLoop. - /// For example, given a*i + b*j + c*k, adding 1 to the coefficient - /// corresponding to the j loop would yield a*i + (b+1)*j + c*k. - const SCEV *addToCoefficient(const SCEV *Expr, - const Loop *TargetLoop, - const SCEV *Value) const; - - /// updateDirection - Update direction vector entry - /// based on the current constraint. - void updateDirection(Dependence::DVEntry &Level, - const Constraint &CurConstraint) const; - - /// Given a linear access function, tries to recover subscripts - /// for each dimension of the array element access. - bool tryDelinearize(Instruction *Src, Instruction *Dst, - SmallVectorImpl &Pair); - - /// Tries to delinearize \p Src and \p Dst access functions for a fixed size - /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to - /// delinearize \p Src and \p Dst separately, - bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst, - const SCEV *SrcAccessFn, - const SCEV *DstAccessFn, - SmallVectorImpl &SrcSubscripts, - SmallVectorImpl &DstSubscripts); - - /// Tries to delinearize access function for a multi-dimensional array with - /// symbolic runtime sizes. - /// Returns true upon success and false otherwise. - bool tryDelinearizeParametricSize( - Instruction *Src, Instruction *Dst, const SCEV *SrcAccessFn, - const SCEV *DstAccessFn, SmallVectorImpl &SrcSubscripts, - SmallVectorImpl &DstSubscripts); - - /// checkSubscript - Helper function for checkSrcSubscript and - /// checkDstSubscript to avoid duplicate code - bool checkSubscript(const SCEV *Expr, const Loop *LoopNest, - SmallBitVector &Loops, bool IsSrc); - }; // class DependenceInfo - - /// AnalysisPass to compute dependence information in a function - class DependenceAnalysis : public AnalysisInfoMixin { - public: - typedef DependenceInfo Result; - LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM); + /// getC - If constraint is a line AX + BY = C, returns C. + /// Otherwise assert. + LLVM_ABI const SCEV *getC() const; - private: - LLVM_ABI static AnalysisKey Key; - friend struct AnalysisInfoMixin; - }; // class DependenceAnalysis + /// getD - If constraint is a distance, returns D. + /// Otherwise assert. + LLVM_ABI const SCEV *getD() const; - /// Printer pass to dump DA results. - struct DependenceAnalysisPrinterPass - : public PassInfoMixin { - DependenceAnalysisPrinterPass(raw_ostream &OS, - bool NormalizeResults = false) - : OS(OS), NormalizeResults(NormalizeResults) {} + /// getAssociatedLoop - Returns the loop associated with this constraint. + LLVM_ABI const Loop *getAssociatedLoop() const; - LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + /// setPoint - Change a constraint to Point. + LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y, + const Loop *CurrentLoop); - static bool isRequired() { return true; } + /// setLine - Change a constraint to Line. + LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C, + const Loop *CurrentLoop); - private: - raw_ostream &OS; - bool NormalizeResults; - }; // class DependenceAnalysisPrinterPass + /// setDistance - Change a constraint to Distance. + LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop); - /// Legacy pass manager pass to access dependence information - class LLVM_ABI DependenceAnalysisWrapperPass : public FunctionPass { - public: - static char ID; // Class identification, replacement for typeinfo - DependenceAnalysisWrapperPass(); + /// setEmpty - Change a constraint to Empty. + LLVM_ABI void setEmpty(); - bool runOnFunction(Function &F) override; - void releaseMemory() override; - void getAnalysisUsage(AnalysisUsage &) const override; - void print(raw_ostream &, const Module * = nullptr) const override; - DependenceInfo &getDI() const; + /// setAny - Change a constraint to Any. + LLVM_ABI void setAny(ScalarEvolution *SE); - private: - std::unique_ptr info; - }; // class DependenceAnalysisWrapperPass + /// dump - For debugging purposes. Dumps the constraint + /// out to OS. + LLVM_ABI void dump(raw_ostream &OS) const; + }; + + /// establishNestingLevels - Examines the loop nesting of the Src and Dst + /// instructions and establishes their shared loops. Sets the variables + /// CommonLevels, SrcLevels, and MaxLevels. + /// The source and destination instructions needn't be contained in the same + /// loop. The routine establishNestingLevels finds the level of most deeply + /// nested loop that contains them both, CommonLevels. An instruction that's + /// not contained in a loop is at level = 0. MaxLevels is equal to the level + /// of the source plus the level of the destination, minus CommonLevels. + /// This lets us allocate vectors MaxLevels in length, with room for every + /// distinct loop referenced in both the source and destination subscripts. + /// The variable SrcLevels is the nesting depth of the source instruction. + /// It's used to help calculate distinct loops referenced by the destination. + /// Here's the map from loops to levels: + /// 0 - unused + /// 1 - outermost common loop + /// ... - other common loops + /// CommonLevels - innermost common loop + /// ... - loops containing Src but not Dst + /// SrcLevels - innermost loop containing Src but not Dst + /// ... - loops containing Dst but not Src + /// MaxLevels - innermost loop containing Dst but not Src + /// Consider the follow code fragment: + /// for (a = ...) { + /// for (b = ...) { + /// for (c = ...) { + /// for (d = ...) { + /// A[] = ...; + /// } + /// } + /// for (e = ...) { + /// for (f = ...) { + /// for (g = ...) { + /// ... = A[]; + /// } + /// } + /// } + /// } + /// } + /// If we're looking at the possibility of a dependence between the store + /// to A (the Src) and the load from A (the Dst), we'll note that they + /// have 2 loops in common, so CommonLevels will equal 2 and the direction + /// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7. + /// A map from loop names to level indices would look like + /// a - 1 + /// b - 2 = CommonLevels + /// c - 3 + /// d - 4 = SrcLevels + /// e - 5 + /// f - 6 + /// g - 7 = MaxLevels + void establishNestingLevels(const Instruction *Src, const Instruction *Dst); + + unsigned CommonLevels, SrcLevels, MaxLevels; + + /// mapSrcLoop - Given one of the loops containing the source, return + /// its level index in our numbering scheme. + unsigned mapSrcLoop(const Loop *SrcLoop) const; + + /// mapDstLoop - Given one of the loops containing the destination, + /// return its level index in our numbering scheme. + unsigned mapDstLoop(const Loop *DstLoop) const; + + /// isLoopInvariant - Returns true if Expression is loop invariant + /// in LoopNest. + bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const; + + /// Makes sure all subscript pairs share the same integer type by + /// sign-extending as necessary. + /// Sign-extending a subscript is safe because getelementptr assumes the + /// array subscripts are signed. + void unifySubscriptType(ArrayRef Pairs); + + /// removeMatchingExtensions - Examines a subscript pair. + /// If the source and destination are identically sign (or zero) + /// extended, it strips off the extension in an effort to + /// simplify the actual analysis. + void removeMatchingExtensions(Subscript *Pair); + + /// collectCommonLoops - Finds the set of loops from the LoopNest that + /// have a level <= CommonLevels and are referred to by the SCEV Expression. + void collectCommonLoops(const SCEV *Expression, const Loop *LoopNest, + SmallBitVector &Loops) const; + + /// checkSrcSubscript - Examines the SCEV Src, returning true iff it's + /// linear. Collect the set of loops mentioned by Src. + bool checkSrcSubscript(const SCEV *Src, const Loop *LoopNest, + SmallBitVector &Loops); + + /// checkDstSubscript - Examines the SCEV Dst, returning true iff it's + /// linear. Collect the set of loops mentioned by Dst. + bool checkDstSubscript(const SCEV *Dst, const Loop *LoopNest, + SmallBitVector &Loops); + + /// isKnownPredicate - Compare X and Y using the predicate Pred. + /// Basically a wrapper for SCEV::isKnownPredicate, + /// but tries harder, especially in the presence of sign and zero + /// extensions and symbolics. + bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X, + const SCEV *Y) const; + + /// isKnownLessThan - Compare to see if S is less than Size + /// Another wrapper for isKnownNegative(S - max(Size, 1)) with some extra + /// checking if S is an AddRec and we can prove lessthan using the loop + /// bounds. + bool isKnownLessThan(const SCEV *S, const SCEV *Size) const; + + /// isKnownNonNegative - Compare to see if S is known not to be negative + /// Uses the fact that S comes from Ptr, which may be an inbound GEP, + /// Proving there is no wrapping going on. + bool isKnownNonNegative(const SCEV *S, const Value *Ptr) const; + + /// collectUpperBound - All subscripts are the same type (on my machine, + /// an i64). The loop bound may be a smaller type. collectUpperBound + /// find the bound, if available, and zero extends it to the Type T. + /// (I zero extend since the bound should always be >= 0.) + /// If no upper bound is available, return NULL. + const SCEV *collectUpperBound(const Loop *l, Type *T) const; + + /// collectConstantUpperBound - Calls collectUpperBound(), then + /// attempts to cast it to SCEVConstant. If the cast fails, + /// returns NULL. + const SCEVConstant *collectConstantUpperBound(const Loop *l, Type *T) const; + + /// classifyPair - Examines the subscript pair (the Src and Dst SCEVs) + /// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear. + /// Collects the associated loops in a set. + Subscript::ClassificationKind + classifyPair(const SCEV *Src, const Loop *SrcLoopNest, const SCEV *Dst, + const Loop *DstLoopNest, SmallBitVector &Loops); + + /// testZIV - Tests the ZIV subscript pair (Src and Dst) for dependence. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// If the dependence isn't proven to exist, + /// marks the Result as inconsistent. + bool testZIV(const SCEV *Src, const SCEV *Dst, FullDependence &Result) const; + + /// testSIV - Tests the SIV subscript pair (Src and Dst) for dependence. + /// Things of the form [c1 + a1*i] and [c2 + a2*j], where + /// i and j are induction variables, c1 and c2 are loop invariant, + /// and a1 and a2 are constant. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction vector entry and, when possible, + /// the distance vector entry. + /// If the dependence isn't proven to exist, + /// marks the Result as inconsistent. + bool testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, + FullDependence &Result, Constraint &NewConstraint, + const SCEV *&SplitIter) const; + + /// testRDIV - Tests the RDIV subscript pair (Src and Dst) for dependence. + /// Things of the form [c1 + a1*i] and [c2 + a2*j] + /// where i and j are induction variables, c1 and c2 are loop invariant, + /// and a1 and a2 are constant. + /// With minor algebra, this test can also be used for things like + /// [c1 + a1*i + a2*j][c2]. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Marks the Result as inconsistent. + bool testRDIV(const SCEV *Src, const SCEV *Dst, FullDependence &Result) const; + + /// testMIV - Tests the MIV subscript pair (Src and Dst) for dependence. + /// Returns true if dependence disproved. + /// Can sometimes refine direction vectors. + bool testMIV(const SCEV *Src, const SCEV *Dst, const SmallBitVector &Loops, + FullDependence &Result) const; + + /// strongSIVtest - Tests the strong SIV subscript pair (Src and Dst) + /// for dependence. + /// Things of the form [c1 + a*i] and [c2 + a*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction and distance. + bool strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint) const; + + /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1 + a*i] and [c2 - a*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + /// Marks the dependence as splitable. + bool weakCrossingSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint, + const SCEV *&SplitIter) const; + + /// ExactSIVtest - Tests the SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1 + a1*i] and [c2 + a2*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a1 and a2 are constant. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + bool exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, + const SCEV *SrcConst, const SCEV *DstConst, + const Loop *CurrentLoop, unsigned Level, + FullDependence &Result, Constraint &NewConstraint) const; + + /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1] and [c2 + a*i], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant. See also weakZeroDstSIVtest. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + /// If loop peeling will break the dependence, mark appropriately. + bool weakZeroSrcSIVtest(const SCEV *DstCoeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint) const; + + /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair + /// (Src and Dst) for dependence. + /// Things of the form [c1 + a*i] and [c2], + /// where i is an induction variable, c1 and c2 are loop invariant, + /// and a is a constant. See also weakZeroSrcSIVtest. + /// Returns true if any possible dependence is disproved. + /// If there might be a dependence, returns false. + /// Sets appropriate direction entry. + /// Set consistent to false. + /// If loop peeling will break the dependence, mark appropriately. + bool weakZeroDstSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst, + const SCEV *DstConst, const Loop *CurrentLoop, + unsigned Level, FullDependence &Result, + Constraint &NewConstraint) const; + + /// exactRDIVtest - Tests the RDIV subscript pair for dependence. + /// Things of the form [c1 + a*i] and [c2 + b*j], + /// where i and j are induction variable, c1 and c2 are loop invariant, + /// and a and b are constants. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Works in some cases that symbolicRDIVtest doesn't, + /// and vice versa. + bool exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, + const SCEV *SrcConst, const SCEV *DstConst, + const Loop *SrcLoop, const Loop *DstLoop, + FullDependence &Result) const; + + /// symbolicRDIVtest - Tests the RDIV subscript pair for dependence. + /// Things of the form [c1 + a*i] and [c2 + b*j], + /// where i and j are induction variable, c1 and c2 are loop invariant, + /// and a and b are constants. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Works in some cases that exactRDIVtest doesn't, + /// and vice versa. Can also be used as a backup for + /// ordinary SIV tests. + bool symbolicRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, + const SCEV *SrcConst, const SCEV *DstConst, + const Loop *SrcLoop, const Loop *DstLoop) const; + + /// gcdMIVtest - Tests an MIV subscript pair for dependence. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Can sometimes disprove the equal direction for 1 or more loops. + // Can handle some symbolics that even the SIV tests don't get, + /// so we use it as a backup for everything. + bool gcdMIVtest(const SCEV *Src, const SCEV *Dst, + FullDependence &Result) const; + + /// banerjeeMIVtest - Tests an MIV subscript pair for dependence. + /// Returns true if any possible dependence is disproved. + /// Marks the result as inconsistent. + /// Computes directions. + bool banerjeeMIVtest(const SCEV *Src, const SCEV *Dst, + const SmallBitVector &Loops, + FullDependence &Result) const; - /// createDependenceAnalysisPass - This creates an instance of the - /// DependenceAnalysis wrapper pass. - LLVM_ABI FunctionPass *createDependenceAnalysisWrapperPass(); + /// collectCoeffInfo - Walks through the subscript, collecting each + /// coefficient, the associated loop bounds, and recording its positive and + /// negative parts for later use. + CoefficientInfo *collectCoeffInfo(const SCEV *Subscript, bool SrcFlag, + const SCEV *&Constant) const; + + /// getPositivePart - X^+ = max(X, 0). + const SCEV *getPositivePart(const SCEV *X) const; + + /// getNegativePart - X^- = min(X, 0). + const SCEV *getNegativePart(const SCEV *X) const; + + /// getLowerBound - Looks through all the bounds info and + /// computes the lower bound given the current direction settings + /// at each level. + const SCEV *getLowerBound(BoundInfo *Bound) const; + + /// getUpperBound - Looks through all the bounds info and + /// computes the upper bound given the current direction settings + /// at each level. + const SCEV *getUpperBound(BoundInfo *Bound) const; + + /// exploreDirections - Hierarchically expands the direction vector + /// search space, combining the directions of discovered dependences + /// in the DirSet field of Bound. Returns the number of distinct + /// dependences discovered. If the dependence is disproved, + /// it will return 0. + unsigned exploreDirections(unsigned Level, CoefficientInfo *A, + CoefficientInfo *B, BoundInfo *Bound, + const SmallBitVector &Loops, + unsigned &DepthExpanded, const SCEV *Delta) const; + + /// testBounds - Returns true iff the current bounds are plausible. + bool testBounds(unsigned char DirKind, unsigned Level, BoundInfo *Bound, + const SCEV *Delta) const; + + /// findBoundsALL - Computes the upper and lower bounds for level K + /// using the * direction. Records them in Bound. + void findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// findBoundsLT - Computes the upper and lower bounds for level K + /// using the < direction. Records them in Bound. + void findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// findBoundsGT - Computes the upper and lower bounds for level K + /// using the > direction. Records them in Bound. + void findBoundsGT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// findBoundsEQ - Computes the upper and lower bounds for level K + /// using the = direction. Records them in Bound. + void findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, + unsigned K) const; + + /// intersectConstraints - Updates X with the intersection + /// of the Constraints X and Y. Returns true if X has changed. + bool intersectConstraints(Constraint *X, const Constraint *Y); + + /// propagate - Review the constraints, looking for opportunities + /// to simplify a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + /// If the simplification isn't exact (that is, if it is conservative + /// in terms of dependence), set consistent to false. + bool propagate(const SCEV *&Src, const SCEV *&Dst, SmallBitVector &Loops, + SmallVectorImpl &Constraints, bool &Consistent); + + /// propagateDistance - Attempt to propagate a distance + /// constraint into a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + /// If the simplification isn't exact (that is, if it is conservative + /// in terms of dependence), set consistent to false. + bool propagateDistance(const SCEV *&Src, const SCEV *&Dst, + Constraint &CurConstraint, bool &Consistent); + + /// propagatePoint - Attempt to propagate a point + /// constraint into a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + bool propagatePoint(const SCEV *&Src, const SCEV *&Dst, + Constraint &CurConstraint); + + /// propagateLine - Attempt to propagate a line + /// constraint into a subscript pair (Src and Dst). + /// Return true if some simplification occurs. + /// If the simplification isn't exact (that is, if it is conservative + /// in terms of dependence), set consistent to false. + bool propagateLine(const SCEV *&Src, const SCEV *&Dst, + Constraint &CurConstraint, bool &Consistent); + + /// findCoefficient - Given a linear SCEV, + /// return the coefficient corresponding to specified loop. + /// If there isn't one, return the SCEV constant 0. + /// For example, given a*i + b*j + c*k, returning the coefficient + /// corresponding to the j loop would yield b. + const SCEV *findCoefficient(const SCEV *Expr, const Loop *TargetLoop) const; + + /// zeroCoefficient - Given a linear SCEV, + /// return the SCEV given by zeroing out the coefficient + /// corresponding to the specified loop. + /// For example, given a*i + b*j + c*k, zeroing the coefficient + /// corresponding to the j loop would yield a*i + c*k. + const SCEV *zeroCoefficient(const SCEV *Expr, const Loop *TargetLoop) const; + + /// addToCoefficient - Given a linear SCEV Expr, + /// return the SCEV given by adding some Value to the + /// coefficient corresponding to the specified TargetLoop. + /// For example, given a*i + b*j + c*k, adding 1 to the coefficient + /// corresponding to the j loop would yield a*i + (b+1)*j + c*k. + const SCEV *addToCoefficient(const SCEV *Expr, const Loop *TargetLoop, + const SCEV *Value) const; + + /// updateDirection - Update direction vector entry + /// based on the current constraint. + void updateDirection(Dependence::DVEntry &Level, + const Constraint &CurConstraint) const; + + /// Given a linear access function, tries to recover subscripts + /// for each dimension of the array element access. + bool tryDelinearize(Instruction *Src, Instruction *Dst, + SmallVectorImpl &Pair); + + /// Tries to delinearize \p Src and \p Dst access functions for a fixed size + /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to + /// delinearize \p Src and \p Dst separately, + bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst, + const SCEV *SrcAccessFn, const SCEV *DstAccessFn, + SmallVectorImpl &SrcSubscripts, + SmallVectorImpl &DstSubscripts); + + /// Tries to delinearize access function for a multi-dimensional array with + /// symbolic runtime sizes. + /// Returns true upon success and false otherwise. + bool + tryDelinearizeParametricSize(Instruction *Src, Instruction *Dst, + const SCEV *SrcAccessFn, const SCEV *DstAccessFn, + SmallVectorImpl &SrcSubscripts, + SmallVectorImpl &DstSubscripts); + + /// checkSubscript - Helper function for checkSrcSubscript and + /// checkDstSubscript to avoid duplicate code + bool checkSubscript(const SCEV *Expr, const Loop *LoopNest, + SmallBitVector &Loops, bool IsSrc); +}; // class DependenceInfo + +/// AnalysisPass to compute dependence information in a function +class DependenceAnalysis : public AnalysisInfoMixin { +public: + typedef DependenceInfo Result; + LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM); + +private: + LLVM_ABI static AnalysisKey Key; + friend struct AnalysisInfoMixin; +}; // class DependenceAnalysis + +/// Printer pass to dump DA results. +struct DependenceAnalysisPrinterPass + : public PassInfoMixin { + DependenceAnalysisPrinterPass(raw_ostream &OS, bool NormalizeResults = false) + : OS(OS), NormalizeResults(NormalizeResults) {} + + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); + + static bool isRequired() { return true; } + +private: + raw_ostream &OS; + bool NormalizeResults; +}; // class DependenceAnalysisPrinterPass + +/// Legacy pass manager pass to access dependence information +class LLVM_ABI DependenceAnalysisWrapperPass : public FunctionPass { +public: + static char ID; // Class identification, replacement for typeinfo + DependenceAnalysisWrapperPass(); + + bool runOnFunction(Function &F) override; + void releaseMemory() override; + void getAnalysisUsage(AnalysisUsage &) const override; + void print(raw_ostream &, const Module * = nullptr) const override; + DependenceInfo &getDI() const; + +private: + std::unique_ptr info; +}; // class DependenceAnalysisWrapperPass + +/// createDependenceAnalysisPass - This creates an instance of the +/// DependenceAnalysis wrapper pass. +LLVM_ABI FunctionPass *createDependenceAnalysisWrapperPass(); } // namespace llvm diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp index f1473b2694ca4..256befa566534 100644 --- a/llvm/lib/Analysis/DependenceAnalysis.cpp +++ b/llvm/lib/Analysis/DependenceAnalysis.cpp @@ -180,8 +180,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA, for (inst_iterator SrcI = inst_begin(F), SrcE = inst_end(F); SrcI != SrcE; ++SrcI) { if (SrcI->mayReadOrWriteMemory()) { - for (inst_iterator DstI = SrcI, DstE = inst_end(F); - DstI != DstE; ++DstI) { + for (inst_iterator DstI = SrcI, DstE = inst_end(F); DstI != DstE; + ++DstI) { if (DstI->mayReadOrWriteMemory()) { OS << "Src:" << *SrcI << " --> Dst:" << *DstI << "\n"; OS << " da analyze - "; @@ -203,7 +203,7 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA, // Normalize negative direction vectors if required by clients. if (NormalizeResults && D->normalize(&SE)) - OS << "normalized - "; + OS << "normalized - "; D->dump(OS); for (unsigned Level = 1; Level <= D->getLevels(); Level++) { if (D->isSplitable(Level)) { @@ -227,8 +227,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA, void DependenceAnalysisWrapperPass::print(raw_ostream &OS, const Module *) const { - dumpExampleDependence(OS, info.get(), - getAnalysis().getSE(), false); + dumpExampleDependence( + OS, info.get(), getAnalysis().getSE(), false); } PreservedAnalyses @@ -249,33 +249,26 @@ bool Dependence::isInput() const { return Src->mayReadFromMemory() && Dst->mayReadFromMemory(); } - // Returns true if this is an output dependence. bool Dependence::isOutput() const { return Src->mayWriteToMemory() && Dst->mayWriteToMemory(); } - // Returns true if this is an flow (aka true) dependence. bool Dependence::isFlow() const { return Src->mayWriteToMemory() && Dst->mayReadFromMemory(); } - // Returns true if this is an anti dependence. bool Dependence::isAnti() const { return Src->mayReadFromMemory() && Dst->mayWriteToMemory(); } - // Returns true if a particular level is scalar; that is, // if no subscript in the source or destination mention the induction // variable associated with the loop at this level. // Leave this out of line, so it will serve as a virtual method anchor -bool Dependence::isScalar(unsigned level) const { - return false; -} - +bool Dependence::isScalar(unsigned level) const { return false; } //===----------------------------------------------------------------------===// // FullDependence methods @@ -338,8 +331,7 @@ bool FullDependence::normalize(ScalarEvolution *SE) { DV[Level - 1].Direction = RevDirection; // Reverse the dependence distance as well. if (DV[Level - 1].Distance != nullptr) - DV[Level - 1].Distance = - SE->getNegativeSCEV(DV[Level - 1].Distance); + DV[Level - 1].Distance = SE->getNegativeSCEV(DV[Level - 1].Distance); } LLVM_DEBUG(dbgs() << "After normalizing negative direction vectors:\n"; @@ -355,14 +347,12 @@ unsigned FullDependence::getDirection(unsigned Level) const { return DV[Level - 1].Direction; } - // Returns the distance (or NULL) associated with a particular level. const SCEV *FullDependence::getDistance(unsigned Level) const { assert(0 < Level && Level <= Levels && "Level out of range"); return DV[Level - 1].Distance; } - // Returns true if a particular level is scalar; that is, // if no subscript in the source or destination mention the induction // variable associated with the loop at this level. @@ -371,7 +361,6 @@ bool FullDependence::isScalar(unsigned Level) const { return DV[Level - 1].Scalar; } - // Returns true if peeling the first iteration from this loop // will break this dependence. bool FullDependence::isPeelFirst(unsigned Level) const { @@ -379,7 +368,6 @@ bool FullDependence::isPeelFirst(unsigned Level) const { return DV[Level - 1].PeelFirst; } - // Returns true if peeling the last iteration from this loop // will break this dependence. bool FullDependence::isPeelLast(unsigned Level) const { @@ -387,14 +375,12 @@ bool FullDependence::isPeelLast(unsigned Level) const { return DV[Level - 1].PeelLast; } - // Returns true if splitting this loop will break the dependence. bool FullDependence::isSplitable(unsigned Level) const { assert(0 < Level && Level <= Levels && "Level out of range"); return DV[Level - 1].Splitable; } - //===----------------------------------------------------------------------===// // DependenceInfo::Constraint methods @@ -405,7 +391,6 @@ const SCEV *DependenceInfo::Constraint::getX() const { return A; } - // If constraint is a point , returns Y. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getY() const { @@ -413,7 +398,6 @@ const SCEV *DependenceInfo::Constraint::getY() const { return B; } - // If constraint is a line AX + BY = C, returns A. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getA() const { @@ -422,7 +406,6 @@ const SCEV *DependenceInfo::Constraint::getA() const { return A; } - // If constraint is a line AX + BY = C, returns B. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getB() const { @@ -431,7 +414,6 @@ const SCEV *DependenceInfo::Constraint::getB() const { return B; } - // If constraint is a line AX + BY = C, returns C. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getC() const { @@ -440,7 +422,6 @@ const SCEV *DependenceInfo::Constraint::getC() const { return C; } - // If constraint is a distance, returns D. // Otherwise assert. const SCEV *DependenceInfo::Constraint::getD() const { @@ -448,7 +429,6 @@ const SCEV *DependenceInfo::Constraint::getD() const { return SE->getNegativeSCEV(C); } - // Returns the loop associated with this constraint. const Loop *DependenceInfo::Constraint::getAssociatedLoop() const { assert((Kind == Distance || Kind == Line || Kind == Point) && @@ -499,17 +479,16 @@ LLVM_DUMP_METHOD void DependenceInfo::Constraint::dump(raw_ostream &OS) const { else if (isPoint()) OS << " Point is <" << *getX() << ", " << *getY() << ">\n"; else if (isDistance()) - OS << " Distance is " << *getD() << - " (" << *getA() << "*X + " << *getB() << "*Y = " << *getC() << ")\n"; + OS << " Distance is " << *getD() << " (" << *getA() << "*X + " << *getB() + << "*Y = " << *getC() << ")\n"; else if (isLine()) - OS << " Line is " << *getA() << "*X + " << - *getB() << "*Y = " << *getC() << "\n"; + OS << " Line is " << *getA() << "*X + " << *getB() << "*Y = " << *getC() + << "\n"; else llvm_unreachable("unknown constraint type in Constraint::dump"); } #endif - // Updates X with the intersection // of the Constraints X and Y. Returns true if X has changed. // Corresponds to Figure 4 from the paper @@ -591,15 +570,14 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { const SCEV *A1B2 = SE->getMulExpr(X->getA(), Y->getB()); const SCEV *A2B1 = SE->getMulExpr(Y->getA(), X->getB()); const SCEVConstant *C1A2_C2A1 = - dyn_cast(SE->getMinusSCEV(C1A2, C2A1)); + dyn_cast(SE->getMinusSCEV(C1A2, C2A1)); const SCEVConstant *C1B2_C2B1 = - dyn_cast(SE->getMinusSCEV(C1B2, C2B1)); + dyn_cast(SE->getMinusSCEV(C1B2, C2B1)); const SCEVConstant *A1B2_A2B1 = - dyn_cast(SE->getMinusSCEV(A1B2, A2B1)); + dyn_cast(SE->getMinusSCEV(A1B2, A2B1)); const SCEVConstant *A2B1_A1B2 = - dyn_cast(SE->getMinusSCEV(A2B1, A1B2)); - if (!C1B2_C2B1 || !C1A2_C2A1 || - !A1B2_A2B1 || !A2B1_A1B2) + dyn_cast(SE->getMinusSCEV(A2B1, A1B2)); + if (!C1B2_C2B1 || !C1A2_C2A1 || !A1B2_A2B1 || !A2B1_A1B2) return false; APInt Xtop = C1B2_C2B1->getAPInt(); APInt Xbot = A1B2_A2B1->getAPInt(); @@ -626,8 +604,8 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { ++DeltaSuccesses; return true; } - if (const SCEVConstant *CUB = - collectConstantUpperBound(X->getAssociatedLoop(), Prod1->getType())) { + if (const SCEVConstant *CUB = collectConstantUpperBound( + X->getAssociatedLoop(), Prod1->getType())) { const APInt &UpperBound = CUB->getAPInt(); LLVM_DEBUG(dbgs() << "\t\tupper bound = " << UpperBound << "\n"); if (Xq.sgt(UpperBound) || Yq.sgt(UpperBound)) { @@ -636,8 +614,7 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { return true; } } - X->setPoint(SE->getConstant(Xq), - SE->getConstant(Yq), + X->setPoint(SE->getConstant(Xq), SE->getConstant(Yq), X->getAssociatedLoop()); ++DeltaSuccesses; return true; @@ -667,7 +644,6 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) { return false; } - //===----------------------------------------------------------------------===// // DependenceInfo methods @@ -737,8 +713,7 @@ void Dependence::dump(raw_ostream &OS) const { // tbaa, non-overlapping regions etc), then it is known there is no dependecy. // Otherwise the underlying objects are checked to see if they point to // different identifiable objects. -static AliasResult underlyingObjectsAlias(AAResults *AA, - const DataLayout &DL, +static AliasResult underlyingObjectsAlias(AAResults *AA, const DataLayout &DL, const MemoryLocation &LocA, const MemoryLocation &LocB) { // Check the original locations (minus size) for noalias, which can happen for @@ -773,8 +748,7 @@ static AliasResult underlyingObjectsAlias(AAResults *AA, // Returns true if the load or store can be analyzed. Atomic and volatile // operations have properties which this analysis does not understand. -static -bool isLoadOrStore(const Instruction *I) { +static bool isLoadOrStore(const Instruction *I) { if (const LoadInst *LI = dyn_cast(I)) return LI->isUnordered(); else if (const StoreInst *SI = dyn_cast(I)) @@ -782,7 +756,6 @@ bool isLoadOrStore(const Instruction *I) { return false; } - // Examines the loop nesting of the Src and Dst // instructions and establishes their shared loops. Sets the variables // CommonLevels, SrcLevels, and MaxLevels. @@ -860,14 +833,12 @@ void DependenceInfo::establishNestingLevels(const Instruction *Src, MaxLevels -= CommonLevels; } - // Given one of the loops containing the source, return // its level index in our numbering scheme. unsigned DependenceInfo::mapSrcLoop(const Loop *SrcLoop) const { return SrcLoop->getLoopDepth(); } - // Given one of the loops containing the destination, // return its level index in our numbering scheme. unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const { @@ -880,7 +851,6 @@ unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const { return D; } - // Returns true if Expression is loop invariant in LoopNest. bool DependenceInfo::isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const { @@ -896,8 +866,6 @@ bool DependenceInfo::isLoopInvariant(const SCEV *Expression, return SE->isLoopInvariant(Expression, LoopNest->getOutermostLoop()); } - - // Finds the set of loops from the LoopNest that // have a level <= CommonLevels and are referred to by the SCEV Expression. void DependenceInfo::collectCommonLoops(const SCEV *Expression, @@ -924,9 +892,9 @@ void DependenceInfo::unifySubscriptType(ArrayRef Pairs) { IntegerType *SrcTy = dyn_cast(Src->getType()); IntegerType *DstTy = dyn_cast(Dst->getType()); if (SrcTy == nullptr || DstTy == nullptr) { - assert(SrcTy == DstTy && "This function only unify integer types and " - "expect Src and Dst share the same type " - "otherwise."); + assert(SrcTy == DstTy && + "This function only unify integer types and " + "expect Src and Dst share the same type otherwise."); continue; } if (SrcTy->getBitWidth() > widestWidthSeen) { @@ -939,7 +907,6 @@ void DependenceInfo::unifySubscriptType(ArrayRef Pairs) { } } - assert(widestWidthSeen > 0); // Now extend each pair to the widest seen. @@ -949,9 +916,9 @@ void DependenceInfo::unifySubscriptType(ArrayRef Pairs) { IntegerType *SrcTy = dyn_cast(Src->getType()); IntegerType *DstTy = dyn_cast(Dst->getType()); if (SrcTy == nullptr || DstTy == nullptr) { - assert(SrcTy == DstTy && "This function only unify integer types and " - "expect Src and Dst share the same type " - "otherwise."); + assert(SrcTy == DstTy && + "This function only unify integer types and " + "expect Src and Dst share the same type otherwise."); continue; } if (SrcTy->getBitWidth() < widestWidthSeen) @@ -1028,7 +995,6 @@ bool DependenceInfo::checkDstSubscript(const SCEV *Dst, const Loop *LoopNest, return checkSubscript(Dst, LoopNest, Loops, false); } - // Examines the subscript pair (the Src and Dst SCEVs) // and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear. // Collects the associated loops in a set. @@ -1049,14 +1015,12 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest, return Subscript::ZIV; if (N == 1) return Subscript::SIV; - if (N == 2 && (SrcLoops.count() == 0 || - DstLoops.count() == 0 || + if (N == 2 && (SrcLoops.count() == 0 || DstLoops.count() == 0 || (SrcLoops.count() == 1 && DstLoops.count() == 1))) return Subscript::RDIV; return Subscript::MIV; } - // A wrapper around SCEV::isKnownPredicate. // Looks for cases where we're interested in comparing for equality. // If both X and Y have been identically sign or zero extended, @@ -1069,12 +1033,9 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest, // involving symbolics. bool DependenceInfo::isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X, const SCEV *Y) const { - if (Pred == CmpInst::ICMP_EQ || - Pred == CmpInst::ICMP_NE) { - if ((isa(X) && - isa(Y)) || - (isa(X) && - isa(Y))) { + if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { + if ((isa(X) && isa(Y)) || + (isa(X) && isa(Y))) { const SCEVIntegralCastExpr *CX = cast(X); const SCEVIntegralCastExpr *CY = cast(Y); const SCEV *Xop = CX->getOperand(); @@ -1111,7 +1072,10 @@ bool DependenceInfo::isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X, } } -/// Compare to see if S is less than Size, using isKnownNegative(S - max(Size, 1)) +/// Compare to see if S is less than Size, using +/// +/// isKnownNegative(S - max(Size, 1)) +/// /// with some extra checking if S is an AddRec and we can prove less-than using /// the loop bounds. bool DependenceInfo::isKnownLessThan(const SCEV *S, const SCEV *Size) const { @@ -1178,7 +1142,6 @@ const SCEV *DependenceInfo::collectUpperBound(const Loop *L, Type *T) const { return nullptr; } - // Calls collectUpperBound(), then attempts to cast it to SCEVConstant. // If the cast fails, returns NULL. const SCEVConstant *DependenceInfo::collectConstantUpperBound(const Loop *L, @@ -1188,7 +1151,6 @@ const SCEVConstant *DependenceInfo::collectConstantUpperBound(const Loop *L, return nullptr; } - // testZIV - // When we have a pair of subscripts of the form [c1] and [c2], // where c1 and c2 are both loop invariant, we attack it using @@ -1218,7 +1180,6 @@ bool DependenceInfo::testZIV(const SCEV *Src, const SCEV *Dst, return false; // possibly dependent } - // strongSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.1 // @@ -1270,9 +1231,9 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound); LLVM_DEBUG(dbgs() << ", " << *UpperBound->getType() << "\n"); const SCEV *AbsDelta = - SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta); + SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta); const SCEV *AbsCoeff = - SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff); + SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff); const SCEV *Product = SE->getMulExpr(UpperBound, AbsCoeff); if (isKnownPredicate(CmpInst::ICMP_SGT, AbsDelta, Product)) { // Distance greater than trip count - no dependence @@ -1286,7 +1247,7 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, if (isa(Delta) && isa(Coeff)) { APInt ConstDelta = cast(Delta)->getAPInt(); APInt ConstCoeff = cast(Coeff)->getAPInt(); - APInt Distance = ConstDelta; // these need to be initialized + APInt Distance = ConstDelta; // these need to be initialized APInt Remainder = ConstDelta; APInt::sdivrem(ConstDelta, ConstCoeff, Distance, Remainder); LLVM_DEBUG(dbgs() << "\t Distance = " << Distance << "\n"); @@ -1307,29 +1268,25 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, else Result.DV[Level].Direction &= Dependence::DVEntry::EQ; ++StrongSIVsuccesses; - } - else if (Delta->isZero()) { + } else if (Delta->isZero()) { // since 0/X == 0 Result.DV[Level].Distance = Delta; NewConstraint.setDistance(Delta, CurLoop); Result.DV[Level].Direction &= Dependence::DVEntry::EQ; ++StrongSIVsuccesses; - } - else { + } else { if (Coeff->isOne()) { LLVM_DEBUG(dbgs() << "\t Distance = " << *Delta << "\n"); Result.DV[Level].Distance = Delta; // since X/1 == X NewConstraint.setDistance(Delta, CurLoop); - } - else { + } else { Result.Consistent = false; - NewConstraint.setLine(Coeff, - SE->getNegativeSCEV(Coeff), + NewConstraint.setLine(Coeff, SE->getNegativeSCEV(Coeff), SE->getNegativeSCEV(Delta), CurLoop); } // maybe we can get a useful direction - bool DeltaMaybeZero = !SE->isKnownNonZero(Delta); + bool DeltaMaybeZero = !SE->isKnownNonZero(Delta); bool DeltaMaybePositive = !SE->isKnownNonPositive(Delta); bool DeltaMaybeNegative = !SE->isKnownNonNegative(Delta); bool CoeffMaybePositive = !SE->isKnownNonPositive(Coeff); @@ -1353,7 +1310,6 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst, return false; } - // weakCrossingSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.2 // @@ -1447,8 +1403,8 @@ bool DependenceInfo::weakCrossingSIVtest( if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) { LLVM_DEBUG(dbgs() << "\t UpperBound = " << *UpperBound << "\n"); const SCEV *ConstantTwo = SE->getConstant(UpperBound->getType(), 2); - const SCEV *ML = SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound), - ConstantTwo); + const SCEV *ML = + SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound), ConstantTwo); LLVM_DEBUG(dbgs() << "\t ML = " << *ML << "\n"); if (isKnownPredicate(CmpInst::ICMP_SGT, Delta, ML)) { // Delta too big, no dependence @@ -1498,7 +1454,6 @@ bool DependenceInfo::weakCrossingSIVtest( return false; } - // Kirch's algorithm, from // // Optimizing Supercompilers for Supercomputers @@ -1519,9 +1474,11 @@ static bool findGCD(unsigned Bits, const APInt &AM, const APInt &BM, APInt R = G0; APInt::sdivrem(G0, G1, Q, R); while (R != 0) { + // clang-format off APInt A2 = A0 - Q*A1; A0 = A1; A1 = A2; APInt B2 = B0 - Q*B1; B0 = B1; B1 = B2; G0 = G1; G1 = R; + // clang-format on APInt::sdivrem(G0, G1, Q, R); } G = G1; @@ -1543,8 +1500,7 @@ static APInt floorOfQuotient(const APInt &A, const APInt &B) { APInt::sdivrem(A, B, Q, R); if (R == 0) return Q; - if ((A.sgt(0) && B.sgt(0)) || - (A.slt(0) && B.slt(0))) + if ((A.sgt(0) && B.sgt(0)) || (A.slt(0) && B.slt(0))) return Q; else return Q - 1; @@ -1556,8 +1512,7 @@ static APInt ceilingOfQuotient(const APInt &A, const APInt &B) { APInt::sdivrem(A, B, Q, R); if (R == 0) return Q; - if ((A.sgt(0) && B.sgt(0)) || - (A.slt(0) && B.slt(0))) + if ((A.sgt(0) && B.sgt(0)) || (A.slt(0) && B.slt(0))) return Q + 1; else return Q; @@ -1733,17 +1688,14 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, return Result.DV[Level].Direction == Dependence::DVEntry::NONE; } - // Return true if the divisor evenly divides the dividend. -static -bool isRemainderZero(const SCEVConstant *Dividend, - const SCEVConstant *Divisor) { +static bool isRemainderZero(const SCEVConstant *Dividend, + const SCEVConstant *Divisor) { const APInt &ConstDividend = Dividend->getAPInt(); const APInt &ConstDivisor = Divisor->getAPInt(); return ConstDividend.srem(ConstDivisor) == 0; } - // weakZeroSrcSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.2 // @@ -1807,11 +1759,11 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff, const SCEVConstant *ConstCoeff = dyn_cast(DstCoeff); if (!ConstCoeff) return false; - const SCEV *AbsCoeff = - SE->isKnownNegative(ConstCoeff) ? - SE->getNegativeSCEV(ConstCoeff) : ConstCoeff; + const SCEV *AbsCoeff = SE->isKnownNegative(ConstCoeff) + ? SE->getNegativeSCEV(ConstCoeff) + : ConstCoeff; const SCEV *NewDelta = - SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; + SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; // check that Delta/SrcCoeff < iteration count // really check NewDelta < count*AbsCoeff @@ -1853,7 +1805,6 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff, return false; } - // weakZeroDstSIVtest - // From the paper, Practical Dependence Testing, Section 4.2.2 // @@ -1916,11 +1867,11 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff, const SCEVConstant *ConstCoeff = dyn_cast(SrcCoeff); if (!ConstCoeff) return false; - const SCEV *AbsCoeff = - SE->isKnownNegative(ConstCoeff) ? - SE->getNegativeSCEV(ConstCoeff) : ConstCoeff; + const SCEV *AbsCoeff = SE->isKnownNegative(ConstCoeff) + ? SE->getNegativeSCEV(ConstCoeff) + : ConstCoeff; const SCEV *NewDelta = - SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; + SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta; // check that Delta/SrcCoeff < iteration count // really check NewDelta < count*AbsCoeff @@ -1962,7 +1913,6 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff, return false; } - // exactRDIVtest - Tests the RDIV subscript pair for dependence. // Things of the form [c1 + a*i] and [c2 + b*j], // where i and j are induction variable, c1 and c2 are loop invariant, @@ -2084,7 +2034,6 @@ bool DependenceInfo::exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff, return TL.sgt(TU); } - // symbolicRDIVtest - // In Section 4.5 of the Practical Dependence Testing paper,the authors // introduce a special case of Banerjee's Inequalities (also called the @@ -2167,8 +2116,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, return true; } } - } - else if (SE->isKnownNonPositive(A2)) { + } else if (SE->isKnownNonPositive(A2)) { // a1 >= 0 && a2 <= 0 if (N1 && N2) { // make sure that c2 - c1 <= a1*N1 - a2*N2 @@ -2187,8 +2135,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, return true; } } - } - else if (SE->isKnownNonPositive(A1)) { + } else if (SE->isKnownNonPositive(A1)) { if (SE->isKnownNonNegative(A2)) { // a1 <= 0 && a2 >= 0 if (N1 && N2) { @@ -2207,8 +2154,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, ++SymbolicRDIVindependence; return true; } - } - else if (SE->isKnownNonPositive(A2)) { + } else if (SE->isKnownNonPositive(A2)) { // a1 <= 0 && a2 <= 0 if (N1) { // make sure that a1*N1 <= c2 - c1 @@ -2233,7 +2179,6 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2, return false; } - // testSIV - // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 - a2*i] // where i is an induction variable, c1 and c2 are loop invariant, and a1 and @@ -2260,17 +2205,17 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, Level = mapSrcLoop(CurLoop); bool disproven; if (SrcCoeff == DstCoeff) - disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, - Level, Result, NewConstraint); + disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level, + Result, NewConstraint); else if (SrcCoeff == SE->getNegativeSCEV(DstCoeff)) disproven = weakCrossingSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level, Result, NewConstraint, SplitIter); else disproven = exactSIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, Level, Result, NewConstraint); - return disproven || - gcdMIVtest(Src, Dst, Result) || - symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, CurLoop); + return disproven || gcdMIVtest(Src, Dst, Result) || + symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, + CurLoop); } if (SrcAddRec) { const SCEV *SrcConst = SrcAddRec->getStart(); @@ -2278,9 +2223,9 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, const SCEV *DstConst = Dst; const Loop *CurLoop = SrcAddRec->getLoop(); Level = mapSrcLoop(CurLoop); - return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, - Level, Result, NewConstraint) || - gcdMIVtest(Src, Dst, Result); + return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level, + Result, NewConstraint) || + gcdMIVtest(Src, Dst, Result); } if (DstAddRec) { const SCEV *DstConst = DstAddRec->getStart(); @@ -2288,15 +2233,14 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level, const SCEV *SrcConst = Src; const Loop *CurLoop = DstAddRec->getLoop(); Level = mapDstLoop(CurLoop); - return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst, - CurLoop, Level, Result, NewConstraint) || - gcdMIVtest(Src, Dst, Result); + return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst, CurLoop, Level, + Result, NewConstraint) || + gcdMIVtest(Src, Dst, Result); } llvm_unreachable("SIV test expected at least one AddRec"); return false; } - // testRDIV - // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*j] // where i and j are induction variables, c1 and c2 are loop invariant, @@ -2333,46 +2277,37 @@ bool DependenceInfo::testRDIV(const SCEV *Src, const SCEV *Dst, DstConst = DstAddRec->getStart(); DstCoeff = DstAddRec->getStepRecurrence(*SE); DstLoop = DstAddRec->getLoop(); - } - else if (SrcAddRec) { + } else if (SrcAddRec) { if (const SCEVAddRecExpr *tmpAddRec = - dyn_cast(SrcAddRec->getStart())) { + dyn_cast(SrcAddRec->getStart())) { SrcConst = tmpAddRec->getStart(); SrcCoeff = tmpAddRec->getStepRecurrence(*SE); SrcLoop = tmpAddRec->getLoop(); DstConst = Dst; DstCoeff = SE->getNegativeSCEV(SrcAddRec->getStepRecurrence(*SE)); DstLoop = SrcAddRec->getLoop(); - } - else + } else llvm_unreachable("RDIV reached by surprising SCEVs"); - } - else if (DstAddRec) { + } else if (DstAddRec) { if (const SCEVAddRecExpr *tmpAddRec = - dyn_cast(DstAddRec->getStart())) { + dyn_cast(DstAddRec->getStart())) { DstConst = tmpAddRec->getStart(); DstCoeff = tmpAddRec->getStepRecurrence(*SE); DstLoop = tmpAddRec->getLoop(); SrcConst = Src; SrcCoeff = SE->getNegativeSCEV(DstAddRec->getStepRecurrence(*SE)); SrcLoop = DstAddRec->getLoop(); - } - else + } else llvm_unreachable("RDIV reached by surprising SCEVs"); - } - else + } else llvm_unreachable("RDIV expected at least one AddRec"); - return exactRDIVtest(SrcCoeff, DstCoeff, - SrcConst, DstConst, - SrcLoop, DstLoop, + return exactRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, SrcLoop, DstLoop, Result) || - gcdMIVtest(Src, Dst, Result) || - symbolicRDIVtest(SrcCoeff, DstCoeff, - SrcConst, DstConst, - SrcLoop, DstLoop); + gcdMIVtest(Src, Dst, Result) || + symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, SrcLoop, + DstLoop); } - // Tests the single-subscript MIV pair (Src and Dst) for dependence. // Return true if dependence disproved. // Can sometimes refine direction vectors. @@ -2383,7 +2318,7 @@ bool DependenceInfo::testMIV(const SCEV *Src, const SCEV *Dst, LLVM_DEBUG(dbgs() << " dst = " << *Dst << "\n"); Result.Consistent = false; return gcdMIVtest(Src, Dst, Result) || - banerjeeMIVtest(Src, Dst, Loops, Result); + banerjeeMIVtest(Src, Dst, Loops, Result); } // Given a product, e.g., 10*X*Y, returns the first constant operand, @@ -2428,7 +2363,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, // we can't quit the loop just because the GCD == 1. const SCEV *Coefficients = Src; while (const SCEVAddRecExpr *AddRec = - dyn_cast(Coefficients)) { + dyn_cast(Coefficients)) { const SCEV *Coeff = AddRec->getStepRecurrence(*SE); // If the coefficient is the product of a constant and other stuff, // we can use the constant in the GCD computation. @@ -2446,7 +2381,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, // we can't quit the loop just because the GCD == 1. Coefficients = Dst; while (const SCEVAddRecExpr *AddRec = - dyn_cast(Coefficients)) { + dyn_cast(Coefficients)) { const SCEV *Coeff = AddRec->getStepRecurrence(*SE); // If the coefficient is the product of a constant and other stuff, // we can use the constant in the GCD computation. @@ -2468,16 +2403,14 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, if (isa(Operand)) { assert(!Constant && "Surprised to find multiple constants"); Constant = cast(Operand); - } - else if (const SCEVMulExpr *Product = dyn_cast(Operand)) { + } else if (const SCEVMulExpr *Product = dyn_cast(Operand)) { // Search for constant operand to participate in GCD; // If none found; return false. std::optional ConstOp = getConstantPart(Product); if (!ConstOp) return false; ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD, ConstOp->abs()); - } - else + } else return false; } } @@ -2512,7 +2445,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, bool Improved = false; Coefficients = Src; while (const SCEVAddRecExpr *AddRec = - dyn_cast(Coefficients)) { + dyn_cast(Coefficients)) { Coefficients = AddRec->getStart(); const Loop *CurLoop = AddRec->getLoop(); RunningGCD = ExtraGCD; @@ -2578,7 +2511,6 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst, return false; } - //===----------------------------------------------------------------------===// // banerjeeMIVtest - // Use Banerjee's Inequalities to test an MIV subscript pair. @@ -2652,8 +2584,8 @@ bool DependenceInfo::banerjeeMIVtest(const SCEV *Src, const SCEV *Dst, if (testBounds(Dependence::DVEntry::ALL, 0, Bound, Delta)) { // Explore the direction vector hierarchy. unsigned DepthExpanded = 0; - unsigned NewDeps = exploreDirections(1, A, B, Bound, - Loops, DepthExpanded, Delta); + unsigned NewDeps = + exploreDirections(1, A, B, Bound, Loops, DepthExpanded, Delta); if (NewDeps > 0) { bool Improved = false; for (unsigned K = 1; K <= CommonLevels; ++K) { @@ -2670,23 +2602,20 @@ bool DependenceInfo::banerjeeMIVtest(const SCEV *Src, const SCEV *Dst, } if (Improved) ++BanerjeeSuccesses; - } - else { + } else { ++BanerjeeIndependence; Disproved = true; } - } - else { + } else { ++BanerjeeIndependence; Disproved = true; } - delete [] Bound; - delete [] A; - delete [] B; + delete[] Bound; + delete[] A; + delete[] B; return Disproved; } - // Hierarchically expands the direction vector // search space, combining the directions of discovered dependences // in the DirSet field of Bound. Returns the number of distinct @@ -2788,27 +2717,26 @@ unsigned DependenceInfo::exploreDirections(unsigned Level, CoefficientInfo *A, // test bounds for <, *, *, ... if (testBounds(Dependence::DVEntry::LT, Level, Bound, Delta)) - NewDeps += exploreDirections(Level + 1, A, B, Bound, - Loops, DepthExpanded, Delta); + NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); // Test bounds for =, *, *, ... if (testBounds(Dependence::DVEntry::EQ, Level, Bound, Delta)) - NewDeps += exploreDirections(Level + 1, A, B, Bound, - Loops, DepthExpanded, Delta); + NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); // test bounds for >, *, *, ... if (testBounds(Dependence::DVEntry::GT, Level, Bound, Delta)) - NewDeps += exploreDirections(Level + 1, A, B, Bound, - Loops, DepthExpanded, Delta); + NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); Bound[Level].Direction = Dependence::DVEntry::ALL; return NewDeps; - } - else - return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, Delta); + } else + return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, + Delta); } - // Returns true iff the current bounds are plausible. bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level, BoundInfo *Bound, const SCEV *Delta) const { @@ -2822,7 +2750,6 @@ bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level, return true; } - // Computes the upper and lower bounds for level K // using the * direction. Records them in Bound. // Wolfe gives the equations @@ -2840,17 +2767,16 @@ bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level, // and the upper bound is always >= 0. void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::ALL] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::ALL] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::ALL] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::ALL] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { - Bound[K].Lower[Dependence::DVEntry::ALL] = - SE->getMulExpr(SE->getMinusSCEV(A[K].NegPart, B[K].PosPart), - Bound[K].Iterations); - Bound[K].Upper[Dependence::DVEntry::ALL] = - SE->getMulExpr(SE->getMinusSCEV(A[K].PosPart, B[K].NegPart), - Bound[K].Iterations); - } - else { + Bound[K].Lower[Dependence::DVEntry::ALL] = SE->getMulExpr( + SE->getMinusSCEV(A[K].NegPart, B[K].PosPart), Bound[K].Iterations); + Bound[K].Upper[Dependence::DVEntry::ALL] = SE->getMulExpr( + SE->getMinusSCEV(A[K].PosPart, B[K].NegPart), Bound[K].Iterations); + } else { // If the difference is 0, we won't need to know the number of iterations. if (isKnownPredicate(CmpInst::ICMP_EQ, A[K].NegPart, B[K].PosPart)) Bound[K].Lower[Dependence::DVEntry::ALL] = @@ -2861,7 +2787,6 @@ void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, } } - // Computes the upper and lower bounds for level K // using the = direction. Records them in Bound. // Wolfe gives the equations @@ -2879,18 +2804,19 @@ void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, // and the upper bound is always >= 0. void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::EQ] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::EQ] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::EQ] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::EQ] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff); const SCEV *NegativePart = getNegativePart(Delta); Bound[K].Lower[Dependence::DVEntry::EQ] = - SE->getMulExpr(NegativePart, Bound[K].Iterations); + SE->getMulExpr(NegativePart, Bound[K].Iterations); const SCEV *PositivePart = getPositivePart(Delta); Bound[K].Upper[Dependence::DVEntry::EQ] = - SE->getMulExpr(PositivePart, Bound[K].Iterations); - } - else { + SE->getMulExpr(PositivePart, Bound[K].Iterations); + } else { // If the positive/negative part of the difference is 0, // we won't need to know the number of iterations. const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff); @@ -2903,7 +2829,6 @@ void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, } } - // Computes the upper and lower bounds for level K // using the < direction. Records them in Bound. // Wolfe gives the equations @@ -2919,35 +2844,35 @@ void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, // We must be careful to handle the case where the upper bound is unknown. void DependenceInfo::findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::LT] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::LT] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::LT] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::LT] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { const SCEV *Iter_1 = SE->getMinusSCEV( Bound[K].Iterations, SE->getOne(Bound[K].Iterations->getType())); const SCEV *NegPart = - getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); + getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); Bound[K].Lower[Dependence::DVEntry::LT] = - SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff); + SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff); const SCEV *PosPart = - getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); + getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); Bound[K].Upper[Dependence::DVEntry::LT] = - SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff); - } - else { + SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff); + } else { // If the positive/negative part of the difference is 0, // we won't need to know the number of iterations. const SCEV *NegPart = - getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); + getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff)); if (NegPart->isZero()) Bound[K].Lower[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff); const SCEV *PosPart = - getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); + getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff)); if (PosPart->isZero()) Bound[K].Upper[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff); } } - // Computes the upper and lower bounds for level K // using the > direction. Records them in Bound. // Wolfe gives the equations @@ -2963,45 +2888,45 @@ void DependenceInfo::findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, // We must be careful to handle the case where the upper bound is unknown. void DependenceInfo::findBoundsGT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound, unsigned K) const { - Bound[K].Lower[Dependence::DVEntry::GT] = nullptr; // Default value = -infinity. - Bound[K].Upper[Dependence::DVEntry::GT] = nullptr; // Default value = +infinity. + Bound[K].Lower[Dependence::DVEntry::GT] = + nullptr; // Default value = -infinity. + Bound[K].Upper[Dependence::DVEntry::GT] = + nullptr; // Default value = +infinity. if (Bound[K].Iterations) { const SCEV *Iter_1 = SE->getMinusSCEV( Bound[K].Iterations, SE->getOne(Bound[K].Iterations->getType())); const SCEV *NegPart = - getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); + getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); Bound[K].Lower[Dependence::DVEntry::GT] = - SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff); + SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff); const SCEV *PosPart = - getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); + getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); Bound[K].Upper[Dependence::DVEntry::GT] = - SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff); - } - else { + SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff); + } else { // If the positive/negative part of the difference is 0, // we won't need to know the number of iterations. - const SCEV *NegPart = getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); + const SCEV *NegPart = + getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart)); if (NegPart->isZero()) Bound[K].Lower[Dependence::DVEntry::GT] = A[K].Coeff; - const SCEV *PosPart = getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); + const SCEV *PosPart = + getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart)); if (PosPart->isZero()) Bound[K].Upper[Dependence::DVEntry::GT] = A[K].Coeff; } } - // X^+ = max(X, 0) const SCEV *DependenceInfo::getPositivePart(const SCEV *X) const { return SE->getSMaxExpr(X, SE->getZero(X->getType())); } - // X^- = min(X, 0) const SCEV *DependenceInfo::getNegativePart(const SCEV *X) const { return SE->getSMinExpr(X, SE->getZero(X->getType())); } - // Walks through the subscript, // collecting each coefficient, the associated loop bounds, // and recording its positive and negative parts for later use. @@ -3046,7 +2971,6 @@ DependenceInfo::collectCoeffInfo(const SCEV *Subscript, bool SrcFlag, return CI; } - // Looks through all the bounds info and // computes the lower bound given the current direction settings // at each level. If the lower bound for any level is -inf, @@ -3062,7 +2986,6 @@ const SCEV *DependenceInfo::getLowerBound(BoundInfo *Bound) const { return Sum; } - // Looks through all the bounds info and // computes the upper bound given the current direction settings // at each level. If the upper bound at any level is +inf, @@ -3078,7 +3001,6 @@ const SCEV *DependenceInfo::getUpperBound(BoundInfo *Bound) const { return Sum; } - //===----------------------------------------------------------------------===// // Constraint manipulation for Delta test. @@ -3098,7 +3020,6 @@ const SCEV *DependenceInfo::findCoefficient(const SCEV *Expr, return findCoefficient(AddRec->getStart(), TargetLoop); } - // Given a linear SCEV, // return the SCEV given by zeroing out the coefficient // corresponding to the specified loop. @@ -3112,12 +3033,10 @@ const SCEV *DependenceInfo::zeroCoefficient(const SCEV *Expr, if (AddRec->getLoop() == TargetLoop) return AddRec->getStart(); return SE->getAddRecExpr(zeroCoefficient(AddRec->getStart(), TargetLoop), - AddRec->getStepRecurrence(*SE), - AddRec->getLoop(), + AddRec->getStepRecurrence(*SE), AddRec->getLoop(), AddRec->getNoWrapFlags()); } - // Given a linear SCEV Expr, // return the SCEV given by adding some Value to the // coefficient corresponding to the specified TargetLoop. @@ -3128,17 +3047,13 @@ const SCEV *DependenceInfo::addToCoefficient(const SCEV *Expr, const SCEV *Value) const { const SCEVAddRecExpr *AddRec = dyn_cast(Expr); if (!AddRec) // create a new addRec - return SE->getAddRecExpr(Expr, - Value, - TargetLoop, + return SE->getAddRecExpr(Expr, Value, TargetLoop, SCEV::FlagAnyWrap); // Worst case, with no info. if (AddRec->getLoop() == TargetLoop) { const SCEV *Sum = SE->getAddExpr(AddRec->getStepRecurrence(*SE), Value); if (Sum->isZero()) return AddRec->getStart(); - return SE->getAddRecExpr(AddRec->getStart(), - Sum, - AddRec->getLoop(), + return SE->getAddRecExpr(AddRec->getStart(), Sum, AddRec->getLoop(), AddRec->getNoWrapFlags()); } if (SE->isLoopInvariant(AddRec, TargetLoop)) @@ -3149,7 +3064,6 @@ const SCEV *DependenceInfo::addToCoefficient(const SCEV *Expr, AddRec->getNoWrapFlags()); } - // Review the constraints, looking for opportunities // to simplify a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3178,7 +3092,6 @@ bool DependenceInfo::propagate(const SCEV *&Src, const SCEV *&Dst, return Result; } - // Attempt to propagate a distance // constraint into a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3204,7 +3117,6 @@ bool DependenceInfo::propagateDistance(const SCEV *&Src, const SCEV *&Dst, return true; } - // Attempt to propagate a line // constraint into a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3224,22 +3136,22 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, if (A->isZero()) { const SCEVConstant *Bconst = dyn_cast(B); const SCEVConstant *Cconst = dyn_cast(C); - if (!Bconst || !Cconst) return false; + if (!Bconst || !Cconst) + return false; APInt Beta = Bconst->getAPInt(); APInt Charlie = Cconst->getAPInt(); APInt CdivB = Charlie.sdiv(Beta); assert(Charlie.srem(Beta) == 0 && "C should be evenly divisible by B"); const SCEV *AP_K = findCoefficient(Dst, CurLoop); - // Src = SE->getAddExpr(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB))); Src = SE->getMinusSCEV(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB))); Dst = zeroCoefficient(Dst, CurLoop); if (!findCoefficient(Src, CurLoop)->isZero()) Consistent = false; - } - else if (B->isZero()) { + } else if (B->isZero()) { const SCEVConstant *Aconst = dyn_cast(A); const SCEVConstant *Cconst = dyn_cast(C); - if (!Aconst || !Cconst) return false; + if (!Aconst || !Cconst) + return false; APInt Alpha = Aconst->getAPInt(); APInt Charlie = Cconst->getAPInt(); APInt CdivA = Charlie.sdiv(Alpha); @@ -3249,11 +3161,11 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, Src = zeroCoefficient(Src, CurLoop); if (!findCoefficient(Dst, CurLoop)->isZero()) Consistent = false; - } - else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) { + } else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) { const SCEVConstant *Aconst = dyn_cast(A); const SCEVConstant *Cconst = dyn_cast(C); - if (!Aconst || !Cconst) return false; + if (!Aconst || !Cconst) + return false; APInt Alpha = Aconst->getAPInt(); APInt Charlie = Cconst->getAPInt(); APInt CdivA = Charlie.sdiv(Alpha); @@ -3264,8 +3176,7 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, Dst = addToCoefficient(Dst, CurLoop, A_K); if (!findCoefficient(Dst, CurLoop)->isZero()) Consistent = false; - } - else { + } else { // paper is incorrect here, or perhaps just misleading const SCEV *A_K = findCoefficient(Src, CurLoop); Src = SE->getMulExpr(Src, A); @@ -3281,7 +3192,6 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst, return true; } - // Attempt to propagate a point // constraint into a subscript pair (Src and Dst). // Return true if some simplification occurs. @@ -3302,7 +3212,6 @@ bool DependenceInfo::propagatePoint(const SCEV *&Src, const SCEV *&Dst, return true; } - // Update direction vector entry based on the current constraint. void DependenceInfo::updateDirection(Dependence::DVEntry &Level, const Constraint &CurConstraint) const { @@ -3322,34 +3231,28 @@ void DependenceInfo::updateDirection(Dependence::DVEntry &Level, if (!SE->isKnownNonNegative(Level.Distance)) // if may be negative NewDirection |= Dependence::DVEntry::GT; Level.Direction &= NewDirection; - } - else if (CurConstraint.isLine()) { + } else if (CurConstraint.isLine()) { Level.Scalar = false; Level.Distance = nullptr; // direction should be accurate - } - else if (CurConstraint.isPoint()) { + } else if (CurConstraint.isPoint()) { Level.Scalar = false; Level.Distance = nullptr; unsigned NewDirection = Dependence::DVEntry::NONE; - if (!isKnownPredicate(CmpInst::ICMP_NE, - CurConstraint.getY(), + if (!isKnownPredicate(CmpInst::ICMP_NE, CurConstraint.getY(), CurConstraint.getX())) // if X may be = Y NewDirection |= Dependence::DVEntry::EQ; - if (!isKnownPredicate(CmpInst::ICMP_SLE, - CurConstraint.getY(), + if (!isKnownPredicate(CmpInst::ICMP_SLE, CurConstraint.getY(), CurConstraint.getX())) // if Y may be > X NewDirection |= Dependence::DVEntry::LT; - if (!isKnownPredicate(CmpInst::ICMP_SGE, - CurConstraint.getY(), + if (!isKnownPredicate(CmpInst::ICMP_SGE, CurConstraint.getY(), CurConstraint.getX())) // if Y may be < X NewDirection |= Dependence::DVEntry::GT; Level.Direction &= NewDirection; - } - else + } else llvm_unreachable("constraint has unexpected kind"); } @@ -3425,7 +3328,7 @@ bool DependenceInfo::tryDelinearizeFixedSize( dyn_cast(SE->getPointerBase(DstAccessFn)); assert(SrcBase && DstBase && SrcBase == DstBase && "expected src and dst scev unknowns to be equal"); - }); + }); SmallVector SrcSizes; SmallVector DstSizes; @@ -3737,9 +3640,8 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, Pair[P].Group.resize(Pairs); removeMatchingExtensions(&Pair[P]); Pair[P].Classification = - classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), - Pair[P].Dst, LI->getLoopFor(Dst->getParent()), - Pair[P].Loops); + classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst, + LI->getLoopFor(Dst->getParent()), Pair[P].Loops); Pair[P].GroupLoops = Pair[P].Loops; Pair[P].Group.set(P); LLVM_DEBUG(dbgs() << " subscript " << P << "\n"); @@ -3814,18 +3716,15 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, if (Pair[SI].Classification == Subscript::NonLinear) { // ignore these, but collect loops for later ++NonlinearSubscriptPairs; - collectCommonLoops(Pair[SI].Src, - LI->getLoopFor(Src->getParent()), + collectCommonLoops(Pair[SI].Src, LI->getLoopFor(Src->getParent()), Pair[SI].Loops); - collectCommonLoops(Pair[SI].Dst, - LI->getLoopFor(Dst->getParent()), + collectCommonLoops(Pair[SI].Dst, LI->getLoopFor(Dst->getParent()), Pair[SI].Loops); Result.Consistent = false; } else if (Pair[SI].Classification == Subscript::ZIV) { // always separable Separable.set(SI); - } - else { + } else { // SIV, RDIV, or MIV, so check for coupled group bool Done = true; for (unsigned SJ = SI + 1; SJ < Pairs; ++SJ) { @@ -3843,8 +3742,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, if (Pair[SI].Group.count() == 1) { Separable.set(SI); ++SeparableSubscriptPairs; - } - else { + } else { Coupled.set(SI); ++CoupledSubscriptPairs; } @@ -3950,10 +3848,9 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, Constraints, Result.Consistent)) { LLVM_DEBUG(dbgs() << "\t Changed\n"); ++DeltaPropagations; - Pair[SJ].Classification = - classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()), - Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()), - Pair[SJ].Loops); + Pair[SJ].Classification = classifyPair( + Pair[SJ].Src, LI->getLoopFor(Src->getParent()), Pair[SJ].Dst, + LI->getLoopFor(Dst->getParent()), Pair[SJ].Loops); switch (Pair[SJ].Classification) { case Subscript::ZIV: LLVM_DEBUG(dbgs() << "ZIV\n"); @@ -3995,8 +3892,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, LLVM_DEBUG(dbgs() << "MIV test\n"); if (testMIV(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops, Result)) return nullptr; - } - else + } else llvm_unreachable("expected only MIV subscripts at this point"); } @@ -4052,8 +3948,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, break; } } - } - else { + } else { // On the other hand, if all directions are equal and there's no // loop-independent dependence possible, then no dependence exists. bool AllEqual = true; @@ -4158,9 +4053,8 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, Pair[P].Group.resize(Pairs); removeMatchingExtensions(&Pair[P]); Pair[P].Classification = - classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), - Pair[P].Dst, LI->getLoopFor(Dst->getParent()), - Pair[P].Loops); + classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst, + LI->getLoopFor(Dst->getParent()), Pair[P].Loops); Pair[P].GroupLoops = Pair[P].Loops; Pair[P].Group.set(P); } @@ -4172,15 +4066,12 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, for (unsigned SI = 0; SI < Pairs; ++SI) { if (Pair[SI].Classification == Subscript::NonLinear) { // ignore these, but collect loops for later - collectCommonLoops(Pair[SI].Src, - LI->getLoopFor(Src->getParent()), + collectCommonLoops(Pair[SI].Src, LI->getLoopFor(Src->getParent()), Pair[SI].Loops); - collectCommonLoops(Pair[SI].Dst, - LI->getLoopFor(Dst->getParent()), + collectCommonLoops(Pair[SI].Dst, LI->getLoopFor(Dst->getParent()), Pair[SI].Loops); Result.Consistent = false; - } - else if (Pair[SI].Classification == Subscript::ZIV) + } else if (Pair[SI].Classification == Subscript::ZIV) Separable.set(SI); else { // SIV, RDIV, or MIV, so check for coupled group @@ -4214,8 +4105,8 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep, case Subscript::SIV: { unsigned Level; const SCEV *SplitIter = nullptr; - (void) testSIV(Pair[SI].Src, Pair[SI].Dst, Level, - Result, NewConstraint, SplitIter); + (void)testSIV(Pair[SI].Src, Pair[SI].Dst, Level, Result, NewConstraint, + SplitIter); if (Level == SplitLevel) { assert(SplitIter != nullptr); return SplitIter;