Skip to content

Commit 451912a

Browse files
ruilingruiling
authored
[MachineScheduler] Make cluster check more efficient (#150884)
1 parent 1194353 commit 451912a

File tree

5 files changed

+79
-46
lines changed

5 files changed

+79
-46
lines changed

‎llvm/include/llvm/CodeGen/MachineScheduler.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1303,8 +1303,8 @@ class LLVM_ABI GenericScheduler : public GenericSchedulerBase {
13031303
SchedBoundary Top;
13041304
SchedBoundary Bot;
13051305

1306-
ClusterInfo *TopCluster;
1307-
ClusterInfo *BotCluster;
1306+
unsigned TopClusterID;
1307+
unsigned BotClusterID;
13081308

13091309
/// Candidate last picked from Top boundary.
13101310
SchedCandidate TopCand;
@@ -1346,8 +1346,8 @@ class LLVM_ABI PostGenericScheduler : public GenericSchedulerBase {
13461346
/// Candidate last picked from Bot boundary.
13471347
SchedCandidate BotCand;
13481348

1349-
ClusterInfo *TopCluster;
1350-
ClusterInfo *BotCluster;
1349+
unsigned TopClusterID;
1350+
unsigned BotClusterID;
13511351

13521352
public:
13531353
PostGenericScheduler(const MachineSchedContext *C)

‎llvm/include/llvm/CodeGen/ScheduleDAG.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,11 @@ class TargetRegisterInfo;
240240
typedef SmallSet<SUnit *, 8> ClusterInfo;
241241
constexpr unsigned InvalidClusterId = ~0u;
242242

243+
/// Return whether the input cluster ID's are the same and valid.
244+
inline bool isTheSameCluster(unsigned A, unsigned B) {
245+
return A != InvalidClusterId && A == B;
246+
}
247+
243248
/// Scheduling unit. This is a node in the scheduling DAG.
244249
class SUnit {
245250
private:

‎llvm/lib/CodeGen/MachineScheduler.cpp

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3676,8 +3676,8 @@ void GenericScheduler::initialize(ScheduleDAGMI *dag) {
36763676
TopCand.SU = nullptr;
36773677
BotCand.SU = nullptr;
36783678

3679-
TopCluster = nullptr;
3680-
BotCluster = nullptr;
3679+
TopClusterID = InvalidClusterId;
3680+
BotClusterID = InvalidClusterId;
36813681
}
36823682

36833683
/// Initialize the per-region scheduling policy.
@@ -3988,10 +3988,14 @@ bool GenericScheduler::tryCandidate(SchedCandidate &Cand,
39883988
// This is a best effort to set things up for a post-RA pass. Optimizations
39893989
// like generating loads of multiple registers should ideally be done within
39903990
// the scheduler pass by combining the loads during DAG postprocessing.
3991-
const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
3992-
const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
3993-
if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
3994-
CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
3991+
unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
3992+
unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
3993+
bool CandIsClusterSucc =
3994+
isTheSameCluster(CandZoneCluster, Cand.SU->ParentClusterIdx);
3995+
bool TryCandIsClusterSucc =
3996+
isTheSameCluster(TryCandZoneCluster, TryCand.SU->ParentClusterIdx);
3997+
3998+
if (tryGreater(TryCandIsClusterSucc, CandIsClusterSucc, TryCand, Cand,
39953999
Cluster))
39964000
return TryCand.Reason != NoCand;
39974001

@@ -4251,24 +4255,30 @@ void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
42514255
void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
42524256
if (IsTopNode) {
42534257
SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
4254-
TopCluster = DAG->getCluster(SU->ParentClusterIdx);
4255-
LLVM_DEBUG(if (TopCluster) {
4256-
dbgs() << " Top Cluster: ";
4257-
for (auto *N : *TopCluster)
4258-
dbgs() << N->NodeNum << '\t';
4259-
dbgs() << '\n';
4258+
TopClusterID = SU->ParentClusterIdx;
4259+
LLVM_DEBUG({
4260+
if (TopClusterID != InvalidClusterId) {
4261+
ClusterInfo *TopCluster = DAG->getCluster(TopClusterID);
4262+
dbgs() << " Top Cluster: ";
4263+
for (auto *N : *TopCluster)
4264+
dbgs() << N->NodeNum << '\t';
4265+
dbgs() << '\n';
4266+
}
42604267
});
42614268
Top.bumpNode(SU);
42624269
if (SU->hasPhysRegUses)
42634270
reschedulePhysReg(SU, true);
42644271
} else {
42654272
SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
4266-
BotCluster = DAG->getCluster(SU->ParentClusterIdx);
4267-
LLVM_DEBUG(if (BotCluster) {
4268-
dbgs() << " Bot Cluster: ";
4269-
for (auto *N : *BotCluster)
4270-
dbgs() << N->NodeNum << '\t';
4271-
dbgs() << '\n';
4273+
BotClusterID = SU->ParentClusterIdx;
4274+
LLVM_DEBUG({
4275+
if (BotClusterID != InvalidClusterId) {
4276+
ClusterInfo *BotCluster = DAG->getCluster(BotClusterID);
4277+
dbgs() << " Bot Cluster: ";
4278+
for (auto *N : *BotCluster)
4279+
dbgs() << N->NodeNum << '\t';
4280+
dbgs() << '\n';
4281+
}
42724282
});
42734283
Bot.bumpNode(SU);
42744284
if (SU->hasPhysRegDefs)
@@ -4306,8 +4316,8 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) {
43064316
if (!Bot.HazardRec) {
43074317
Bot.HazardRec = DAG->TII->CreateTargetMIHazardRecognizer(Itin, DAG);
43084318
}
4309-
TopCluster = nullptr;
4310-
BotCluster = nullptr;
4319+
TopClusterID = InvalidClusterId;
4320+
BotClusterID = InvalidClusterId;
43114321
}
43124322

43134323
void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin,
@@ -4373,10 +4383,14 @@ bool PostGenericScheduler::tryCandidate(SchedCandidate &Cand,
43734383
return TryCand.Reason != NoCand;
43744384

43754385
// Keep clustered nodes together.
4376-
const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
4377-
const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
4378-
if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
4379-
CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
4386+
unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
4387+
unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
4388+
bool CandIsClusterSucc =
4389+
isTheSameCluster(CandZoneCluster, Cand.SU->ParentClusterIdx);
4390+
bool TryCandIsClusterSucc =
4391+
isTheSameCluster(TryCandZoneCluster, TryCand.SU->ParentClusterIdx);
4392+
4393+
if (tryGreater(TryCandIsClusterSucc, CandIsClusterSucc, TryCand, Cand,
43804394
Cluster))
43814395
return TryCand.Reason != NoCand;
43824396
// Avoid critical resource consumption and balance the schedule.
@@ -4575,11 +4589,11 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
45754589
void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
45764590
if (IsTopNode) {
45774591
SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
4578-
TopCluster = DAG->getCluster(SU->ParentClusterIdx);
4592+
TopClusterID = SU->ParentClusterIdx;
45794593
Top.bumpNode(SU);
45804594
} else {
45814595
SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
4582-
BotCluster = DAG->getCluster(SU->ParentClusterIdx);
4596+
BotClusterID = SU->ParentClusterIdx;
45834597
Bot.bumpNode(SU);
45844598
}
45854599
}

‎llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -592,10 +592,13 @@ bool GCNMaxILPSchedStrategy::tryCandidate(SchedCandidate &Cand,
592592
// This is a best effort to set things up for a post-RA pass. Optimizations
593593
// like generating loads of multiple registers should ideally be done within
594594
// the scheduler pass by combining the loads during DAG postprocessing.
595-
const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
596-
const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
597-
if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
598-
CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
595+
unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
596+
unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
597+
bool CandIsClusterSucc =
598+
isTheSameCluster(CandZoneCluster, Cand.SU->ParentClusterIdx);
599+
bool TryCandIsClusterSucc =
600+
isTheSameCluster(TryCandZoneCluster, TryCand.SU->ParentClusterIdx);
601+
if (tryGreater(TryCandIsClusterSucc, CandIsClusterSucc, TryCand, Cand,
599602
Cluster))
600603
return TryCand.Reason != NoCand;
601604

@@ -666,10 +669,13 @@ bool GCNMaxMemoryClauseSchedStrategy::tryCandidate(SchedCandidate &Cand,
666669

667670
// MaxMemoryClause-specific: We prioritize clustered instructions as we would
668671
// get more benefit from clausing these memory instructions.
669-
const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
670-
const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
671-
if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
672-
CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
672+
unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
673+
unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
674+
bool CandIsClusterSucc =
675+
isTheSameCluster(CandZoneCluster, Cand.SU->ParentClusterIdx);
676+
bool TryCandIsClusterSucc =
677+
isTheSameCluster(TryCandZoneCluster, TryCand.SU->ParentClusterIdx);
678+
if (tryGreater(TryCandIsClusterSucc, CandIsClusterSucc, TryCand, Cand,
673679
Cluster))
674680
return TryCand.Reason != NoCand;
675681

‎llvm/lib/Target/PowerPC/PPCMachineScheduler.cpp

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -100,10 +100,14 @@ bool PPCPreRASchedStrategy::tryCandidate(SchedCandidate &Cand,
100100
// This is a best effort to set things up for a post-RA pass. Optimizations
101101
// like generating loads of multiple registers should ideally be done within
102102
// the scheduler pass by combining the loads during DAG postprocessing.
103-
const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
104-
const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
105-
if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
106-
CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
103+
unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
104+
unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
105+
bool CandIsClusterSucc =
106+
isTheSameCluster(CandZoneCluster, Cand.SU->ParentClusterIdx);
107+
bool TryCandIsClusterSucc =
108+
isTheSameCluster(TryCandZoneCluster, TryCand.SU->ParentClusterIdx);
109+
110+
if (tryGreater(TryCandIsClusterSucc, CandIsClusterSucc, TryCand, Cand,
107111
Cluster))
108112
return TryCand.Reason != NoCand;
109113

@@ -189,10 +193,14 @@ bool PPCPostRASchedStrategy::tryCandidate(SchedCandidate &Cand,
189193
return TryCand.Reason != NoCand;
190194

191195
// Keep clustered nodes together.
192-
const ClusterInfo *CandCluster = Cand.AtTop ? TopCluster : BotCluster;
193-
const ClusterInfo *TryCandCluster = TryCand.AtTop ? TopCluster : BotCluster;
194-
if (tryGreater(TryCandCluster && TryCandCluster->contains(TryCand.SU),
195-
CandCluster && CandCluster->contains(Cand.SU), TryCand, Cand,
196+
unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
197+
unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
198+
bool CandIsClusterSucc =
199+
isTheSameCluster(CandZoneCluster, Cand.SU->ParentClusterIdx);
200+
bool TryCandIsClusterSucc =
201+
isTheSameCluster(TryCandZoneCluster, TryCand.SU->ParentClusterIdx);
202+
203+
if (tryGreater(TryCandIsClusterSucc, CandIsClusterSucc, TryCand, Cand,
196204
Cluster))
197205
return TryCand.Reason != NoCand;
198206

0 commit comments

Comments
 (0)