ScheduleDAG.cpp 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. //===- ScheduleDAG.cpp - Implement the ScheduleDAG class ------------------===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. //
  9. /// \file Implements the ScheduleDAG class, which is a base class used by
  10. /// scheduling implementation classes.
  11. //
  12. //===----------------------------------------------------------------------===//
  13. #include "llvm/CodeGen/ScheduleDAG.h"
  14. #include "llvm/ADT/STLExtras.h"
  15. #include "llvm/ADT/SmallVector.h"
  16. #include "llvm/ADT/Statistic.h"
  17. #include "llvm/ADT/iterator_range.h"
  18. #include "llvm/CodeGen/MachineFunction.h"
  19. #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
  20. #include "llvm/CodeGen/SelectionDAGNodes.h"
  21. #include "llvm/CodeGen/TargetInstrInfo.h"
  22. #include "llvm/CodeGen/TargetRegisterInfo.h"
  23. #include "llvm/CodeGen/TargetSubtargetInfo.h"
  24. #include "llvm/Config/llvm-config.h"
  25. #include "llvm/Support/CommandLine.h"
  26. #include "llvm/Support/Compiler.h"
  27. #include "llvm/Support/Debug.h"
  28. #include "llvm/Support/raw_ostream.h"
  29. #include <algorithm>
  30. #include <cassert>
  31. #include <iterator>
  32. #include <limits>
  33. #include <utility>
  34. #include <vector>
  35. using namespace llvm;
  36. #define DEBUG_TYPE "pre-RA-sched"
  37. STATISTIC(NumNewPredsAdded, "Number of times a single predecessor was added");
  38. STATISTIC(NumTopoInits,
  39. "Number of times the topological order has been recomputed");
  40. #ifndef NDEBUG
  41. static cl::opt<bool> StressSchedOpt(
  42. "stress-sched", cl::Hidden, cl::init(false),
  43. cl::desc("Stress test instruction scheduling"));
  44. #endif
  45. void SchedulingPriorityQueue::anchor() {}
  46. ScheduleDAG::ScheduleDAG(MachineFunction &mf)
  47. : TM(mf.getTarget()), TII(mf.getSubtarget().getInstrInfo()),
  48. TRI(mf.getSubtarget().getRegisterInfo()), MF(mf),
  49. MRI(mf.getRegInfo()) {
  50. #ifndef NDEBUG
  51. StressSched = StressSchedOpt;
  52. #endif
  53. }
  54. ScheduleDAG::~ScheduleDAG() = default;
  55. void ScheduleDAG::clearDAG() {
  56. SUnits.clear();
  57. EntrySU = SUnit();
  58. ExitSU = SUnit();
  59. }
  60. const MCInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const {
  61. if (!Node || !Node->isMachineOpcode()) return nullptr;
  62. return &TII->get(Node->getMachineOpcode());
  63. }
  64. LLVM_DUMP_METHOD void SDep::dump(const TargetRegisterInfo *TRI) const {
  65. switch (getKind()) {
  66. case Data: dbgs() << "Data"; break;
  67. case Anti: dbgs() << "Anti"; break;
  68. case Output: dbgs() << "Out "; break;
  69. case Order: dbgs() << "Ord "; break;
  70. }
  71. switch (getKind()) {
  72. case Data:
  73. dbgs() << " Latency=" << getLatency();
  74. if (TRI && isAssignedRegDep())
  75. dbgs() << " Reg=" << printReg(getReg(), TRI);
  76. break;
  77. case Anti:
  78. case Output:
  79. dbgs() << " Latency=" << getLatency();
  80. break;
  81. case Order:
  82. dbgs() << " Latency=" << getLatency();
  83. switch(Contents.OrdKind) {
  84. case Barrier: dbgs() << " Barrier"; break;
  85. case MayAliasMem:
  86. case MustAliasMem: dbgs() << " Memory"; break;
  87. case Artificial: dbgs() << " Artificial"; break;
  88. case Weak: dbgs() << " Weak"; break;
  89. case Cluster: dbgs() << " Cluster"; break;
  90. }
  91. break;
  92. }
  93. }
  94. bool SUnit::addPred(const SDep &D, bool Required) {
  95. // If this node already has this dependence, don't add a redundant one.
  96. for (SDep &PredDep : Preds) {
  97. // Zero-latency weak edges may be added purely for heuristic ordering. Don't
  98. // add them if another kind of edge already exists.
  99. if (!Required && PredDep.getSUnit() == D.getSUnit())
  100. return false;
  101. if (PredDep.overlaps(D)) {
  102. // Extend the latency if needed. Equivalent to
  103. // removePred(PredDep) + addPred(D).
  104. if (PredDep.getLatency() < D.getLatency()) {
  105. SUnit *PredSU = PredDep.getSUnit();
  106. // Find the corresponding successor in N.
  107. SDep ForwardD = PredDep;
  108. ForwardD.setSUnit(this);
  109. for (SDep &SuccDep : PredSU->Succs) {
  110. if (SuccDep == ForwardD) {
  111. SuccDep.setLatency(D.getLatency());
  112. break;
  113. }
  114. }
  115. PredDep.setLatency(D.getLatency());
  116. }
  117. return false;
  118. }
  119. }
  120. // Now add a corresponding succ to N.
  121. SDep P = D;
  122. P.setSUnit(this);
  123. SUnit *N = D.getSUnit();
  124. // Update the bookkeeping.
  125. if (D.getKind() == SDep::Data) {
  126. assert(NumPreds < std::numeric_limits<unsigned>::max() &&
  127. "NumPreds will overflow!");
  128. assert(N->NumSuccs < std::numeric_limits<unsigned>::max() &&
  129. "NumSuccs will overflow!");
  130. ++NumPreds;
  131. ++N->NumSuccs;
  132. }
  133. if (!N->isScheduled) {
  134. if (D.isWeak()) {
  135. ++WeakPredsLeft;
  136. }
  137. else {
  138. assert(NumPredsLeft < std::numeric_limits<unsigned>::max() &&
  139. "NumPredsLeft will overflow!");
  140. ++NumPredsLeft;
  141. }
  142. }
  143. if (!isScheduled) {
  144. if (D.isWeak()) {
  145. ++N->WeakSuccsLeft;
  146. }
  147. else {
  148. assert(N->NumSuccsLeft < std::numeric_limits<unsigned>::max() &&
  149. "NumSuccsLeft will overflow!");
  150. ++N->NumSuccsLeft;
  151. }
  152. }
  153. Preds.push_back(D);
  154. N->Succs.push_back(P);
  155. if (P.getLatency() != 0) {
  156. this->setDepthDirty();
  157. N->setHeightDirty();
  158. }
  159. return true;
  160. }
  161. void SUnit::removePred(const SDep &D) {
  162. // Find the matching predecessor.
  163. SmallVectorImpl<SDep>::iterator I = llvm::find(Preds, D);
  164. if (I == Preds.end())
  165. return;
  166. // Find the corresponding successor in N.
  167. SDep P = D;
  168. P.setSUnit(this);
  169. SUnit *N = D.getSUnit();
  170. SmallVectorImpl<SDep>::iterator Succ = llvm::find(N->Succs, P);
  171. assert(Succ != N->Succs.end() && "Mismatching preds / succs lists!");
  172. N->Succs.erase(Succ);
  173. Preds.erase(I);
  174. // Update the bookkeeping.
  175. if (P.getKind() == SDep::Data) {
  176. assert(NumPreds > 0 && "NumPreds will underflow!");
  177. assert(N->NumSuccs > 0 && "NumSuccs will underflow!");
  178. --NumPreds;
  179. --N->NumSuccs;
  180. }
  181. if (!N->isScheduled) {
  182. if (D.isWeak())
  183. --WeakPredsLeft;
  184. else {
  185. assert(NumPredsLeft > 0 && "NumPredsLeft will underflow!");
  186. --NumPredsLeft;
  187. }
  188. }
  189. if (!isScheduled) {
  190. if (D.isWeak())
  191. --N->WeakSuccsLeft;
  192. else {
  193. assert(N->NumSuccsLeft > 0 && "NumSuccsLeft will underflow!");
  194. --N->NumSuccsLeft;
  195. }
  196. }
  197. if (P.getLatency() != 0) {
  198. this->setDepthDirty();
  199. N->setHeightDirty();
  200. }
  201. }
  202. void SUnit::setDepthDirty() {
  203. if (!isDepthCurrent) return;
  204. SmallVector<SUnit*, 8> WorkList;
  205. WorkList.push_back(this);
  206. do {
  207. SUnit *SU = WorkList.pop_back_val();
  208. SU->isDepthCurrent = false;
  209. for (SDep &SuccDep : SU->Succs) {
  210. SUnit *SuccSU = SuccDep.getSUnit();
  211. if (SuccSU->isDepthCurrent)
  212. WorkList.push_back(SuccSU);
  213. }
  214. } while (!WorkList.empty());
  215. }
  216. void SUnit::setHeightDirty() {
  217. if (!isHeightCurrent) return;
  218. SmallVector<SUnit*, 8> WorkList;
  219. WorkList.push_back(this);
  220. do {
  221. SUnit *SU = WorkList.pop_back_val();
  222. SU->isHeightCurrent = false;
  223. for (SDep &PredDep : SU->Preds) {
  224. SUnit *PredSU = PredDep.getSUnit();
  225. if (PredSU->isHeightCurrent)
  226. WorkList.push_back(PredSU);
  227. }
  228. } while (!WorkList.empty());
  229. }
  230. void SUnit::setDepthToAtLeast(unsigned NewDepth) {
  231. if (NewDepth <= getDepth())
  232. return;
  233. setDepthDirty();
  234. Depth = NewDepth;
  235. isDepthCurrent = true;
  236. }
  237. void SUnit::setHeightToAtLeast(unsigned NewHeight) {
  238. if (NewHeight <= getHeight())
  239. return;
  240. setHeightDirty();
  241. Height = NewHeight;
  242. isHeightCurrent = true;
  243. }
  244. /// Calculates the maximal path from the node to the exit.
  245. void SUnit::ComputeDepth() {
  246. SmallVector<SUnit*, 8> WorkList;
  247. WorkList.push_back(this);
  248. do {
  249. SUnit *Cur = WorkList.back();
  250. bool Done = true;
  251. unsigned MaxPredDepth = 0;
  252. for (const SDep &PredDep : Cur->Preds) {
  253. SUnit *PredSU = PredDep.getSUnit();
  254. if (PredSU->isDepthCurrent)
  255. MaxPredDepth = std::max(MaxPredDepth,
  256. PredSU->Depth + PredDep.getLatency());
  257. else {
  258. Done = false;
  259. WorkList.push_back(PredSU);
  260. }
  261. }
  262. if (Done) {
  263. WorkList.pop_back();
  264. if (MaxPredDepth != Cur->Depth) {
  265. Cur->setDepthDirty();
  266. Cur->Depth = MaxPredDepth;
  267. }
  268. Cur->isDepthCurrent = true;
  269. }
  270. } while (!WorkList.empty());
  271. }
  272. /// Calculates the maximal path from the node to the entry.
  273. void SUnit::ComputeHeight() {
  274. SmallVector<SUnit*, 8> WorkList;
  275. WorkList.push_back(this);
  276. do {
  277. SUnit *Cur = WorkList.back();
  278. bool Done = true;
  279. unsigned MaxSuccHeight = 0;
  280. for (const SDep &SuccDep : Cur->Succs) {
  281. SUnit *SuccSU = SuccDep.getSUnit();
  282. if (SuccSU->isHeightCurrent)
  283. MaxSuccHeight = std::max(MaxSuccHeight,
  284. SuccSU->Height + SuccDep.getLatency());
  285. else {
  286. Done = false;
  287. WorkList.push_back(SuccSU);
  288. }
  289. }
  290. if (Done) {
  291. WorkList.pop_back();
  292. if (MaxSuccHeight != Cur->Height) {
  293. Cur->setHeightDirty();
  294. Cur->Height = MaxSuccHeight;
  295. }
  296. Cur->isHeightCurrent = true;
  297. }
  298. } while (!WorkList.empty());
  299. }
  300. void SUnit::biasCriticalPath() {
  301. if (NumPreds < 2)
  302. return;
  303. SUnit::pred_iterator BestI = Preds.begin();
  304. unsigned MaxDepth = BestI->getSUnit()->getDepth();
  305. for (SUnit::pred_iterator I = std::next(BestI), E = Preds.end(); I != E;
  306. ++I) {
  307. if (I->getKind() == SDep::Data && I->getSUnit()->getDepth() > MaxDepth)
  308. BestI = I;
  309. }
  310. if (BestI != Preds.begin())
  311. std::swap(*Preds.begin(), *BestI);
  312. }
  313. #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
  314. LLVM_DUMP_METHOD void SUnit::dumpAttributes() const {
  315. dbgs() << " # preds left : " << NumPredsLeft << "\n";
  316. dbgs() << " # succs left : " << NumSuccsLeft << "\n";
  317. if (WeakPredsLeft)
  318. dbgs() << " # weak preds left : " << WeakPredsLeft << "\n";
  319. if (WeakSuccsLeft)
  320. dbgs() << " # weak succs left : " << WeakSuccsLeft << "\n";
  321. dbgs() << " # rdefs left : " << NumRegDefsLeft << "\n";
  322. dbgs() << " Latency : " << Latency << "\n";
  323. dbgs() << " Depth : " << getDepth() << "\n";
  324. dbgs() << " Height : " << getHeight() << "\n";
  325. }
  326. LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeName(const SUnit &SU) const {
  327. if (&SU == &EntrySU)
  328. dbgs() << "EntrySU";
  329. else if (&SU == &ExitSU)
  330. dbgs() << "ExitSU";
  331. else
  332. dbgs() << "SU(" << SU.NodeNum << ")";
  333. }
  334. LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeAll(const SUnit &SU) const {
  335. dumpNode(SU);
  336. SU.dumpAttributes();
  337. if (SU.Preds.size() > 0) {
  338. dbgs() << " Predecessors:\n";
  339. for (const SDep &Dep : SU.Preds) {
  340. dbgs() << " ";
  341. dumpNodeName(*Dep.getSUnit());
  342. dbgs() << ": ";
  343. Dep.dump(TRI);
  344. dbgs() << '\n';
  345. }
  346. }
  347. if (SU.Succs.size() > 0) {
  348. dbgs() << " Successors:\n";
  349. for (const SDep &Dep : SU.Succs) {
  350. dbgs() << " ";
  351. dumpNodeName(*Dep.getSUnit());
  352. dbgs() << ": ";
  353. Dep.dump(TRI);
  354. dbgs() << '\n';
  355. }
  356. }
  357. }
  358. #endif
  359. #ifndef NDEBUG
  360. unsigned ScheduleDAG::VerifyScheduledDAG(bool isBottomUp) {
  361. bool AnyNotSched = false;
  362. unsigned DeadNodes = 0;
  363. for (const SUnit &SUnit : SUnits) {
  364. if (!SUnit.isScheduled) {
  365. if (SUnit.NumPreds == 0 && SUnit.NumSuccs == 0) {
  366. ++DeadNodes;
  367. continue;
  368. }
  369. if (!AnyNotSched)
  370. dbgs() << "*** Scheduling failed! ***\n";
  371. dumpNode(SUnit);
  372. dbgs() << "has not been scheduled!\n";
  373. AnyNotSched = true;
  374. }
  375. if (SUnit.isScheduled &&
  376. (isBottomUp ? SUnit.getHeight() : SUnit.getDepth()) >
  377. unsigned(std::numeric_limits<int>::max())) {
  378. if (!AnyNotSched)
  379. dbgs() << "*** Scheduling failed! ***\n";
  380. dumpNode(SUnit);
  381. dbgs() << "has an unexpected "
  382. << (isBottomUp ? "Height" : "Depth") << " value!\n";
  383. AnyNotSched = true;
  384. }
  385. if (isBottomUp) {
  386. if (SUnit.NumSuccsLeft != 0) {
  387. if (!AnyNotSched)
  388. dbgs() << "*** Scheduling failed! ***\n";
  389. dumpNode(SUnit);
  390. dbgs() << "has successors left!\n";
  391. AnyNotSched = true;
  392. }
  393. } else {
  394. if (SUnit.NumPredsLeft != 0) {
  395. if (!AnyNotSched)
  396. dbgs() << "*** Scheduling failed! ***\n";
  397. dumpNode(SUnit);
  398. dbgs() << "has predecessors left!\n";
  399. AnyNotSched = true;
  400. }
  401. }
  402. }
  403. assert(!AnyNotSched);
  404. return SUnits.size() - DeadNodes;
  405. }
  406. #endif
  407. void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() {
  408. // The idea of the algorithm is taken from
  409. // "Online algorithms for managing the topological order of
  410. // a directed acyclic graph" by David J. Pearce and Paul H.J. Kelly
  411. // This is the MNR algorithm, which was first introduced by
  412. // A. Marchetti-Spaccamela, U. Nanni and H. Rohnert in
  413. // "Maintaining a topological order under edge insertions".
  414. //
  415. // Short description of the algorithm:
  416. //
  417. // Topological ordering, ord, of a DAG maps each node to a topological
  418. // index so that for all edges X->Y it is the case that ord(X) < ord(Y).
  419. //
  420. // This means that if there is a path from the node X to the node Z,
  421. // then ord(X) < ord(Z).
  422. //
  423. // This property can be used to check for reachability of nodes:
  424. // if Z is reachable from X, then an insertion of the edge Z->X would
  425. // create a cycle.
  426. //
  427. // The algorithm first computes a topological ordering for the DAG by
  428. // initializing the Index2Node and Node2Index arrays and then tries to keep
  429. // the ordering up-to-date after edge insertions by reordering the DAG.
  430. //
  431. // On insertion of the edge X->Y, the algorithm first marks by calling DFS
  432. // the nodes reachable from Y, and then shifts them using Shift to lie
  433. // immediately after X in Index2Node.
  434. // Cancel pending updates, mark as valid.
  435. Dirty = false;
  436. Updates.clear();
  437. unsigned DAGSize = SUnits.size();
  438. std::vector<SUnit*> WorkList;
  439. WorkList.reserve(DAGSize);
  440. Index2Node.resize(DAGSize);
  441. Node2Index.resize(DAGSize);
  442. // Initialize the data structures.
  443. if (ExitSU)
  444. WorkList.push_back(ExitSU);
  445. for (SUnit &SU : SUnits) {
  446. int NodeNum = SU.NodeNum;
  447. unsigned Degree = SU.Succs.size();
  448. // Temporarily use the Node2Index array as scratch space for degree counts.
  449. Node2Index[NodeNum] = Degree;
  450. // Is it a node without dependencies?
  451. if (Degree == 0) {
  452. assert(SU.Succs.empty() && "SUnit should have no successors");
  453. // Collect leaf nodes.
  454. WorkList.push_back(&SU);
  455. }
  456. }
  457. int Id = DAGSize;
  458. while (!WorkList.empty()) {
  459. SUnit *SU = WorkList.back();
  460. WorkList.pop_back();
  461. if (SU->NodeNum < DAGSize)
  462. Allocate(SU->NodeNum, --Id);
  463. for (const SDep &PredDep : SU->Preds) {
  464. SUnit *SU = PredDep.getSUnit();
  465. if (SU->NodeNum < DAGSize && !--Node2Index[SU->NodeNum])
  466. // If all dependencies of the node are processed already,
  467. // then the node can be computed now.
  468. WorkList.push_back(SU);
  469. }
  470. }
  471. Visited.resize(DAGSize);
  472. NumTopoInits++;
  473. #ifndef NDEBUG
  474. // Check correctness of the ordering
  475. for (SUnit &SU : SUnits) {
  476. for (const SDep &PD : SU.Preds) {
  477. assert(Node2Index[SU.NodeNum] > Node2Index[PD.getSUnit()->NodeNum] &&
  478. "Wrong topological sorting");
  479. }
  480. }
  481. #endif
  482. }
  483. void ScheduleDAGTopologicalSort::FixOrder() {
  484. // Recompute from scratch after new nodes have been added.
  485. if (Dirty) {
  486. InitDAGTopologicalSorting();
  487. return;
  488. }
  489. // Otherwise apply updates one-by-one.
  490. for (auto &U : Updates)
  491. AddPred(U.first, U.second);
  492. Updates.clear();
  493. }
  494. void ScheduleDAGTopologicalSort::AddPredQueued(SUnit *Y, SUnit *X) {
  495. // Recomputing the order from scratch is likely more efficient than applying
  496. // updates one-by-one for too many updates. The current cut-off is arbitrarily
  497. // chosen.
  498. Dirty = Dirty || Updates.size() > 10;
  499. if (Dirty)
  500. return;
  501. Updates.emplace_back(Y, X);
  502. }
  503. void ScheduleDAGTopologicalSort::AddPred(SUnit *Y, SUnit *X) {
  504. int UpperBound, LowerBound;
  505. LowerBound = Node2Index[Y->NodeNum];
  506. UpperBound = Node2Index[X->NodeNum];
  507. bool HasLoop = false;
  508. // Is Ord(X) < Ord(Y) ?
  509. if (LowerBound < UpperBound) {
  510. // Update the topological order.
  511. Visited.reset();
  512. DFS(Y, UpperBound, HasLoop);
  513. assert(!HasLoop && "Inserted edge creates a loop!");
  514. // Recompute topological indexes.
  515. Shift(Visited, LowerBound, UpperBound);
  516. }
  517. NumNewPredsAdded++;
  518. }
  519. void ScheduleDAGTopologicalSort::RemovePred(SUnit *M, SUnit *N) {
  520. // InitDAGTopologicalSorting();
  521. }
  522. void ScheduleDAGTopologicalSort::DFS(const SUnit *SU, int UpperBound,
  523. bool &HasLoop) {
  524. std::vector<const SUnit*> WorkList;
  525. WorkList.reserve(SUnits.size());
  526. WorkList.push_back(SU);
  527. do {
  528. SU = WorkList.back();
  529. WorkList.pop_back();
  530. Visited.set(SU->NodeNum);
  531. for (const SDep &SuccDep : llvm::reverse(SU->Succs)) {
  532. unsigned s = SuccDep.getSUnit()->NodeNum;
  533. // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
  534. if (s >= Node2Index.size())
  535. continue;
  536. if (Node2Index[s] == UpperBound) {
  537. HasLoop = true;
  538. return;
  539. }
  540. // Visit successors if not already and in affected region.
  541. if (!Visited.test(s) && Node2Index[s] < UpperBound) {
  542. WorkList.push_back(SuccDep.getSUnit());
  543. }
  544. }
  545. } while (!WorkList.empty());
  546. }
  547. std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
  548. const SUnit &TargetSU,
  549. bool &Success) {
  550. std::vector<const SUnit*> WorkList;
  551. int LowerBound = Node2Index[StartSU.NodeNum];
  552. int UpperBound = Node2Index[TargetSU.NodeNum];
  553. bool Found = false;
  554. BitVector VisitedBack;
  555. std::vector<int> Nodes;
  556. if (LowerBound > UpperBound) {
  557. Success = false;
  558. return Nodes;
  559. }
  560. WorkList.reserve(SUnits.size());
  561. Visited.reset();
  562. // Starting from StartSU, visit all successors up
  563. // to UpperBound.
  564. WorkList.push_back(&StartSU);
  565. do {
  566. const SUnit *SU = WorkList.back();
  567. WorkList.pop_back();
  568. for (const SDep &SD : llvm::reverse(SU->Succs)) {
  569. const SUnit *Succ = SD.getSUnit();
  570. unsigned s = Succ->NodeNum;
  571. // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
  572. if (Succ->isBoundaryNode())
  573. continue;
  574. if (Node2Index[s] == UpperBound) {
  575. Found = true;
  576. continue;
  577. }
  578. // Visit successors if not already and in affected region.
  579. if (!Visited.test(s) && Node2Index[s] < UpperBound) {
  580. Visited.set(s);
  581. WorkList.push_back(Succ);
  582. }
  583. }
  584. } while (!WorkList.empty());
  585. if (!Found) {
  586. Success = false;
  587. return Nodes;
  588. }
  589. WorkList.clear();
  590. VisitedBack.resize(SUnits.size());
  591. Found = false;
  592. // Starting from TargetSU, visit all predecessors up
  593. // to LowerBound. SUs that are visited by the two
  594. // passes are added to Nodes.
  595. WorkList.push_back(&TargetSU);
  596. do {
  597. const SUnit *SU = WorkList.back();
  598. WorkList.pop_back();
  599. for (const SDep &SD : llvm::reverse(SU->Preds)) {
  600. const SUnit *Pred = SD.getSUnit();
  601. unsigned s = Pred->NodeNum;
  602. // Edges to non-SUnits are allowed but ignored (e.g. EntrySU).
  603. if (Pred->isBoundaryNode())
  604. continue;
  605. if (Node2Index[s] == LowerBound) {
  606. Found = true;
  607. continue;
  608. }
  609. if (!VisitedBack.test(s) && Visited.test(s)) {
  610. VisitedBack.set(s);
  611. WorkList.push_back(Pred);
  612. Nodes.push_back(s);
  613. }
  614. }
  615. } while (!WorkList.empty());
  616. assert(Found && "Error in SUnit Graph!");
  617. Success = true;
  618. return Nodes;
  619. }
  620. void ScheduleDAGTopologicalSort::Shift(BitVector& Visited, int LowerBound,
  621. int UpperBound) {
  622. std::vector<int> L;
  623. int shift = 0;
  624. int i;
  625. for (i = LowerBound; i <= UpperBound; ++i) {
  626. // w is node at topological index i.
  627. int w = Index2Node[i];
  628. if (Visited.test(w)) {
  629. // Unmark.
  630. Visited.reset(w);
  631. L.push_back(w);
  632. shift = shift + 1;
  633. } else {
  634. Allocate(w, i - shift);
  635. }
  636. }
  637. for (unsigned LI : L) {
  638. Allocate(LI, i - shift);
  639. i = i + 1;
  640. }
  641. }
  642. bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *TargetSU, SUnit *SU) {
  643. FixOrder();
  644. // Is SU reachable from TargetSU via successor edges?
  645. if (IsReachable(SU, TargetSU))
  646. return true;
  647. for (const SDep &PredDep : TargetSU->Preds)
  648. if (PredDep.isAssignedRegDep() &&
  649. IsReachable(SU, PredDep.getSUnit()))
  650. return true;
  651. return false;
  652. }
  653. void ScheduleDAGTopologicalSort::AddSUnitWithoutPredecessors(const SUnit *SU) {
  654. assert(SU->NodeNum == Index2Node.size() && "Node cannot be added at the end");
  655. assert(SU->NumPreds == 0 && "Can only add SU's with no predecessors");
  656. Node2Index.push_back(Index2Node.size());
  657. Index2Node.push_back(SU->NodeNum);
  658. Visited.resize(Node2Index.size());
  659. }
  660. bool ScheduleDAGTopologicalSort::IsReachable(const SUnit *SU,
  661. const SUnit *TargetSU) {
  662. FixOrder();
  663. // If insertion of the edge SU->TargetSU would create a cycle
  664. // then there is a path from TargetSU to SU.
  665. int UpperBound, LowerBound;
  666. LowerBound = Node2Index[TargetSU->NodeNum];
  667. UpperBound = Node2Index[SU->NodeNum];
  668. bool HasLoop = false;
  669. // Is Ord(TargetSU) < Ord(SU) ?
  670. if (LowerBound < UpperBound) {
  671. Visited.reset();
  672. // There may be a path from TargetSU to SU. Check for it.
  673. DFS(TargetSU, UpperBound, HasLoop);
  674. }
  675. return HasLoop;
  676. }
  677. void ScheduleDAGTopologicalSort::Allocate(int n, int index) {
  678. Node2Index[n] = index;
  679. Index2Node[index] = n;
  680. }
  681. ScheduleDAGTopologicalSort::
  682. ScheduleDAGTopologicalSort(std::vector<SUnit> &sunits, SUnit *exitsu)
  683. : SUnits(sunits), ExitSU(exitsu) {}
  684. ScheduleHazardRecognizer::~ScheduleHazardRecognizer() = default;