LoopUtils.h 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561
  1. #pragma once
  2. #ifdef __GNUC__
  3. #pragma GCC diagnostic push
  4. #pragma GCC diagnostic ignored "-Wunused-parameter"
  5. #endif
  6. //===- llvm/Transforms/Utils/LoopUtils.h - Loop utilities -------*- C++ -*-===//
  7. //
  8. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  9. // See https://llvm.org/LICENSE.txt for license information.
  10. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  11. //
  12. //===----------------------------------------------------------------------===//
  13. //
  14. // This file defines some loop transformation utilities.
  15. //
  16. //===----------------------------------------------------------------------===//
  17. #ifndef LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
  18. #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
  19. #include "llvm/Analysis/IVDescriptors.h"
  20. #include "llvm/Analysis/LoopAccessAnalysis.h"
  21. #include "llvm/Transforms/Utils/ValueMapper.h"
  22. namespace llvm {
  23. template <typename T> class DomTreeNodeBase;
  24. using DomTreeNode = DomTreeNodeBase<BasicBlock>;
  25. class AssumptionCache;
  26. class StringRef;
  27. class AnalysisUsage;
  28. class TargetTransformInfo;
  29. class AAResults;
  30. class BasicBlock;
  31. class ICFLoopSafetyInfo;
  32. class IRBuilderBase;
  33. class Loop;
  34. class LoopInfo;
  35. class MemoryAccess;
  36. class MemorySSA;
  37. class MemorySSAUpdater;
  38. class OptimizationRemarkEmitter;
  39. class PredIteratorCache;
  40. class ScalarEvolution;
  41. class SCEV;
  42. class SCEVExpander;
  43. class TargetLibraryInfo;
  44. class LPPassManager;
  45. class Instruction;
  46. struct RuntimeCheckingPtrGroup;
  47. typedef std::pair<const RuntimeCheckingPtrGroup *,
  48. const RuntimeCheckingPtrGroup *>
  49. RuntimePointerCheck;
  50. template <typename T, unsigned N> class SmallSetVector;
  51. template <typename T, unsigned N> class SmallPriorityWorklist;
  52. BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
  53. MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
  54. /// Ensure that all exit blocks of the loop are dedicated exits.
  55. ///
  56. /// For any loop exit block with non-loop predecessors, we split the loop
  57. /// predecessors to use a dedicated loop exit block. We update the dominator
  58. /// tree and loop info if provided, and will preserve LCSSA if requested.
  59. bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
  60. MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
  61. /// Ensures LCSSA form for every instruction from the Worklist in the scope of
  62. /// innermost containing loop.
  63. ///
  64. /// For the given instruction which have uses outside of the loop, an LCSSA PHI
  65. /// node is inserted and the uses outside the loop are rewritten to use this
  66. /// node.
  67. ///
  68. /// LoopInfo and DominatorTree are required and, since the routine makes no
  69. /// changes to CFG, preserved.
  70. ///
  71. /// Returns true if any modifications are made.
  72. ///
  73. /// This function may introduce unused PHI nodes. If \p PHIsToRemove is not
  74. /// nullptr, those are added to it (before removing, the caller has to check if
  75. /// they still do not have any uses). Otherwise the PHIs are directly removed.
  76. bool formLCSSAForInstructions(
  77. SmallVectorImpl<Instruction *> &Worklist, const DominatorTree &DT,
  78. const LoopInfo &LI, ScalarEvolution *SE, IRBuilderBase &Builder,
  79. SmallVectorImpl<PHINode *> *PHIsToRemove = nullptr);
  80. /// Put loop into LCSSA form.
  81. ///
  82. /// Looks at all instructions in the loop which have uses outside of the
  83. /// current loop. For each, an LCSSA PHI node is inserted and the uses outside
  84. /// the loop are rewritten to use this node. Sub-loops must be in LCSSA form
  85. /// already.
  86. ///
  87. /// LoopInfo and DominatorTree are required and preserved.
  88. ///
  89. /// If ScalarEvolution is passed in, it will be preserved.
  90. ///
  91. /// Returns true if any modifications are made to the loop.
  92. bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
  93. ScalarEvolution *SE);
  94. /// Put a loop nest into LCSSA form.
  95. ///
  96. /// This recursively forms LCSSA for a loop nest.
  97. ///
  98. /// LoopInfo and DominatorTree are required and preserved.
  99. ///
  100. /// If ScalarEvolution is passed in, it will be preserved.
  101. ///
  102. /// Returns true if any modifications are made to the loop.
  103. bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
  104. ScalarEvolution *SE);
  105. /// Flags controlling how much is checked when sinking or hoisting
  106. /// instructions. The number of memory access in the loop (and whether there
  107. /// are too many) is determined in the constructors when using MemorySSA.
  108. class SinkAndHoistLICMFlags {
  109. public:
  110. // Explicitly set limits.
  111. SinkAndHoistLICMFlags(unsigned LicmMssaOptCap,
  112. unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
  113. Loop *L = nullptr, MemorySSA *MSSA = nullptr);
  114. // Use default limits.
  115. SinkAndHoistLICMFlags(bool IsSink, Loop *L = nullptr,
  116. MemorySSA *MSSA = nullptr);
  117. void setIsSink(bool B) { IsSink = B; }
  118. bool getIsSink() { return IsSink; }
  119. bool tooManyMemoryAccesses() { return NoOfMemAccTooLarge; }
  120. bool tooManyClobberingCalls() { return LicmMssaOptCounter >= LicmMssaOptCap; }
  121. void incrementClobberingCalls() { ++LicmMssaOptCounter; }
  122. protected:
  123. bool NoOfMemAccTooLarge = false;
  124. unsigned LicmMssaOptCounter = 0;
  125. unsigned LicmMssaOptCap;
  126. unsigned LicmMssaNoAccForPromotionCap;
  127. bool IsSink;
  128. };
  129. /// Walk the specified region of the CFG (defined by all blocks
  130. /// dominated by the specified block, and that are in the current loop) in
  131. /// reverse depth first order w.r.t the DominatorTree. This allows us to visit
  132. /// uses before definitions, allowing us to sink a loop body in one pass without
  133. /// iteration. Takes DomTreeNode, AAResults, LoopInfo, DominatorTree,
  134. /// TargetLibraryInfo, Loop, AliasSet information for all
  135. /// instructions of the loop and loop safety information as
  136. /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
  137. /// \p CurLoop is a loop to do sinking on. \p OutermostLoop is used only when
  138. /// this function is called by \p sinkRegionForLoopNest.
  139. bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
  140. TargetLibraryInfo *, TargetTransformInfo *, Loop *CurLoop,
  141. MemorySSAUpdater &, ICFLoopSafetyInfo *,
  142. SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *,
  143. Loop *OutermostLoop = nullptr);
  144. /// Call sinkRegion on loops contained within the specified loop
  145. /// in order from innermost to outermost.
  146. bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *,
  147. DominatorTree *, TargetLibraryInfo *,
  148. TargetTransformInfo *, Loop *, MemorySSAUpdater &,
  149. ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &,
  150. OptimizationRemarkEmitter *);
  151. /// Walk the specified region of the CFG (defined by all blocks
  152. /// dominated by the specified block, and that are in the current loop) in depth
  153. /// first order w.r.t the DominatorTree. This allows us to visit definitions
  154. /// before uses, allowing us to hoist a loop body in one pass without iteration.
  155. /// Takes DomTreeNode, AAResults, LoopInfo, DominatorTree,
  156. /// TargetLibraryInfo, Loop, AliasSet information for all
  157. /// instructions of the loop and loop safety information as arguments.
  158. /// Diagnostics is emitted via \p ORE. It returns changed status.
  159. /// \p AllowSpeculation is whether values should be hoisted even if they are not
  160. /// guaranteed to execute in the loop, but are safe to speculatively execute.
  161. bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
  162. AssumptionCache *, TargetLibraryInfo *, Loop *,
  163. MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *,
  164. SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool,
  165. bool AllowSpeculation);
  166. /// This function deletes dead loops. The caller of this function needs to
  167. /// guarantee that the loop is infact dead.
  168. /// The function requires a bunch or prerequisites to be present:
  169. /// - The loop needs to be in LCSSA form
  170. /// - The loop needs to have a Preheader
  171. /// - A unique dedicated exit block must exist
  172. ///
  173. /// This also updates the relevant analysis information in \p DT, \p SE, \p LI
  174. /// and \p MSSA if pointers to those are provided.
  175. /// It also updates the loop PM if an updater struct is provided.
  176. void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
  177. LoopInfo *LI, MemorySSA *MSSA = nullptr);
  178. /// Remove the backedge of the specified loop. Handles loop nests and general
  179. /// loop structures subject to the precondition that the loop has no parent
  180. /// loop and has a single latch block. Preserves all listed analyses.
  181. void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
  182. LoopInfo &LI, MemorySSA *MSSA);
  183. /// Try to promote memory values to scalars by sinking stores out of
  184. /// the loop and moving loads to before the loop. We do this by looping over
  185. /// the stores in the loop, looking for stores to Must pointers which are
  186. /// loop invariant. It takes a set of must-alias values, Loop exit blocks
  187. /// vector, loop exit blocks insertion point vector, PredIteratorCache,
  188. /// LoopInfo, DominatorTree, Loop, AliasSet information for all instructions
  189. /// of the loop and loop safety information as arguments.
  190. /// Diagnostics is emitted via \p ORE. It returns changed status.
  191. /// \p AllowSpeculation is whether values should be hoisted even if they are not
  192. /// guaranteed to execute in the loop, but are safe to speculatively execute.
  193. bool promoteLoopAccessesToScalars(
  194. const SmallSetVector<Value *, 8> &, SmallVectorImpl<BasicBlock *> &,
  195. SmallVectorImpl<Instruction *> &, SmallVectorImpl<MemoryAccess *> &,
  196. PredIteratorCache &, LoopInfo *, DominatorTree *, AssumptionCache *AC,
  197. const TargetLibraryInfo *, TargetTransformInfo *, Loop *,
  198. MemorySSAUpdater &, ICFLoopSafetyInfo *, OptimizationRemarkEmitter *,
  199. bool AllowSpeculation, bool HasReadsOutsideSet);
  200. /// Does a BFS from a given node to all of its children inside a given loop.
  201. /// The returned vector of nodes includes the starting point.
  202. SmallVector<DomTreeNode *, 16> collectChildrenInLoop(DomTreeNode *N,
  203. const Loop *CurLoop);
  204. /// Returns the instructions that use values defined in the loop.
  205. SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L);
  206. /// Find a combination of metadata ("llvm.loop.vectorize.width" and
  207. /// "llvm.loop.vectorize.scalable.enable") for a loop and use it to construct a
  208. /// ElementCount. If the metadata "llvm.loop.vectorize.width" cannot be found
  209. /// then std::nullopt is returned.
  210. std::optional<ElementCount>
  211. getOptionalElementCountLoopAttribute(const Loop *TheLoop);
  212. /// Create a new loop identifier for a loop created from a loop transformation.
  213. ///
  214. /// @param OrigLoopID The loop ID of the loop before the transformation.
  215. /// @param FollowupAttrs List of attribute names that contain attributes to be
  216. /// added to the new loop ID.
  217. /// @param InheritOptionsAttrsPrefix Selects which attributes should be inherited
  218. /// from the original loop. The following values
  219. /// are considered:
  220. /// nullptr : Inherit all attributes from @p OrigLoopID.
  221. /// "" : Do not inherit any attribute from @p OrigLoopID; only use
  222. /// those specified by a followup attribute.
  223. /// "<prefix>": Inherit all attributes except those which start with
  224. /// <prefix>; commonly used to remove metadata for the
  225. /// applied transformation.
  226. /// @param AlwaysNew If true, do not try to reuse OrigLoopID and never return
  227. /// std::nullopt.
  228. ///
  229. /// @return The loop ID for the after-transformation loop. The following values
  230. /// can be returned:
  231. /// std::nullopt : No followup attribute was found; it is up to the
  232. /// transformation to choose attributes that make sense.
  233. /// @p OrigLoopID: The original identifier can be reused.
  234. /// nullptr : The new loop has no attributes.
  235. /// MDNode* : A new unique loop identifier.
  236. std::optional<MDNode *>
  237. makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef<StringRef> FollowupAttrs,
  238. const char *InheritOptionsAttrsPrefix = "",
  239. bool AlwaysNew = false);
  240. /// Look for the loop attribute that disables all transformation heuristic.
  241. bool hasDisableAllTransformsHint(const Loop *L);
  242. /// Look for the loop attribute that disables the LICM transformation heuristics.
  243. bool hasDisableLICMTransformsHint(const Loop *L);
  244. /// The mode sets how eager a transformation should be applied.
  245. enum TransformationMode {
  246. /// The pass can use heuristics to determine whether a transformation should
  247. /// be applied.
  248. TM_Unspecified,
  249. /// The transformation should be applied without considering a cost model.
  250. TM_Enable,
  251. /// The transformation should not be applied.
  252. TM_Disable,
  253. /// Force is a flag and should not be used alone.
  254. TM_Force = 0x04,
  255. /// The transformation was directed by the user, e.g. by a #pragma in
  256. /// the source code. If the transformation could not be applied, a
  257. /// warning should be emitted.
  258. TM_ForcedByUser = TM_Enable | TM_Force,
  259. /// The transformation must not be applied. For instance, `#pragma clang loop
  260. /// unroll(disable)` explicitly forbids any unrolling to take place. Unlike
  261. /// general loop metadata, it must not be dropped. Most passes should not
  262. /// behave differently under TM_Disable and TM_SuppressedByUser.
  263. TM_SuppressedByUser = TM_Disable | TM_Force
  264. };
  265. /// @{
  266. /// Get the mode for LLVM's supported loop transformations.
  267. TransformationMode hasUnrollTransformation(const Loop *L);
  268. TransformationMode hasUnrollAndJamTransformation(const Loop *L);
  269. TransformationMode hasVectorizeTransformation(const Loop *L);
  270. TransformationMode hasDistributeTransformation(const Loop *L);
  271. TransformationMode hasLICMVersioningTransformation(const Loop *L);
  272. /// @}
  273. /// Set input string into loop metadata by keeping other values intact.
  274. /// If the string is already in loop metadata update value if it is
  275. /// different.
  276. void addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
  277. unsigned V = 0);
  278. /// Returns a loop's estimated trip count based on branch weight metadata.
  279. /// In addition if \p EstimatedLoopInvocationWeight is not null it is
  280. /// initialized with weight of loop's latch leading to the exit.
  281. /// Returns 0 when the count is estimated to be 0, or std::nullopt when a
  282. /// meaningful estimate can not be made.
  283. std::optional<unsigned>
  284. getLoopEstimatedTripCount(Loop *L,
  285. unsigned *EstimatedLoopInvocationWeight = nullptr);
  286. /// Set a loop's branch weight metadata to reflect that loop has \p
  287. /// EstimatedTripCount iterations and \p EstimatedLoopInvocationWeight exits
  288. /// through latch. Returns true if metadata is successfully updated, false
  289. /// otherwise. Note that loop must have a latch block which controls loop exit
  290. /// in order to succeed.
  291. bool setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount,
  292. unsigned EstimatedLoopInvocationWeight);
  293. /// Check inner loop (L) backedge count is known to be invariant on all
  294. /// iterations of its outer loop. If the loop has no parent, this is trivially
  295. /// true.
  296. bool hasIterationCountInvariantInParent(Loop *L, ScalarEvolution &SE);
  297. /// Helper to consistently add the set of standard passes to a loop pass's \c
  298. /// AnalysisUsage.
  299. ///
  300. /// All loop passes should call this as part of implementing their \c
  301. /// getAnalysisUsage.
  302. void getLoopAnalysisUsage(AnalysisUsage &AU);
  303. /// Returns true if is legal to hoist or sink this instruction disregarding the
  304. /// possible introduction of faults. Reasoning about potential faulting
  305. /// instructions is the responsibility of the caller since it is challenging to
  306. /// do efficiently from within this routine.
  307. /// \p TargetExecutesOncePerLoop is true only when it is guaranteed that the
  308. /// target executes at most once per execution of the loop body. This is used
  309. /// to assess the legality of duplicating atomic loads. Generally, this is
  310. /// true when moving out of loop and not true when moving into loops.
  311. /// If \p ORE is set use it to emit optimization remarks.
  312. bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
  313. Loop *CurLoop, MemorySSAUpdater &MSSAU,
  314. bool TargetExecutesOncePerLoop,
  315. SinkAndHoistLICMFlags &LICMFlags,
  316. OptimizationRemarkEmitter *ORE = nullptr);
  317. /// Returns the comparison predicate used when expanding a min/max reduction.
  318. CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK);
  319. /// See RecurrenceDescriptor::isSelectCmpPattern for a description of the
  320. /// pattern we are trying to match. In this pattern we are only ever selecting
  321. /// between two values: 1) an initial PHI start value, and 2) a loop invariant
  322. /// value. This function uses \p LoopExitInst to determine 2), which we then use
  323. /// to select between \p Left and \p Right. Any lane value in \p Left that
  324. /// matches 2) will be merged into \p Right.
  325. Value *createSelectCmpOp(IRBuilderBase &Builder, Value *StartVal, RecurKind RK,
  326. Value *Left, Value *Right);
  327. /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
  328. /// The Builder's fast-math-flags must be set to propagate the expected values.
  329. Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
  330. Value *Right);
  331. /// Generates an ordered vector reduction using extracts to reduce the value.
  332. Value *getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src,
  333. unsigned Op, RecurKind MinMaxKind = RecurKind::None);
  334. /// Generates a vector reduction using shufflevectors to reduce the value.
  335. /// Fast-math-flags are propagated using the IRBuilder's setting.
  336. Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op,
  337. RecurKind MinMaxKind = RecurKind::None);
  338. /// Create a target reduction of the given vector. The reduction operation
  339. /// is described by the \p Opcode parameter. min/max reductions require
  340. /// additional information supplied in \p RdxKind.
  341. /// The target is queried to determine if intrinsics or shuffle sequences are
  342. /// required to implement the reduction.
  343. /// Fast-math-flags are propagated using the IRBuilder's setting.
  344. Value *createSimpleTargetReduction(IRBuilderBase &B,
  345. const TargetTransformInfo *TTI, Value *Src,
  346. RecurKind RdxKind);
  347. /// Create a target reduction of the given vector \p Src for a reduction of the
  348. /// kind RecurKind::SelectICmp or RecurKind::SelectFCmp. The reduction operation
  349. /// is described by \p Desc.
  350. Value *createSelectCmpTargetReduction(IRBuilderBase &B,
  351. const TargetTransformInfo *TTI,
  352. Value *Src,
  353. const RecurrenceDescriptor &Desc,
  354. PHINode *OrigPhi);
  355. /// Create a generic target reduction using a recurrence descriptor \p Desc
  356. /// The target is queried to determine if intrinsics or shuffle sequences are
  357. /// required to implement the reduction.
  358. /// Fast-math-flags are propagated using the RecurrenceDescriptor.
  359. Value *createTargetReduction(IRBuilderBase &B, const TargetTransformInfo *TTI,
  360. const RecurrenceDescriptor &Desc, Value *Src,
  361. PHINode *OrigPhi = nullptr);
  362. /// Create an ordered reduction intrinsic using the given recurrence
  363. /// descriptor \p Desc.
  364. Value *createOrderedReduction(IRBuilderBase &B,
  365. const RecurrenceDescriptor &Desc, Value *Src,
  366. Value *Start);
  367. /// Get the intersection (logical and) of all of the potential IR flags
  368. /// of each scalar operation (VL) that will be converted into a vector (I).
  369. /// If OpValue is non-null, we only consider operations similar to OpValue
  370. /// when intersecting.
  371. /// Flag set: NSW, NUW (if IncludeWrapFlags is true), exact, and all of
  372. /// fast-math.
  373. void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr,
  374. bool IncludeWrapFlags = true);
  375. /// Returns true if we can prove that \p S is defined and always negative in
  376. /// loop \p L.
  377. bool isKnownNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE);
  378. /// Returns true if we can prove that \p S is defined and always non-negative in
  379. /// loop \p L.
  380. bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L,
  381. ScalarEvolution &SE);
  382. /// Returns true if \p S is defined and never is equal to signed/unsigned max.
  383. bool cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
  384. bool Signed);
  385. /// Returns true if \p S is defined and never is equal to signed/unsigned min.
  386. bool cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
  387. bool Signed);
  388. enum ReplaceExitVal {
  389. NeverRepl,
  390. OnlyCheapRepl,
  391. NoHardUse,
  392. UnusedIndVarInLoop,
  393. AlwaysRepl
  394. };
  395. /// If the final value of any expressions that are recurrent in the loop can
  396. /// be computed, substitute the exit values from the loop into any instructions
  397. /// outside of the loop that use the final values of the current expressions.
  398. /// Return the number of loop exit values that have been replaced, and the
  399. /// corresponding phi node will be added to DeadInsts.
  400. int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI,
  401. ScalarEvolution *SE, const TargetTransformInfo *TTI,
  402. SCEVExpander &Rewriter, DominatorTree *DT,
  403. ReplaceExitVal ReplaceExitValue,
  404. SmallVector<WeakTrackingVH, 16> &DeadInsts);
  405. /// Set weights for \p UnrolledLoop and \p RemainderLoop based on weights for
  406. /// \p OrigLoop and the following distribution of \p OrigLoop iteration among \p
  407. /// UnrolledLoop and \p RemainderLoop. \p UnrolledLoop receives weights that
  408. /// reflect TC/UF iterations, and \p RemainderLoop receives weights that reflect
  409. /// the remaining TC%UF iterations.
  410. ///
  411. /// Note that \p OrigLoop may be equal to either \p UnrolledLoop or \p
  412. /// RemainderLoop in which case weights for \p OrigLoop are updated accordingly.
  413. /// Note also behavior is undefined if \p UnrolledLoop and \p RemainderLoop are
  414. /// equal. \p UF must be greater than zero.
  415. /// If \p OrigLoop has no profile info associated nothing happens.
  416. ///
  417. /// This utility may be useful for such optimizations as unroller and
  418. /// vectorizer as it's typical transformation for them.
  419. void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop,
  420. Loop *RemainderLoop, uint64_t UF);
  421. /// Utility that implements appending of loops onto a worklist given a range.
  422. /// We want to process loops in postorder, but the worklist is a LIFO data
  423. /// structure, so we append to it in *reverse* postorder.
  424. /// For trees, a preorder traversal is a viable reverse postorder, so we
  425. /// actually append using a preorder walk algorithm.
  426. template <typename RangeT>
  427. void appendLoopsToWorklist(RangeT &&, SmallPriorityWorklist<Loop *, 4> &);
  428. /// Utility that implements appending of loops onto a worklist given a range.
  429. /// It has the same behavior as appendLoopsToWorklist, but assumes the range of
  430. /// loops has already been reversed, so it processes loops in the given order.
  431. template <typename RangeT>
  432. void appendReversedLoopsToWorklist(RangeT &&,
  433. SmallPriorityWorklist<Loop *, 4> &);
  434. /// Utility that implements appending of loops onto a worklist given LoopInfo.
  435. /// Calls the templated utility taking a Range of loops, handing it the Loops
  436. /// in LoopInfo, iterated in reverse. This is because the loops are stored in
  437. /// RPO w.r.t. the control flow graph in LoopInfo. For the purpose of unrolling,
  438. /// loop deletion, and LICM, we largely want to work forward across the CFG so
  439. /// that we visit defs before uses and can propagate simplifications from one
  440. /// loop nest into the next. Calls appendReversedLoopsToWorklist with the
  441. /// already reversed loops in LI.
  442. /// FIXME: Consider changing the order in LoopInfo.
  443. void appendLoopsToWorklist(LoopInfo &, SmallPriorityWorklist<Loop *, 4> &);
  444. /// Recursively clone the specified loop and all of its children,
  445. /// mapping the blocks with the specified map.
  446. Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM,
  447. LoopInfo *LI, LPPassManager *LPM);
  448. /// Add code that checks at runtime if the accessed arrays in \p PointerChecks
  449. /// overlap. Returns the final comparator value or NULL if no check is needed.
  450. Value *
  451. addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
  452. const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
  453. SCEVExpander &Expander);
  454. Value *addDiffRuntimeChecks(
  455. Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
  456. function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC);
  457. /// Struct to hold information about a partially invariant condition.
  458. struct IVConditionInfo {
  459. /// Instructions that need to be duplicated and checked for the unswitching
  460. /// condition.
  461. SmallVector<Instruction *> InstToDuplicate;
  462. /// Constant to indicate for which value the condition is invariant.
  463. Constant *KnownValue = nullptr;
  464. /// True if the partially invariant path is no-op (=does not have any
  465. /// side-effects and no loop value is used outside the loop).
  466. bool PathIsNoop = true;
  467. /// If the partially invariant path reaches a single exit block, ExitForPath
  468. /// is set to that block. Otherwise it is nullptr.
  469. BasicBlock *ExitForPath = nullptr;
  470. };
  471. /// Check if the loop header has a conditional branch that is not
  472. /// loop-invariant, because it involves load instructions. If all paths from
  473. /// either the true or false successor to the header or loop exists do not
  474. /// modify the memory feeding the condition, perform 'partial unswitching'. That
  475. /// is, duplicate the instructions feeding the condition in the pre-header. Then
  476. /// unswitch on the duplicated condition. The condition is now known in the
  477. /// unswitched version for the 'invariant' path through the original loop.
  478. ///
  479. /// If the branch condition of the header is partially invariant, return a pair
  480. /// containing the instructions to duplicate and a boolean Constant to update
  481. /// the condition in the loops created for the true or false successors.
  482. std::optional<IVConditionInfo> hasPartialIVCondition(const Loop &L,
  483. unsigned MSSAThreshold,
  484. const MemorySSA &MSSA,
  485. AAResults &AA);
  486. } // end namespace llvm
  487. #endif // LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
  488. #ifdef __GNUC__
  489. #pragma GCC diagnostic pop
  490. #endif