FunctionAttrs.cpp 71 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054
  1. //===- FunctionAttrs.cpp - Pass which marks functions attributes ----------===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. //
  9. /// \file
  10. /// This file implements interprocedural passes which walk the
  11. /// call-graph deducing and/or propagating function attributes.
  12. //
  13. //===----------------------------------------------------------------------===//
  14. #include "llvm/Transforms/IPO/FunctionAttrs.h"
  15. #include "llvm/ADT/ArrayRef.h"
  16. #include "llvm/ADT/DenseMap.h"
  17. #include "llvm/ADT/SCCIterator.h"
  18. #include "llvm/ADT/STLExtras.h"
  19. #include "llvm/ADT/SetVector.h"
  20. #include "llvm/ADT/SmallPtrSet.h"
  21. #include "llvm/ADT/SmallSet.h"
  22. #include "llvm/ADT/SmallVector.h"
  23. #include "llvm/ADT/Statistic.h"
  24. #include "llvm/Analysis/AssumptionCache.h"
  25. #include "llvm/Analysis/BasicAliasAnalysis.h"
  26. #include "llvm/Analysis/CFG.h"
  27. #include "llvm/Analysis/CGSCCPassManager.h"
  28. #include "llvm/Analysis/CallGraph.h"
  29. #include "llvm/Analysis/CallGraphSCCPass.h"
  30. #include "llvm/Analysis/CaptureTracking.h"
  31. #include "llvm/Analysis/LazyCallGraph.h"
  32. #include "llvm/Analysis/MemoryBuiltins.h"
  33. #include "llvm/Analysis/MemoryLocation.h"
  34. #include "llvm/Analysis/ValueTracking.h"
  35. #include "llvm/IR/Argument.h"
  36. #include "llvm/IR/Attributes.h"
  37. #include "llvm/IR/BasicBlock.h"
  38. #include "llvm/IR/Constant.h"
  39. #include "llvm/IR/Constants.h"
  40. #include "llvm/IR/Function.h"
  41. #include "llvm/IR/InstIterator.h"
  42. #include "llvm/IR/InstrTypes.h"
  43. #include "llvm/IR/Instruction.h"
  44. #include "llvm/IR/Instructions.h"
  45. #include "llvm/IR/IntrinsicInst.h"
  46. #include "llvm/IR/Metadata.h"
  47. #include "llvm/IR/PassManager.h"
  48. #include "llvm/IR/Type.h"
  49. #include "llvm/IR/Use.h"
  50. #include "llvm/IR/User.h"
  51. #include "llvm/IR/Value.h"
  52. #include "llvm/InitializePasses.h"
  53. #include "llvm/Pass.h"
  54. #include "llvm/Support/Casting.h"
  55. #include "llvm/Support/CommandLine.h"
  56. #include "llvm/Support/Compiler.h"
  57. #include "llvm/Support/Debug.h"
  58. #include "llvm/Support/ErrorHandling.h"
  59. #include "llvm/Support/raw_ostream.h"
  60. #include "llvm/Transforms/IPO.h"
  61. #include "llvm/Transforms/Utils/Local.h"
  62. #include <cassert>
  63. #include <iterator>
  64. #include <map>
  65. #include <vector>
  66. using namespace llvm;
  67. #define DEBUG_TYPE "function-attrs"
  68. STATISTIC(NumReadNone, "Number of functions marked readnone");
  69. STATISTIC(NumReadOnly, "Number of functions marked readonly");
  70. STATISTIC(NumWriteOnly, "Number of functions marked writeonly");
  71. STATISTIC(NumNoCapture, "Number of arguments marked nocapture");
  72. STATISTIC(NumReturned, "Number of arguments marked returned");
  73. STATISTIC(NumReadNoneArg, "Number of arguments marked readnone");
  74. STATISTIC(NumReadOnlyArg, "Number of arguments marked readonly");
  75. STATISTIC(NumWriteOnlyArg, "Number of arguments marked writeonly");
  76. STATISTIC(NumNoAlias, "Number of function returns marked noalias");
  77. STATISTIC(NumNonNullReturn, "Number of function returns marked nonnull");
  78. STATISTIC(NumNoRecurse, "Number of functions marked as norecurse");
  79. STATISTIC(NumNoUnwind, "Number of functions marked as nounwind");
  80. STATISTIC(NumNoFree, "Number of functions marked as nofree");
  81. STATISTIC(NumWillReturn, "Number of functions marked as willreturn");
  82. STATISTIC(NumNoSync, "Number of functions marked as nosync");
  83. STATISTIC(NumThinLinkNoRecurse,
  84. "Number of functions marked as norecurse during thinlink");
  85. STATISTIC(NumThinLinkNoUnwind,
  86. "Number of functions marked as nounwind during thinlink");
  87. static cl::opt<bool> EnableNonnullArgPropagation(
  88. "enable-nonnull-arg-prop", cl::init(true), cl::Hidden,
  89. cl::desc("Try to propagate nonnull argument attributes from callsites to "
  90. "caller functions."));
  91. static cl::opt<bool> DisableNoUnwindInference(
  92. "disable-nounwind-inference", cl::Hidden,
  93. cl::desc("Stop inferring nounwind attribute during function-attrs pass"));
  94. static cl::opt<bool> DisableNoFreeInference(
  95. "disable-nofree-inference", cl::Hidden,
  96. cl::desc("Stop inferring nofree attribute during function-attrs pass"));
  97. static cl::opt<bool> DisableThinLTOPropagation(
  98. "disable-thinlto-funcattrs", cl::init(true), cl::Hidden,
  99. cl::desc("Don't propagate function-attrs in thinLTO"));
  100. namespace {
  101. using SCCNodeSet = SmallSetVector<Function *, 8>;
  102. } // end anonymous namespace
  103. /// Returns the memory access attribute for function F using AAR for AA results,
  104. /// where SCCNodes is the current SCC.
  105. ///
  106. /// If ThisBody is true, this function may examine the function body and will
  107. /// return a result pertaining to this copy of the function. If it is false, the
  108. /// result will be based only on AA results for the function declaration; it
  109. /// will be assumed that some other (perhaps less optimized) version of the
  110. /// function may be selected at link time.
  111. static MemoryAccessKind checkFunctionMemoryAccess(Function &F, bool ThisBody,
  112. AAResults &AAR,
  113. const SCCNodeSet &SCCNodes) {
  114. FunctionModRefBehavior MRB = AAR.getModRefBehavior(&F);
  115. if (MRB == FMRB_DoesNotAccessMemory)
  116. // Already perfect!
  117. return MAK_ReadNone;
  118. if (!ThisBody) {
  119. if (AliasAnalysis::onlyReadsMemory(MRB))
  120. return MAK_ReadOnly;
  121. if (AliasAnalysis::onlyWritesMemory(MRB))
  122. return MAK_WriteOnly;
  123. // Conservatively assume it reads and writes to memory.
  124. return MAK_MayWrite;
  125. }
  126. // Scan the function body for instructions that may read or write memory.
  127. bool ReadsMemory = false;
  128. bool WritesMemory = false;
  129. for (Instruction &I : instructions(F)) {
  130. // Some instructions can be ignored even if they read or write memory.
  131. // Detect these now, skipping to the next instruction if one is found.
  132. if (auto *Call = dyn_cast<CallBase>(&I)) {
  133. // Ignore calls to functions in the same SCC, as long as the call sites
  134. // don't have operand bundles. Calls with operand bundles are allowed to
  135. // have memory effects not described by the memory effects of the call
  136. // target.
  137. if (!Call->hasOperandBundles() && Call->getCalledFunction() &&
  138. SCCNodes.count(Call->getCalledFunction()))
  139. continue;
  140. FunctionModRefBehavior MRB = AAR.getModRefBehavior(Call);
  141. ModRefInfo MRI = createModRefInfo(MRB);
  142. // If the call doesn't access memory, we're done.
  143. if (isNoModRef(MRI))
  144. continue;
  145. // A pseudo probe call shouldn't change any function attribute since it
  146. // doesn't translate to a real instruction. It comes with a memory access
  147. // tag to prevent itself being removed by optimizations and not block
  148. // other instructions being optimized.
  149. if (isa<PseudoProbeInst>(I))
  150. continue;
  151. if (!AliasAnalysis::onlyAccessesArgPointees(MRB)) {
  152. // The call could access any memory. If that includes writes, note it.
  153. if (isModSet(MRI))
  154. WritesMemory = true;
  155. // If it reads, note it.
  156. if (isRefSet(MRI))
  157. ReadsMemory = true;
  158. continue;
  159. }
  160. // Check whether all pointer arguments point to local memory, and
  161. // ignore calls that only access local memory.
  162. for (const Use &U : Call->args()) {
  163. const Value *Arg = U;
  164. if (!Arg->getType()->isPtrOrPtrVectorTy())
  165. continue;
  166. MemoryLocation Loc =
  167. MemoryLocation::getBeforeOrAfter(Arg, I.getAAMetadata());
  168. // Skip accesses to local or constant memory as they don't impact the
  169. // externally visible mod/ref behavior.
  170. if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
  171. continue;
  172. if (isModSet(MRI))
  173. // Writes non-local memory.
  174. WritesMemory = true;
  175. if (isRefSet(MRI))
  176. // Ok, it reads non-local memory.
  177. ReadsMemory = true;
  178. }
  179. continue;
  180. } else if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
  181. // Ignore non-volatile loads from local memory. (Atomic is okay here.)
  182. if (!LI->isVolatile()) {
  183. MemoryLocation Loc = MemoryLocation::get(LI);
  184. if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
  185. continue;
  186. }
  187. } else if (StoreInst *SI = dyn_cast<StoreInst>(&I)) {
  188. // Ignore non-volatile stores to local memory. (Atomic is okay here.)
  189. if (!SI->isVolatile()) {
  190. MemoryLocation Loc = MemoryLocation::get(SI);
  191. if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
  192. continue;
  193. }
  194. } else if (VAArgInst *VI = dyn_cast<VAArgInst>(&I)) {
  195. // Ignore vaargs on local memory.
  196. MemoryLocation Loc = MemoryLocation::get(VI);
  197. if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
  198. continue;
  199. }
  200. // Any remaining instructions need to be taken seriously! Check if they
  201. // read or write memory.
  202. //
  203. // Writes memory, remember that.
  204. WritesMemory |= I.mayWriteToMemory();
  205. // If this instruction may read memory, remember that.
  206. ReadsMemory |= I.mayReadFromMemory();
  207. }
  208. if (WritesMemory) {
  209. if (!ReadsMemory)
  210. return MAK_WriteOnly;
  211. else
  212. return MAK_MayWrite;
  213. }
  214. return ReadsMemory ? MAK_ReadOnly : MAK_ReadNone;
  215. }
  216. MemoryAccessKind llvm::computeFunctionBodyMemoryAccess(Function &F,
  217. AAResults &AAR) {
  218. return checkFunctionMemoryAccess(F, /*ThisBody=*/true, AAR, {});
  219. }
  220. /// Deduce readonly/readnone attributes for the SCC.
  221. template <typename AARGetterT>
  222. static void addReadAttrs(const SCCNodeSet &SCCNodes, AARGetterT &&AARGetter,
  223. SmallSet<Function *, 8> &Changed) {
  224. // Check if any of the functions in the SCC read or write memory. If they
  225. // write memory then they can't be marked readnone or readonly.
  226. bool ReadsMemory = false;
  227. bool WritesMemory = false;
  228. for (Function *F : SCCNodes) {
  229. // Call the callable parameter to look up AA results for this function.
  230. AAResults &AAR = AARGetter(*F);
  231. // Non-exact function definitions may not be selected at link time, and an
  232. // alternative version that writes to memory may be selected. See the
  233. // comment on GlobalValue::isDefinitionExact for more details.
  234. switch (checkFunctionMemoryAccess(*F, F->hasExactDefinition(),
  235. AAR, SCCNodes)) {
  236. case MAK_MayWrite:
  237. return;
  238. case MAK_ReadOnly:
  239. ReadsMemory = true;
  240. break;
  241. case MAK_WriteOnly:
  242. WritesMemory = true;
  243. break;
  244. case MAK_ReadNone:
  245. // Nothing to do!
  246. break;
  247. }
  248. }
  249. // If the SCC contains both functions that read and functions that write, then
  250. // we cannot add readonly attributes.
  251. if (ReadsMemory && WritesMemory)
  252. return;
  253. // Success! Functions in this SCC do not access memory, or only read memory.
  254. // Give them the appropriate attribute.
  255. for (Function *F : SCCNodes) {
  256. if (F->doesNotAccessMemory())
  257. // Already perfect!
  258. continue;
  259. if (F->onlyReadsMemory() && ReadsMemory)
  260. // No change.
  261. continue;
  262. if (F->onlyWritesMemory() && WritesMemory)
  263. continue;
  264. Changed.insert(F);
  265. // Clear out any existing attributes.
  266. AttributeMask AttrsToRemove;
  267. AttrsToRemove.addAttribute(Attribute::ReadOnly);
  268. AttrsToRemove.addAttribute(Attribute::ReadNone);
  269. AttrsToRemove.addAttribute(Attribute::WriteOnly);
  270. if (!WritesMemory && !ReadsMemory) {
  271. // Clear out any "access range attributes" if readnone was deduced.
  272. AttrsToRemove.addAttribute(Attribute::ArgMemOnly);
  273. AttrsToRemove.addAttribute(Attribute::InaccessibleMemOnly);
  274. AttrsToRemove.addAttribute(Attribute::InaccessibleMemOrArgMemOnly);
  275. }
  276. F->removeFnAttrs(AttrsToRemove);
  277. // Add in the new attribute.
  278. if (WritesMemory && !ReadsMemory)
  279. F->addFnAttr(Attribute::WriteOnly);
  280. else
  281. F->addFnAttr(ReadsMemory ? Attribute::ReadOnly : Attribute::ReadNone);
  282. if (WritesMemory && !ReadsMemory)
  283. ++NumWriteOnly;
  284. else if (ReadsMemory)
  285. ++NumReadOnly;
  286. else
  287. ++NumReadNone;
  288. }
  289. }
  290. // Compute definitive function attributes for a function taking into account
  291. // prevailing definitions and linkage types
  292. static FunctionSummary *calculatePrevailingSummary(
  293. ValueInfo VI,
  294. DenseMap<ValueInfo, FunctionSummary *> &CachedPrevailingSummary,
  295. function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
  296. IsPrevailing) {
  297. if (CachedPrevailingSummary.count(VI))
  298. return CachedPrevailingSummary[VI];
  299. /// At this point, prevailing symbols have been resolved. The following leads
  300. /// to returning a conservative result:
  301. /// - Multiple instances with local linkage. Normally local linkage would be
  302. /// unique per module
  303. /// as the GUID includes the module path. We could have a guid alias if
  304. /// there wasn't any distinguishing path when each file was compiled, but
  305. /// that should be rare so we'll punt on those.
  306. /// These next 2 cases should not happen and will assert:
  307. /// - Multiple instances with external linkage. This should be caught in
  308. /// symbol resolution
  309. /// - Non-existent FunctionSummary for Aliasee. This presents a hole in our
  310. /// knowledge meaning we have to go conservative.
  311. /// Otherwise, we calculate attributes for a function as:
  312. /// 1. If we have a local linkage, take its attributes. If there's somehow
  313. /// multiple, bail and go conservative.
  314. /// 2. If we have an external/WeakODR/LinkOnceODR linkage check that it is
  315. /// prevailing, take its attributes.
  316. /// 3. If we have a Weak/LinkOnce linkage the copies can have semantic
  317. /// differences. However, if the prevailing copy is known it will be used
  318. /// so take its attributes. If the prevailing copy is in a native file
  319. /// all IR copies will be dead and propagation will go conservative.
  320. /// 4. AvailableExternally summaries without a prevailing copy are known to
  321. /// occur in a couple of circumstances:
  322. /// a. An internal function gets imported due to its caller getting
  323. /// imported, it becomes AvailableExternally but no prevailing
  324. /// definition exists. Because it has to get imported along with its
  325. /// caller the attributes will be captured by propagating on its
  326. /// caller.
  327. /// b. C++11 [temp.explicit]p10 can generate AvailableExternally
  328. /// definitions of explicitly instanced template declarations
  329. /// for inlining which are ultimately dropped from the TU. Since this
  330. /// is localized to the TU the attributes will have already made it to
  331. /// the callers.
  332. /// These are edge cases and already captured by their callers so we
  333. /// ignore these for now. If they become relevant to optimize in the
  334. /// future this can be revisited.
  335. /// 5. Otherwise, go conservative.
  336. CachedPrevailingSummary[VI] = nullptr;
  337. FunctionSummary *Local = nullptr;
  338. FunctionSummary *Prevailing = nullptr;
  339. for (const auto &GVS : VI.getSummaryList()) {
  340. if (!GVS->isLive())
  341. continue;
  342. FunctionSummary *FS = dyn_cast<FunctionSummary>(GVS->getBaseObject());
  343. // Virtual and Unknown (e.g. indirect) calls require going conservative
  344. if (!FS || FS->fflags().HasUnknownCall)
  345. return nullptr;
  346. const auto &Linkage = GVS->linkage();
  347. if (GlobalValue::isLocalLinkage(Linkage)) {
  348. if (Local) {
  349. LLVM_DEBUG(
  350. dbgs()
  351. << "ThinLTO FunctionAttrs: Multiple Local Linkage, bailing on "
  352. "function "
  353. << VI.name() << " from " << FS->modulePath() << ". Previous module "
  354. << Local->modulePath() << "\n");
  355. return nullptr;
  356. }
  357. Local = FS;
  358. } else if (GlobalValue::isExternalLinkage(Linkage)) {
  359. assert(IsPrevailing(VI.getGUID(), GVS.get()));
  360. Prevailing = FS;
  361. break;
  362. } else if (GlobalValue::isWeakODRLinkage(Linkage) ||
  363. GlobalValue::isLinkOnceODRLinkage(Linkage) ||
  364. GlobalValue::isWeakAnyLinkage(Linkage) ||
  365. GlobalValue::isLinkOnceAnyLinkage(Linkage)) {
  366. if (IsPrevailing(VI.getGUID(), GVS.get())) {
  367. Prevailing = FS;
  368. break;
  369. }
  370. } else if (GlobalValue::isAvailableExternallyLinkage(Linkage)) {
  371. // TODO: Handle these cases if they become meaningful
  372. continue;
  373. }
  374. }
  375. if (Local) {
  376. assert(!Prevailing);
  377. CachedPrevailingSummary[VI] = Local;
  378. } else if (Prevailing) {
  379. assert(!Local);
  380. CachedPrevailingSummary[VI] = Prevailing;
  381. }
  382. return CachedPrevailingSummary[VI];
  383. }
  384. bool llvm::thinLTOPropagateFunctionAttrs(
  385. ModuleSummaryIndex &Index,
  386. function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
  387. IsPrevailing) {
  388. // TODO: implement addNoAliasAttrs once
  389. // there's more information about the return type in the summary
  390. if (DisableThinLTOPropagation)
  391. return false;
  392. DenseMap<ValueInfo, FunctionSummary *> CachedPrevailingSummary;
  393. bool Changed = false;
  394. auto PropagateAttributes = [&](std::vector<ValueInfo> &SCCNodes) {
  395. // Assume we can propagate unless we discover otherwise
  396. FunctionSummary::FFlags InferredFlags;
  397. InferredFlags.NoRecurse = (SCCNodes.size() == 1);
  398. InferredFlags.NoUnwind = true;
  399. for (auto &V : SCCNodes) {
  400. FunctionSummary *CallerSummary =
  401. calculatePrevailingSummary(V, CachedPrevailingSummary, IsPrevailing);
  402. // Function summaries can fail to contain information such as declarations
  403. if (!CallerSummary)
  404. return;
  405. if (CallerSummary->fflags().MayThrow)
  406. InferredFlags.NoUnwind = false;
  407. for (const auto &Callee : CallerSummary->calls()) {
  408. FunctionSummary *CalleeSummary = calculatePrevailingSummary(
  409. Callee.first, CachedPrevailingSummary, IsPrevailing);
  410. if (!CalleeSummary)
  411. return;
  412. if (!CalleeSummary->fflags().NoRecurse)
  413. InferredFlags.NoRecurse = false;
  414. if (!CalleeSummary->fflags().NoUnwind)
  415. InferredFlags.NoUnwind = false;
  416. if (!InferredFlags.NoUnwind && !InferredFlags.NoRecurse)
  417. break;
  418. }
  419. }
  420. if (InferredFlags.NoUnwind || InferredFlags.NoRecurse) {
  421. Changed = true;
  422. for (auto &V : SCCNodes) {
  423. if (InferredFlags.NoRecurse) {
  424. LLVM_DEBUG(dbgs() << "ThinLTO FunctionAttrs: Propagated NoRecurse to "
  425. << V.name() << "\n");
  426. ++NumThinLinkNoRecurse;
  427. }
  428. if (InferredFlags.NoUnwind) {
  429. LLVM_DEBUG(dbgs() << "ThinLTO FunctionAttrs: Propagated NoUnwind to "
  430. << V.name() << "\n");
  431. ++NumThinLinkNoUnwind;
  432. }
  433. for (auto &S : V.getSummaryList()) {
  434. if (auto *FS = dyn_cast<FunctionSummary>(S.get())) {
  435. if (InferredFlags.NoRecurse)
  436. FS->setNoRecurse();
  437. if (InferredFlags.NoUnwind)
  438. FS->setNoUnwind();
  439. }
  440. }
  441. }
  442. }
  443. };
  444. // Call propagation functions on each SCC in the Index
  445. for (scc_iterator<ModuleSummaryIndex *> I = scc_begin(&Index); !I.isAtEnd();
  446. ++I) {
  447. std::vector<ValueInfo> Nodes(*I);
  448. PropagateAttributes(Nodes);
  449. }
  450. return Changed;
  451. }
  452. namespace {
  453. /// For a given pointer Argument, this retains a list of Arguments of functions
  454. /// in the same SCC that the pointer data flows into. We use this to build an
  455. /// SCC of the arguments.
  456. struct ArgumentGraphNode {
  457. Argument *Definition;
  458. SmallVector<ArgumentGraphNode *, 4> Uses;
  459. };
  460. class ArgumentGraph {
  461. // We store pointers to ArgumentGraphNode objects, so it's important that
  462. // that they not move around upon insert.
  463. using ArgumentMapTy = std::map<Argument *, ArgumentGraphNode>;
  464. ArgumentMapTy ArgumentMap;
  465. // There is no root node for the argument graph, in fact:
  466. // void f(int *x, int *y) { if (...) f(x, y); }
  467. // is an example where the graph is disconnected. The SCCIterator requires a
  468. // single entry point, so we maintain a fake ("synthetic") root node that
  469. // uses every node. Because the graph is directed and nothing points into
  470. // the root, it will not participate in any SCCs (except for its own).
  471. ArgumentGraphNode SyntheticRoot;
  472. public:
  473. ArgumentGraph() { SyntheticRoot.Definition = nullptr; }
  474. using iterator = SmallVectorImpl<ArgumentGraphNode *>::iterator;
  475. iterator begin() { return SyntheticRoot.Uses.begin(); }
  476. iterator end() { return SyntheticRoot.Uses.end(); }
  477. ArgumentGraphNode *getEntryNode() { return &SyntheticRoot; }
  478. ArgumentGraphNode *operator[](Argument *A) {
  479. ArgumentGraphNode &Node = ArgumentMap[A];
  480. Node.Definition = A;
  481. SyntheticRoot.Uses.push_back(&Node);
  482. return &Node;
  483. }
  484. };
  485. /// This tracker checks whether callees are in the SCC, and if so it does not
  486. /// consider that a capture, instead adding it to the "Uses" list and
  487. /// continuing with the analysis.
  488. struct ArgumentUsesTracker : public CaptureTracker {
  489. ArgumentUsesTracker(const SCCNodeSet &SCCNodes) : SCCNodes(SCCNodes) {}
  490. void tooManyUses() override { Captured = true; }
  491. bool captured(const Use *U) override {
  492. CallBase *CB = dyn_cast<CallBase>(U->getUser());
  493. if (!CB) {
  494. Captured = true;
  495. return true;
  496. }
  497. Function *F = CB->getCalledFunction();
  498. if (!F || !F->hasExactDefinition() || !SCCNodes.count(F)) {
  499. Captured = true;
  500. return true;
  501. }
  502. assert(!CB->isCallee(U) && "callee operand reported captured?");
  503. const unsigned UseIndex = CB->getDataOperandNo(U);
  504. if (UseIndex >= CB->arg_size()) {
  505. // Data operand, but not a argument operand -- must be a bundle operand
  506. assert(CB->hasOperandBundles() && "Must be!");
  507. // CaptureTracking told us that we're being captured by an operand bundle
  508. // use. In this case it does not matter if the callee is within our SCC
  509. // or not -- we've been captured in some unknown way, and we have to be
  510. // conservative.
  511. Captured = true;
  512. return true;
  513. }
  514. if (UseIndex >= F->arg_size()) {
  515. assert(F->isVarArg() && "More params than args in non-varargs call");
  516. Captured = true;
  517. return true;
  518. }
  519. Uses.push_back(&*std::next(F->arg_begin(), UseIndex));
  520. return false;
  521. }
  522. // True only if certainly captured (used outside our SCC).
  523. bool Captured = false;
  524. // Uses within our SCC.
  525. SmallVector<Argument *, 4> Uses;
  526. const SCCNodeSet &SCCNodes;
  527. };
  528. } // end anonymous namespace
  529. namespace llvm {
  530. template <> struct GraphTraits<ArgumentGraphNode *> {
  531. using NodeRef = ArgumentGraphNode *;
  532. using ChildIteratorType = SmallVectorImpl<ArgumentGraphNode *>::iterator;
  533. static NodeRef getEntryNode(NodeRef A) { return A; }
  534. static ChildIteratorType child_begin(NodeRef N) { return N->Uses.begin(); }
  535. static ChildIteratorType child_end(NodeRef N) { return N->Uses.end(); }
  536. };
  537. template <>
  538. struct GraphTraits<ArgumentGraph *> : public GraphTraits<ArgumentGraphNode *> {
  539. static NodeRef getEntryNode(ArgumentGraph *AG) { return AG->getEntryNode(); }
  540. static ChildIteratorType nodes_begin(ArgumentGraph *AG) {
  541. return AG->begin();
  542. }
  543. static ChildIteratorType nodes_end(ArgumentGraph *AG) { return AG->end(); }
  544. };
  545. } // end namespace llvm
  546. /// Returns Attribute::None, Attribute::ReadOnly or Attribute::ReadNone.
  547. static Attribute::AttrKind
  548. determinePointerAccessAttrs(Argument *A,
  549. const SmallPtrSet<Argument *, 8> &SCCNodes) {
  550. SmallVector<Use *, 32> Worklist;
  551. SmallPtrSet<Use *, 32> Visited;
  552. // inalloca arguments are always clobbered by the call.
  553. if (A->hasInAllocaAttr() || A->hasPreallocatedAttr())
  554. return Attribute::None;
  555. bool IsRead = false;
  556. bool IsWrite = false;
  557. for (Use &U : A->uses()) {
  558. Visited.insert(&U);
  559. Worklist.push_back(&U);
  560. }
  561. while (!Worklist.empty()) {
  562. if (IsWrite && IsRead)
  563. // No point in searching further..
  564. return Attribute::None;
  565. Use *U = Worklist.pop_back_val();
  566. Instruction *I = cast<Instruction>(U->getUser());
  567. switch (I->getOpcode()) {
  568. case Instruction::BitCast:
  569. case Instruction::GetElementPtr:
  570. case Instruction::PHI:
  571. case Instruction::Select:
  572. case Instruction::AddrSpaceCast:
  573. // The original value is not read/written via this if the new value isn't.
  574. for (Use &UU : I->uses())
  575. if (Visited.insert(&UU).second)
  576. Worklist.push_back(&UU);
  577. break;
  578. case Instruction::Call:
  579. case Instruction::Invoke: {
  580. CallBase &CB = cast<CallBase>(*I);
  581. if (CB.isCallee(U)) {
  582. IsRead = true;
  583. // Note that indirect calls do not capture, see comment in
  584. // CaptureTracking for context
  585. continue;
  586. }
  587. // Given we've explictily handled the callee operand above, what's left
  588. // must be a data operand (e.g. argument or operand bundle)
  589. const unsigned UseIndex = CB.getDataOperandNo(U);
  590. if (!CB.doesNotCapture(UseIndex)) {
  591. if (!CB.onlyReadsMemory())
  592. // If the callee can save a copy into other memory, then simply
  593. // scanning uses of the call is insufficient. We have no way
  594. // of tracking copies of the pointer through memory to see
  595. // if a reloaded copy is written to, thus we must give up.
  596. return Attribute::None;
  597. // Push users for processing once we finish this one
  598. if (!I->getType()->isVoidTy())
  599. for (Use &UU : I->uses())
  600. if (Visited.insert(&UU).second)
  601. Worklist.push_back(&UU);
  602. }
  603. if (CB.doesNotAccessMemory())
  604. continue;
  605. if (Function *F = CB.getCalledFunction())
  606. if (CB.isArgOperand(U) && UseIndex < F->arg_size() &&
  607. SCCNodes.count(F->getArg(UseIndex)))
  608. // This is an argument which is part of the speculative SCC. Note
  609. // that only operands corresponding to formal arguments of the callee
  610. // can participate in the speculation.
  611. break;
  612. // The accessors used on call site here do the right thing for calls and
  613. // invokes with operand bundles.
  614. if (CB.doesNotAccessMemory(UseIndex)) {
  615. /* nop */
  616. } else if (CB.onlyReadsMemory() || CB.onlyReadsMemory(UseIndex)) {
  617. IsRead = true;
  618. } else if (CB.hasFnAttr(Attribute::WriteOnly) ||
  619. CB.dataOperandHasImpliedAttr(UseIndex, Attribute::WriteOnly)) {
  620. IsWrite = true;
  621. } else {
  622. return Attribute::None;
  623. }
  624. break;
  625. }
  626. case Instruction::Load:
  627. // A volatile load has side effects beyond what readonly can be relied
  628. // upon.
  629. if (cast<LoadInst>(I)->isVolatile())
  630. return Attribute::None;
  631. IsRead = true;
  632. break;
  633. case Instruction::Store:
  634. if (cast<StoreInst>(I)->getValueOperand() == *U)
  635. // untrackable capture
  636. return Attribute::None;
  637. // A volatile store has side effects beyond what writeonly can be relied
  638. // upon.
  639. if (cast<StoreInst>(I)->isVolatile())
  640. return Attribute::None;
  641. IsWrite = true;
  642. break;
  643. case Instruction::ICmp:
  644. case Instruction::Ret:
  645. break;
  646. default:
  647. return Attribute::None;
  648. }
  649. }
  650. if (IsWrite && IsRead)
  651. return Attribute::None;
  652. else if (IsRead)
  653. return Attribute::ReadOnly;
  654. else if (IsWrite)
  655. return Attribute::WriteOnly;
  656. else
  657. return Attribute::ReadNone;
  658. }
  659. /// Deduce returned attributes for the SCC.
  660. static void addArgumentReturnedAttrs(const SCCNodeSet &SCCNodes,
  661. SmallSet<Function *, 8> &Changed) {
  662. // Check each function in turn, determining if an argument is always returned.
  663. for (Function *F : SCCNodes) {
  664. // We can infer and propagate function attributes only when we know that the
  665. // definition we'll get at link time is *exactly* the definition we see now.
  666. // For more details, see GlobalValue::mayBeDerefined.
  667. if (!F->hasExactDefinition())
  668. continue;
  669. if (F->getReturnType()->isVoidTy())
  670. continue;
  671. // There is nothing to do if an argument is already marked as 'returned'.
  672. if (llvm::any_of(F->args(),
  673. [](const Argument &Arg) { return Arg.hasReturnedAttr(); }))
  674. continue;
  675. auto FindRetArg = [&]() -> Value * {
  676. Value *RetArg = nullptr;
  677. for (BasicBlock &BB : *F)
  678. if (auto *Ret = dyn_cast<ReturnInst>(BB.getTerminator())) {
  679. // Note that stripPointerCasts should look through functions with
  680. // returned arguments.
  681. Value *RetVal = Ret->getReturnValue()->stripPointerCasts();
  682. if (!isa<Argument>(RetVal) || RetVal->getType() != F->getReturnType())
  683. return nullptr;
  684. if (!RetArg)
  685. RetArg = RetVal;
  686. else if (RetArg != RetVal)
  687. return nullptr;
  688. }
  689. return RetArg;
  690. };
  691. if (Value *RetArg = FindRetArg()) {
  692. auto *A = cast<Argument>(RetArg);
  693. A->addAttr(Attribute::Returned);
  694. ++NumReturned;
  695. Changed.insert(F);
  696. }
  697. }
  698. }
  699. /// If a callsite has arguments that are also arguments to the parent function,
  700. /// try to propagate attributes from the callsite's arguments to the parent's
  701. /// arguments. This may be important because inlining can cause information loss
  702. /// when attribute knowledge disappears with the inlined call.
  703. static bool addArgumentAttrsFromCallsites(Function &F) {
  704. if (!EnableNonnullArgPropagation)
  705. return false;
  706. bool Changed = false;
  707. // For an argument attribute to transfer from a callsite to the parent, the
  708. // call must be guaranteed to execute every time the parent is called.
  709. // Conservatively, just check for calls in the entry block that are guaranteed
  710. // to execute.
  711. // TODO: This could be enhanced by testing if the callsite post-dominates the
  712. // entry block or by doing simple forward walks or backward walks to the
  713. // callsite.
  714. BasicBlock &Entry = F.getEntryBlock();
  715. for (Instruction &I : Entry) {
  716. if (auto *CB = dyn_cast<CallBase>(&I)) {
  717. if (auto *CalledFunc = CB->getCalledFunction()) {
  718. for (auto &CSArg : CalledFunc->args()) {
  719. if (!CSArg.hasNonNullAttr(/* AllowUndefOrPoison */ false))
  720. continue;
  721. // If the non-null callsite argument operand is an argument to 'F'
  722. // (the caller) and the call is guaranteed to execute, then the value
  723. // must be non-null throughout 'F'.
  724. auto *FArg = dyn_cast<Argument>(CB->getArgOperand(CSArg.getArgNo()));
  725. if (FArg && !FArg->hasNonNullAttr()) {
  726. FArg->addAttr(Attribute::NonNull);
  727. Changed = true;
  728. }
  729. }
  730. }
  731. }
  732. if (!isGuaranteedToTransferExecutionToSuccessor(&I))
  733. break;
  734. }
  735. return Changed;
  736. }
  737. static bool addAccessAttr(Argument *A, Attribute::AttrKind R) {
  738. assert((R == Attribute::ReadOnly || R == Attribute::ReadNone ||
  739. R == Attribute::WriteOnly)
  740. && "Must be an access attribute.");
  741. assert(A && "Argument must not be null.");
  742. // If the argument already has the attribute, nothing needs to be done.
  743. if (A->hasAttribute(R))
  744. return false;
  745. // Otherwise, remove potentially conflicting attribute, add the new one,
  746. // and update statistics.
  747. A->removeAttr(Attribute::WriteOnly);
  748. A->removeAttr(Attribute::ReadOnly);
  749. A->removeAttr(Attribute::ReadNone);
  750. A->addAttr(R);
  751. if (R == Attribute::ReadOnly)
  752. ++NumReadOnlyArg;
  753. else if (R == Attribute::WriteOnly)
  754. ++NumWriteOnlyArg;
  755. else
  756. ++NumReadNoneArg;
  757. return true;
  758. }
  759. /// Deduce nocapture attributes for the SCC.
  760. static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
  761. SmallSet<Function *, 8> &Changed) {
  762. ArgumentGraph AG;
  763. // Check each function in turn, determining which pointer arguments are not
  764. // captured.
  765. for (Function *F : SCCNodes) {
  766. // We can infer and propagate function attributes only when we know that the
  767. // definition we'll get at link time is *exactly* the definition we see now.
  768. // For more details, see GlobalValue::mayBeDerefined.
  769. if (!F->hasExactDefinition())
  770. continue;
  771. if (addArgumentAttrsFromCallsites(*F))
  772. Changed.insert(F);
  773. // Functions that are readonly (or readnone) and nounwind and don't return
  774. // a value can't capture arguments. Don't analyze them.
  775. if (F->onlyReadsMemory() && F->doesNotThrow() &&
  776. F->getReturnType()->isVoidTy()) {
  777. for (Function::arg_iterator A = F->arg_begin(), E = F->arg_end(); A != E;
  778. ++A) {
  779. if (A->getType()->isPointerTy() && !A->hasNoCaptureAttr()) {
  780. A->addAttr(Attribute::NoCapture);
  781. ++NumNoCapture;
  782. Changed.insert(F);
  783. }
  784. }
  785. continue;
  786. }
  787. for (Function::arg_iterator A = F->arg_begin(), E = F->arg_end(); A != E;
  788. ++A) {
  789. if (!A->getType()->isPointerTy())
  790. continue;
  791. bool HasNonLocalUses = false;
  792. if (!A->hasNoCaptureAttr()) {
  793. ArgumentUsesTracker Tracker(SCCNodes);
  794. PointerMayBeCaptured(&*A, &Tracker);
  795. if (!Tracker.Captured) {
  796. if (Tracker.Uses.empty()) {
  797. // If it's trivially not captured, mark it nocapture now.
  798. A->addAttr(Attribute::NoCapture);
  799. ++NumNoCapture;
  800. Changed.insert(F);
  801. } else {
  802. // If it's not trivially captured and not trivially not captured,
  803. // then it must be calling into another function in our SCC. Save
  804. // its particulars for Argument-SCC analysis later.
  805. ArgumentGraphNode *Node = AG[&*A];
  806. for (Argument *Use : Tracker.Uses) {
  807. Node->Uses.push_back(AG[Use]);
  808. if (Use != &*A)
  809. HasNonLocalUses = true;
  810. }
  811. }
  812. }
  813. // Otherwise, it's captured. Don't bother doing SCC analysis on it.
  814. }
  815. if (!HasNonLocalUses && !A->onlyReadsMemory()) {
  816. // Can we determine that it's readonly/readnone/writeonly without doing
  817. // an SCC? Note that we don't allow any calls at all here, or else our
  818. // result will be dependent on the iteration order through the
  819. // functions in the SCC.
  820. SmallPtrSet<Argument *, 8> Self;
  821. Self.insert(&*A);
  822. Attribute::AttrKind R = determinePointerAccessAttrs(&*A, Self);
  823. if (R != Attribute::None)
  824. if (addAccessAttr(A, R))
  825. Changed.insert(F);
  826. }
  827. }
  828. }
  829. // The graph we've collected is partial because we stopped scanning for
  830. // argument uses once we solved the argument trivially. These partial nodes
  831. // show up as ArgumentGraphNode objects with an empty Uses list, and for
  832. // these nodes the final decision about whether they capture has already been
  833. // made. If the definition doesn't have a 'nocapture' attribute by now, it
  834. // captures.
  835. for (scc_iterator<ArgumentGraph *> I = scc_begin(&AG); !I.isAtEnd(); ++I) {
  836. const std::vector<ArgumentGraphNode *> &ArgumentSCC = *I;
  837. if (ArgumentSCC.size() == 1) {
  838. if (!ArgumentSCC[0]->Definition)
  839. continue; // synthetic root node
  840. // eg. "void f(int* x) { if (...) f(x); }"
  841. if (ArgumentSCC[0]->Uses.size() == 1 &&
  842. ArgumentSCC[0]->Uses[0] == ArgumentSCC[0]) {
  843. Argument *A = ArgumentSCC[0]->Definition;
  844. A->addAttr(Attribute::NoCapture);
  845. ++NumNoCapture;
  846. Changed.insert(A->getParent());
  847. // Infer the access attributes given the new nocapture one
  848. SmallPtrSet<Argument *, 8> Self;
  849. Self.insert(&*A);
  850. Attribute::AttrKind R = determinePointerAccessAttrs(&*A, Self);
  851. if (R != Attribute::None)
  852. addAccessAttr(A, R);
  853. }
  854. continue;
  855. }
  856. bool SCCCaptured = false;
  857. for (auto I = ArgumentSCC.begin(), E = ArgumentSCC.end();
  858. I != E && !SCCCaptured; ++I) {
  859. ArgumentGraphNode *Node = *I;
  860. if (Node->Uses.empty()) {
  861. if (!Node->Definition->hasNoCaptureAttr())
  862. SCCCaptured = true;
  863. }
  864. }
  865. if (SCCCaptured)
  866. continue;
  867. SmallPtrSet<Argument *, 8> ArgumentSCCNodes;
  868. // Fill ArgumentSCCNodes with the elements of the ArgumentSCC. Used for
  869. // quickly looking up whether a given Argument is in this ArgumentSCC.
  870. for (ArgumentGraphNode *I : ArgumentSCC) {
  871. ArgumentSCCNodes.insert(I->Definition);
  872. }
  873. for (auto I = ArgumentSCC.begin(), E = ArgumentSCC.end();
  874. I != E && !SCCCaptured; ++I) {
  875. ArgumentGraphNode *N = *I;
  876. for (ArgumentGraphNode *Use : N->Uses) {
  877. Argument *A = Use->Definition;
  878. if (A->hasNoCaptureAttr() || ArgumentSCCNodes.count(A))
  879. continue;
  880. SCCCaptured = true;
  881. break;
  882. }
  883. }
  884. if (SCCCaptured)
  885. continue;
  886. for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) {
  887. Argument *A = ArgumentSCC[i]->Definition;
  888. A->addAttr(Attribute::NoCapture);
  889. ++NumNoCapture;
  890. Changed.insert(A->getParent());
  891. }
  892. // We also want to compute readonly/readnone/writeonly. With a small number
  893. // of false negatives, we can assume that any pointer which is captured
  894. // isn't going to be provably readonly or readnone, since by definition
  895. // we can't analyze all uses of a captured pointer.
  896. //
  897. // The false negatives happen when the pointer is captured by a function
  898. // that promises readonly/readnone behaviour on the pointer, then the
  899. // pointer's lifetime ends before anything that writes to arbitrary memory.
  900. // Also, a readonly/readnone pointer may be returned, but returning a
  901. // pointer is capturing it.
  902. auto meetAccessAttr = [](Attribute::AttrKind A, Attribute::AttrKind B) {
  903. if (A == B)
  904. return A;
  905. if (A == Attribute::ReadNone)
  906. return B;
  907. if (B == Attribute::ReadNone)
  908. return A;
  909. return Attribute::None;
  910. };
  911. Attribute::AttrKind AccessAttr = Attribute::ReadNone;
  912. for (unsigned i = 0, e = ArgumentSCC.size();
  913. i != e && AccessAttr != Attribute::None; ++i) {
  914. Argument *A = ArgumentSCC[i]->Definition;
  915. Attribute::AttrKind K = determinePointerAccessAttrs(A, ArgumentSCCNodes);
  916. AccessAttr = meetAccessAttr(AccessAttr, K);
  917. }
  918. if (AccessAttr != Attribute::None) {
  919. for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) {
  920. Argument *A = ArgumentSCC[i]->Definition;
  921. if (addAccessAttr(A, AccessAttr))
  922. Changed.insert(A->getParent());
  923. }
  924. }
  925. }
  926. }
  927. /// Tests whether a function is "malloc-like".
  928. ///
  929. /// A function is "malloc-like" if it returns either null or a pointer that
  930. /// doesn't alias any other pointer visible to the caller.
  931. static bool isFunctionMallocLike(Function *F, const SCCNodeSet &SCCNodes) {
  932. SmallSetVector<Value *, 8> FlowsToReturn;
  933. for (BasicBlock &BB : *F)
  934. if (ReturnInst *Ret = dyn_cast<ReturnInst>(BB.getTerminator()))
  935. FlowsToReturn.insert(Ret->getReturnValue());
  936. for (unsigned i = 0; i != FlowsToReturn.size(); ++i) {
  937. Value *RetVal = FlowsToReturn[i];
  938. if (Constant *C = dyn_cast<Constant>(RetVal)) {
  939. if (!C->isNullValue() && !isa<UndefValue>(C))
  940. return false;
  941. continue;
  942. }
  943. if (isa<Argument>(RetVal))
  944. return false;
  945. if (Instruction *RVI = dyn_cast<Instruction>(RetVal))
  946. switch (RVI->getOpcode()) {
  947. // Extend the analysis by looking upwards.
  948. case Instruction::BitCast:
  949. case Instruction::GetElementPtr:
  950. case Instruction::AddrSpaceCast:
  951. FlowsToReturn.insert(RVI->getOperand(0));
  952. continue;
  953. case Instruction::Select: {
  954. SelectInst *SI = cast<SelectInst>(RVI);
  955. FlowsToReturn.insert(SI->getTrueValue());
  956. FlowsToReturn.insert(SI->getFalseValue());
  957. continue;
  958. }
  959. case Instruction::PHI: {
  960. PHINode *PN = cast<PHINode>(RVI);
  961. for (Value *IncValue : PN->incoming_values())
  962. FlowsToReturn.insert(IncValue);
  963. continue;
  964. }
  965. // Check whether the pointer came from an allocation.
  966. case Instruction::Alloca:
  967. break;
  968. case Instruction::Call:
  969. case Instruction::Invoke: {
  970. CallBase &CB = cast<CallBase>(*RVI);
  971. if (CB.hasRetAttr(Attribute::NoAlias))
  972. break;
  973. if (CB.getCalledFunction() && SCCNodes.count(CB.getCalledFunction()))
  974. break;
  975. LLVM_FALLTHROUGH;
  976. }
  977. default:
  978. return false; // Did not come from an allocation.
  979. }
  980. if (PointerMayBeCaptured(RetVal, false, /*StoreCaptures=*/false))
  981. return false;
  982. }
  983. return true;
  984. }
  985. /// Deduce noalias attributes for the SCC.
  986. static void addNoAliasAttrs(const SCCNodeSet &SCCNodes,
  987. SmallSet<Function *, 8> &Changed) {
  988. // Check each function in turn, determining which functions return noalias
  989. // pointers.
  990. for (Function *F : SCCNodes) {
  991. // Already noalias.
  992. if (F->returnDoesNotAlias())
  993. continue;
  994. // We can infer and propagate function attributes only when we know that the
  995. // definition we'll get at link time is *exactly* the definition we see now.
  996. // For more details, see GlobalValue::mayBeDerefined.
  997. if (!F->hasExactDefinition())
  998. return;
  999. // We annotate noalias return values, which are only applicable to
  1000. // pointer types.
  1001. if (!F->getReturnType()->isPointerTy())
  1002. continue;
  1003. if (!isFunctionMallocLike(F, SCCNodes))
  1004. return;
  1005. }
  1006. for (Function *F : SCCNodes) {
  1007. if (F->returnDoesNotAlias() ||
  1008. !F->getReturnType()->isPointerTy())
  1009. continue;
  1010. F->setReturnDoesNotAlias();
  1011. ++NumNoAlias;
  1012. Changed.insert(F);
  1013. }
  1014. }
  1015. /// Tests whether this function is known to not return null.
  1016. ///
  1017. /// Requires that the function returns a pointer.
  1018. ///
  1019. /// Returns true if it believes the function will not return a null, and sets
  1020. /// \p Speculative based on whether the returned conclusion is a speculative
  1021. /// conclusion due to SCC calls.
  1022. static bool isReturnNonNull(Function *F, const SCCNodeSet &SCCNodes,
  1023. bool &Speculative) {
  1024. assert(F->getReturnType()->isPointerTy() &&
  1025. "nonnull only meaningful on pointer types");
  1026. Speculative = false;
  1027. SmallSetVector<Value *, 8> FlowsToReturn;
  1028. for (BasicBlock &BB : *F)
  1029. if (auto *Ret = dyn_cast<ReturnInst>(BB.getTerminator()))
  1030. FlowsToReturn.insert(Ret->getReturnValue());
  1031. auto &DL = F->getParent()->getDataLayout();
  1032. for (unsigned i = 0; i != FlowsToReturn.size(); ++i) {
  1033. Value *RetVal = FlowsToReturn[i];
  1034. // If this value is locally known to be non-null, we're good
  1035. if (isKnownNonZero(RetVal, DL))
  1036. continue;
  1037. // Otherwise, we need to look upwards since we can't make any local
  1038. // conclusions.
  1039. Instruction *RVI = dyn_cast<Instruction>(RetVal);
  1040. if (!RVI)
  1041. return false;
  1042. switch (RVI->getOpcode()) {
  1043. // Extend the analysis by looking upwards.
  1044. case Instruction::BitCast:
  1045. case Instruction::GetElementPtr:
  1046. case Instruction::AddrSpaceCast:
  1047. FlowsToReturn.insert(RVI->getOperand(0));
  1048. continue;
  1049. case Instruction::Select: {
  1050. SelectInst *SI = cast<SelectInst>(RVI);
  1051. FlowsToReturn.insert(SI->getTrueValue());
  1052. FlowsToReturn.insert(SI->getFalseValue());
  1053. continue;
  1054. }
  1055. case Instruction::PHI: {
  1056. PHINode *PN = cast<PHINode>(RVI);
  1057. for (int i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
  1058. FlowsToReturn.insert(PN->getIncomingValue(i));
  1059. continue;
  1060. }
  1061. case Instruction::Call:
  1062. case Instruction::Invoke: {
  1063. CallBase &CB = cast<CallBase>(*RVI);
  1064. Function *Callee = CB.getCalledFunction();
  1065. // A call to a node within the SCC is assumed to return null until
  1066. // proven otherwise
  1067. if (Callee && SCCNodes.count(Callee)) {
  1068. Speculative = true;
  1069. continue;
  1070. }
  1071. return false;
  1072. }
  1073. default:
  1074. return false; // Unknown source, may be null
  1075. };
  1076. llvm_unreachable("should have either continued or returned");
  1077. }
  1078. return true;
  1079. }
  1080. /// Deduce nonnull attributes for the SCC.
  1081. static void addNonNullAttrs(const SCCNodeSet &SCCNodes,
  1082. SmallSet<Function *, 8> &Changed) {
  1083. // Speculative that all functions in the SCC return only nonnull
  1084. // pointers. We may refute this as we analyze functions.
  1085. bool SCCReturnsNonNull = true;
  1086. // Check each function in turn, determining which functions return nonnull
  1087. // pointers.
  1088. for (Function *F : SCCNodes) {
  1089. // Already nonnull.
  1090. if (F->getAttributes().hasRetAttr(Attribute::NonNull))
  1091. continue;
  1092. // We can infer and propagate function attributes only when we know that the
  1093. // definition we'll get at link time is *exactly* the definition we see now.
  1094. // For more details, see GlobalValue::mayBeDerefined.
  1095. if (!F->hasExactDefinition())
  1096. return;
  1097. // We annotate nonnull return values, which are only applicable to
  1098. // pointer types.
  1099. if (!F->getReturnType()->isPointerTy())
  1100. continue;
  1101. bool Speculative = false;
  1102. if (isReturnNonNull(F, SCCNodes, Speculative)) {
  1103. if (!Speculative) {
  1104. // Mark the function eagerly since we may discover a function
  1105. // which prevents us from speculating about the entire SCC
  1106. LLVM_DEBUG(dbgs() << "Eagerly marking " << F->getName()
  1107. << " as nonnull\n");
  1108. F->addRetAttr(Attribute::NonNull);
  1109. ++NumNonNullReturn;
  1110. Changed.insert(F);
  1111. }
  1112. continue;
  1113. }
  1114. // At least one function returns something which could be null, can't
  1115. // speculate any more.
  1116. SCCReturnsNonNull = false;
  1117. }
  1118. if (SCCReturnsNonNull) {
  1119. for (Function *F : SCCNodes) {
  1120. if (F->getAttributes().hasRetAttr(Attribute::NonNull) ||
  1121. !F->getReturnType()->isPointerTy())
  1122. continue;
  1123. LLVM_DEBUG(dbgs() << "SCC marking " << F->getName() << " as nonnull\n");
  1124. F->addRetAttr(Attribute::NonNull);
  1125. ++NumNonNullReturn;
  1126. Changed.insert(F);
  1127. }
  1128. }
  1129. }
  1130. namespace {
  1131. /// Collects a set of attribute inference requests and performs them all in one
  1132. /// go on a single SCC Node. Inference involves scanning function bodies
  1133. /// looking for instructions that violate attribute assumptions.
  1134. /// As soon as all the bodies are fine we are free to set the attribute.
  1135. /// Customization of inference for individual attributes is performed by
  1136. /// providing a handful of predicates for each attribute.
  1137. class AttributeInferer {
  1138. public:
  1139. /// Describes a request for inference of a single attribute.
  1140. struct InferenceDescriptor {
  1141. /// Returns true if this function does not have to be handled.
  1142. /// General intent for this predicate is to provide an optimization
  1143. /// for functions that do not need this attribute inference at all
  1144. /// (say, for functions that already have the attribute).
  1145. std::function<bool(const Function &)> SkipFunction;
  1146. /// Returns true if this instruction violates attribute assumptions.
  1147. std::function<bool(Instruction &)> InstrBreaksAttribute;
  1148. /// Sets the inferred attribute for this function.
  1149. std::function<void(Function &)> SetAttribute;
  1150. /// Attribute we derive.
  1151. Attribute::AttrKind AKind;
  1152. /// If true, only "exact" definitions can be used to infer this attribute.
  1153. /// See GlobalValue::isDefinitionExact.
  1154. bool RequiresExactDefinition;
  1155. InferenceDescriptor(Attribute::AttrKind AK,
  1156. std::function<bool(const Function &)> SkipFunc,
  1157. std::function<bool(Instruction &)> InstrScan,
  1158. std::function<void(Function &)> SetAttr,
  1159. bool ReqExactDef)
  1160. : SkipFunction(SkipFunc), InstrBreaksAttribute(InstrScan),
  1161. SetAttribute(SetAttr), AKind(AK),
  1162. RequiresExactDefinition(ReqExactDef) {}
  1163. };
  1164. private:
  1165. SmallVector<InferenceDescriptor, 4> InferenceDescriptors;
  1166. public:
  1167. void registerAttrInference(InferenceDescriptor AttrInference) {
  1168. InferenceDescriptors.push_back(AttrInference);
  1169. }
  1170. void run(const SCCNodeSet &SCCNodes, SmallSet<Function *, 8> &Changed);
  1171. };
  1172. /// Perform all the requested attribute inference actions according to the
  1173. /// attribute predicates stored before.
  1174. void AttributeInferer::run(const SCCNodeSet &SCCNodes,
  1175. SmallSet<Function *, 8> &Changed) {
  1176. SmallVector<InferenceDescriptor, 4> InferInSCC = InferenceDescriptors;
  1177. // Go through all the functions in SCC and check corresponding attribute
  1178. // assumptions for each of them. Attributes that are invalid for this SCC
  1179. // will be removed from InferInSCC.
  1180. for (Function *F : SCCNodes) {
  1181. // No attributes whose assumptions are still valid - done.
  1182. if (InferInSCC.empty())
  1183. return;
  1184. // Check if our attributes ever need scanning/can be scanned.
  1185. llvm::erase_if(InferInSCC, [F](const InferenceDescriptor &ID) {
  1186. if (ID.SkipFunction(*F))
  1187. return false;
  1188. // Remove from further inference (invalidate) when visiting a function
  1189. // that has no instructions to scan/has an unsuitable definition.
  1190. return F->isDeclaration() ||
  1191. (ID.RequiresExactDefinition && !F->hasExactDefinition());
  1192. });
  1193. // For each attribute still in InferInSCC that doesn't explicitly skip F,
  1194. // set up the F instructions scan to verify assumptions of the attribute.
  1195. SmallVector<InferenceDescriptor, 4> InferInThisFunc;
  1196. llvm::copy_if(
  1197. InferInSCC, std::back_inserter(InferInThisFunc),
  1198. [F](const InferenceDescriptor &ID) { return !ID.SkipFunction(*F); });
  1199. if (InferInThisFunc.empty())
  1200. continue;
  1201. // Start instruction scan.
  1202. for (Instruction &I : instructions(*F)) {
  1203. llvm::erase_if(InferInThisFunc, [&](const InferenceDescriptor &ID) {
  1204. if (!ID.InstrBreaksAttribute(I))
  1205. return false;
  1206. // Remove attribute from further inference on any other functions
  1207. // because attribute assumptions have just been violated.
  1208. llvm::erase_if(InferInSCC, [&ID](const InferenceDescriptor &D) {
  1209. return D.AKind == ID.AKind;
  1210. });
  1211. // Remove attribute from the rest of current instruction scan.
  1212. return true;
  1213. });
  1214. if (InferInThisFunc.empty())
  1215. break;
  1216. }
  1217. }
  1218. if (InferInSCC.empty())
  1219. return;
  1220. for (Function *F : SCCNodes)
  1221. // At this point InferInSCC contains only functions that were either:
  1222. // - explicitly skipped from scan/inference, or
  1223. // - verified to have no instructions that break attribute assumptions.
  1224. // Hence we just go and force the attribute for all non-skipped functions.
  1225. for (auto &ID : InferInSCC) {
  1226. if (ID.SkipFunction(*F))
  1227. continue;
  1228. Changed.insert(F);
  1229. ID.SetAttribute(*F);
  1230. }
  1231. }
  1232. struct SCCNodesResult {
  1233. SCCNodeSet SCCNodes;
  1234. bool HasUnknownCall;
  1235. };
  1236. } // end anonymous namespace
  1237. /// Helper for non-Convergent inference predicate InstrBreaksAttribute.
  1238. static bool InstrBreaksNonConvergent(Instruction &I,
  1239. const SCCNodeSet &SCCNodes) {
  1240. const CallBase *CB = dyn_cast<CallBase>(&I);
  1241. // Breaks non-convergent assumption if CS is a convergent call to a function
  1242. // not in the SCC.
  1243. return CB && CB->isConvergent() &&
  1244. !SCCNodes.contains(CB->getCalledFunction());
  1245. }
  1246. /// Helper for NoUnwind inference predicate InstrBreaksAttribute.
  1247. static bool InstrBreaksNonThrowing(Instruction &I, const SCCNodeSet &SCCNodes) {
  1248. if (!I.mayThrow())
  1249. return false;
  1250. if (const auto *CI = dyn_cast<CallInst>(&I)) {
  1251. if (Function *Callee = CI->getCalledFunction()) {
  1252. // I is a may-throw call to a function inside our SCC. This doesn't
  1253. // invalidate our current working assumption that the SCC is no-throw; we
  1254. // just have to scan that other function.
  1255. if (SCCNodes.contains(Callee))
  1256. return false;
  1257. }
  1258. }
  1259. return true;
  1260. }
  1261. /// Helper for NoFree inference predicate InstrBreaksAttribute.
  1262. static bool InstrBreaksNoFree(Instruction &I, const SCCNodeSet &SCCNodes) {
  1263. CallBase *CB = dyn_cast<CallBase>(&I);
  1264. if (!CB)
  1265. return false;
  1266. if (CB->hasFnAttr(Attribute::NoFree))
  1267. return false;
  1268. // Speculatively assume in SCC.
  1269. if (Function *Callee = CB->getCalledFunction())
  1270. if (SCCNodes.contains(Callee))
  1271. return false;
  1272. return true;
  1273. }
  1274. /// Attempt to remove convergent function attribute when possible.
  1275. ///
  1276. /// Returns true if any changes to function attributes were made.
  1277. static void inferConvergent(const SCCNodeSet &SCCNodes,
  1278. SmallSet<Function *, 8> &Changed) {
  1279. AttributeInferer AI;
  1280. // Request to remove the convergent attribute from all functions in the SCC
  1281. // if every callsite within the SCC is not convergent (except for calls
  1282. // to functions within the SCC).
  1283. // Note: Removal of the attr from the callsites will happen in
  1284. // InstCombineCalls separately.
  1285. AI.registerAttrInference(AttributeInferer::InferenceDescriptor{
  1286. Attribute::Convergent,
  1287. // Skip non-convergent functions.
  1288. [](const Function &F) { return !F.isConvergent(); },
  1289. // Instructions that break non-convergent assumption.
  1290. [SCCNodes](Instruction &I) {
  1291. return InstrBreaksNonConvergent(I, SCCNodes);
  1292. },
  1293. [](Function &F) {
  1294. LLVM_DEBUG(dbgs() << "Removing convergent attr from fn " << F.getName()
  1295. << "\n");
  1296. F.setNotConvergent();
  1297. },
  1298. /* RequiresExactDefinition= */ false});
  1299. // Perform all the requested attribute inference actions.
  1300. AI.run(SCCNodes, Changed);
  1301. }
  1302. /// Infer attributes from all functions in the SCC by scanning every
  1303. /// instruction for compliance to the attribute assumptions. Currently it
  1304. /// does:
  1305. /// - addition of NoUnwind attribute
  1306. ///
  1307. /// Returns true if any changes to function attributes were made.
  1308. static void inferAttrsFromFunctionBodies(const SCCNodeSet &SCCNodes,
  1309. SmallSet<Function *, 8> &Changed) {
  1310. AttributeInferer AI;
  1311. if (!DisableNoUnwindInference)
  1312. // Request to infer nounwind attribute for all the functions in the SCC if
  1313. // every callsite within the SCC is not throwing (except for calls to
  1314. // functions within the SCC). Note that nounwind attribute suffers from
  1315. // derefinement - results may change depending on how functions are
  1316. // optimized. Thus it can be inferred only from exact definitions.
  1317. AI.registerAttrInference(AttributeInferer::InferenceDescriptor{
  1318. Attribute::NoUnwind,
  1319. // Skip non-throwing functions.
  1320. [](const Function &F) { return F.doesNotThrow(); },
  1321. // Instructions that break non-throwing assumption.
  1322. [&SCCNodes](Instruction &I) {
  1323. return InstrBreaksNonThrowing(I, SCCNodes);
  1324. },
  1325. [](Function &F) {
  1326. LLVM_DEBUG(dbgs()
  1327. << "Adding nounwind attr to fn " << F.getName() << "\n");
  1328. F.setDoesNotThrow();
  1329. ++NumNoUnwind;
  1330. },
  1331. /* RequiresExactDefinition= */ true});
  1332. if (!DisableNoFreeInference)
  1333. // Request to infer nofree attribute for all the functions in the SCC if
  1334. // every callsite within the SCC does not directly or indirectly free
  1335. // memory (except for calls to functions within the SCC). Note that nofree
  1336. // attribute suffers from derefinement - results may change depending on
  1337. // how functions are optimized. Thus it can be inferred only from exact
  1338. // definitions.
  1339. AI.registerAttrInference(AttributeInferer::InferenceDescriptor{
  1340. Attribute::NoFree,
  1341. // Skip functions known not to free memory.
  1342. [](const Function &F) { return F.doesNotFreeMemory(); },
  1343. // Instructions that break non-deallocating assumption.
  1344. [&SCCNodes](Instruction &I) {
  1345. return InstrBreaksNoFree(I, SCCNodes);
  1346. },
  1347. [](Function &F) {
  1348. LLVM_DEBUG(dbgs()
  1349. << "Adding nofree attr to fn " << F.getName() << "\n");
  1350. F.setDoesNotFreeMemory();
  1351. ++NumNoFree;
  1352. },
  1353. /* RequiresExactDefinition= */ true});
  1354. // Perform all the requested attribute inference actions.
  1355. AI.run(SCCNodes, Changed);
  1356. }
  1357. static void addNoRecurseAttrs(const SCCNodeSet &SCCNodes,
  1358. SmallSet<Function *, 8> &Changed) {
  1359. // Try and identify functions that do not recurse.
  1360. // If the SCC contains multiple nodes we know for sure there is recursion.
  1361. if (SCCNodes.size() != 1)
  1362. return;
  1363. Function *F = *SCCNodes.begin();
  1364. if (!F || !F->hasExactDefinition() || F->doesNotRecurse())
  1365. return;
  1366. // If all of the calls in F are identifiable and are to norecurse functions, F
  1367. // is norecurse. This check also detects self-recursion as F is not currently
  1368. // marked norecurse, so any called from F to F will not be marked norecurse.
  1369. for (auto &BB : *F)
  1370. for (auto &I : BB.instructionsWithoutDebug())
  1371. if (auto *CB = dyn_cast<CallBase>(&I)) {
  1372. Function *Callee = CB->getCalledFunction();
  1373. if (!Callee || Callee == F || !Callee->doesNotRecurse())
  1374. // Function calls a potentially recursive function.
  1375. return;
  1376. }
  1377. // Every call was to a non-recursive function other than this function, and
  1378. // we have no indirect recursion as the SCC size is one. This function cannot
  1379. // recurse.
  1380. F->setDoesNotRecurse();
  1381. ++NumNoRecurse;
  1382. Changed.insert(F);
  1383. }
  1384. static bool instructionDoesNotReturn(Instruction &I) {
  1385. if (auto *CB = dyn_cast<CallBase>(&I))
  1386. return CB->hasFnAttr(Attribute::NoReturn);
  1387. return false;
  1388. }
  1389. // A basic block can only return if it terminates with a ReturnInst and does not
  1390. // contain calls to noreturn functions.
  1391. static bool basicBlockCanReturn(BasicBlock &BB) {
  1392. if (!isa<ReturnInst>(BB.getTerminator()))
  1393. return false;
  1394. return none_of(BB, instructionDoesNotReturn);
  1395. }
  1396. // FIXME: this doesn't handle recursion.
  1397. static bool canReturn(Function &F) {
  1398. SmallVector<BasicBlock *, 16> Worklist;
  1399. SmallPtrSet<BasicBlock *, 16> Visited;
  1400. Visited.insert(&F.front());
  1401. Worklist.push_back(&F.front());
  1402. do {
  1403. BasicBlock *BB = Worklist.pop_back_val();
  1404. if (basicBlockCanReturn(*BB))
  1405. return true;
  1406. for (BasicBlock *Succ : successors(BB))
  1407. if (Visited.insert(Succ).second)
  1408. Worklist.push_back(Succ);
  1409. } while (!Worklist.empty());
  1410. return false;
  1411. }
  1412. // Set the noreturn function attribute if possible.
  1413. static void addNoReturnAttrs(const SCCNodeSet &SCCNodes,
  1414. SmallSet<Function *, 8> &Changed) {
  1415. for (Function *F : SCCNodes) {
  1416. if (!F || !F->hasExactDefinition() || F->hasFnAttribute(Attribute::Naked) ||
  1417. F->doesNotReturn())
  1418. continue;
  1419. if (!canReturn(*F)) {
  1420. F->setDoesNotReturn();
  1421. Changed.insert(F);
  1422. }
  1423. }
  1424. }
  1425. static bool functionWillReturn(const Function &F) {
  1426. // We can infer and propagate function attributes only when we know that the
  1427. // definition we'll get at link time is *exactly* the definition we see now.
  1428. // For more details, see GlobalValue::mayBeDerefined.
  1429. if (!F.hasExactDefinition())
  1430. return false;
  1431. // Must-progress function without side-effects must return.
  1432. if (F.mustProgress() && F.onlyReadsMemory())
  1433. return true;
  1434. // Can only analyze functions with a definition.
  1435. if (F.isDeclaration())
  1436. return false;
  1437. // Functions with loops require more sophisticated analysis, as the loop
  1438. // may be infinite. For now, don't try to handle them.
  1439. SmallVector<std::pair<const BasicBlock *, const BasicBlock *>> Backedges;
  1440. FindFunctionBackedges(F, Backedges);
  1441. if (!Backedges.empty())
  1442. return false;
  1443. // If there are no loops, then the function is willreturn if all calls in
  1444. // it are willreturn.
  1445. return all_of(instructions(F), [](const Instruction &I) {
  1446. return I.willReturn();
  1447. });
  1448. }
  1449. // Set the willreturn function attribute if possible.
  1450. static void addWillReturn(const SCCNodeSet &SCCNodes,
  1451. SmallSet<Function *, 8> &Changed) {
  1452. for (Function *F : SCCNodes) {
  1453. if (!F || F->willReturn() || !functionWillReturn(*F))
  1454. continue;
  1455. F->setWillReturn();
  1456. NumWillReturn++;
  1457. Changed.insert(F);
  1458. }
  1459. }
  1460. // Return true if this is an atomic which has an ordering stronger than
  1461. // unordered. Note that this is different than the predicate we use in
  1462. // Attributor. Here we chose to be conservative and consider monotonic
  1463. // operations potentially synchronizing. We generally don't do much with
  1464. // monotonic operations, so this is simply risk reduction.
  1465. static bool isOrderedAtomic(Instruction *I) {
  1466. if (!I->isAtomic())
  1467. return false;
  1468. if (auto *FI = dyn_cast<FenceInst>(I))
  1469. // All legal orderings for fence are stronger than monotonic.
  1470. return FI->getSyncScopeID() != SyncScope::SingleThread;
  1471. else if (isa<AtomicCmpXchgInst>(I) || isa<AtomicRMWInst>(I))
  1472. return true;
  1473. else if (auto *SI = dyn_cast<StoreInst>(I))
  1474. return !SI->isUnordered();
  1475. else if (auto *LI = dyn_cast<LoadInst>(I))
  1476. return !LI->isUnordered();
  1477. else {
  1478. llvm_unreachable("unknown atomic instruction?");
  1479. }
  1480. }
  1481. static bool InstrBreaksNoSync(Instruction &I, const SCCNodeSet &SCCNodes) {
  1482. // Volatile may synchronize
  1483. if (I.isVolatile())
  1484. return true;
  1485. // An ordered atomic may synchronize. (See comment about on monotonic.)
  1486. if (isOrderedAtomic(&I))
  1487. return true;
  1488. auto *CB = dyn_cast<CallBase>(&I);
  1489. if (!CB)
  1490. // Non call site cases covered by the two checks above
  1491. return false;
  1492. if (CB->hasFnAttr(Attribute::NoSync))
  1493. return false;
  1494. // Non volatile memset/memcpy/memmoves are nosync
  1495. // NOTE: Only intrinsics with volatile flags should be handled here. All
  1496. // others should be marked in Intrinsics.td.
  1497. if (auto *MI = dyn_cast<MemIntrinsic>(&I))
  1498. if (!MI->isVolatile())
  1499. return false;
  1500. // Speculatively assume in SCC.
  1501. if (Function *Callee = CB->getCalledFunction())
  1502. if (SCCNodes.contains(Callee))
  1503. return false;
  1504. return true;
  1505. }
  1506. // Infer the nosync attribute.
  1507. static void addNoSyncAttr(const SCCNodeSet &SCCNodes,
  1508. SmallSet<Function *, 8> &Changed) {
  1509. AttributeInferer AI;
  1510. AI.registerAttrInference(AttributeInferer::InferenceDescriptor{
  1511. Attribute::NoSync,
  1512. // Skip already marked functions.
  1513. [](const Function &F) { return F.hasNoSync(); },
  1514. // Instructions that break nosync assumption.
  1515. [&SCCNodes](Instruction &I) {
  1516. return InstrBreaksNoSync(I, SCCNodes);
  1517. },
  1518. [](Function &F) {
  1519. LLVM_DEBUG(dbgs()
  1520. << "Adding nosync attr to fn " << F.getName() << "\n");
  1521. F.setNoSync();
  1522. ++NumNoSync;
  1523. },
  1524. /* RequiresExactDefinition= */ true});
  1525. AI.run(SCCNodes, Changed);
  1526. }
  1527. static SCCNodesResult createSCCNodeSet(ArrayRef<Function *> Functions) {
  1528. SCCNodesResult Res;
  1529. Res.HasUnknownCall = false;
  1530. for (Function *F : Functions) {
  1531. if (!F || F->hasOptNone() || F->hasFnAttribute(Attribute::Naked) ||
  1532. F->isPresplitCoroutine()) {
  1533. // Treat any function we're trying not to optimize as if it were an
  1534. // indirect call and omit it from the node set used below.
  1535. Res.HasUnknownCall = true;
  1536. continue;
  1537. }
  1538. // Track whether any functions in this SCC have an unknown call edge.
  1539. // Note: if this is ever a performance hit, we can common it with
  1540. // subsequent routines which also do scans over the instructions of the
  1541. // function.
  1542. if (!Res.HasUnknownCall) {
  1543. for (Instruction &I : instructions(*F)) {
  1544. if (auto *CB = dyn_cast<CallBase>(&I)) {
  1545. if (!CB->getCalledFunction()) {
  1546. Res.HasUnknownCall = true;
  1547. break;
  1548. }
  1549. }
  1550. }
  1551. }
  1552. Res.SCCNodes.insert(F);
  1553. }
  1554. return Res;
  1555. }
  1556. template <typename AARGetterT>
  1557. static SmallSet<Function *, 8>
  1558. deriveAttrsInPostOrder(ArrayRef<Function *> Functions, AARGetterT &&AARGetter) {
  1559. SCCNodesResult Nodes = createSCCNodeSet(Functions);
  1560. // Bail if the SCC only contains optnone functions.
  1561. if (Nodes.SCCNodes.empty())
  1562. return {};
  1563. SmallSet<Function *, 8> Changed;
  1564. addArgumentReturnedAttrs(Nodes.SCCNodes, Changed);
  1565. addReadAttrs(Nodes.SCCNodes, AARGetter, Changed);
  1566. addArgumentAttrs(Nodes.SCCNodes, Changed);
  1567. inferConvergent(Nodes.SCCNodes, Changed);
  1568. addNoReturnAttrs(Nodes.SCCNodes, Changed);
  1569. addWillReturn(Nodes.SCCNodes, Changed);
  1570. // If we have no external nodes participating in the SCC, we can deduce some
  1571. // more precise attributes as well.
  1572. if (!Nodes.HasUnknownCall) {
  1573. addNoAliasAttrs(Nodes.SCCNodes, Changed);
  1574. addNonNullAttrs(Nodes.SCCNodes, Changed);
  1575. inferAttrsFromFunctionBodies(Nodes.SCCNodes, Changed);
  1576. addNoRecurseAttrs(Nodes.SCCNodes, Changed);
  1577. }
  1578. addNoSyncAttr(Nodes.SCCNodes, Changed);
  1579. // Finally, infer the maximal set of attributes from the ones we've inferred
  1580. // above. This is handling the cases where one attribute on a signature
  1581. // implies another, but for implementation reasons the inference rule for
  1582. // the later is missing (or simply less sophisticated).
  1583. for (Function *F : Nodes.SCCNodes)
  1584. if (F)
  1585. if (inferAttributesFromOthers(*F))
  1586. Changed.insert(F);
  1587. return Changed;
  1588. }
  1589. PreservedAnalyses PostOrderFunctionAttrsPass::run(LazyCallGraph::SCC &C,
  1590. CGSCCAnalysisManager &AM,
  1591. LazyCallGraph &CG,
  1592. CGSCCUpdateResult &) {
  1593. FunctionAnalysisManager &FAM =
  1594. AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
  1595. // We pass a lambda into functions to wire them up to the analysis manager
  1596. // for getting function analyses.
  1597. auto AARGetter = [&](Function &F) -> AAResults & {
  1598. return FAM.getResult<AAManager>(F);
  1599. };
  1600. SmallVector<Function *, 8> Functions;
  1601. for (LazyCallGraph::Node &N : C) {
  1602. Functions.push_back(&N.getFunction());
  1603. }
  1604. auto ChangedFunctions = deriveAttrsInPostOrder(Functions, AARGetter);
  1605. if (ChangedFunctions.empty())
  1606. return PreservedAnalyses::all();
  1607. // Invalidate analyses for modified functions so that we don't have to
  1608. // invalidate all analyses for all functions in this SCC.
  1609. PreservedAnalyses FuncPA;
  1610. // We haven't changed the CFG for modified functions.
  1611. FuncPA.preserveSet<CFGAnalyses>();
  1612. for (Function *Changed : ChangedFunctions) {
  1613. FAM.invalidate(*Changed, FuncPA);
  1614. // Also invalidate any direct callers of changed functions since analyses
  1615. // may care about attributes of direct callees. For example, MemorySSA cares
  1616. // about whether or not a call's callee modifies memory and queries that
  1617. // through function attributes.
  1618. for (auto *U : Changed->users()) {
  1619. if (auto *Call = dyn_cast<CallBase>(U)) {
  1620. if (Call->getCalledFunction() == Changed)
  1621. FAM.invalidate(*Call->getFunction(), FuncPA);
  1622. }
  1623. }
  1624. }
  1625. PreservedAnalyses PA;
  1626. // We have not added or removed functions.
  1627. PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
  1628. // We already invalidated all relevant function analyses above.
  1629. PA.preserveSet<AllAnalysesOn<Function>>();
  1630. return PA;
  1631. }
  1632. namespace {
  1633. struct PostOrderFunctionAttrsLegacyPass : public CallGraphSCCPass {
  1634. // Pass identification, replacement for typeid
  1635. static char ID;
  1636. PostOrderFunctionAttrsLegacyPass() : CallGraphSCCPass(ID) {
  1637. initializePostOrderFunctionAttrsLegacyPassPass(
  1638. *PassRegistry::getPassRegistry());
  1639. }
  1640. bool runOnSCC(CallGraphSCC &SCC) override;
  1641. void getAnalysisUsage(AnalysisUsage &AU) const override {
  1642. AU.setPreservesCFG();
  1643. AU.addRequired<AssumptionCacheTracker>();
  1644. getAAResultsAnalysisUsage(AU);
  1645. CallGraphSCCPass::getAnalysisUsage(AU);
  1646. }
  1647. };
  1648. } // end anonymous namespace
  1649. char PostOrderFunctionAttrsLegacyPass::ID = 0;
  1650. INITIALIZE_PASS_BEGIN(PostOrderFunctionAttrsLegacyPass, "function-attrs",
  1651. "Deduce function attributes", false, false)
  1652. INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
  1653. INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
  1654. INITIALIZE_PASS_END(PostOrderFunctionAttrsLegacyPass, "function-attrs",
  1655. "Deduce function attributes", false, false)
  1656. Pass *llvm::createPostOrderFunctionAttrsLegacyPass() {
  1657. return new PostOrderFunctionAttrsLegacyPass();
  1658. }
  1659. template <typename AARGetterT>
  1660. static bool runImpl(CallGraphSCC &SCC, AARGetterT AARGetter) {
  1661. SmallVector<Function *, 8> Functions;
  1662. for (CallGraphNode *I : SCC) {
  1663. Functions.push_back(I->getFunction());
  1664. }
  1665. return !deriveAttrsInPostOrder(Functions, AARGetter).empty();
  1666. }
  1667. bool PostOrderFunctionAttrsLegacyPass::runOnSCC(CallGraphSCC &SCC) {
  1668. if (skipSCC(SCC))
  1669. return false;
  1670. return runImpl(SCC, LegacyAARGetter(*this));
  1671. }
  1672. namespace {
  1673. struct ReversePostOrderFunctionAttrsLegacyPass : public ModulePass {
  1674. // Pass identification, replacement for typeid
  1675. static char ID;
  1676. ReversePostOrderFunctionAttrsLegacyPass() : ModulePass(ID) {
  1677. initializeReversePostOrderFunctionAttrsLegacyPassPass(
  1678. *PassRegistry::getPassRegistry());
  1679. }
  1680. bool runOnModule(Module &M) override;
  1681. void getAnalysisUsage(AnalysisUsage &AU) const override {
  1682. AU.setPreservesCFG();
  1683. AU.addRequired<CallGraphWrapperPass>();
  1684. AU.addPreserved<CallGraphWrapperPass>();
  1685. }
  1686. };
  1687. } // end anonymous namespace
  1688. char ReversePostOrderFunctionAttrsLegacyPass::ID = 0;
  1689. INITIALIZE_PASS_BEGIN(ReversePostOrderFunctionAttrsLegacyPass,
  1690. "rpo-function-attrs", "Deduce function attributes in RPO",
  1691. false, false)
  1692. INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
  1693. INITIALIZE_PASS_END(ReversePostOrderFunctionAttrsLegacyPass,
  1694. "rpo-function-attrs", "Deduce function attributes in RPO",
  1695. false, false)
  1696. Pass *llvm::createReversePostOrderFunctionAttrsPass() {
  1697. return new ReversePostOrderFunctionAttrsLegacyPass();
  1698. }
  1699. static bool addNoRecurseAttrsTopDown(Function &F) {
  1700. // We check the preconditions for the function prior to calling this to avoid
  1701. // the cost of building up a reversible post-order list. We assert them here
  1702. // to make sure none of the invariants this relies on were violated.
  1703. assert(!F.isDeclaration() && "Cannot deduce norecurse without a definition!");
  1704. assert(!F.doesNotRecurse() &&
  1705. "This function has already been deduced as norecurs!");
  1706. assert(F.hasInternalLinkage() &&
  1707. "Can only do top-down deduction for internal linkage functions!");
  1708. // If F is internal and all of its uses are calls from a non-recursive
  1709. // functions, then none of its calls could in fact recurse without going
  1710. // through a function marked norecurse, and so we can mark this function too
  1711. // as norecurse. Note that the uses must actually be calls -- otherwise
  1712. // a pointer to this function could be returned from a norecurse function but
  1713. // this function could be recursively (indirectly) called. Note that this
  1714. // also detects if F is directly recursive as F is not yet marked as
  1715. // a norecurse function.
  1716. for (auto *U : F.users()) {
  1717. auto *I = dyn_cast<Instruction>(U);
  1718. if (!I)
  1719. return false;
  1720. CallBase *CB = dyn_cast<CallBase>(I);
  1721. if (!CB || !CB->getParent()->getParent()->doesNotRecurse())
  1722. return false;
  1723. }
  1724. F.setDoesNotRecurse();
  1725. ++NumNoRecurse;
  1726. return true;
  1727. }
  1728. static bool deduceFunctionAttributeInRPO(Module &M, CallGraph &CG) {
  1729. // We only have a post-order SCC traversal (because SCCs are inherently
  1730. // discovered in post-order), so we accumulate them in a vector and then walk
  1731. // it in reverse. This is simpler than using the RPO iterator infrastructure
  1732. // because we need to combine SCC detection and the PO walk of the call
  1733. // graph. We can also cheat egregiously because we're primarily interested in
  1734. // synthesizing norecurse and so we can only save the singular SCCs as SCCs
  1735. // with multiple functions in them will clearly be recursive.
  1736. SmallVector<Function *, 16> Worklist;
  1737. for (scc_iterator<CallGraph *> I = scc_begin(&CG); !I.isAtEnd(); ++I) {
  1738. if (I->size() != 1)
  1739. continue;
  1740. Function *F = I->front()->getFunction();
  1741. if (F && !F->isDeclaration() && !F->doesNotRecurse() &&
  1742. F->hasInternalLinkage())
  1743. Worklist.push_back(F);
  1744. }
  1745. bool Changed = false;
  1746. for (auto *F : llvm::reverse(Worklist))
  1747. Changed |= addNoRecurseAttrsTopDown(*F);
  1748. return Changed;
  1749. }
  1750. bool ReversePostOrderFunctionAttrsLegacyPass::runOnModule(Module &M) {
  1751. if (skipModule(M))
  1752. return false;
  1753. auto &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
  1754. return deduceFunctionAttributeInRPO(M, CG);
  1755. }
  1756. PreservedAnalyses
  1757. ReversePostOrderFunctionAttrsPass::run(Module &M, ModuleAnalysisManager &AM) {
  1758. auto &CG = AM.getResult<CallGraphAnalysis>(M);
  1759. if (!deduceFunctionAttributeInRPO(M, CG))
  1760. return PreservedAnalyses::all();
  1761. PreservedAnalyses PA;
  1762. PA.preserve<CallGraphAnalysis>();
  1763. return PA;
  1764. }