AArch64PostLegalizerLowering.cpp 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070
  1. //=== AArch64PostLegalizerLowering.cpp --------------------------*- C++ -*-===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. ///
  9. /// \file
  10. /// Post-legalization lowering for instructions.
  11. ///
  12. /// This is used to offload pattern matching from the selector.
  13. ///
  14. /// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually
  15. /// a G_ZIP, G_UZP, etc.
  16. ///
  17. /// General optimization combines should be handled by either the
  18. /// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
  19. ///
  20. //===----------------------------------------------------------------------===//
  21. #include "AArch64GlobalISelUtils.h"
  22. #include "AArch64Subtarget.h"
  23. #include "AArch64TargetMachine.h"
  24. #include "GISel/AArch64LegalizerInfo.h"
  25. #include "MCTargetDesc/AArch64MCTargetDesc.h"
  26. #include "TargetInfo/AArch64TargetInfo.h"
  27. #include "Utils/AArch64BaseInfo.h"
  28. #include "llvm/CodeGen/GlobalISel/Combiner.h"
  29. #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
  30. #include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
  31. #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
  32. #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
  33. #include "llvm/CodeGen/GlobalISel/Utils.h"
  34. #include "llvm/CodeGen/MachineFunctionPass.h"
  35. #include "llvm/CodeGen/MachineInstrBuilder.h"
  36. #include "llvm/CodeGen/MachineRegisterInfo.h"
  37. #include "llvm/CodeGen/TargetOpcodes.h"
  38. #include "llvm/CodeGen/TargetPassConfig.h"
  39. #include "llvm/IR/InstrTypes.h"
  40. #include "llvm/InitializePasses.h"
  41. #include "llvm/Support/Debug.h"
  42. #include "llvm/Support/ErrorHandling.h"
  43. #define DEBUG_TYPE "aarch64-postlegalizer-lowering"
  44. using namespace llvm;
  45. using namespace MIPatternMatch;
  46. using namespace AArch64GISelUtils;
  47. /// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
  48. ///
  49. /// Used for matching target-supported shuffles before codegen.
  50. struct ShuffleVectorPseudo {
  51. unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1)
  52. Register Dst; ///< Destination register.
  53. SmallVector<SrcOp, 2> SrcOps; ///< Source registers.
  54. ShuffleVectorPseudo(unsigned Opc, Register Dst,
  55. std::initializer_list<SrcOp> SrcOps)
  56. : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
  57. ShuffleVectorPseudo() {}
  58. };
  59. /// Check if a vector shuffle corresponds to a REV instruction with the
  60. /// specified blocksize.
  61. static bool isREVMask(ArrayRef<int> M, unsigned EltSize, unsigned NumElts,
  62. unsigned BlockSize) {
  63. assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
  64. "Only possible block sizes for REV are: 16, 32, 64");
  65. assert(EltSize != 64 && "EltSize cannot be 64 for REV mask.");
  66. unsigned BlockElts = M[0] + 1;
  67. // If the first shuffle index is UNDEF, be optimistic.
  68. if (M[0] < 0)
  69. BlockElts = BlockSize / EltSize;
  70. if (BlockSize <= EltSize || BlockSize != BlockElts * EltSize)
  71. return false;
  72. for (unsigned i = 0; i < NumElts; ++i) {
  73. // Ignore undef indices.
  74. if (M[i] < 0)
  75. continue;
  76. if (static_cast<unsigned>(M[i]) !=
  77. (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
  78. return false;
  79. }
  80. return true;
  81. }
  82. /// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts.
  83. /// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p WhichResult.
  84. static bool isTRNMask(ArrayRef<int> M, unsigned NumElts,
  85. unsigned &WhichResult) {
  86. if (NumElts % 2 != 0)
  87. return false;
  88. WhichResult = (M[0] == 0 ? 0 : 1);
  89. for (unsigned i = 0; i < NumElts; i += 2) {
  90. if ((M[i] >= 0 && static_cast<unsigned>(M[i]) != i + WhichResult) ||
  91. (M[i + 1] >= 0 &&
  92. static_cast<unsigned>(M[i + 1]) != i + NumElts + WhichResult))
  93. return false;
  94. }
  95. return true;
  96. }
  97. /// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector
  98. /// sources of the shuffle are different.
  99. static Optional<std::pair<bool, uint64_t>> getExtMask(ArrayRef<int> M,
  100. unsigned NumElts) {
  101. // Look for the first non-undef element.
  102. auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
  103. if (FirstRealElt == M.end())
  104. return None;
  105. // Use APInt to handle overflow when calculating expected element.
  106. unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
  107. APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1);
  108. // The following shuffle indices must be the successive elements after the
  109. // first real element.
  110. if (any_of(
  111. make_range(std::next(FirstRealElt), M.end()),
  112. [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
  113. return None;
  114. // The index of an EXT is the first element if it is not UNDEF.
  115. // Watch out for the beginning UNDEFs. The EXT index should be the expected
  116. // value of the first element. E.g.
  117. // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
  118. // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
  119. // ExpectedElt is the last mask index plus 1.
  120. uint64_t Imm = ExpectedElt.getZExtValue();
  121. bool ReverseExt = false;
  122. // There are two difference cases requiring to reverse input vectors.
  123. // For example, for vector <4 x i32> we have the following cases,
  124. // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
  125. // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
  126. // For both cases, we finally use mask <5, 6, 7, 0>, which requires
  127. // to reverse two input vectors.
  128. if (Imm < NumElts)
  129. ReverseExt = true;
  130. else
  131. Imm -= NumElts;
  132. return std::make_pair(ReverseExt, Imm);
  133. }
  134. /// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts.
  135. /// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p WhichResult.
  136. static bool isUZPMask(ArrayRef<int> M, unsigned NumElts,
  137. unsigned &WhichResult) {
  138. WhichResult = (M[0] == 0 ? 0 : 1);
  139. for (unsigned i = 0; i != NumElts; ++i) {
  140. // Skip undef indices.
  141. if (M[i] < 0)
  142. continue;
  143. if (static_cast<unsigned>(M[i]) != 2 * i + WhichResult)
  144. return false;
  145. }
  146. return true;
  147. }
  148. /// \return true if \p M is a zip mask for a shuffle vector of \p NumElts.
  149. /// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p WhichResult.
  150. static bool isZipMask(ArrayRef<int> M, unsigned NumElts,
  151. unsigned &WhichResult) {
  152. if (NumElts % 2 != 0)
  153. return false;
  154. // 0 means use ZIP1, 1 means use ZIP2.
  155. WhichResult = (M[0] == 0 ? 0 : 1);
  156. unsigned Idx = WhichResult * NumElts / 2;
  157. for (unsigned i = 0; i != NumElts; i += 2) {
  158. if ((M[i] >= 0 && static_cast<unsigned>(M[i]) != Idx) ||
  159. (M[i + 1] >= 0 && static_cast<unsigned>(M[i + 1]) != Idx + NumElts))
  160. return false;
  161. Idx += 1;
  162. }
  163. return true;
  164. }
  165. /// Helper function for matchINS.
  166. ///
  167. /// \returns a value when \p M is an ins mask for \p NumInputElements.
  168. ///
  169. /// First element of the returned pair is true when the produced
  170. /// G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.
  171. ///
  172. /// Second element is the destination lane for the G_INSERT_VECTOR_ELT.
  173. static Optional<std::pair<bool, int>> isINSMask(ArrayRef<int> M,
  174. int NumInputElements) {
  175. if (M.size() != static_cast<size_t>(NumInputElements))
  176. return None;
  177. int NumLHSMatch = 0, NumRHSMatch = 0;
  178. int LastLHSMismatch = -1, LastRHSMismatch = -1;
  179. for (int Idx = 0; Idx < NumInputElements; ++Idx) {
  180. if (M[Idx] == -1) {
  181. ++NumLHSMatch;
  182. ++NumRHSMatch;
  183. continue;
  184. }
  185. M[Idx] == Idx ? ++NumLHSMatch : LastLHSMismatch = Idx;
  186. M[Idx] == Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch = Idx;
  187. }
  188. const int NumNeededToMatch = NumInputElements - 1;
  189. if (NumLHSMatch == NumNeededToMatch)
  190. return std::make_pair(true, LastLHSMismatch);
  191. if (NumRHSMatch == NumNeededToMatch)
  192. return std::make_pair(false, LastRHSMismatch);
  193. return None;
  194. }
  195. /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a
  196. /// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc.
  197. static bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI,
  198. ShuffleVectorPseudo &MatchInfo) {
  199. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  200. ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
  201. Register Dst = MI.getOperand(0).getReg();
  202. Register Src = MI.getOperand(1).getReg();
  203. LLT Ty = MRI.getType(Dst);
  204. unsigned EltSize = Ty.getScalarSizeInBits();
  205. // Element size for a rev cannot be 64.
  206. if (EltSize == 64)
  207. return false;
  208. unsigned NumElts = Ty.getNumElements();
  209. // Try to produce G_REV64
  210. if (isREVMask(ShuffleMask, EltSize, NumElts, 64)) {
  211. MatchInfo = ShuffleVectorPseudo(AArch64::G_REV64, Dst, {Src});
  212. return true;
  213. }
  214. // TODO: Produce G_REV32 and G_REV16 once we have proper legalization support.
  215. // This should be identical to above, but with a constant 32 and constant
  216. // 16.
  217. return false;
  218. }
  219. /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
  220. /// a G_TRN1 or G_TRN2 instruction.
  221. static bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI,
  222. ShuffleVectorPseudo &MatchInfo) {
  223. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  224. unsigned WhichResult;
  225. ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
  226. Register Dst = MI.getOperand(0).getReg();
  227. unsigned NumElts = MRI.getType(Dst).getNumElements();
  228. if (!isTRNMask(ShuffleMask, NumElts, WhichResult))
  229. return false;
  230. unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
  231. Register V1 = MI.getOperand(1).getReg();
  232. Register V2 = MI.getOperand(2).getReg();
  233. MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
  234. return true;
  235. }
  236. /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
  237. /// a G_UZP1 or G_UZP2 instruction.
  238. ///
  239. /// \param [in] MI - The shuffle vector instruction.
  240. /// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success.
  241. static bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI,
  242. ShuffleVectorPseudo &MatchInfo) {
  243. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  244. unsigned WhichResult;
  245. ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
  246. Register Dst = MI.getOperand(0).getReg();
  247. unsigned NumElts = MRI.getType(Dst).getNumElements();
  248. if (!isUZPMask(ShuffleMask, NumElts, WhichResult))
  249. return false;
  250. unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
  251. Register V1 = MI.getOperand(1).getReg();
  252. Register V2 = MI.getOperand(2).getReg();
  253. MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
  254. return true;
  255. }
  256. static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,
  257. ShuffleVectorPseudo &MatchInfo) {
  258. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  259. unsigned WhichResult;
  260. ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
  261. Register Dst = MI.getOperand(0).getReg();
  262. unsigned NumElts = MRI.getType(Dst).getNumElements();
  263. if (!isZipMask(ShuffleMask, NumElts, WhichResult))
  264. return false;
  265. unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
  266. Register V1 = MI.getOperand(1).getReg();
  267. Register V2 = MI.getOperand(2).getReg();
  268. MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
  269. return true;
  270. }
  271. /// Helper function for matchDup.
  272. static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI,
  273. MachineRegisterInfo &MRI,
  274. ShuffleVectorPseudo &MatchInfo) {
  275. if (Lane != 0)
  276. return false;
  277. // Try to match a vector splat operation into a dup instruction.
  278. // We're looking for this pattern:
  279. //
  280. // %scalar:gpr(s64) = COPY $x0
  281. // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF
  282. // %cst0:gpr(s32) = G_CONSTANT i32 0
  283. // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)
  284. // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)
  285. // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef, %zerovec(<2 x s32>)
  286. //
  287. // ...into:
  288. // %splat = G_DUP %scalar
  289. // Begin matching the insert.
  290. auto *InsMI = getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT,
  291. MI.getOperand(1).getReg(), MRI);
  292. if (!InsMI)
  293. return false;
  294. // Match the undef vector operand.
  295. if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),
  296. MRI))
  297. return false;
  298. // Match the index constant 0.
  299. if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ZeroInt()))
  300. return false;
  301. MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(),
  302. {InsMI->getOperand(2).getReg()});
  303. return true;
  304. }
  305. /// Helper function for matchDup.
  306. static bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
  307. MachineRegisterInfo &MRI,
  308. ShuffleVectorPseudo &MatchInfo) {
  309. assert(Lane >= 0 && "Expected positive lane?");
  310. // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
  311. // lane's definition directly.
  312. auto *BuildVecMI = getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,
  313. MI.getOperand(1).getReg(), MRI);
  314. if (!BuildVecMI)
  315. return false;
  316. Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();
  317. MatchInfo =
  318. ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg});
  319. return true;
  320. }
  321. static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
  322. ShuffleVectorPseudo &MatchInfo) {
  323. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  324. auto MaybeLane = getSplatIndex(MI);
  325. if (!MaybeLane)
  326. return false;
  327. int Lane = *MaybeLane;
  328. // If this is undef splat, generate it via "just" vdup, if possible.
  329. if (Lane < 0)
  330. Lane = 0;
  331. if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))
  332. return true;
  333. if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))
  334. return true;
  335. return false;
  336. }
  337. static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
  338. ShuffleVectorPseudo &MatchInfo) {
  339. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  340. Register Dst = MI.getOperand(0).getReg();
  341. auto ExtInfo = getExtMask(MI.getOperand(3).getShuffleMask(),
  342. MRI.getType(Dst).getNumElements());
  343. if (!ExtInfo)
  344. return false;
  345. bool ReverseExt;
  346. uint64_t Imm;
  347. std::tie(ReverseExt, Imm) = *ExtInfo;
  348. Register V1 = MI.getOperand(1).getReg();
  349. Register V2 = MI.getOperand(2).getReg();
  350. if (ReverseExt)
  351. std::swap(V1, V2);
  352. uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;
  353. Imm *= ExtFactor;
  354. MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm});
  355. return true;
  356. }
  357. /// Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
  358. /// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR.
  359. static bool applyShuffleVectorPseudo(MachineInstr &MI,
  360. ShuffleVectorPseudo &MatchInfo) {
  361. MachineIRBuilder MIRBuilder(MI);
  362. MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps);
  363. MI.eraseFromParent();
  364. return true;
  365. }
  366. /// Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
  367. /// Special-cased because the constant operand must be emitted as a G_CONSTANT
  368. /// for the imported tablegen patterns to work.
  369. static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
  370. MachineIRBuilder MIRBuilder(MI);
  371. // Tablegen patterns expect an i32 G_CONSTANT as the final op.
  372. auto Cst =
  373. MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm());
  374. MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst},
  375. {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
  376. MI.eraseFromParent();
  377. return true;
  378. }
  379. /// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
  380. /// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
  381. ///
  382. /// e.g.
  383. /// %shuf = G_SHUFFLE_VECTOR %left, %right, shufflemask(0, 0)
  384. ///
  385. /// Can be represented as
  386. ///
  387. /// %extract = G_EXTRACT_VECTOR_ELT %left, 0
  388. /// %ins = G_INSERT_VECTOR_ELT %left, %extract, 1
  389. ///
  390. static bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI,
  391. std::tuple<Register, int, Register, int> &MatchInfo) {
  392. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  393. ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
  394. Register Dst = MI.getOperand(0).getReg();
  395. int NumElts = MRI.getType(Dst).getNumElements();
  396. auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts);
  397. if (!DstIsLeftAndDstLane)
  398. return false;
  399. bool DstIsLeft;
  400. int DstLane;
  401. std::tie(DstIsLeft, DstLane) = *DstIsLeftAndDstLane;
  402. Register Left = MI.getOperand(1).getReg();
  403. Register Right = MI.getOperand(2).getReg();
  404. Register DstVec = DstIsLeft ? Left : Right;
  405. Register SrcVec = Left;
  406. int SrcLane = ShuffleMask[DstLane];
  407. if (SrcLane >= NumElts) {
  408. SrcVec = Right;
  409. SrcLane -= NumElts;
  410. }
  411. MatchInfo = std::make_tuple(DstVec, DstLane, SrcVec, SrcLane);
  412. return true;
  413. }
  414. static bool applyINS(MachineInstr &MI, MachineRegisterInfo &MRI,
  415. MachineIRBuilder &Builder,
  416. std::tuple<Register, int, Register, int> &MatchInfo) {
  417. Builder.setInstrAndDebugLoc(MI);
  418. Register Dst = MI.getOperand(0).getReg();
  419. auto ScalarTy = MRI.getType(Dst).getElementType();
  420. Register DstVec, SrcVec;
  421. int DstLane, SrcLane;
  422. std::tie(DstVec, DstLane, SrcVec, SrcLane) = MatchInfo;
  423. auto SrcCst = Builder.buildConstant(LLT::scalar(64), SrcLane);
  424. auto Extract = Builder.buildExtractVectorElement(ScalarTy, SrcVec, SrcCst);
  425. auto DstCst = Builder.buildConstant(LLT::scalar(64), DstLane);
  426. Builder.buildInsertVectorElement(Dst, DstVec, Extract, DstCst);
  427. MI.eraseFromParent();
  428. return true;
  429. }
  430. /// isVShiftRImm - Check if this is a valid vector for the immediate
  431. /// operand of a vector shift right operation. The value must be in the range:
  432. /// 1 <= Value <= ElementBits for a right shift.
  433. static bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty,
  434. int64_t &Cnt) {
  435. assert(Ty.isVector() && "vector shift count is not a vector type");
  436. MachineInstr *MI = MRI.getVRegDef(Reg);
  437. auto Cst = getAArch64VectorSplatScalar(*MI, MRI);
  438. if (!Cst)
  439. return false;
  440. Cnt = *Cst;
  441. int64_t ElementBits = Ty.getScalarSizeInBits();
  442. return Cnt >= 1 && Cnt <= ElementBits;
  443. }
  444. /// Match a vector G_ASHR or G_LSHR with a valid immediate shift.
  445. static bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,
  446. int64_t &Imm) {
  447. assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
  448. MI.getOpcode() == TargetOpcode::G_LSHR);
  449. LLT Ty = MRI.getType(MI.getOperand(1).getReg());
  450. if (!Ty.isVector())
  451. return false;
  452. return isVShiftRImm(MI.getOperand(2).getReg(), MRI, Ty, Imm);
  453. }
  454. static bool applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,
  455. int64_t &Imm) {
  456. unsigned Opc = MI.getOpcode();
  457. assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
  458. unsigned NewOpc =
  459. Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
  460. MachineIRBuilder MIB(MI);
  461. auto ImmDef = MIB.buildConstant(LLT::scalar(32), Imm);
  462. MIB.buildInstr(NewOpc, {MI.getOperand(0)}, {MI.getOperand(1), ImmDef});
  463. MI.eraseFromParent();
  464. return true;
  465. }
  466. /// Determine if it is possible to modify the \p RHS and predicate \p P of a
  467. /// G_ICMP instruction such that the right-hand side is an arithmetic immediate.
  468. ///
  469. /// \returns A pair containing the updated immediate and predicate which may
  470. /// be used to optimize the instruction.
  471. ///
  472. /// \note This assumes that the comparison has been legalized.
  473. Optional<std::pair<uint64_t, CmpInst::Predicate>>
  474. tryAdjustICmpImmAndPred(Register RHS, CmpInst::Predicate P,
  475. const MachineRegisterInfo &MRI) {
  476. const auto &Ty = MRI.getType(RHS);
  477. if (Ty.isVector())
  478. return None;
  479. unsigned Size = Ty.getSizeInBits();
  480. assert((Size == 32 || Size == 64) && "Expected 32 or 64 bit compare only?");
  481. // If the RHS is not a constant, or the RHS is already a valid arithmetic
  482. // immediate, then there is nothing to change.
  483. auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, MRI);
  484. if (!ValAndVReg)
  485. return None;
  486. uint64_t C = ValAndVReg->Value.getZExtValue();
  487. if (isLegalArithImmed(C))
  488. return None;
  489. // We have a non-arithmetic immediate. Check if adjusting the immediate and
  490. // adjusting the predicate will result in a legal arithmetic immediate.
  491. switch (P) {
  492. default:
  493. return None;
  494. case CmpInst::ICMP_SLT:
  495. case CmpInst::ICMP_SGE:
  496. // Check for
  497. //
  498. // x slt c => x sle c - 1
  499. // x sge c => x sgt c - 1
  500. //
  501. // When c is not the smallest possible negative number.
  502. if ((Size == 64 && static_cast<int64_t>(C) == INT64_MIN) ||
  503. (Size == 32 && static_cast<int32_t>(C) == INT32_MIN))
  504. return None;
  505. P = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;
  506. C -= 1;
  507. break;
  508. case CmpInst::ICMP_ULT:
  509. case CmpInst::ICMP_UGE:
  510. // Check for
  511. //
  512. // x ult c => x ule c - 1
  513. // x uge c => x ugt c - 1
  514. //
  515. // When c is not zero.
  516. if (C == 0)
  517. return None;
  518. P = (P == CmpInst::ICMP_ULT) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
  519. C -= 1;
  520. break;
  521. case CmpInst::ICMP_SLE:
  522. case CmpInst::ICMP_SGT:
  523. // Check for
  524. //
  525. // x sle c => x slt c + 1
  526. // x sgt c => s sge c + 1
  527. //
  528. // When c is not the largest possible signed integer.
  529. if ((Size == 32 && static_cast<int32_t>(C) == INT32_MAX) ||
  530. (Size == 64 && static_cast<int64_t>(C) == INT64_MAX))
  531. return None;
  532. P = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;
  533. C += 1;
  534. break;
  535. case CmpInst::ICMP_ULE:
  536. case CmpInst::ICMP_UGT:
  537. // Check for
  538. //
  539. // x ule c => x ult c + 1
  540. // x ugt c => s uge c + 1
  541. //
  542. // When c is not the largest possible unsigned integer.
  543. if ((Size == 32 && static_cast<uint32_t>(C) == UINT32_MAX) ||
  544. (Size == 64 && C == UINT64_MAX))
  545. return None;
  546. P = (P == CmpInst::ICMP_ULE) ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
  547. C += 1;
  548. break;
  549. }
  550. // Check if the new constant is valid, and return the updated constant and
  551. // predicate if it is.
  552. if (Size == 32)
  553. C = static_cast<uint32_t>(C);
  554. if (!isLegalArithImmed(C))
  555. return None;
  556. return {{C, P}};
  557. }
  558. /// Determine whether or not it is possible to update the RHS and predicate of
  559. /// a G_ICMP instruction such that the RHS will be selected as an arithmetic
  560. /// immediate.
  561. ///
  562. /// \p MI - The G_ICMP instruction
  563. /// \p MatchInfo - The new RHS immediate and predicate on success
  564. ///
  565. /// See tryAdjustICmpImmAndPred for valid transformations.
  566. bool matchAdjustICmpImmAndPred(
  567. MachineInstr &MI, const MachineRegisterInfo &MRI,
  568. std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
  569. assert(MI.getOpcode() == TargetOpcode::G_ICMP);
  570. Register RHS = MI.getOperand(3).getReg();
  571. auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
  572. if (auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(RHS, Pred, MRI)) {
  573. MatchInfo = *MaybeNewImmAndPred;
  574. return true;
  575. }
  576. return false;
  577. }
  578. bool applyAdjustICmpImmAndPred(
  579. MachineInstr &MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
  580. MachineIRBuilder &MIB, GISelChangeObserver &Observer) {
  581. MIB.setInstrAndDebugLoc(MI);
  582. MachineOperand &RHS = MI.getOperand(3);
  583. MachineRegisterInfo &MRI = *MIB.getMRI();
  584. auto Cst = MIB.buildConstant(MRI.cloneVirtualRegister(RHS.getReg()),
  585. MatchInfo.first);
  586. Observer.changingInstr(MI);
  587. RHS.setReg(Cst->getOperand(0).getReg());
  588. MI.getOperand(1).setPredicate(MatchInfo.second);
  589. Observer.changedInstr(MI);
  590. return true;
  591. }
  592. bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
  593. std::pair<unsigned, int> &MatchInfo) {
  594. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  595. Register Src1Reg = MI.getOperand(1).getReg();
  596. const LLT SrcTy = MRI.getType(Src1Reg);
  597. const LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
  598. auto LaneIdx = getSplatIndex(MI);
  599. if (!LaneIdx)
  600. return false;
  601. // The lane idx should be within the first source vector.
  602. if (*LaneIdx >= SrcTy.getNumElements())
  603. return false;
  604. if (DstTy != SrcTy)
  605. return false;
  606. LLT ScalarTy = SrcTy.getElementType();
  607. unsigned ScalarSize = ScalarTy.getSizeInBits();
  608. unsigned Opc = 0;
  609. switch (SrcTy.getNumElements()) {
  610. case 2:
  611. if (ScalarSize == 64)
  612. Opc = AArch64::G_DUPLANE64;
  613. else if (ScalarSize == 32)
  614. Opc = AArch64::G_DUPLANE32;
  615. break;
  616. case 4:
  617. if (ScalarSize == 32)
  618. Opc = AArch64::G_DUPLANE32;
  619. break;
  620. case 8:
  621. if (ScalarSize == 16)
  622. Opc = AArch64::G_DUPLANE16;
  623. break;
  624. case 16:
  625. if (ScalarSize == 8)
  626. Opc = AArch64::G_DUPLANE8;
  627. break;
  628. default:
  629. break;
  630. }
  631. if (!Opc)
  632. return false;
  633. MatchInfo.first = Opc;
  634. MatchInfo.second = *LaneIdx;
  635. return true;
  636. }
  637. bool applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
  638. MachineIRBuilder &B, std::pair<unsigned, int> &MatchInfo) {
  639. assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
  640. Register Src1Reg = MI.getOperand(1).getReg();
  641. const LLT SrcTy = MRI.getType(Src1Reg);
  642. B.setInstrAndDebugLoc(MI);
  643. auto Lane = B.buildConstant(LLT::scalar(64), MatchInfo.second);
  644. Register DupSrc = MI.getOperand(1).getReg();
  645. // For types like <2 x s32>, we can use G_DUPLANE32, with a <4 x s32> source.
  646. // To do this, we can use a G_CONCAT_VECTORS to do the widening.
  647. if (SrcTy == LLT::fixed_vector(2, LLT::scalar(32))) {
  648. assert(MRI.getType(MI.getOperand(0).getReg()).getNumElements() == 2 &&
  649. "Unexpected dest elements");
  650. auto Undef = B.buildUndef(SrcTy);
  651. DupSrc = B.buildConcatVectors(
  652. SrcTy.changeElementCount(ElementCount::getFixed(4)),
  653. {Src1Reg, Undef.getReg(0)})
  654. .getReg(0);
  655. }
  656. B.buildInstr(MatchInfo.first, {MI.getOperand(0).getReg()}, {DupSrc, Lane});
  657. MI.eraseFromParent();
  658. return true;
  659. }
  660. static bool matchBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI) {
  661. assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
  662. auto Splat = getAArch64VectorSplat(MI, MRI);
  663. if (!Splat)
  664. return false;
  665. if (Splat->isReg())
  666. return true;
  667. // Later, during selection, we'll try to match imported patterns using
  668. // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower
  669. // G_BUILD_VECTORs which could match those patterns.
  670. int64_t Cst = Splat->getCst();
  671. return (Cst != 0 && Cst != -1);
  672. }
  673. static bool applyBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI,
  674. MachineIRBuilder &B) {
  675. B.setInstrAndDebugLoc(MI);
  676. B.buildInstr(AArch64::G_DUP, {MI.getOperand(0).getReg()},
  677. {MI.getOperand(1).getReg()});
  678. MI.eraseFromParent();
  679. return true;
  680. }
  681. /// \returns how many instructions would be saved by folding a G_ICMP's shift
  682. /// and/or extension operations.
  683. static unsigned getCmpOperandFoldingProfit(Register CmpOp,
  684. const MachineRegisterInfo &MRI) {
  685. // No instructions to save if there's more than one use or no uses.
  686. if (!MRI.hasOneNonDBGUse(CmpOp))
  687. return 0;
  688. // FIXME: This is duplicated with the selector. (See: selectShiftedRegister)
  689. auto IsSupportedExtend = [&](const MachineInstr &MI) {
  690. if (MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
  691. return true;
  692. if (MI.getOpcode() != TargetOpcode::G_AND)
  693. return false;
  694. auto ValAndVReg =
  695. getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
  696. if (!ValAndVReg)
  697. return false;
  698. uint64_t Mask = ValAndVReg->Value.getZExtValue();
  699. return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
  700. };
  701. MachineInstr *Def = getDefIgnoringCopies(CmpOp, MRI);
  702. if (IsSupportedExtend(*Def))
  703. return 1;
  704. unsigned Opc = Def->getOpcode();
  705. if (Opc != TargetOpcode::G_SHL && Opc != TargetOpcode::G_ASHR &&
  706. Opc != TargetOpcode::G_LSHR)
  707. return 0;
  708. auto MaybeShiftAmt =
  709. getIConstantVRegValWithLookThrough(Def->getOperand(2).getReg(), MRI);
  710. if (!MaybeShiftAmt)
  711. return 0;
  712. uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
  713. MachineInstr *ShiftLHS =
  714. getDefIgnoringCopies(Def->getOperand(1).getReg(), MRI);
  715. // Check if we can fold an extend and a shift.
  716. // FIXME: This is duplicated with the selector. (See:
  717. // selectArithExtendedRegister)
  718. if (IsSupportedExtend(*ShiftLHS))
  719. return (ShiftAmt <= 4) ? 2 : 1;
  720. LLT Ty = MRI.getType(Def->getOperand(0).getReg());
  721. if (Ty.isVector())
  722. return 0;
  723. unsigned ShiftSize = Ty.getSizeInBits();
  724. if ((ShiftSize == 32 && ShiftAmt <= 31) ||
  725. (ShiftSize == 64 && ShiftAmt <= 63))
  726. return 1;
  727. return 0;
  728. }
  729. /// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
  730. /// instruction \p MI.
  731. static bool trySwapICmpOperands(MachineInstr &MI,
  732. const MachineRegisterInfo &MRI) {
  733. assert(MI.getOpcode() == TargetOpcode::G_ICMP);
  734. // Swap the operands if it would introduce a profitable folding opportunity.
  735. // (e.g. a shift + extend).
  736. //
  737. // For example:
  738. // lsl w13, w11, #1
  739. // cmp w13, w12
  740. // can be turned into:
  741. // cmp w12, w11, lsl #1
  742. // Don't swap if there's a constant on the RHS, because we know we can fold
  743. // that.
  744. Register RHS = MI.getOperand(3).getReg();
  745. auto RHSCst = getIConstantVRegValWithLookThrough(RHS, MRI);
  746. if (RHSCst && isLegalArithImmed(RHSCst->Value.getSExtValue()))
  747. return false;
  748. Register LHS = MI.getOperand(2).getReg();
  749. auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
  750. auto GetRegForProfit = [&](Register Reg) {
  751. MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
  752. return isCMN(Def, Pred, MRI) ? Def->getOperand(2).getReg() : Reg;
  753. };
  754. // Don't have a constant on the RHS. If we swap the LHS and RHS of the
  755. // compare, would we be able to fold more instructions?
  756. Register TheLHS = GetRegForProfit(LHS);
  757. Register TheRHS = GetRegForProfit(RHS);
  758. // If the LHS is more likely to give us a folding opportunity, then swap the
  759. // LHS and RHS.
  760. return (getCmpOperandFoldingProfit(TheLHS, MRI) >
  761. getCmpOperandFoldingProfit(TheRHS, MRI));
  762. }
  763. static bool applySwapICmpOperands(MachineInstr &MI,
  764. GISelChangeObserver &Observer) {
  765. auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
  766. Register LHS = MI.getOperand(2).getReg();
  767. Register RHS = MI.getOperand(3).getReg();
  768. Observer.changedInstr(MI);
  769. MI.getOperand(1).setPredicate(CmpInst::getSwappedPredicate(Pred));
  770. MI.getOperand(2).setReg(RHS);
  771. MI.getOperand(3).setReg(LHS);
  772. Observer.changedInstr(MI);
  773. return true;
  774. }
  775. /// \returns a function which builds a vector floating point compare instruction
  776. /// for a condition code \p CC.
  777. /// \param [in] IsZero - True if the comparison is against 0.
  778. /// \param [in] NoNans - True if the target has NoNansFPMath.
  779. static std::function<Register(MachineIRBuilder &)>
  780. getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool IsZero,
  781. bool NoNans, MachineRegisterInfo &MRI) {
  782. LLT DstTy = MRI.getType(LHS);
  783. assert(DstTy.isVector() && "Expected vector types only?");
  784. assert(DstTy == MRI.getType(RHS) && "Src and Dst types must match!");
  785. switch (CC) {
  786. default:
  787. llvm_unreachable("Unexpected condition code!");
  788. case AArch64CC::NE:
  789. return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
  790. auto FCmp = IsZero
  791. ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS})
  792. : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS});
  793. return MIB.buildNot(DstTy, FCmp).getReg(0);
  794. };
  795. case AArch64CC::EQ:
  796. return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
  797. return IsZero
  798. ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS}).getReg(0)
  799. : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS})
  800. .getReg(0);
  801. };
  802. case AArch64CC::GE:
  803. return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
  804. return IsZero
  805. ? MIB.buildInstr(AArch64::G_FCMGEZ, {DstTy}, {LHS}).getReg(0)
  806. : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {LHS, RHS})
  807. .getReg(0);
  808. };
  809. case AArch64CC::GT:
  810. return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
  811. return IsZero
  812. ? MIB.buildInstr(AArch64::G_FCMGTZ, {DstTy}, {LHS}).getReg(0)
  813. : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {LHS, RHS})
  814. .getReg(0);
  815. };
  816. case AArch64CC::LS:
  817. return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
  818. return IsZero
  819. ? MIB.buildInstr(AArch64::G_FCMLEZ, {DstTy}, {LHS}).getReg(0)
  820. : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {RHS, LHS})
  821. .getReg(0);
  822. };
  823. case AArch64CC::MI:
  824. return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
  825. return IsZero
  826. ? MIB.buildInstr(AArch64::G_FCMLTZ, {DstTy}, {LHS}).getReg(0)
  827. : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {RHS, LHS})
  828. .getReg(0);
  829. };
  830. }
  831. }
  832. /// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
  833. static bool lowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
  834. MachineIRBuilder &MIB) {
  835. assert(MI.getOpcode() == TargetOpcode::G_FCMP);
  836. const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();
  837. Register Dst = MI.getOperand(0).getReg();
  838. LLT DstTy = MRI.getType(Dst);
  839. if (!DstTy.isVector() || !ST.hasNEON())
  840. return false;
  841. const auto Pred =
  842. static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
  843. Register LHS = MI.getOperand(2).getReg();
  844. // TODO: Handle v4s16 case.
  845. unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
  846. if (EltSize != 32 && EltSize != 64)
  847. return false;
  848. Register RHS = MI.getOperand(3).getReg();
  849. auto Splat = getAArch64VectorSplat(*MRI.getVRegDef(RHS), MRI);
  850. // Compares against 0 have special target-specific pseudos.
  851. bool IsZero = Splat && Splat->isCst() && Splat->getCst() == 0;
  852. bool Invert;
  853. AArch64CC::CondCode CC, CC2;
  854. changeVectorFCMPPredToAArch64CC(Pred, CC, CC2, Invert);
  855. bool NoNans = ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
  856. // Instead of having an apply function, just build here to simplify things.
  857. MIB.setInstrAndDebugLoc(MI);
  858. auto Cmp = getVectorFCMP(CC, LHS, RHS, IsZero, NoNans, MRI);
  859. Register CmpRes;
  860. if (CC2 == AArch64CC::AL)
  861. CmpRes = Cmp(MIB);
  862. else {
  863. auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, IsZero, NoNans, MRI);
  864. auto Cmp2Dst = Cmp2(MIB);
  865. auto Cmp1Dst = Cmp(MIB);
  866. CmpRes = MIB.buildOr(DstTy, Cmp1Dst, Cmp2Dst).getReg(0);
  867. }
  868. if (Invert)
  869. CmpRes = MIB.buildNot(DstTy, CmpRes).getReg(0);
  870. MRI.replaceRegWith(Dst, CmpRes);
  871. MI.eraseFromParent();
  872. return false;
  873. }
  874. static bool matchFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
  875. Register &SrcReg) {
  876. assert(MI.getOpcode() == TargetOpcode::G_STORE);
  877. Register DstReg = MI.getOperand(0).getReg();
  878. if (MRI.getType(DstReg).isVector())
  879. return false;
  880. // Match a store of a truncate.
  881. if (!mi_match(DstReg, MRI, m_GTrunc(m_Reg(SrcReg))))
  882. return false;
  883. // Only form truncstores for value types of max 64b.
  884. return MRI.getType(SrcReg).getSizeInBits() <= 64;
  885. }
  886. static bool applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
  887. MachineIRBuilder &B,
  888. GISelChangeObserver &Observer,
  889. Register &SrcReg) {
  890. assert(MI.getOpcode() == TargetOpcode::G_STORE);
  891. Observer.changingInstr(MI);
  892. MI.getOperand(0).setReg(SrcReg);
  893. Observer.changedInstr(MI);
  894. return true;
  895. }
  896. #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
  897. #include "AArch64GenPostLegalizeGILowering.inc"
  898. #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
  899. namespace {
  900. #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
  901. #include "AArch64GenPostLegalizeGILowering.inc"
  902. #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
  903. class AArch64PostLegalizerLoweringInfo : public CombinerInfo {
  904. public:
  905. AArch64GenPostLegalizerLoweringHelperRuleConfig GeneratedRuleCfg;
  906. AArch64PostLegalizerLoweringInfo(bool OptSize, bool MinSize)
  907. : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
  908. /*LegalizerInfo*/ nullptr, /*OptEnabled = */ true, OptSize,
  909. MinSize) {
  910. if (!GeneratedRuleCfg.parseCommandLineOption())
  911. report_fatal_error("Invalid rule identifier");
  912. }
  913. virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
  914. MachineIRBuilder &B) const override;
  915. };
  916. bool AArch64PostLegalizerLoweringInfo::combine(GISelChangeObserver &Observer,
  917. MachineInstr &MI,
  918. MachineIRBuilder &B) const {
  919. CombinerHelper Helper(Observer, B);
  920. AArch64GenPostLegalizerLoweringHelper Generated(GeneratedRuleCfg);
  921. return Generated.tryCombineAll(Observer, MI, B, Helper);
  922. }
  923. #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP
  924. #include "AArch64GenPostLegalizeGILowering.inc"
  925. #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP
  926. class AArch64PostLegalizerLowering : public MachineFunctionPass {
  927. public:
  928. static char ID;
  929. AArch64PostLegalizerLowering();
  930. StringRef getPassName() const override {
  931. return "AArch64PostLegalizerLowering";
  932. }
  933. bool runOnMachineFunction(MachineFunction &MF) override;
  934. void getAnalysisUsage(AnalysisUsage &AU) const override;
  935. };
  936. } // end anonymous namespace
  937. void AArch64PostLegalizerLowering::getAnalysisUsage(AnalysisUsage &AU) const {
  938. AU.addRequired<TargetPassConfig>();
  939. AU.setPreservesCFG();
  940. getSelectionDAGFallbackAnalysisUsage(AU);
  941. MachineFunctionPass::getAnalysisUsage(AU);
  942. }
  943. AArch64PostLegalizerLowering::AArch64PostLegalizerLowering()
  944. : MachineFunctionPass(ID) {
  945. initializeAArch64PostLegalizerLoweringPass(*PassRegistry::getPassRegistry());
  946. }
  947. bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {
  948. if (MF.getProperties().hasProperty(
  949. MachineFunctionProperties::Property::FailedISel))
  950. return false;
  951. assert(MF.getProperties().hasProperty(
  952. MachineFunctionProperties::Property::Legalized) &&
  953. "Expected a legalized function?");
  954. auto *TPC = &getAnalysis<TargetPassConfig>();
  955. const Function &F = MF.getFunction();
  956. AArch64PostLegalizerLoweringInfo PCInfo(F.hasOptSize(), F.hasMinSize());
  957. Combiner C(PCInfo, TPC);
  958. return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr);
  959. }
  960. char AArch64PostLegalizerLowering::ID = 0;
  961. INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE,
  962. "Lower AArch64 MachineInstrs after legalization", false,
  963. false)
  964. INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
  965. INITIALIZE_PASS_END(AArch64PostLegalizerLowering, DEBUG_TYPE,
  966. "Lower AArch64 MachineInstrs after legalization", false,
  967. false)
  968. namespace llvm {
  969. FunctionPass *createAArch64PostLegalizerLowering() {
  970. return new AArch64PostLegalizerLowering();
  971. }
  972. } // end namespace llvm