yql_library_compiler.cpp 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. #include "yql_library_compiler.h"
  2. #include "yql_expr_optimize.h"
  3. #include <yql/essentials/sql/sql.h>
  4. #include <util/system/file.h>
  5. #include <unordered_set>
  6. #include <unordered_map>
  7. namespace NYql {
  8. namespace {
  9. bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges, TNodeSet& visited, TNodeSet& parents)
  10. {
  11. if (!node.ChildrenSize()) {
  12. return true;
  13. }
  14. const auto pair = parents.emplace(&node);
  15. if (!pair.second) {
  16. return false;
  17. }
  18. if (!visited.emplace(&node).second) {
  19. parents.erase(pair.first);
  20. return true;
  21. }
  22. for (ui32 i = 0U; i < node.ChildrenSize(); ++i) {
  23. auto& child = node.ChildRef(i);
  24. if (const auto it = replaces.find(node.Child(i)); replaces.cend() != it) {
  25. child = it->second;
  26. hasChanges = true;
  27. }
  28. if (!ReplaceNodes(*node.Child(i), replaces, hasChanges, visited, parents)) {
  29. child.Reset();
  30. return false;
  31. }
  32. }
  33. parents.erase(pair.first);
  34. return true;
  35. }
  36. bool ReplaceNodes(TExprNode& node, const TNodeOnNodeOwnedMap& replaces, bool& hasChanges) {
  37. TNodeSet visited;
  38. TNodeSet parents;
  39. return ReplaceNodes(node, replaces, hasChanges, visited, parents);
  40. }
  41. TString Load(const TString& path)
  42. {
  43. TFile file(path, EOpenModeFlag::RdOnly);
  44. if (file.GetLength() <= 0)
  45. return TString();
  46. std::vector<TString::value_type> buffer(file.GetLength());
  47. file.Load(buffer.data(), buffer.size());
  48. return TString(buffer.data(), buffer.size());
  49. }
  50. }
  51. bool OptimizeLibrary(TLibraryCohesion& cohesion, TExprContext& ctx) {
  52. TExprNode::TListType tupleItems;
  53. for (const auto& x : cohesion.Exports.Symbols()) {
  54. tupleItems.push_back(x.second);
  55. }
  56. auto root = ctx.NewList(TPositionHandle(), std::move(tupleItems));
  57. for (;;) {
  58. auto status = ExpandApply(root, root, ctx);
  59. if (status == IGraphTransformer::TStatus::Error) {
  60. return false;
  61. }
  62. if (status == IGraphTransformer::TStatus::Ok) {
  63. ctx.Step.Repeat(TExprStep::ExpandApplyForLambdas);
  64. break;
  65. }
  66. }
  67. ui32 index = 0;
  68. for (auto& x : cohesion.Exports.Symbols(ctx)) {
  69. x.second = root->ChildPtr(index++);
  70. }
  71. return true;
  72. }
  73. bool CompileLibrary(const TString& alias, const TString& script, TExprContext& ctx, TLibraryCohesion& cohesion, bool optimize)
  74. {
  75. TAstParseResult res;
  76. if (alias.EndsWith(".sql")) {
  77. NSQLTranslation::TTranslationSettings translationSettings;
  78. translationSettings.SyntaxVersion = 1;
  79. translationSettings.Mode = NSQLTranslation::ESqlMode::LIBRARY;
  80. res = NSQLTranslation::SqlToYql(script, translationSettings);
  81. } else {
  82. res = ParseAst(script, nullptr, alias);
  83. }
  84. if (!res.IsOk()) {
  85. for (const auto& originalError : res.Issues) {
  86. TIssue error(originalError);
  87. TStringBuilder message;
  88. message << error.GetMessage() << " (at " << alias << ")";
  89. error.SetMessage(message);
  90. ctx.AddError(error);
  91. }
  92. return false;
  93. }
  94. if (!CompileExpr(*res.Root, cohesion, ctx))
  95. return false;
  96. if (!optimize) {
  97. return true;
  98. }
  99. return OptimizeLibrary(cohesion, ctx);
  100. }
  101. bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const TModulesTable* loadedModules) {
  102. std::function<const TExportTable*(const TString&)> f = [loadedModules](const TString& normalizedModuleName) -> const TExportTable* {
  103. if (!loadedModules) {
  104. return nullptr;
  105. }
  106. return loadedModules->FindPtr(normalizedModuleName);
  107. };
  108. return LinkLibraries(libs, ctx, ctxToClone, f);
  109. }
  110. bool LinkLibraries(THashMap<TString, TLibraryCohesion>& libs, TExprContext& ctx, TExprContext& ctxToClone, const std::function<const TExportTable*(const TString&)>& module2ExportTable)
  111. {
  112. TNodeOnNodeOwnedMap clones, replaces;
  113. for (const auto& lib : libs) {
  114. for (const auto& import : lib.second.Imports) {
  115. if (import.first->Dead()) {
  116. continue;
  117. }
  118. if (import.second.first == lib.first) {
  119. ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()),
  120. TStringBuilder() << "Library '" << lib.first << "' tries to import itself."));
  121. return false;
  122. }
  123. const auto* exportTable = module2ExportTable(TModuleResolver::NormalizeModuleName(import.second.first));
  124. const bool externalModule = exportTable;
  125. if (!exportTable) {
  126. if (const auto it = libs.find(import.second.first); libs.cend() != it) {
  127. exportTable = &it->second.Exports;
  128. }
  129. }
  130. if (!exportTable) {
  131. ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()),
  132. TStringBuilder() << "Library '" << lib.first << "' has unresolved dependency from '" << import.second.first << "'."));
  133. return false;
  134. }
  135. if (const auto ex = exportTable->Symbols().find(import.second.second); exportTable->Symbols().cend() != ex) {
  136. replaces[import.first] = externalModule ? ctxToClone.DeepCopy(*ex->second, exportTable->ExprCtx(), clones, true, false) : ex->second;
  137. } else {
  138. ctx.AddError(TIssue(ctxToClone.GetPosition(import.first->Pos()),
  139. TStringBuilder() << "Library '" << lib.first << "' has unresolved symbol '" << import.second.second << "' from '" << import.second.first << "'."));
  140. return false;
  141. }
  142. }
  143. }
  144. if (!replaces.empty()) {
  145. for (auto& lib : libs) {
  146. for (auto& expo : lib.second.Exports.Symbols(lib.second.Exports.ExprCtx())) {
  147. if (const auto find = replaces.find(expo.second.Get()); replaces.cend() != find)
  148. expo.second = find->second;
  149. }
  150. }
  151. }
  152. for (bool hasChanges = !replaces.empty(); hasChanges;) {
  153. hasChanges = false;
  154. for (const auto& lib : libs) {
  155. for (const auto& expo : lib.second.Exports.Symbols()) {
  156. if (!ReplaceNodes(*expo.second, replaces, hasChanges)) {
  157. ctx.AddError(TIssue(ctxToClone.GetPosition(expo.second->Pos()),
  158. TStringBuilder() << "Cross reference detected under '" << expo.first << "' in '" << lib.first << "'."));
  159. return false;
  160. }
  161. }
  162. }
  163. }
  164. return true;
  165. }
  166. bool CompileLibraries(const TUserDataTable& userData, TExprContext& ctx, TModulesTable& modules, bool optimize)
  167. {
  168. THashMap<TString, TLibraryCohesion> libs;
  169. for (const auto& data : userData) {
  170. if (data.first.IsFile() && data.second.Usage.Test(EUserDataBlockUsage::Library)) {
  171. TString libraryData;
  172. const TString& alias = data.first.Alias();
  173. if (data.second.Type == EUserDataType::PATH) {
  174. libraryData = Load(data.second.Data);
  175. } else if (data.second.Type == EUserDataType::RAW_INLINE_DATA) {
  176. libraryData = data.second.Data;
  177. }
  178. if (!libraryData.empty()) {
  179. if (CompileLibrary(alias, libraryData, ctx, libs[alias], optimize))
  180. modules[TModuleResolver::NormalizeModuleName(alias)] = libs[alias].Exports;
  181. else
  182. return false;
  183. }
  184. }
  185. }
  186. return LinkLibraries(libs, ctx, ctx);
  187. }
  188. }