ymake_conf.py 99 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648
  1. #!/usr/bin/env python
  2. # coding=utf-8
  3. from __future__ import print_function
  4. import base64
  5. import itertools
  6. import json
  7. import logging
  8. import optparse
  9. import os
  10. import posixpath
  11. import re
  12. import subprocess
  13. import sys
  14. import tempfile
  15. import six
  16. from functools import total_ordering
  17. logger = logging.getLogger(__name__ if __name__ != '__main__' else 'ymake_conf.py')
  18. class WindowsVersion(object):
  19. """
  20. Predefined values for _WIN32_WINNT macro.
  21. This macro specifies minimal Windows version required by the binary being build.
  22. A complete list of the values supported by the Windows SDK can be found at
  23. https://docs.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt
  24. """
  25. Windows07 = '0x0601'
  26. Windows08 = '0x0602'
  27. Windows10 = '0x0A00'
  28. # This is default Android API level unless `-DANDROID_API` is specified in cmdline
  29. # Android api level can be resolved to Android version using
  30. # https://android.googlesource.com/platform/bionic/+/master/docs/status.md
  31. ANDROID_API_DEFAULT = 21
  32. # This is default Linux SDK unless `-DOS_SDK` is specified in cmdline
  33. LINUX_SDK_DEFAULT = "ubuntu-16"
  34. MACOS_VERSION_MIN = "11.0"
  35. MACOS_VERSION_MIN_AS_INT = "110000"
  36. IOS_VERSION_MIN = "13.0"
  37. WINDOWS_VERSION_MIN = WindowsVersion.Windows07
  38. def init_logger(verbose):
  39. logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)
  40. def find_conf(conf_file):
  41. script_dir = os.path.dirname(__file__)
  42. full_path = os.path.join(script_dir, conf_file)
  43. if os.path.exists(full_path):
  44. return full_path
  45. return None
  46. # All paths in build graph shall be in posix format.
  47. # This ensures that UIDs of cross and native builds won't diverge
  48. # due to paths difference. This function is to be used to fix paths produced
  49. # by os.path.join on Windows.
  50. #
  51. # FIXME: upon complete switch to Python3 os.path.join may be replaced by posixpath.join
  52. # and this function will become unnecessary
  53. def win_path_fix(path):
  54. return path if sys.platform != 'win32' else path.replace('\\', '/')
  55. class DebugString(object):
  56. def __init__(self, get_string_func):
  57. self.get_string_func = get_string_func
  58. def __str__(self):
  59. return self.get_string_func()
  60. class ConfigureError(Exception):
  61. pass
  62. @total_ordering
  63. class Platform(object):
  64. def __init__(self, name, os, arch):
  65. """
  66. :type name: str
  67. :type os: str
  68. :type arch: str
  69. """
  70. self.name = name
  71. self.os = self._parse_os(os)
  72. self.arch = arch.lower()
  73. self.is_i386 = self.arch in ('i386', 'x86')
  74. self.is_i686 = self.arch == 'i686'
  75. self.is_x86 = self.is_i386 or self.is_i686
  76. self.is_x86_64 = self.arch in ('x86_64', 'amd64')
  77. self.is_intel = self.is_x86 or self.is_x86_64
  78. self.is_armv7 = self.arch in ('armv7', 'armv7a', 'armv7ahf', 'armv7a_neon', 'arm', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53')
  79. self.is_armv8 = self.arch in ('armv8', 'armv8a', 'arm64', 'aarch64', 'armv8a_cortex_a35', 'armv8a_cortex_a53')
  80. self.is_armv8m = self.arch in ('armv8m_cortex_m33', 'armv8m_cortex_m23')
  81. self.is_armv7em = self.arch in ('armv7em_cortex_m4', 'armv7em_cortex_m7')
  82. self.is_arm64 = self.arch in ('arm64',)
  83. self.is_arm = self.is_armv7 or self.is_armv8 or self.is_armv8m or self.is_armv7em
  84. self.is_armv7_neon = self.arch in ('armv7a_neon', 'armv7ahf', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53')
  85. self.is_armv7hf = self.arch in ('armv7ahf', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53')
  86. self.is_armv5te = self.arch in ('armv5te_arm968e_s')
  87. self.is_rv32imc = self.arch in ('riscv32_esp',)
  88. self.is_riscv32 = self.is_rv32imc
  89. self.is_nds32 = self.arch in ('nds32le_elf_mculib_v5f',)
  90. self.is_tc32 = self.arch in ('tc32_elf',)
  91. self.is_xtensa_hifi4 = self.arch == 'xtensa_hifi4'
  92. self.is_xtensa_hifi5 = self.arch == 'xtensa_hifi5'
  93. self.is_xtensa = self.is_xtensa_hifi4 or self.is_xtensa_hifi5
  94. self.armv7_float_abi = None
  95. if self.is_armv7:
  96. if self.is_armv7hf:
  97. self.armv7_float_abi = 'hard'
  98. else:
  99. self.armv7_float_abi = 'softfp'
  100. self.is_cortex_a9 = self.arch in ('armv7a_cortex_a9',)
  101. self.is_cortex_a35 = self.arch in ('armv7ahf_cortex_a35', 'armv8a_cortex_a35')
  102. self.is_cortex_a53 = self.arch in ('armv7ahf_cortex_a53', 'armv8a_cortex_a53')
  103. self.is_cortex_m33 = self.arch in ('armv8m_cortex_m33',)
  104. self.is_cortex_m23 = self.arch in ('armv8m_cortex_m23',)
  105. self.is_cortex_m4 = self.arch in ('armv7em_cortex_m4',)
  106. self.is_cortex_m7 = self.arch in ('armv7em_cortex_m7')
  107. self.is_arm968e_s = self.arch in ('armv5te_arm968e_s')
  108. self.is_power8le = self.arch == 'ppc64le'
  109. self.is_power9le = self.arch == 'power9le'
  110. self.is_powerpc = self.is_power8le or self.is_power9le
  111. self.is_wasm64 = self.arch == 'wasm64'
  112. self.is_wasm = self.is_wasm64
  113. self.is_32_bit = self.is_x86 or self.is_armv5te or self.is_armv7 or self.is_armv8m or self.is_riscv32 or self.is_nds32 or self.is_armv7em or self.is_xtensa or self.is_tc32
  114. self.is_64_bit = self.is_x86_64 or self.is_armv8 or self.is_powerpc or self.is_wasm64
  115. assert self.is_32_bit or self.is_64_bit
  116. assert not (self.is_32_bit and self.is_64_bit)
  117. self.is_linux = self.os == 'linux' or 'yocto' in self.os
  118. self.is_linux_x86_64 = self.is_linux and self.is_x86_64
  119. self.is_linux_armv8 = self.is_linux and self.is_armv8
  120. self.is_linux_armv7 = self.is_linux and self.is_armv7
  121. self.is_linux_power8le = self.is_linux and self.is_power8le
  122. self.is_linux_power9le = self.is_linux and self.is_power9le
  123. self.is_linux_powerpc = self.is_linux_power8le or self.is_linux_power9le
  124. self.is_macos = self.os == 'macos'
  125. self.is_macos_x86_64 = self.is_macos and self.is_x86_64
  126. self.is_macos_arm64 = self.is_macos and self.is_arm64
  127. self.is_iossim = self.os == 'iossim' or (self.os == 'ios' and self.is_intel)
  128. self.is_ios = self.os == 'ios' or self.is_iossim
  129. self.is_apple = self.is_macos or self.is_ios
  130. self.is_windows = self.os == 'windows'
  131. self.is_windows_x86_64 = self.is_windows and self.is_x86_64
  132. self.is_android = self.os == 'android'
  133. if self.is_android:
  134. self.android_api = int(preset('ANDROID_API', ANDROID_API_DEFAULT))
  135. self.is_cygwin = self.os == 'cygwin'
  136. self.is_yocto = self.os == 'yocto'
  137. self.is_emscripten = self.os == 'emscripten'
  138. self.is_none = self.os == 'none'
  139. self.is_posix = self.is_linux or self.is_apple or self.is_android or self.is_cygwin or self.is_yocto
  140. @staticmethod
  141. def from_json(data):
  142. name = data.get('visible_name', data['toolchain'])
  143. return Platform(name, os=data['os'], arch=data['arch'])
  144. @property
  145. def os_variables(self):
  146. # 'LINUX' variable, for backward compatibility
  147. yield self.os.upper()
  148. # 'OS_LINUX' variable
  149. yield 'OS_{}'.format(self.os.upper())
  150. # yocto is linux
  151. if 'yocto' in self.os:
  152. yield 'LINUX'
  153. yield 'OS_LINUX'
  154. if self.is_macos:
  155. yield 'DARWIN'
  156. yield 'OS_DARWIN'
  157. if self.is_iossim:
  158. yield 'IOS'
  159. yield 'OS_IOS'
  160. yield 'OS_IOSSIM'
  161. @property
  162. def arch_variables(self):
  163. return select_multiple((
  164. (self.is_i386 or self.is_i686, 'ARCH_I386'),
  165. (self.is_i686, 'ARCH_I686'),
  166. (self.is_x86_64, 'ARCH_X86_64'),
  167. (self.is_armv7, 'ARCH_ARM7'),
  168. (self.is_armv7_neon, 'ARCH_ARM7_NEON'),
  169. (self.is_armv8, 'ARCH_ARM64'),
  170. (self.is_armv8m, 'ARCH_ARM8M'),
  171. (self.is_armv7em, 'ARCH_ARM7EM'),
  172. (self.is_armv5te, 'ARCH_ARM5TE'),
  173. (self.is_arm, 'ARCH_ARM'),
  174. (self.is_linux_armv8 or self.is_macos_arm64, 'ARCH_AARCH64'),
  175. (self.is_powerpc, 'ARCH_PPC64LE'),
  176. (self.is_power8le, 'ARCH_POWER8LE'),
  177. (self.is_power9le, 'ARCH_POWER9LE'),
  178. (self.is_riscv32, 'ARCH_RISCV32'),
  179. (self.is_xtensa_hifi4, 'ARCH_XTENSA_HIFI4'),
  180. (self.is_xtensa_hifi5, 'ARCH_XTENSA_HIFI5'),
  181. (self.is_xtensa, 'ARCH_XTENSA'),
  182. (self.is_nds32, 'ARCH_NDS32'),
  183. (self.is_tc32, 'ARCH_TC32'),
  184. (self.is_wasm64, 'ARCH_WASM64'),
  185. (self.is_32_bit, 'ARCH_TYPE_32'),
  186. (self.is_64_bit, 'ARCH_TYPE_64'),
  187. ))
  188. @property
  189. def library_path_variables(self):
  190. return ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']
  191. def find_in_dict(self, dict_, default=None):
  192. if dict_ is None:
  193. return default
  194. for key in six.iterkeys(dict_):
  195. if self._parse_os(key) == self.os:
  196. return dict_[key]
  197. return default
  198. @property
  199. def os_compat(self):
  200. if self.is_macos:
  201. return 'DARWIN'
  202. else:
  203. return self.os.upper()
  204. @property
  205. def canonized_platform(self):
  206. os = None
  207. if self.os == 'macos':
  208. os = 'darwin'
  209. elif self.os == 'windows':
  210. os = 'win32'
  211. else:
  212. os = self.os
  213. if self.arch != 'x86_64':
  214. return '-'.join([os, self.arch])
  215. else:
  216. return os
  217. def exe(self, *paths):
  218. return posixpath.join(*paths)
  219. def __str__(self):
  220. return '{name}-{os}-{arch}'.format(name=self.name, os=self.os, arch=self.arch)
  221. def __eq__(self, other):
  222. return (self.name, self.os, self.arch) == (other.name, other.os, other.arch)
  223. def __lt__(self, other):
  224. return (self.name, self.os, self.arch) < (other.name, other.os, other.arch)
  225. def __hash__(self):
  226. return hash((self.name, self.os, self.arch))
  227. @staticmethod
  228. def _parse_os(os):
  229. os = os.lower()
  230. if os == 'darwin':
  231. return 'macos'
  232. if os in ('win', 'win32', 'win64'):
  233. return 'windows'
  234. if os.startswith('cygwin'):
  235. return 'cygwin'
  236. return os
  237. def which(prog):
  238. if os.path.exists(prog) and os.access(prog, os.X_OK):
  239. return prog
  240. # Ищем в $PATH только простые команды, без путей.
  241. if os.path.dirname(prog) != '':
  242. return None
  243. path = os.getenv('PATH', '')
  244. pathext = os.environ.get('PATHEXT')
  245. # На Windows %PATHEXT% указывает на список расширений, которые нужно проверять
  246. # при поиске команды в пути. Точное совпадение без расширения имеет приоритет.
  247. pathext = [''] if pathext is None else [''] + pathext.lower().split(os.pathsep)
  248. for dir_ in path.split(os.path.pathsep):
  249. for ext in pathext:
  250. p = os.path.join(dir_, prog + ext)
  251. if os.path.exists(p) and os.path.isfile(p) and os.access(p, os.X_OK):
  252. return win_path_fix(p)
  253. return None
  254. def get_stdout(command):
  255. stdout, code = get_stdout_and_code(command)
  256. return stdout if code == 0 else None
  257. def get_stdout_and_code(command):
  258. # noinspection PyBroadException
  259. try:
  260. process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  261. stdout, _ = process.communicate()
  262. return six.ensure_str(stdout), process.returncode
  263. except Exception as e:
  264. logger.info("While run: `%s`", e)
  265. return None, None
  266. def to_strings(o):
  267. if isinstance(o, (list, tuple)):
  268. for s in o:
  269. for ss in to_strings(s):
  270. yield ss
  271. else:
  272. if o is not None:
  273. if isinstance(o, bool):
  274. yield 'yes' if o else 'no'
  275. elif isinstance(o, (str, int)):
  276. yield str(o)
  277. else:
  278. raise ConfigureError('Unexpected value {} {}'.format(type(o), o))
  279. def emit(key, *value):
  280. print('{0}={1}'.format(key, ' '.join(to_strings(value))))
  281. def emit_with_comment(comment, key, *value):
  282. print('# {}'.format(comment))
  283. emit(key, *value)
  284. def emit_with_ignore_comment(key, *value):
  285. emit_with_comment('IGNORE YMAKE CONF CONTEXT', key, *value)
  286. def append(key, *value):
  287. print('{0}+={1}'.format(key, ' '.join(to_strings(value))))
  288. def emit_big(text):
  289. prefix = None
  290. first = True
  291. for line in text.split('\n'):
  292. if prefix is None:
  293. if not line:
  294. continue
  295. prefix = 0
  296. while prefix < len(line) and line[prefix] == ' ':
  297. prefix += 1
  298. if first: # Be pretty, prepend an empty line before the output
  299. print()
  300. first = False
  301. print(line[prefix:])
  302. class Variables(dict):
  303. def emit(self, with_ignore_comment=[]):
  304. with_ignore_comment_set = set(with_ignore_comment)
  305. for k in sorted(self.keys()):
  306. if k in with_ignore_comment_set:
  307. emit_with_ignore_comment(k, self[k])
  308. else:
  309. emit(k, self[k])
  310. def update_from_presets(self):
  311. for k in six.iterkeys(self):
  312. v = preset(k)
  313. if v is not None:
  314. self[k] = v
  315. def reset_if_any(self, value_check=None, reset_value=None):
  316. if value_check is None:
  317. def value_check(v_):
  318. return v_ is None
  319. if any(map(value_check, six.itervalues(self))):
  320. for k in six.iterkeys(self):
  321. self[k] = reset_value
  322. def format_env(env, list_separator=':'):
  323. def format_value(value):
  324. return value if isinstance(value, str) else ('\\' + list_separator).join(value)
  325. def format(kv):
  326. return '${env:"%s=%s"}' % (kv[0], format_value(kv[1]))
  327. return ' '.join(map(format, sorted(six.iteritems(env))))
  328. # TODO(somov): Проверить, используется ли это. Может быть, выпилить.
  329. def userify_presets(presets, keys):
  330. for key in keys:
  331. user_key = 'USER_{}'.format(key)
  332. values = [presets.pop(key, None), presets.get(user_key)]
  333. presets[user_key] = ' '.join(filter(None, values))
  334. def preset(key, default=None):
  335. return opts().presets.get(key, default)
  336. def is_positive(key):
  337. return is_positive_str(preset(key, ''))
  338. def is_positive_str(s):
  339. return s.lower() in ('yes', 'true', 'on', '1')
  340. def is_negative(key):
  341. return is_negative_str(preset(key, ''))
  342. def is_negative_str(s):
  343. return s.lower() in ('no', 'false', 'off', '0')
  344. def to_bool(s, default=None):
  345. if isinstance(s, six.string_types):
  346. if is_positive_str(s):
  347. return True
  348. if is_negative_str(s):
  349. return False
  350. if default is None:
  351. raise ConfigureError('{} is not a bool value'.format(s))
  352. return default
  353. def select(selectors, default=None, no_default=False):
  354. for enabled, value in selectors:
  355. if enabled:
  356. return value
  357. if no_default:
  358. raise ConfigureError()
  359. return default
  360. def select_multiple(selectors):
  361. for enabled, value in selectors:
  362. if enabled:
  363. yield value
  364. def unique(it):
  365. known = set()
  366. for i in it:
  367. if i not in known:
  368. known.add(i)
  369. yield i
  370. class Options(object):
  371. def __init__(self, argv):
  372. def parse_presets(raw_presets):
  373. presets = {}
  374. for p in raw_presets:
  375. toks = p.split('=', 1)
  376. name = toks[0]
  377. value = toks[1] if len(toks) >= 2 else ''
  378. presets[name] = value
  379. return presets
  380. parser = optparse.OptionParser(add_help_option=False)
  381. opt_group = optparse.OptionGroup(parser, 'Conf script options')
  382. opt_group.add_option('--toolchain-params', dest='toolchain_params', action='store', help='Set toolchain params via file')
  383. opt_group.add_option('-D', '--preset', dest='presets', action='append', default=[], help='set or override presets')
  384. opt_group.add_option('-l', '--local-distbuild', dest='local_distbuild', action='store_true', default=False, help='conf for local distbuild')
  385. parser.add_option_group(opt_group)
  386. self.options, self.arguments = parser.parse_args(argv)
  387. argv = self.arguments
  388. if len(argv) < 4:
  389. print('Usage: ArcRoot, --BuildType--, Verbosity, [Path to local.ymake]', file=sys.stderr)
  390. sys.exit(1)
  391. self.arcadia_root = argv[1]
  392. init_logger(argv[3] == 'verbose')
  393. # Эти переменные не надо использоваться напрямую. Их значения уже разбираются в других местах.
  394. self.build_type = argv[2].lower()
  395. self.local_distbuild = self.options.local_distbuild
  396. self.toolchain_params = self.options.toolchain_params
  397. self.presets = parse_presets(self.options.presets)
  398. userify_presets(self.presets, ('CFLAGS', 'CXXFLAGS', 'CONLYFLAGS', 'LDFLAGS', 'GO_COMPILE_FLAGS', 'GO_LINK_FLAGS', 'STD_CXX_VERSION'))
  399. Instance = None
  400. def opts():
  401. if Options.Instance is None:
  402. Options.Instance = Options(sys.argv)
  403. return Options.Instance
  404. class Profiler(object):
  405. Generic = 'generic'
  406. GProf = 'gprof'
  407. class Arcadia(object):
  408. def __init__(self, root):
  409. self.root = root
  410. class Build(object):
  411. def __init__(self, arcadia, build_type, toolchain_params, force_ignore_local_files=False):
  412. self.arcadia = arcadia
  413. self.params = self._load_json_from_base64(toolchain_params)
  414. self.build_type = build_type
  415. platform = self.params['platform']
  416. self.host = Platform.from_json(platform['host'])
  417. self.target = Platform.from_json(platform['target'])
  418. self.tc = self._get_toolchain_options()
  419. # TODO(somov): Удалить, когда перестанет использоваться.
  420. self.build_system = 'ymake'
  421. self.ignore_local_files = False
  422. dist_prefix = 'dist-'
  423. if self.build_type.startswith(dist_prefix):
  424. self.build_system = 'distbuild'
  425. self.build_type = self.build_type[len(dist_prefix):]
  426. if force_ignore_local_files:
  427. self.ignore_local_files = True
  428. if self.is_ide_build_type(self.build_type):
  429. self.ignore_local_files = True
  430. self.pic = not is_positive('FORCE_NO_PIC')
  431. @property
  432. def host_target(self):
  433. return self.host, self.target
  434. def print_build(self):
  435. self._print_build_settings()
  436. host_os = System(self.host)
  437. host_os.print_host_settings()
  438. target_os = System(self.target)
  439. target_os.print_target_settings()
  440. if self.pic:
  441. emit('PIC', 'yes')
  442. emit('COMPILER_ID', self.tc.type.upper())
  443. if self.is_valgrind:
  444. emit('WITH_VALGRIND', 'yes')
  445. if self.is_coverage:
  446. emit('_BUILD_COVERAGE', 'yes')
  447. if self.is_debug:
  448. emit('_BUILD_DEBUG', 'yes')
  449. if self.is_fast_debug:
  450. emit('_BUILD_FAST_DEBUG', 'yes')
  451. if self.is_release:
  452. emit('_BUILD_RELEASE', 'yes')
  453. if self.is_sanitized:
  454. emit('_BUILD_SANITIZED', 'yes')
  455. if self.is_size_optimized:
  456. emit('_BUILD_SIZE_OPTIMIZED', 'yes')
  457. toolchain_type, compiler_type, linker_type = Compilers[self.tc.type]
  458. toolchain = toolchain_type(self.tc, self)
  459. compiler = compiler_type(self.tc, self)
  460. linker = linker_type(self.tc, self)
  461. toolchain.print_toolchain()
  462. compiler.print_compiler()
  463. linker.print_linker()
  464. self._print_other_settings(compiler)
  465. def _print_build_settings(self):
  466. emit('BUILD_TYPE', self.build_type.upper())
  467. emit('BT_' + self.build_type.upper().replace('-', '_'), 'yes')
  468. if self.build_system == 'distbuild':
  469. emit('DISTBUILD', 'yes')
  470. elif self.build_system != 'ymake':
  471. raise ConfigureError()
  472. python_bin = preset('BUILD_PYTHON_BIN', '$(PYTHON)/python')
  473. emit('YMAKE_PYTHON', python_bin)
  474. emit('YMAKE_UNPICKLER', python_bin, '$ARCADIA_ROOT/build/plugins/_unpickler.py')
  475. @property
  476. def is_release(self):
  477. # TODO(somov): Проверить, бывают ли тут суффиксы на самом деле
  478. return self.build_type in ('release', 'relwithdebinfo', 'minsizerel', 'profile', 'gprof') or self.build_type.endswith('-release')
  479. @property
  480. def is_debug(self):
  481. return self.build_type in ('debug', 'debugnoasserts', 'fastdebug') or self.build_type.endswith('-debug')
  482. @property
  483. def is_fast_debug(self):
  484. return self.build_type == 'fastdebug'
  485. @property
  486. def is_size_optimized(self):
  487. return self.build_type == 'minsizerel'
  488. @property
  489. def is_coverage(self):
  490. return self.build_type == 'coverage'
  491. @property
  492. def is_sanitized(self):
  493. sanitizer = preset('SANITIZER_TYPE')
  494. return bool(sanitizer) and not is_negative_str(sanitizer)
  495. @property
  496. def is_valgrind(self):
  497. return self.build_type == 'valgrind' or self.build_type == 'valgrind-release'
  498. @property
  499. def is_ide(self):
  500. return self.is_ide_build_type(self.build_type)
  501. @property
  502. def profiler_type(self):
  503. if self.build_type == 'profile':
  504. return Profiler.Generic
  505. elif self.build_type == 'gprof':
  506. return Profiler.GProf
  507. else:
  508. return None
  509. @staticmethod
  510. def is_ide_build_type(build_type):
  511. return build_type == 'nobuild'
  512. def _configure_runtime_versions(self):
  513. try:
  514. res = subprocess.check_output(['xcrun', 'simctl', 'list', '--json', 'runtimes'])
  515. except subprocess.CalledProcessError as exc:
  516. logger.warning(exc)
  517. return
  518. raw_object = json.loads(res)
  519. raw_object = raw_object['runtimes']
  520. for runtime in raw_object:
  521. if runtime['isAvailable']:
  522. if "iOS" in runtime['identifier']:
  523. emit('DEFAULT_IOS_RUNTIME', '{}'.format(runtime['identifier']))
  524. def _get_toolchain_options(self):
  525. type_ = self.params['params']['type']
  526. if self.params['params'].get('local') and type_ == 'xcode':
  527. detector = CompilerDetector()
  528. detector.detect(self.params['params']['c_compiler'], self.params['params']['cxx_compiler'])
  529. emit('LOCAL_XCODE_TOOLS', 'yes')
  530. emit('XCODE', 'yes')
  531. emit('ACTOOL_PATH', self.params['params']['actool'])
  532. emit('IBTOOL_PATH', self.params['params']['ibtool'])
  533. self._configure_runtime_versions()
  534. elif type_ == 'system_cxx':
  535. c_compiler = self.params['params'].get('c_compiler')
  536. cxx_compiler = self.params['params'].get('cxx_compiler')
  537. if is_positive('USE_CLANG_CL'):
  538. c_compiler = 'clang-cl.exe' if c_compiler == 'cl.exe' else c_compiler
  539. cxx_compiler = 'clang-cl.exe' if cxx_compiler == 'cl.exe' else cxx_compiler
  540. detector = CompilerDetector()
  541. detector.detect(c_compiler, cxx_compiler)
  542. type_ = detector.type
  543. else:
  544. detector = None
  545. if type_ == 'msvc':
  546. return MSVCToolchainOptions(self, detector)
  547. else:
  548. return GnuToolchainOptions(self, detector)
  549. def _print_other_settings(self, compiler):
  550. host = self.host
  551. emit('USE_LOCAL_TOOLS', 'no' if self.ignore_local_files else 'yes')
  552. ragel = Ragel()
  553. ragel.configure_toolchain(self, compiler)
  554. ragel.print_variables()
  555. swiftc = SwiftCompiler(self)
  556. swiftc.configure()
  557. swiftc.print_compiler()
  558. if host.is_linux or host.is_macos or host.is_cygwin:
  559. if is_negative('USE_ARCADIA_PYTHON'):
  560. python = Python(self.tc)
  561. python.configure_posix()
  562. python.print_variables()
  563. cuda = Cuda(self)
  564. cuda.print_()
  565. CuDNN(cuda).print_()
  566. if self.ignore_local_files or host.is_windows or is_positive('NO_SVN_DEPENDS'):
  567. emit_with_ignore_comment('SVN_DEPENDS_CACHE__NO_UID__')
  568. else:
  569. emit_with_ignore_comment('SVN_DEPENDS_CACHE__NO_UID__', '${hide;kv:"disable_cache"}')
  570. @staticmethod
  571. def _load_json_from_base64(base64str):
  572. """
  573. :rtype: dict[str, Any]
  574. """
  575. def un_unicode(o):
  576. if isinstance(o, six.text_type):
  577. return six.ensure_str(o)
  578. if isinstance(o, list):
  579. return [un_unicode(oo) for oo in o]
  580. if isinstance(o, dict):
  581. return {un_unicode(k): un_unicode(v) for k, v in six.iteritems(o)}
  582. return o
  583. return un_unicode(json.loads(base64.b64decode(base64str)))
  584. class YMake(object):
  585. def __init__(self, arcadia):
  586. self.arcadia = arcadia
  587. def print_presets(self):
  588. presets = opts().presets
  589. if is_positive_str(presets.get('EXPORT_GRADLE', 'no')):
  590. print('@import "${CONF_ROOT}/conf/export_gradle.yes.conf"')
  591. else:
  592. print('@import "${CONF_ROOT}/conf/export_gradle.no.conf"')
  593. if presets.get('CLANG_COVERAGE', None) is None:
  594. print('@import "${CONF_ROOT}/conf/coverage_full_instrumentation.conf"')
  595. else:
  596. print('@import "${CONF_ROOT}/conf/coverage_selective_instrumentation.conf"')
  597. if presets:
  598. print('# Variables set from command line by -D options')
  599. for key in sorted(presets):
  600. emit(key, opts().presets[key])
  601. @staticmethod
  602. def _print_conf_content(path):
  603. with open(path, 'r') as fin:
  604. print(fin.read())
  605. def print_core_conf(self):
  606. print('@import "${CONF_ROOT}/ymake.core.conf"')
  607. def print_settings(self):
  608. pass
  609. class System(object):
  610. def __init__(self, platform):
  611. """
  612. :type platform: Platform
  613. """
  614. self.platform = platform
  615. def print_windows_target_const(self):
  616. # TODO(somov): Remove this variables, use generic OS/arch variables in makelists.
  617. emit('WINDOWS', 'yes')
  618. emit('WIN32', 'yes')
  619. if self.platform.is_64_bit == 64:
  620. emit('WIN64', 'yes')
  621. def print_nix_target_const(self):
  622. emit('JAVA_INCLUDE', '-I{0}/include -I{0}/include/{1}'.format('/usr/lib/jvm/default-java', self.platform.os_compat))
  623. emit('UNIX', 'yes')
  624. emit('REALPRJNAME')
  625. emit('SONAME')
  626. def print_target_settings(self):
  627. emit('TARGET_PLATFORM', self.platform.os_compat)
  628. emit('CANONIZED_TARGET_PLATFORM', self.platform.canonized_platform)
  629. emit('HARDWARE_ARCH', '32' if self.platform.is_32_bit else '64')
  630. emit('HARDWARE_TYPE', self.platform.arch)
  631. for variable in self.platform.arch_variables:
  632. emit(variable, 'yes')
  633. for variable in self.platform.os_variables:
  634. emit(variable, 'yes')
  635. if self.platform.is_armv7:
  636. emit('ARM7_FLOAT_ABI', self.platform.armv7_float_abi)
  637. if self.platform.is_android:
  638. emit('ANDROID_API', str(self.platform.android_api))
  639. if self.platform.is_posix:
  640. self.print_nix_target_const()
  641. elif self.platform.is_windows:
  642. self.print_windows_target_const()
  643. def print_host_settings(self):
  644. emit('HOST_PLATFORM', self.platform.os_compat)
  645. emit('CANONIZED_HOST_PLATFORM', self.platform.canonized_platform)
  646. for variable in itertools.chain(self.platform.os_variables, self.platform.arch_variables):
  647. emit('HOST_{var}'.format(var=variable), 'yes')
  648. class CompilerDetector(object):
  649. def __init__(self):
  650. self.type = None
  651. self.c_compiler = None
  652. self.cxx_compiler = None
  653. self.version_list = None
  654. @staticmethod
  655. def preprocess_source(compiler, source):
  656. # noinspection PyBroadException
  657. try:
  658. fd, path = tempfile.mkstemp(suffix='.cpp')
  659. try:
  660. with os.fdopen(fd, 'w') as output:
  661. output.write(source)
  662. stdout, code = get_stdout_and_code([compiler, '-E', path])
  663. except Exception as e:
  664. logger.info("While writing: `%s`", e)
  665. finally:
  666. os.remove(path)
  667. return stdout, code
  668. except Exception as e:
  669. logger.debug('Preprocessing failed: %s', e)
  670. return None, None
  671. @staticmethod
  672. def get_compiler_vars(compiler, names):
  673. prefix = '____YA_VAR_'
  674. source = '\n'.join(['{prefix}{name}={name}\n'.format(prefix=prefix, name=n) for n in names])
  675. # Некоторые препроцессоры возвращают ненулевой код возврата. Поэтому его проверять нельзя.
  676. # Мы можем только удостовериться после разбора stdout, что в нём
  677. # присутствовала хотя бы одна подставленная переменная.
  678. # TODO(somov): Исследовать, можно ли проверять ограниченный набор кодов возврата.
  679. # TODO(v-korovin): Нормально прокидывать Exception-ы, оно и так упадёт
  680. stdout, _ = CompilerDetector.preprocess_source(compiler, source)
  681. if stdout is None:
  682. return None
  683. vars_ = {}
  684. for line in six.ensure_str(stdout).split('\n'):
  685. parts = line.split('=', 1)
  686. if len(parts) == 2 and parts[0].startswith(prefix):
  687. name, value = parts[0][len(prefix):], parts[1]
  688. if value == name:
  689. continue # Preprocessor variable was not substituted
  690. vars_[name] = value
  691. return vars_
  692. def detect(self, c_compiler=None, cxx_compiler=None):
  693. c_compiler = c_compiler or os.environ.get('CC')
  694. cxx_compiler = cxx_compiler or os.environ.get('CXX') or c_compiler
  695. c_compiler = c_compiler or cxx_compiler
  696. logger.debug('e=%s', os.environ)
  697. if c_compiler is None:
  698. raise ConfigureError('Custom compiler was requested but not specified')
  699. c_compiler_path = which(c_compiler)
  700. clang_vars = ['__clang_major__', '__clang_minor__', '__clang_patchlevel__']
  701. gcc_vars = ['__GNUC__', '__GNUC_MINOR__', '__GNUC_PATCHLEVEL__']
  702. msvc_vars = ['_MSC_VER']
  703. apple_var = '__apple_build_version__'
  704. compiler_vars = self.get_compiler_vars(c_compiler_path, clang_vars + [apple_var] + gcc_vars + msvc_vars)
  705. if not compiler_vars:
  706. raise ConfigureError('Could not determine custom compiler version: {}'.format(c_compiler))
  707. def version(version_names):
  708. def iter_version():
  709. for name in version_names:
  710. yield int(compiler_vars[name])
  711. # noinspection PyBroadException
  712. try:
  713. return list(iter_version())
  714. except Exception:
  715. return None
  716. clang_version = version(clang_vars)
  717. apple_build = apple_var in compiler_vars
  718. # TODO(somov): Учитывать номера версий сборки Apple компилятора Clang.
  719. _ = apple_build
  720. gcc_version = version(gcc_vars)
  721. msvc_version = version(msvc_vars)
  722. if msvc_version:
  723. logger.debug('Detected MSVC version %s', msvc_version)
  724. self.type = 'msvc'
  725. elif clang_version:
  726. logger.debug('Detected Clang version %s', clang_version)
  727. self.type = 'clang'
  728. elif gcc_version:
  729. logger.debug('Detected GCC version %s', gcc_version)
  730. # TODO(somov): Переименовать в gcc.
  731. self.type = 'gnu'
  732. else:
  733. raise ConfigureError('Could not determine custom compiler type: {}'.format(c_compiler))
  734. self.version_list = clang_version or gcc_version or msvc_version
  735. self.c_compiler = c_compiler_path
  736. self.cxx_compiler = cxx_compiler and which(cxx_compiler) or c_compiler_path
  737. class ToolchainOptions(object):
  738. def __init__(self, build, detector):
  739. """
  740. :type build: Build
  741. """
  742. self.host = build.host
  743. self.target = build.target
  744. tc_json = build.params
  745. logger.debug('Toolchain host %s', self.host)
  746. logger.debug('Toolchain target %s', self.target)
  747. logger.debug('Toolchain json %s', DebugString(lambda: json.dumps(tc_json, indent=4, sort_keys=True)))
  748. self.params = tc_json['params']
  749. self._name = tc_json.get('name', 'theyknow')
  750. if detector:
  751. self.type = detector.type
  752. self.from_arcadia = False
  753. self.c_compiler = detector.c_compiler
  754. self.cxx_compiler = detector.cxx_compiler
  755. self.compiler_version_list = detector.version_list
  756. self.compiler_version = '.'.join(map(lambda part: six.ensure_str(str(part)), self.compiler_version_list))
  757. else:
  758. self.type = self.params['type']
  759. self.from_arcadia = True
  760. self.c_compiler = self.params['c_compiler']
  761. self.cxx_compiler = self.params['cxx_compiler']
  762. # TODO(somov): Требовать номер версии всегда.
  763. self.compiler_version = self.params.get('gcc_version') or self.params.get('version') or '0'
  764. self.compiler_version_list = list(map(int, self.compiler_version.split('.')))
  765. # 'match_root' at this point contains real name for references via toolchain
  766. self.name_marker = '$(%s)' % self.params.get('match_root', self._name.upper())
  767. self.arch_opt = self.params.get('arch_opt', [])
  768. self.triplet_opt = self.params.get('triplet_opt', {})
  769. self.target_opt = self.params.get('target_opt', [])
  770. # TODO(somov): Убрать чтение настройки из os.environ.
  771. self.werror_mode = preset('WERROR_MODE') or os.environ.get('WERROR_MODE') or self.params.get('werror_mode') or 'compiler_specific'
  772. # default C++ standard is set here, some older toolchains might need to redefine it in ya.conf.json
  773. self.cxx_std = self.params.get('cxx_std', 'c++20')
  774. self._env = tc_json.get('env', {})
  775. self.android_ndk_version = self.params.get('android_ndk_version', None)
  776. logger.debug('c_compiler=%s', self.c_compiler)
  777. logger.debug('cxx_compiler=%s', self.cxx_compiler)
  778. self.compiler_platform_projects = self.target.find_in_dict(self.params.get('platform'), [])
  779. def version_at_least(self, *args):
  780. return args <= tuple(self.compiler_version_list)
  781. def version_below(self, *args):
  782. return args > tuple(self.compiler_version_list)
  783. def version_exactly(self, *args):
  784. if not args or len(args) > len(self.compiler_version_list):
  785. return False
  786. for left, right in zip(args, list(self.compiler_version_list)[:len(args)]):
  787. if left != right:
  788. return False
  789. return True
  790. @property
  791. def is_gcc(self):
  792. return self.type == 'gnu'
  793. @property
  794. def is_clang(self):
  795. return self.type in ('clang', 'xcode')
  796. @property
  797. def is_xcode(self):
  798. return self.type == 'xcode'
  799. @property
  800. def is_from_arcadia(self):
  801. return self.from_arcadia
  802. @property
  803. def is_system_cxx(self):
  804. return self._name == "system_cxx"
  805. def get_env(self, convert_list=None):
  806. convert_list = convert_list or (lambda x: x)
  807. r = {}
  808. for k, v in six.iteritems(self._env):
  809. if isinstance(v, str):
  810. r[k] = v
  811. elif isinstance(v, list):
  812. r[k] = convert_list(v)
  813. else:
  814. logger.debug('Unexpected values in environment: %s', self._env)
  815. raise ConfigureError('Internal error')
  816. return r
  817. class GnuToolchainOptions(ToolchainOptions):
  818. def __init__(self, build, detector):
  819. super(GnuToolchainOptions, self).__init__(build, detector)
  820. self.ar = self.params.get('ar')
  821. self.ar_plugin = self.params.get('ar_plugin')
  822. self.strip = self.params.get('strip')
  823. self.objcopy = self.params.get('objcopy')
  824. self.objdump = self.params.get('objdump')
  825. self.isystem = self.params.get('isystem')
  826. self.dwarf_tool = self.target.find_in_dict(self.params.get('dwarf_tool'))
  827. # TODO(somov): Унифицировать формат sys_lib
  828. self.sys_lib = self.params.get('sys_lib', {})
  829. if isinstance(self.sys_lib, dict):
  830. self.sys_lib = self.target.find_in_dict(self.sys_lib, [])
  831. self.os_sdk = preset('OS_SDK') or self._default_os_sdk()
  832. self.os_sdk_local = False
  833. if build.host.is_apple and build.target.is_apple and to_bool(preset('APPLE_SDK_LOCAL'), default=False):
  834. self.os_sdk_local = True
  835. if build.host.is_apple and build.target.is_ios and to_bool(preset('MAPSMOBI_BUILD_TARGET'), default=False):
  836. self.os_sdk_local = True
  837. if self.os_sdk == 'local':
  838. self.os_sdk_local = True
  839. def _default_os_sdk(self):
  840. if self.target.is_linux:
  841. if self.target.is_armv8:
  842. return 'ubuntu-16'
  843. if self.target.is_armv7 and self.target.armv7_float_abi == 'hard':
  844. return 'ubuntu-16'
  845. if self.target.is_armv7 and self.target.armv7_float_abi == 'softfp':
  846. return 'ubuntu-18'
  847. # Default OS SDK for Linux builds
  848. return LINUX_SDK_DEFAULT
  849. class Toolchain(object):
  850. def __init__(self, tc, build):
  851. """
  852. :type tc: ToolchainOptions
  853. :type build: Build
  854. """
  855. self.tc = tc
  856. self.build = build
  857. self.platform_projects = self.tc.compiler_platform_projects
  858. def print_toolchain(self):
  859. if self.platform_projects:
  860. emit('COMPILER_PLATFORM', list(unique(self.platform_projects)))
  861. class Compiler(object):
  862. def __init__(self, tc, compiler_variable):
  863. self.compiler_variable = compiler_variable
  864. self.tc = tc
  865. def print_compiler(self):
  866. # CLANG and CLANG_VER variables
  867. emit(self.compiler_variable, 'yes')
  868. cv = self.tc.compiler_version
  869. if '.' in cv:
  870. cv = cv[:cv.index('.')]
  871. emit('{}_VER'.format(self.compiler_variable), cv)
  872. if self.tc.is_xcode:
  873. emit('XCODE', 'yes')
  874. class GnuToolchain(Toolchain):
  875. def __init__(self, tc, build):
  876. """
  877. :type tc: GnuToolchainOptions
  878. :type build: Build
  879. """
  880. def get_os_sdk(target):
  881. sdk_native_version = 10.11 if not preset('EXPERIMENTAL_MACOS_M1_SUPPORT') else '11.1'
  882. if target.is_macos:
  883. return '$MACOS_SDK_RESOURCE_GLOBAL/MacOSX{}.sdk'.format(sdk_native_version)
  884. elif target.is_yocto:
  885. return '$YOCTO_SDK_RESOURCE_GLOBAL'
  886. return '$OS_SDK_ROOT_RESOURCE_GLOBAL'
  887. super(GnuToolchain, self).__init__(tc, build)
  888. self.tc = tc
  889. host = build.host
  890. target = build.target
  891. self.c_flags_platform = list(tc.target_opt)
  892. self.default_os_sdk_root = get_os_sdk(target)
  893. self.env = self.tc.get_env()
  894. self.env_go = {}
  895. if self.tc.is_clang and not self.tc.is_system_cxx:
  896. self.env_go = {'PATH': ['{}/bin'.format(self.tc.name_marker)]}
  897. if self.tc.is_gcc:
  898. self.env_go = {'PATH': ['{}/gcc/bin'.format(self.tc.name_marker)]}
  899. if 'PATH' in self.env_go:
  900. if target.is_linux:
  901. self.env_go['PATH'].append('$OS_SDK_ROOT_RESOURCE_GLOBAL/usr/bin')
  902. elif target.is_macos:
  903. self.env_go['PATH'].extend([
  904. '$MACOS_SDK_RESOURCE_GLOBAL/usr/bin',
  905. '$GO_FAKE_XCRUN_RESOURCE_GLOBAL',
  906. ])
  907. self.swift_flags_platform = []
  908. self.swift_lib_path = None
  909. if self.tc.is_from_arcadia:
  910. for lib_path in build.host.library_path_variables:
  911. self.env.setdefault(lib_path, []).append('{}/lib'.format(self.tc.name_marker))
  912. swift_target = select(default=None, selectors=[
  913. (target.is_iossim and target.is_x86_64, 'x86_64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  914. (target.is_iossim and target.is_x86, 'i386-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  915. (target.is_iossim and target.is_armv8, 'arm64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  916. (not target.is_iossim and target.is_ios and target.is_armv8, 'arm64-apple-ios9'),
  917. (not target.is_iossim and target.is_ios and target.is_armv7, 'armv7-apple-ios9'),
  918. ])
  919. if swift_target:
  920. self.swift_flags_platform += ['-target', swift_target]
  921. if self.tc.is_from_arcadia:
  922. self.swift_lib_path = select(default=None, selectors=[
  923. (host.is_macos and target.is_iossim, '$SWIFT_XCODE_TOOLCHAIN_ROOT_RESOURCE_GLOBAL/usr/lib/swift/iphonesimulator'),
  924. (host.is_macos and not target.is_iossim and target.is_ios and (target.is_armv8 or target.is_armv7), '$SWIFT_XCODE_TOOLCHAIN_ROOT_RESOURCE_GLOBAL/usr/lib/swift/iphoneos'),
  925. ])
  926. if self.tc.is_clang:
  927. target_triple = self.tc.triplet_opt.get(target.arch, None)
  928. if not target_triple:
  929. target_triple = select(default=None, selectors=[
  930. (target.is_linux and target.is_x86_64, 'x86_64-linux-gnu'),
  931. (target.is_linux and target.is_armv8, 'aarch64-linux-gnu'),
  932. (target.is_linux and target.is_armv7 and target.armv7_float_abi == 'hard', 'armv7-linux-gnueabihf'),
  933. (target.is_linux and target.is_armv7 and target.armv7_float_abi == 'softfp', 'armv7-linux-gnueabi'),
  934. (target.is_linux and target.is_powerpc, 'powerpc64le-linux-gnu'),
  935. (target.is_iossim and target.is_x86_64, 'x86_64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  936. (target.is_iossim and target.is_x86, 'i386-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  937. (target.is_iossim and target.is_armv8, 'arm64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  938. (not target.is_iossim and target.is_ios and target.is_armv8, 'arm64-apple-ios{}'.format(IOS_VERSION_MIN)),
  939. (not target.is_iossim and target.is_ios and target.is_armv7, 'armv7-apple-ios{}'.format(IOS_VERSION_MIN)),
  940. (target.is_apple and target.is_x86, 'i386-apple-darwin14'),
  941. (target.is_apple and target.is_x86_64, 'x86_64-apple-darwin14'),
  942. (target.is_apple and target.is_macos_arm64, 'arm64-apple-macos11'),
  943. (target.is_apple and target.is_armv7, 'armv7-apple-darwin14'),
  944. (target.is_apple and target.is_armv8, 'arm64-apple-darwin14'),
  945. (target.is_yocto and target.is_armv7, 'arm-poky-linux-gnueabi'),
  946. (target.is_android and target.is_x86, 'i686-linux-android'),
  947. (target.is_android and target.is_x86_64, 'x86_64-linux-android'),
  948. (target.is_android and target.is_armv7, 'armv7a-linux-androideabi'),
  949. (target.is_android and target.is_armv8, 'aarch64-linux-android'),
  950. (target.is_emscripten and target.is_wasm64, 'wasm64-unknown-emscripten'),
  951. ])
  952. if target.is_android:
  953. # Android NDK allows specification of API level in target triple, e.g.:
  954. # armv7a-linux-androideabi16, aarch64-linux-android21
  955. target_triple += str(target.android_api)
  956. if target_triple:
  957. self.c_flags_platform.append('--target={}'.format(target_triple))
  958. if self.tc.isystem:
  959. for root in list(self.tc.isystem):
  960. self.c_flags_platform.extend(['-isystem', root])
  961. if target.is_android:
  962. self.c_flags_platform.extend(['-isystem', '{}/sources/cxx-stl/llvm-libc++abi/include'.format(self.tc.name_marker)])
  963. if target.is_cortex_a9:
  964. self.c_flags_platform.append('-mcpu=cortex-a9')
  965. if target.is_cortex_a35:
  966. self.c_flags_platform.append('-mcpu=cortex-a35')
  967. elif target.is_cortex_a53:
  968. self.c_flags_platform.append('-mcpu=cortex-a53')
  969. elif target.is_cortex_m4:
  970. self.c_flags_platform.append('-mcpu=cortex-m4 -mfpu=fpv4-sp-d16')
  971. elif target.is_cortex_m7:
  972. self.c_flags_platform.append('-mcpu=cortex-m7 -mfpu=fpv5-sp-d16')
  973. elif target.is_cortex_m23:
  974. self.c_flags_platform.append('-mcpu=cortex-m23')
  975. elif target.is_cortex_m33:
  976. self.c_flags_platform.append('-mcpu=cortex-m33 -mfpu=fpv5-sp-d16')
  977. elif target.is_armv7_neon:
  978. self.c_flags_platform.append('-mfpu=neon')
  979. elif target.is_arm968e_s:
  980. self.c_flags_platform.append('-march=armv5te -mcpu=arm968e-s -mthumb-interwork -mlittle-endian')
  981. if (target.is_armv7 or target.is_armv8m or target.is_armv7em or target.is_armv5te) and build.is_size_optimized:
  982. # Enable ARM Thumb2 variable-length instruction encoding
  983. # to reduce code size
  984. self.c_flags_platform.append('-mthumb')
  985. if target.is_arm or target.is_powerpc:
  986. # On linux, ARM and PPC default to unsigned char
  987. # However, Arcadia requires char to be signed
  988. self.c_flags_platform.append('-fsigned-char')
  989. if target.is_rv32imc:
  990. self.c_flags_platform.append('-march=rv32imc')
  991. if self.tc.is_clang or self.tc.is_gcc and self.tc.version_at_least(8, 2):
  992. target_flags = select(default=[], selectors=[
  993. (target.is_linux and target.is_power8le, ['-mcpu=power8', '-mtune=power8', '-maltivec']),
  994. (target.is_linux and target.is_power9le, ['-mcpu=power9', '-mtune=power9', '-maltivec']),
  995. (target.is_linux and target.is_armv8, ['-march=armv8a']),
  996. (target.is_macos, ['-mmacosx-version-min={}'.format(MACOS_VERSION_MIN)]),
  997. (target.is_ios and not target.is_iossim, ['-mios-version-min={}'.format(IOS_VERSION_MIN)]),
  998. (target.is_iossim, ['-mios-simulator-version-min={}'.format(IOS_VERSION_MIN)]),
  999. (target.is_android and target.is_armv7, ['-march=armv7-a', '-mfloat-abi=softfp']),
  1000. (target.is_android and target.is_armv8, ['-march=armv8-a']),
  1001. (target.is_yocto and target.is_armv7, ['-march=armv7-a', '-mfpu=neon', '-mfloat-abi=hard', '-mcpu=cortex-a9', '-O1'])
  1002. ])
  1003. if target_flags:
  1004. self.c_flags_platform.extend(target_flags)
  1005. if target.is_ios:
  1006. self.c_flags_platform.append('-D__IOS__=1')
  1007. if target.is_apple:
  1008. self.setup_apple_sdk(target)
  1009. if self.tc.is_from_arcadia or self.tc.is_system_cxx:
  1010. if target.is_linux:
  1011. if not tc.os_sdk_local:
  1012. self.setup_sdk(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL')
  1013. if target.is_x86_64:
  1014. if host.is_linux and not self.tc.is_gcc:
  1015. self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu')
  1016. elif host.is_macos:
  1017. self.setup_tools(project='build/platform/binutils', var='$BINUTILS_ROOT_RESOURCE_GLOBAL', bin='x86_64-linux-gnu/bin', ldlibs=None)
  1018. elif target.is_powerpc:
  1019. self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/x86_64-linux-gnu/powerpc64le-linux-gnu/lib')
  1020. elif target.is_armv8:
  1021. self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu')
  1022. if target.is_yocto:
  1023. self.setup_sdk(project='build/platform/yocto_sdk/yocto_sdk', var='${YOCTO_SDK_ROOT_RESOURCE_GLOBAL}')
  1024. def setup_apple_sdk(self, target):
  1025. if not self.tc.os_sdk_local:
  1026. self.setup_apple_arcadia_sdk(target)
  1027. else:
  1028. self.setup_apple_local_sdk(target)
  1029. def setup_apple_arcadia_sdk(self, target):
  1030. if target.is_ios and not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  1031. self.setup_xcode_sdk(project='build/internal/platform/ios_sdk', var='${IOS_SDK_ROOT_RESOURCE_GLOBAL}')
  1032. self.platform_projects.append('build/internal/platform/macos_system_stl')
  1033. if target.is_macos and not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  1034. self.setup_xcode_sdk(project='build/internal/platform/macos_sdk', var='${MACOS_SDK_RESOURCE_GLOBAL}')
  1035. def setup_apple_local_sdk(self, target):
  1036. def get_output(*args):
  1037. return six.ensure_str(subprocess.check_output(tuple(args))).strip()
  1038. def get_sdk_root(sdk):
  1039. root = self.env.get('SDKROOT')
  1040. if root not in (None, '', ['']):
  1041. return root
  1042. root = os.environ.get('SDKROOT')
  1043. if root:
  1044. return root
  1045. return get_output('xcrun', '-sdk', sdk, '--show-sdk-path')
  1046. if target.is_iossim:
  1047. self.env['SDKROOT'] = get_sdk_root('iphonesimulator')
  1048. elif target.is_ios:
  1049. self.env['SDKROOT'] = get_sdk_root('iphoneos')
  1050. elif target.is_macos:
  1051. self.env['SDKROOT'] = get_sdk_root('macosx')
  1052. def setup_sdk(self, project, var):
  1053. self.platform_projects.append(project)
  1054. self.c_flags_platform.append('--sysroot={}'.format(var))
  1055. self.swift_flags_platform += ['-sdk', var]
  1056. def setup_xcode_sdk(self, project, var):
  1057. self.platform_projects.append(project)
  1058. self.c_flags_platform.append('-isysroot')
  1059. self.c_flags_platform.append(var)
  1060. self.swift_flags_platform += ['-sdk', var]
  1061. # noinspection PyShadowingBuiltins
  1062. def setup_tools(self, project, var, bin, ldlibs):
  1063. self.platform_projects.append(project)
  1064. self.c_flags_platform.append('-B{}/{}'.format(var, bin))
  1065. if ldlibs:
  1066. for lib_path in self.build.host.library_path_variables:
  1067. self.env.setdefault(lib_path, []).append('{}/{}'.format(var, ldlibs))
  1068. def print_toolchain(self):
  1069. super(GnuToolchain, self).print_toolchain()
  1070. emit('TOOLCHAIN_ENV', format_env(self.env, list_separator=':'))
  1071. emit('_GO_TOOLCHAIN_ENV_PATH', format_env(self.env_go, list_separator=':'))
  1072. emit('C_FLAGS_PLATFORM', self.c_flags_platform)
  1073. emit('SWIFT_FLAGS_PLATFORM', self.swift_flags_platform)
  1074. emit('SWIFT_LD_FLAGS', '-L{}'.format(self.swift_lib_path) if self.swift_lib_path else '')
  1075. emit('PERL_SDK', preset('OS_SDK') or self.tc.os_sdk)
  1076. if preset('OS_SDK') is None:
  1077. emit('OS_SDK', self.tc.os_sdk)
  1078. emit('OS_SDK_ROOT', None if self.tc.os_sdk_local else self.default_os_sdk_root)
  1079. class GnuCompiler(Compiler):
  1080. def __init__(self, tc, build):
  1081. """
  1082. :type tc: GnuToolchainOptions
  1083. :type build: Build
  1084. """
  1085. compiler_variable = 'CLANG' if tc.is_clang else 'GCC'
  1086. super(GnuCompiler, self).__init__(tc, compiler_variable)
  1087. self.build = build
  1088. self.host = self.build.host
  1089. self.target = self.build.target
  1090. self.tc = tc
  1091. self.c_foptions = [
  1092. # Enable C++ exceptions (and allow them to be throw through pure C code)
  1093. '-fexceptions',
  1094. # Enable standard-conforming behavior and generate duplicate symbol error in case of duplicated global constants.
  1095. # See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85678#c0
  1096. '-fno-common',
  1097. # Split all functions and data into separate sections for DCE and ICF linker passes
  1098. '-ffunction-sections',
  1099. '-fdata-sections'
  1100. ]
  1101. if self.tc.is_clang and self.target.is_linux:
  1102. # Use .init_array instead of .ctors (default for old clang versions)
  1103. # See: https://maskray.me/blog/2021-11-07-init-ctors-init-array
  1104. self.c_foptions.append('-fuse-init-array')
  1105. if self.target.is_xtensa:
  1106. self.c_foptions.append('-mlongcalls')
  1107. if self.tc.is_clang:
  1108. self.c_foptions += [
  1109. # Set up output colorization
  1110. '-fcolor-diagnostics',
  1111. # Enable aligned allocation
  1112. '-faligned-allocation',
  1113. '-fdebug-default-version=4',
  1114. ]
  1115. elif self.tc.is_gcc:
  1116. if self.target.is_xtensa or self.target.is_tc32:
  1117. # Xtensa and tc32 toolchains does not support this flag
  1118. pass
  1119. else:
  1120. self.c_foptions += [
  1121. # Set up output colorization
  1122. '-fdiagnostics-color=always',
  1123. # It looks like there is no way to enable aligned allocation in gcc
  1124. ]
  1125. self.c_warnings = [
  1126. # Enable default warnings subset
  1127. '-Wall',
  1128. '-Wextra',
  1129. ]
  1130. self.cxx_warnings = [
  1131. # Issue a warning if certain overload is hidden due to inheritance
  1132. '-Woverloaded-virtual',
  1133. ]
  1134. # Disable some warnings which will fail compilation at the time
  1135. self.c_warnings += [
  1136. '-Wno-parentheses',
  1137. ]
  1138. self.c_defines = ['${hide:CPP_FAKEID}']
  1139. if self.target.is_android:
  1140. self.c_defines.append('${hide:ANDROID_FAKEID}')
  1141. self.c_defines.extend([
  1142. '-DARCADIA_ROOT=${ARCADIA_ROOT}',
  1143. '-DARCADIA_BUILD_ROOT=${ARCADIA_BUILD_ROOT}',
  1144. ])
  1145. self.c_defines.extend([
  1146. '-D_THREAD_SAFE', '-D_PTHREADS', '-D_REENTRANT',
  1147. '-D_LARGEFILE_SOURCE', '-D__STDC_CONSTANT_MACROS', '-D__STDC_FORMAT_MACROS',
  1148. ])
  1149. if self.target.is_macos and self.tc.version_below(17):
  1150. # OS-versioning macros for Darwin-based OSes were generalized in
  1151. # https://github.com/llvm/llvm-project/commit/c8e2dd8c6f490b68e41fe663b44535a8a21dfeab
  1152. #
  1153. # This PR was released in llvm-17. Provide similar behaviour manually.
  1154. self.c_defines.append("-D__ENVIRONMENT_OS_VERSION_MIN_REQUIRED__=" + MACOS_VERSION_MIN_AS_INT)
  1155. if not self.target.is_android:
  1156. # There is no usable _FILE_OFFSET_BITS=64 support in Androids until API 21. And it's incomplete until at least API 24.
  1157. # https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md
  1158. # Arcadia have API 16 for 32-bit Androids.
  1159. self.c_defines.append('-D_FILE_OFFSET_BITS=64')
  1160. if self.target.is_linux or self.target.is_android or self.target.is_cygwin or self.target.is_none:
  1161. self.c_defines.append('-D_GNU_SOURCE')
  1162. if self.tc.is_clang and self.target.is_linux and self.target.is_x86_64:
  1163. self.c_defines.append('-D_YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE')
  1164. self.c_flags = ['$CL_DEBUG_INFO', '$CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__']
  1165. self.c_flags += self.tc.arch_opt + ['-pipe']
  1166. self.sfdl_flags = ['-E', '-C', '-x', 'c++']
  1167. if self.target.is_x86:
  1168. self.c_flags.append('-m32')
  1169. if self.target.is_x86_64:
  1170. self.c_flags.append('-m64')
  1171. if self.target.is_wasm64:
  1172. # WebAssembly-specific exception handling flags
  1173. self.c_foptions += [
  1174. '-fwasm-exceptions',
  1175. ]
  1176. self.debug_info_flags = ['-g']
  1177. if self.target.is_linux:
  1178. self.debug_info_flags.append('-ggnu-pubnames')
  1179. self.cross_suffix = '' if is_positive('FORCE_NO_PIC') else '.pic'
  1180. self.optimize = None
  1181. self.configure_build_type()
  1182. if self.tc.is_clang:
  1183. self.sfdl_flags.append('-Qunused-arguments')
  1184. self.c_warnings += [
  1185. '-Wno-implicit-const-int-float-conversion',
  1186. # For nvcc to accept the above.
  1187. '-Wno-unknown-warning-option',
  1188. ]
  1189. self.cxx_warnings += [
  1190. '-Wimport-preprocessor-directive-pedantic',
  1191. '-Wno-ambiguous-reversed-operator',
  1192. '-Wno-defaulted-function-deleted',
  1193. '-Wno-deprecated-anon-enum-enum-conversion',
  1194. '-Wno-deprecated-enum-enum-conversion',
  1195. '-Wno-deprecated-enum-float-conversion',
  1196. '-Wno-deprecated-volatile',
  1197. '-Wno-pessimizing-move',
  1198. '-Wno-return-std-move',
  1199. '-Wno-undefined-var-template',
  1200. ]
  1201. elif self.tc.is_gcc:
  1202. if self.target.is_xtensa:
  1203. # Xtensa toolchain does not support this flags
  1204. pass
  1205. else:
  1206. self.c_foptions.append('-fno-delete-null-pointer-checks')
  1207. self.c_foptions.append('-fabi-version=8')
  1208. def configure_build_type(self):
  1209. if self.build.is_valgrind:
  1210. self.c_defines.append('-DWITH_VALGRIND=1')
  1211. if self.build.is_debug:
  1212. self.c_foptions.append('$FSTACK')
  1213. if self.build.is_fast_debug:
  1214. self.c_flags.append('-Og')
  1215. if self.build.is_release:
  1216. self.c_flags.append('$OPTIMIZE')
  1217. if self.build.is_size_optimized:
  1218. # -Oz is clang's more size-aggressive version of -Os
  1219. # For ARM specifically, clang -Oz is on par with gcc -Os:
  1220. # https://github.com/android/ndk/issues/133#issuecomment-365763507
  1221. if self.tc.is_clang:
  1222. self.optimize = '-Oz'
  1223. else:
  1224. self.optimize = '-Os'
  1225. # Generate sections with address significance tables for ICF linker pass
  1226. if self.tc.is_clang:
  1227. self.c_foptions.extend(['-faddrsig'])
  1228. else:
  1229. self.optimize = '-O3'
  1230. if self.build.profiler_type in (Profiler.Generic, Profiler.GProf):
  1231. self.c_foptions.append('-fno-omit-frame-pointer')
  1232. if self.build.profiler_type == Profiler.GProf:
  1233. self.c_flags.append('-pg')
  1234. def print_compiler(self):
  1235. super(GnuCompiler, self).print_compiler()
  1236. emit('C_COMPILER', '"{}"'.format(self.tc.c_compiler))
  1237. emit('C_COMPILER_OLD_UNQUOTED', self.tc.c_compiler)
  1238. emit('C_COMPILER_OLD', '${quo:C_COMPILER_OLD_UNQUOTED}')
  1239. emit('OPTIMIZE', self.optimize)
  1240. emit('WERROR_MODE', self.tc.werror_mode)
  1241. emit('_C_FLAGS', self.c_flags)
  1242. emit('_C_FOPTIONS', self.c_foptions)
  1243. emit('_STD_CXX_VERSION', preset('USER_STD_CXX_VERSION') or self.tc.cxx_std)
  1244. append('C_DEFINES', self.c_defines)
  1245. append('C_WARNING_OPTS', self.c_warnings)
  1246. append('CXX_WARNING_OPTS', self.cxx_warnings)
  1247. emit('CXX_COMPILER', '"{}"'.format(self.tc.cxx_compiler))
  1248. emit('CXX_COMPILER_OLD_UNQUOTED', format(self.tc.cxx_compiler))
  1249. emit('CXX_COMPILER_OLD', '${quo:CXX_COMPILER_OLD_UNQUOTED}')
  1250. # TODO(somov): Убрать чтение настройки из os.environ
  1251. emit('USE_ARC_PROFILE', 'yes' if preset('USE_ARC_PROFILE') or os.environ.get('USE_ARC_PROFILE') else 'no')
  1252. emit('DEBUG_INFO_FLAGS', self.debug_info_flags)
  1253. if self.build.is_coverage:
  1254. emit('_IS_COVERAGE', 'yes')
  1255. if self.tc.is_clang and self.tc.version_at_least(9):
  1256. emit('_HAS_TIME_TRACE', 'yes')
  1257. print('@import "${CONF_ROOT}/conf/compilers/gnu_compiler.conf"')
  1258. class SwiftCompiler(object):
  1259. def __init__(self, build):
  1260. self.host = build.host
  1261. self.compiler = None
  1262. def configure(self):
  1263. if self.host.is_macos:
  1264. self.compiler = '$SWIFT_XCODE_TOOLCHAIN_ROOT_RESOURCE_GLOBAL/usr/bin/swiftc'
  1265. def print_compiler(self):
  1266. emit('SWIFT_COMPILER', self.compiler or '')
  1267. class Linker(object):
  1268. BFD = 'bfd'
  1269. LLD = 'lld'
  1270. GOLD = 'gold'
  1271. def __init__(self, tc, build):
  1272. """
  1273. :type tc: ToolchainOptions
  1274. :type build: Build
  1275. """
  1276. self.tc = tc
  1277. self.build = build
  1278. self.type = self._get_default_linker_type()
  1279. def _get_default_linker_type(self):
  1280. if not self.tc.is_from_arcadia or is_positive('EXPORT_CMAKE'):
  1281. # External (e.g. system) toolchain: disable linker selection logic
  1282. return None
  1283. if self.build.target.is_android:
  1284. # Android toolchain is NDK, LLD works on all supported platforms
  1285. return Linker.LLD
  1286. elif self.build.target.is_linux or self.build.target.is_macos or self.build.target.is_ios or self.build.target.is_wasm:
  1287. return Linker.LLD
  1288. # There is no linker choice on Windows (link.exe)
  1289. return None
  1290. def print_linker(self):
  1291. self._print_linker_selector()
  1292. def _print_linker_selector(self):
  1293. # if self.type is None then _DEFAULT_LINKER is set to empty string value
  1294. emit('_DEFAULT_LINKER_ID', self.type)
  1295. class LD(Linker):
  1296. def __init__(self, tc, build):
  1297. """
  1298. :type tc: GnuToolchainOptions
  1299. :type build: Build
  1300. """
  1301. super(LD, self).__init__(tc, build)
  1302. self.build = build
  1303. self.host = self.build.host
  1304. self.target = self.build.target
  1305. self.tc = tc
  1306. target = self.target
  1307. self.ar = preset('AR') or self.tc.ar
  1308. self.ar_plugin = self.tc.ar_plugin
  1309. self.strip = self.tc.strip
  1310. self.objcopy = self.tc.objcopy
  1311. self.objdump = self.tc.objdump
  1312. self.musl = Setting('MUSL', convert=to_bool)
  1313. if self.ar is None:
  1314. if self.tc.is_from_arcadia:
  1315. if self.tc.is_clang:
  1316. self.ar = '{}/bin/llvm-ar'.format(self.tc.name_marker)
  1317. if self.tc.is_gcc:
  1318. self.ar = '{}/gcc/bin/gcc-ar'.format(self.tc.name_marker)
  1319. else:
  1320. self.ar = 'ar'
  1321. self.ar_type = 'GNU_AR'
  1322. self.llvm_ar_format = 'None'
  1323. if 'libtool' in self.ar:
  1324. self.ar_type = 'LIBTOOL'
  1325. elif 'llvm-ar' in self.ar:
  1326. self.ar_type = 'LLVM_AR'
  1327. if target.is_apple:
  1328. self.llvm_ar_format = "darwin"
  1329. else:
  1330. self.llvm_ar_format = "gnu"
  1331. self.ld_flags = []
  1332. if self.musl.value:
  1333. self.ld_flags.extend(['-Wl,--no-as-needed'])
  1334. elif target.is_linux:
  1335. self.ld_flags.extend(['-ldl', '-lrt', '-Wl,--no-as-needed'])
  1336. if self.tc.is_gcc:
  1337. self.ld_flags.extend(('-Wl,-Bstatic', '-latomic', '-Wl,-Bdynamic'))
  1338. elif target.is_android:
  1339. self.ld_flags.extend(['-ldl', '-Wl,--no-as-needed'])
  1340. self.ld_sdk = select(default=None, selectors=[
  1341. # -platform_version <platform> <min_version> <sdk_version>
  1342. (target.is_macos, '-Wl,-platform_version,macos,{MACOS_VERSION_MIN},11.0'.format(MACOS_VERSION_MIN=MACOS_VERSION_MIN)),
  1343. (not target.is_iossim and target.is_ios, '-Wl,-platform_version,ios,{IOS_VERSION_MIN},13.1'.format(IOS_VERSION_MIN=IOS_VERSION_MIN)),
  1344. (target.is_iossim, '-Wl,-platform_version,ios-simulator,{IOS_VERSION_MIN},14.5'.format(IOS_VERSION_MIN=IOS_VERSION_MIN)),
  1345. ])
  1346. if self.build.profiler_type == Profiler.GProf:
  1347. self.ld_flags.append('-pg')
  1348. if self.ld_sdk:
  1349. self.ld_flags.append(self.ld_sdk)
  1350. self.sys_lib = self.tc.sys_lib
  1351. if target.is_android:
  1352. # Use toolchain defaults to link with libunwind/clang_rt.builtins
  1353. self.use_stdlib = '-nostdlib++'
  1354. def print_linker(self):
  1355. super(LD, self).print_linker()
  1356. emit('AR_TOOL', self.ar)
  1357. emit('AR_TYPE', self.ar_type)
  1358. emit('_LD_LLVM_AR_FORMAT', self.llvm_ar_format)
  1359. emit('_LD_AR_PLUGIN', self.ar_plugin or 'None')
  1360. emit('STRIP_TOOL_VENDOR', self.strip)
  1361. emit('OBJCOPY_TOOL_VENDOR', self.objcopy)
  1362. emit('OBJDUMP_TOOL_VENDOR', self.objdump)
  1363. emit('_LD_FLAGS', self.ld_flags)
  1364. emit('_LD_SYS_LIB', self.sys_lib)
  1365. emit('LD_SDK_VERSION', self.ld_sdk)
  1366. dwarf_tool = self.tc.dwarf_tool
  1367. if dwarf_tool is None and self.tc.is_clang and (self.target.is_macos or self.target.is_ios):
  1368. dsymutil = '{}/bin/{}dsymutil'.format(self.tc.name_marker, '' if self.tc.version_at_least(7) else 'llvm-')
  1369. dwarf_tool = '${YMAKE_PYTHON} ${input:"build/scripts/run_llvm_dsymutil.py"} ' + dsymutil
  1370. if self.tc.version_at_least(5, 0):
  1371. dwarf_tool += ' -flat'
  1372. if dwarf_tool is not None:
  1373. emit('DWARF_TOOL', dwarf_tool)
  1374. arch_flag = '--arch={arch}'.format(arch=self.target.os_compat)
  1375. emit('_LD_ARCH_FLAG', arch_flag)
  1376. print('@import "${CONF_ROOT}/conf/linkers/ld.conf"')
  1377. class MSVCToolchainOptions(ToolchainOptions):
  1378. def __init__(self, build, detector):
  1379. super(MSVCToolchainOptions, self).__init__(build, detector)
  1380. # C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.14.26428
  1381. self.vc_root = None
  1382. # C:\Program Files (x86)\Windows Kits\10\Include\10.0.14393.0
  1383. self.kit_includes = None
  1384. # C:\Program Files (x86)\Windows Kits\10\Lib\10.0.14393.0
  1385. self.kit_libs = None
  1386. self.under_wine_compiler = self.params.get('wine', False)
  1387. self.under_wine_tools = not build.host.is_windows
  1388. self.under_wine_link = self.under_wine_tools
  1389. self.under_wine_lib = self.under_wine_tools
  1390. self.system_msvc = self.params.get('system_msvc', False)
  1391. self.ide_msvs = self.params.get('ide_msvs', False)
  1392. self.use_clang = self.params.get('use_clang', False)
  1393. self.use_msvc_linker = is_positive('USE_MSVC_LINKER')
  1394. self.use_arcadia_toolchain = self.params.get('use_arcadia_toolchain', False)
  1395. self.sdk_version = None
  1396. if self.ide_msvs:
  1397. bindir = '$(VC_ExecutablePath_x64_x64)\\'
  1398. self.c_compiler = bindir + 'cl.exe'
  1399. self.cxx_compiler = self.c_compiler
  1400. self.link = bindir + 'link.exe'
  1401. self.lib = bindir + 'lib.exe'
  1402. self.masm_compiler = bindir + 'ml64.exe'
  1403. self.vc_root = None
  1404. sdk_dir = '$(WindowsSdkDir)'
  1405. self.sdk_version = '$(WindowsTargetPlatformVersion)'
  1406. self.kit_includes = win_path_fix(os.path.join(sdk_dir, 'Include', self.sdk_version))
  1407. self.kit_libs = win_path_fix(os.path.join(sdk_dir, 'Lib', self.sdk_version))
  1408. elif detector:
  1409. self.use_clang = is_positive('USE_CLANG_CL')
  1410. self.masm_compiler = which('ml64.exe')
  1411. self.link = which('link.exe')
  1412. self.lib = which('lib.exe')
  1413. sdk_dir = os.environ.get('WindowsSdkDir')
  1414. self.sdk_version = os.environ.get('WindowsSDKVersion').replace('\\', '')
  1415. vc_install_dir = os.environ.get('VCToolsInstallDir')
  1416. if any([x is None for x in (sdk_dir, self.sdk_version, vc_install_dir)]):
  1417. raise ConfigureError('No %WindowsSdkDir%, %WindowsSDKVersion% or %VCINSTALLDIR% present. Please, run vcvars64.bat to setup preferred environment.')
  1418. self.vc_root = os.path.normpath(vc_install_dir)
  1419. self.kit_includes = win_path_fix(os.path.normpath(os.path.join(sdk_dir, 'Include', self.sdk_version)))
  1420. self.kit_libs = win_path_fix(os.path.normpath(os.path.join(sdk_dir, 'Lib', self.sdk_version)))
  1421. # TODO(somov): Определять автоматически self.version в этом случае
  1422. else:
  1423. if self.version_at_least(2019):
  1424. self.sdk_version = '10.0.18362.0'
  1425. if is_positive('MSVC20'): # XXX: temporary flag, remove after DTCC-123 is completed
  1426. self.cxx_std = 'c++latest'
  1427. else:
  1428. self.sdk_version = '10.0.16299.0'
  1429. sdk_dir = '$WINDOWS_KITS_RESOURCE_GLOBAL'
  1430. self.vc_root = self.name_marker if not self.use_clang else '$MSVC_FOR_CLANG_RESOURCE_GLOBAL'
  1431. self.kit_includes = win_path_fix(os.path.join(sdk_dir, 'Include', self.sdk_version))
  1432. self.kit_libs = win_path_fix(os.path.join(sdk_dir, 'Lib', self.sdk_version))
  1433. bindir = win_path_fix(os.path.join(self.vc_root, 'bin', 'Hostx64'))
  1434. tools_name = select(selectors=[
  1435. (build.target.is_x86, 'x86'),
  1436. (build.target.is_x86_64, 'x64'),
  1437. (build.target.is_armv7, 'arm'),
  1438. ])
  1439. asm_name = select(selectors=[
  1440. (build.target.is_x86, 'ml.exe'),
  1441. (build.target.is_x86_64, 'ml64.exe'),
  1442. (build.target.is_armv7, 'armasm.exe'),
  1443. ])
  1444. self.masm_compiler = win_path_fix(os.path.join(bindir, tools_name, asm_name))
  1445. if self.use_clang and not self.use_msvc_linker:
  1446. self.link = self.host.exe(self.name_marker, "bin", "lld-link")
  1447. self.under_wine_link = False
  1448. else:
  1449. self.link = win_path_fix(os.path.join(bindir, tools_name, 'link.exe'))
  1450. self.under_wine_link = self.under_wine_tools
  1451. if self.use_clang:
  1452. self.lib = self.host.exe(self.name_marker, "bin", "llvm-lib")
  1453. self.under_wine_lib = False
  1454. else:
  1455. self.lib = win_path_fix(os.path.join(bindir, tools_name, 'lib.exe'))
  1456. self.under_wine_lib = self.under_wine_tools
  1457. logger.debug("link {}; lib {}".format(self.link, self.lib))
  1458. class MSVC(object):
  1459. def __init__(self, tc, build):
  1460. """
  1461. :type tc: MSVCToolchainOptions
  1462. :type build: Build
  1463. """
  1464. if not isinstance(tc, MSVCToolchainOptions):
  1465. raise TypeError('Got {} ({}) instead of an MSVCToolchainOptions'.format(tc, type(tc)))
  1466. self.build = build
  1467. self.tc = tc
  1468. class MSVCToolchain(MSVC, Toolchain):
  1469. def __init__(self, tc, build):
  1470. """
  1471. :type tc: MSVCToolchainOptions
  1472. :param build: Build
  1473. """
  1474. Toolchain.__init__(self, tc, build)
  1475. MSVC.__init__(self, tc, build)
  1476. if self.tc.from_arcadia and not self.tc.ide_msvs:
  1477. if not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  1478. self.platform_projects.append('build/internal/platform/msvc')
  1479. self.platform_projects.append('build/platform/wine')
  1480. def print_toolchain(self):
  1481. super(MSVCToolchain, self).print_toolchain()
  1482. emit('TOOLCHAIN_ENV', format_env(self.tc.get_env(), list_separator=';'))
  1483. if self.tc.sdk_version:
  1484. emit('WINDOWS_KITS_VERSION', self.tc.sdk_version)
  1485. if self.tc.under_wine_tools:
  1486. emit('_UNDER_WINE_TOOLS', 'yes')
  1487. if self.tc.under_wine_link:
  1488. emit('_UNDER_WINE_LINK', 'yes')
  1489. if self.tc.under_wine_lib:
  1490. emit('_UNDER_WINE_LIB', 'yes')
  1491. if self.tc.under_wine_compiler:
  1492. emit('_UNDER_WINE_COMPILER', 'yes')
  1493. if self.tc.use_clang:
  1494. emit('CLANG_CL', 'yes')
  1495. if self.tc.ide_msvs:
  1496. emit('IDE_MSVS', 'yes')
  1497. if self.tc.use_arcadia_toolchain:
  1498. emit('USE_ARCADIA_TOOLCHAIN', 'yes')
  1499. emit('_MSVC_TC_KIT_LIBS', self.tc.kit_libs)
  1500. emit('_MSVC_TC_VC_ROOT', self.tc.vc_root)
  1501. print('@import "${CONF_ROOT}/conf/toolchains/msvc_toolchain.conf"')
  1502. class MSVCCompiler(MSVC, Compiler):
  1503. def __init__(self, tc, build):
  1504. Compiler.__init__(self, tc, 'MSVC')
  1505. MSVC.__init__(self, tc, build)
  1506. def print_compiler(self):
  1507. super(MSVCCompiler, self).print_compiler()
  1508. target = self.build.target
  1509. warns_enabled = [
  1510. 4018, # 'expression' : signed/unsigned mismatch
  1511. 4265, # 'class' : class has virtual functions, but destructor is not virtual
  1512. 4296, # 'operator' : expression is always false
  1513. 4431, # missing type specifier - int assumed
  1514. ]
  1515. warns_as_error = [
  1516. 4013, # 'function' undefined; assuming extern returning int
  1517. ]
  1518. warns_disabled = [
  1519. # While this warning corresponds to enabled-by-default -Wmacro-redefinition,
  1520. # it floods clog with abundant amount of log lines,
  1521. # as yvals_core.h from Windows SDK redefines certain
  1522. # which macros logically belong to libcxx
  1523. 4005, # '__cpp_lib_*': macro redefinition.
  1524. # Ne need to recheck this, but it looks like _CRT_USE_BUILTIN_OFFSETOF still makes sense
  1525. 4117, # macro name '_CRT_USE_BUILTIN_OFFSETOF' is reserved, '#define' ignored
  1526. 4127, # conditional expression is constant
  1527. 4200, # nonstandard extension used : zero-sized array in struct/union
  1528. 4201, # nonstandard extension used : nameless struct/union
  1529. 4351, # elements of array will be default initialized
  1530. 4355, # 'this' : used in base member initializer list
  1531. 4503, # decorated name length exceeded, name was truncated
  1532. 4510, # default constructor could not be generated
  1533. 4511, # copy constructor could not be generated
  1534. 4512, # assignment operator could not be generated
  1535. 4554, # check operator precedence for possible error; use parentheses to clarify precedence
  1536. 4610, # 'object' can never be instantiated - user defined constructor required
  1537. 4706, # assignment within conditional expression
  1538. 4800, # forcing value to bool 'true' or 'false' (performance warning)
  1539. 4996, # The POSIX name for this item is deprecated
  1540. 4714, # function marked as __forceinline not inlined
  1541. 4197, # 'TAtomic' : top-level volatile in cast is ignored
  1542. 4245, # 'initializing' : conversion from 'int' to 'ui32', signed/unsigned mismatch
  1543. 4324, # 'ystd::function<void (uint8_t *)>': structure was padded due to alignment specifier
  1544. 5033, # 'register' is no longer a supported storage class
  1545. ]
  1546. defines = [
  1547. '${hide:CPP_FAKEID}',
  1548. # FIXME: This is quick fix to let catboost build from MSVS IDE
  1549. # This place is questionable overall, see YMAKE-437
  1550. '/DARCADIA_ROOT=' + ('${ARCADIA_ROOT}' if not self.tc.ide_msvs else '.'),
  1551. '/DARCADIA_BUILD_ROOT=' + ('${ARCADIA_BUILD_ROOT}' if not self.tc.ide_msvs else '.'),
  1552. '/DWIN32',
  1553. '/D_WIN32',
  1554. '/D_WINDOWS',
  1555. # Define _CRT_*_NO_WARNINGS macros to prevent ucrt from issuing a warning whenever
  1556. # a POSIX-style function is used instead of the alternative Microsoft suggests as a secure / standard replacement
  1557. # (e. g. `strncpy()` instead of `strncpy_s()`, `access()` instead of `_access()`)
  1558. # For details see:
  1559. # https://docs.microsoft.com/en-us/cpp/c-runtime-library/security-features-in-the-crt
  1560. '/D_CRT_SECURE_NO_WARNINGS',
  1561. '/D_CRT_NONSTDC_NO_WARNINGS',
  1562. # Math constants (such as M_PI, M_E, M_SQRT2) are not defined in standard C / C++
  1563. # In order to get them defined by Windows ucrt library,
  1564. # you must first define _USE_MATH_DEFINES before #including <cmath> or math.h>.
  1565. # (NB: glibc defines these macros whenever _XOPEN_SOURCE is defined)
  1566. '/D_USE_MATH_DEFINES',
  1567. '/D__STDC_CONSTANT_MACROS',
  1568. '/D__STDC_FORMAT_MACROS',
  1569. '/D_USING_V110_SDK71_',
  1570. # Below defines are covered at
  1571. # https://docs.microsoft.com/en-us/windows/win32/winprog/using-the-windows-headers#faster-builds-with-smaller-header-files
  1572. # Exclude APIs such as Cryptography, DDE, RPC, Shell, and Windows Sockets (while including <windows.h>)
  1573. '/DWIN32_LEAN_AND_MEAN',
  1574. # Define NOMINMAX to avoid min() and max() macros definition (while including <windows.h>)
  1575. '/DNOMINMAX',
  1576. ]
  1577. cxx_defines = [
  1578. # Use builtin offsetof implementation
  1579. # instead of a crutcy macro defined in ucrt/stddef.h.
  1580. # The latter can not be used in constexpr statements.
  1581. '/D_CRT_USE_BUILTIN_OFFSETOF',
  1582. ]
  1583. if target.is_x86_64:
  1584. defines.extend(('/D_WIN64', '/DWIN64'))
  1585. if target.is_armv7:
  1586. defines.extend(('/D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE', '/D__arm__'))
  1587. winapi_unicode = False
  1588. flags = [
  1589. '/nologo', '/Zm500', '/GR', '/bigobj', '/FC', '/EHs', '/errorReport:prompt', '$MSVC_INLINE_FLAG', '/utf-8',
  1590. # enable standard conforming mode
  1591. '/permissive-'
  1592. ]
  1593. flags += self.tc.arch_opt
  1594. c_warnings = ['/we{}'.format(code) for code in warns_as_error]
  1595. c_warnings += ['/w1{}'.format(code) for code in warns_enabled]
  1596. c_warnings += ['/wd{}'.format(code) for code in warns_disabled]
  1597. cxx_warnings = []
  1598. if self.tc.use_clang:
  1599. if not self.tc.is_system_cxx:
  1600. flags += [
  1601. # Allow <windows.h> to be included via <Windows.h> in case-sensitive file-systems.
  1602. '-fcase-insensitive-paths',
  1603. ]
  1604. flags += [
  1605. # At the time clang-cl identifies itself as MSVC 19.11:
  1606. # (actual value can be found in clang/lib/Driver/ToolChains/MSVC.cpp, the syntax would be like
  1607. # ```
  1608. # MSVT = VersionTuple(19, 11);
  1609. # ```
  1610. #
  1611. # We override this value to match current value of the actual MSVC being used.
  1612. '-fms-compatibility-version=19.21',
  1613. # for msvc compatibility
  1614. # https://clang.llvm.org/docs/UsersManual.html#microsoft-extensions
  1615. '-fdelayed-template-parsing',
  1616. ]
  1617. if target.is_x86:
  1618. flags.append('-m32')
  1619. elif target.is_x86_64:
  1620. flags.append('-m64')
  1621. c_warnings.extend((
  1622. '-Wno-format',
  1623. '-Wno-parentheses',
  1624. '-Wno-unknown-warning-option',
  1625. ))
  1626. cxx_warnings += [
  1627. '-Wimport-preprocessor-directive-pedantic',
  1628. '-Woverloaded-virtual',
  1629. '-Wno-ambiguous-reversed-operator',
  1630. '-Wno-defaulted-function-deleted',
  1631. '-Wno-deprecated-anon-enum-enum-conversion',
  1632. '-Wno-deprecated-enum-enum-conversion',
  1633. '-Wno-deprecated-enum-float-conversion',
  1634. '-Wno-deprecated-volatile',
  1635. '-Wno-undefined-var-template',
  1636. ]
  1637. if self.tc.ide_msvs:
  1638. cxx_warnings += [
  1639. '-Wno-unused-command-line-argument',
  1640. ]
  1641. defines.append('/D_WIN32_WINNT={0}'.format(WINDOWS_VERSION_MIN))
  1642. if winapi_unicode:
  1643. defines += ['/DUNICODE', '/D_UNICODE']
  1644. else:
  1645. defines += ['/D_MBCS']
  1646. vc_include = win_path_fix(os.path.join(self.tc.vc_root, 'include')) if not self.tc.ide_msvs else "$(VC_VC_IncludePath.Split(';')[0].Replace('\\','/'))"
  1647. if not self.tc.ide_msvs:
  1648. def include_flag(path):
  1649. return '{flag}"{path}"'.format(path=path, flag='/I ' if not self.tc.use_clang else '-imsvc')
  1650. for name in ('shared', 'ucrt', 'um', 'winrt'):
  1651. flags.append(include_flag(win_path_fix(os.path.join(self.tc.kit_includes, name))))
  1652. flags.append(include_flag(vc_include))
  1653. if self.tc.use_clang:
  1654. emit('CLANG_CL', 'yes')
  1655. if self.tc.ide_msvs:
  1656. emit('IDE_MSVS', 'yes')
  1657. if self.tc.use_arcadia_toolchain:
  1658. emit('USE_ARCADIA_TOOLCHAIN', 'yes')
  1659. emit('CXX_COMPILER_UNQUOTED', '"{}"'.format(self.tc.cxx_compiler))
  1660. emit('CXX_COMPILER_OLD_UNQUOTED', self.tc.cxx_compiler)
  1661. emit('C_COMPILER_UNQUOTED', '"{}"'.format(self.tc.c_compiler))
  1662. emit('C_COMPILER_OLD_UNQUOTED', self.tc.c_compiler)
  1663. emit('MASM_COMPILER_UNQUOTED', '"{}"'.format(self.tc.masm_compiler))
  1664. emit('MASM_COMPILER_OLD_UNQUOTED', self.tc.masm_compiler)
  1665. append('C_DEFINES', defines)
  1666. append('C_WARNING_OPTS', c_warnings)
  1667. emit('_CXX_DEFINES', cxx_defines)
  1668. append('CXX_WARNING_OPTS', cxx_warnings)
  1669. if self.build.is_release:
  1670. emit('CFLAGS_PER_TYPE', '$CFLAGS_RELEASE')
  1671. if self.build.is_debug:
  1672. emit('CFLAGS_PER_TYPE', '$CFLAGS_DEBUG')
  1673. if self.build.is_ide:
  1674. emit('CFLAGS_PER_TYPE', '@[debug|$CFLAGS_DEBUG]@[release|$CFLAGS_RELEASE]')
  1675. emit('_STD_CXX_VERSION', preset('USER_STD_CXX_VERSION') or self.tc.cxx_std)
  1676. emit('_MSVC_FLAGS', flags)
  1677. ucrt_include = win_path_fix(os.path.join(self.tc.kit_includes, 'ucrt')) if not self.tc.ide_msvs else "$(UniversalCRT_IncludePath.Split(';')[0].Replace('\\','/'))"
  1678. # clang-cl has '#include_next', and MSVC hasn't. It needs separately specified CRT and VC include directories for libc++ to include second in order standard C and C++ headers.
  1679. if not self.tc.use_clang:
  1680. emit('_CFLAGS_UCRT_VC_INCLUDES', '/DY_UCRT_INCLUDE="%s"' % ucrt_include, '/DY_MSVC_INCLUDE="%s"' % vc_include)
  1681. else:
  1682. emit('_CFLAGS_UCRT_VC_INCLUDES')
  1683. emit('WERROR_MODE', self.tc.werror_mode)
  1684. print('@import "${CONF_ROOT}/conf/compilers/msvc_compiler.conf"')
  1685. class MSVCLinker(MSVC, Linker):
  1686. def __init__(self, tc, build):
  1687. MSVC.__init__(self, tc, build)
  1688. Linker.__init__(self, tc, build)
  1689. def print_linker(self):
  1690. super(MSVCLinker, self).print_linker()
  1691. linker = self.tc.link
  1692. linker_lib = self.tc.lib
  1693. emit('_MSVC_LIB', '"{}"'.format(linker_lib))
  1694. emit('_MSVC_LIB_OLD_UNQUOTED', linker_lib)
  1695. emit('_MSVC_LIB_OLD', '${quo:_MSVC_LIB_OLD_UNQUOTED}')
  1696. emit('_MSVC_LINK', '"{}"'.format(linker))
  1697. emit('_MSVC_LINK_OLD_UNQUOTED', linker)
  1698. emit('_MSVC_LINK_OLD', '${quo:_MSVC_LINK_OLD_UNQUOTED}')
  1699. if self.build.is_release:
  1700. emit('LINK_EXE_FLAGS_PER_TYPE', '$LINK_EXE_FLAGS_RELEASE')
  1701. if self.build.is_debug:
  1702. emit('LINK_EXE_FLAGS_PER_TYPE', '$LINK_EXE_FLAGS_DEBUG')
  1703. if self.build.is_ide and self.tc.ide_msvs:
  1704. emit('LINK_EXE_FLAGS_PER_TYPE', '@[debug|$LINK_EXE_FLAGS_DEBUG]@[release|$LINK_EXE_FLAGS_RELEASE]')
  1705. print('@import "${CONF_ROOT}/conf/linkers/msvc_linker.conf"')
  1706. # TODO(somov): Rename!
  1707. Compilers = {
  1708. 'gnu': (GnuToolchain, GnuCompiler, LD),
  1709. 'clang': (GnuToolchain, GnuCompiler, LD),
  1710. 'xcode': (GnuToolchain, GnuCompiler, LD),
  1711. 'msvc': (MSVCToolchain, MSVCCompiler, MSVCLinker),
  1712. }
  1713. class Ragel(object):
  1714. def __init__(self):
  1715. self.rlgen_flags = []
  1716. self.ragel_flags = []
  1717. self.ragel6_flags = []
  1718. def configure_toolchain(self, build, compiler):
  1719. if isinstance(compiler, MSVCCompiler):
  1720. self.set_default_flags(optimized=False)
  1721. elif isinstance(compiler, GnuCompiler):
  1722. self.set_default_flags(optimized=build.is_release and not build.is_sanitized)
  1723. else:
  1724. raise ConfigureError('Unexpected compiler {}'.format(compiler))
  1725. def set_default_flags(self, optimized):
  1726. if optimized:
  1727. self.rlgen_flags.append('-G2')
  1728. self.ragel6_flags.append('-CG2')
  1729. else:
  1730. self.rlgen_flags.append('-T0')
  1731. self.ragel6_flags.append('-CT0')
  1732. def print_variables(self):
  1733. emit('RLGEN_FLAGS', self.rlgen_flags)
  1734. emit('RAGEL_FLAGS', self.ragel_flags)
  1735. emit('RAGEL6_FLAGS', self.ragel6_flags)
  1736. class Python(object):
  1737. def __init__(self, tc):
  1738. self.python = None
  1739. self.flags = None
  1740. self.ldflags = None
  1741. self.libraries = None
  1742. self.includes = None
  1743. self.tc = tc
  1744. def configure_posix(self, python=None, python_config=None):
  1745. self.check_configuration()
  1746. python = python or preset('PYTHON_BIN') or which('python')
  1747. python_config = python_config or preset('PYTHON_CONFIG') or which('python-config')
  1748. if python is None or python_config is None:
  1749. return
  1750. # python-config dumps each option on one line in the specified order
  1751. config = get_stdout([python_config, '--cflags', '--ldflags', '--includes']) or ''
  1752. config = config.split('\n')
  1753. if len(config) < 3:
  1754. return
  1755. self.python = python
  1756. self.flags = config[0]
  1757. self.ldflags = config[1]
  1758. self.includes = config[2]
  1759. # Do not split libraries from ldflags.
  1760. # They are not used separately and get overriden together, so it is safe.
  1761. # TODO(somov): Удалить эту переменную и PYTHON_LIBRARIES из makelist-ов.
  1762. self.libraries = ''
  1763. def check_configuration(self):
  1764. if preset('USE_ARCADIA_PYTHON') == 'no':
  1765. # Set USE_LOCAL_PYTHON to enable configuration
  1766. # when we still using fixed OS SDK,
  1767. # but at the same time using Python from the local system.
  1768. #
  1769. # This configuration is not guaranteed to work,
  1770. # for example, if local and fixed OS SDKs differ much.
  1771. if preset('USE_LOCAL_PYTHON') == 'yes':
  1772. return
  1773. if not preset('USE_SYSTEM_PYTHON') and not self.tc.os_sdk_local:
  1774. raise Exception("Use fixed python (see https://clubs.at.yandex-team.ru/arcadia/15392) or set OS_SDK=local flag")
  1775. def print_variables(self):
  1776. variables = Variables({
  1777. 'PYTHON_BIN': self.python,
  1778. 'PYTHON_FLAGS': self.flags,
  1779. 'PYTHON_LDFLAGS': self.ldflags,
  1780. 'PYTHON_LIBRARIES': self.libraries,
  1781. 'PYTHON_INCLUDE': self.includes
  1782. })
  1783. variables.update_from_presets()
  1784. variables.reset_if_any(reset_value='PYTHON-NOT-FOUND')
  1785. variables.emit()
  1786. class Setting(object):
  1787. def __init__(self, key, auto=None, convert=None, rewrite=False):
  1788. self.key = key
  1789. self.auto = auto
  1790. self.convert = convert
  1791. self.preset = preset(key)
  1792. self.from_user = self.preset is not None
  1793. self.rewrite = rewrite
  1794. self._value = Setting.no_value
  1795. @property
  1796. def value(self):
  1797. if self._value is Setting.no_value:
  1798. self._value = self.calculate_value()
  1799. return self._value
  1800. def calculate_value(self):
  1801. if not self.from_user:
  1802. return self.auto if not callable(self.auto) else self.auto()
  1803. else:
  1804. return self.preset if not self.convert else self.convert(self.preset)
  1805. @value.setter
  1806. def value(self, value):
  1807. if self.from_user:
  1808. raise ConfigureError("Variable {key} already set by user to {old}. Can not change it's value to {new}".format(key=self.key, old=self._value, new=value))
  1809. self._value = value
  1810. def emit(self):
  1811. if not self.from_user or self.rewrite:
  1812. emit(self.key, self.value)
  1813. no_value = object()
  1814. class Cuda(object):
  1815. def __init__(self, build):
  1816. """
  1817. :type build: Build
  1818. """
  1819. self.build = build
  1820. self.have_cuda = Setting('HAVE_CUDA', auto=self.auto_have_cuda, convert=to_bool)
  1821. self.cuda_root = Setting('CUDA_ROOT')
  1822. self.cuda_target_root = Setting('CUDA_TARGET_ROOT')
  1823. self.cuda_version = Setting('CUDA_VERSION', auto=self.auto_cuda_version, convert=self.convert_major_version, rewrite=True)
  1824. self.cuda_architectures = Setting('CUDA_ARCHITECTURES', auto=self.auto_cuda_architectures, rewrite=True)
  1825. self.use_arcadia_cuda = Setting('USE_ARCADIA_CUDA', auto=self.auto_use_arcadia_cuda, convert=to_bool)
  1826. self.use_arcadia_cuda_host_compiler = Setting('USE_ARCADIA_CUDA_HOST_COMPILER', auto=self.auto_use_arcadia_cuda_host_compiler, convert=to_bool)
  1827. self.cuda_use_clang = Setting('CUDA_USE_CLANG', auto=False, convert=to_bool)
  1828. self.cuda_host_compiler = Setting('CUDA_HOST_COMPILER', auto=self.auto_cuda_host_compiler)
  1829. self.cuda_host_compiler_env = Setting('CUDA_HOST_COMPILER_ENV')
  1830. self.cuda_host_msvc_version = Setting('CUDA_HOST_MSVC_VERSION')
  1831. self.cuda_nvcc_flags = Setting('CUDA_NVCC_FLAGS', auto=[])
  1832. self.peerdirs = ['build/platform/cuda']
  1833. self.nvcc_flags = [
  1834. # Compress fatbinary to reduce size of .nv_fatbin and prevent problems with linking
  1835. #
  1836. # Idea comes from many resources, one of them is https://discourse.llvm.org/t/lld-relocation-overflows-and-nv-fatbin/58889/6
  1837. # Some sources suggest using `-Xfatbin=-compress-all`, other suggest using `-Xcuda-fatbinary --compress-all`
  1838. # We will use the same flag as in nixpkgs
  1839. # (https://github.com/NixOS/nixpkgs/pull/220402/files#diff-a38e6c4e8421c03dc6c2a60c9a172ceb4059048b65798e5d4a400a7a4a5720ffR167)
  1840. "-Xfatbin=-compress-all",
  1841. # Allow __host__, __device__ annotations in lambda declaration.
  1842. "--expt-extended-lambda",
  1843. # Allow host code to invoke __device__ constexpr functions and vice versa
  1844. "--expt-relaxed-constexpr",
  1845. # Allow to use newer compilers than CUDA Toolkit officially supports
  1846. "--allow-unsupported-compiler",
  1847. # Set paths explicitly
  1848. "--dont-use-profile",
  1849. "--libdevice-directory=$CUDA_ROOT/nvvm/libdevice",
  1850. ]
  1851. if not self.have_cuda.value:
  1852. return
  1853. if self.cuda_host_compiler.value:
  1854. self.nvcc_flags.append('--compiler-bindir=$CUDA_HOST_COMPILER')
  1855. if self.use_arcadia_cuda.value:
  1856. self.cuda_root.value = '$CUDA_RESOURCE_GLOBAL'
  1857. host, target = self.build.host_target
  1858. if host == target:
  1859. self.cuda_target_root.value = '$CUDA_RESOURCE_GLOBAL'
  1860. else:
  1861. self.cuda_target_root.value = '$CUDA_TARGET_RESOURCE_GLOBAL'
  1862. if self.build.target.is_linux_x86_64 and self.build.tc.is_clang:
  1863. # TODO(somov): Эта настройка должна приезжать сюда автоматически из другого места
  1864. self.nvcc_flags.append('-I$OS_SDK_ROOT/usr/include/x86_64-linux-gnu')
  1865. def print_(self):
  1866. self.print_variables()
  1867. self.print_macros()
  1868. print('@import "${CONF_ROOT}/conf/compilers/nvcc.conf"')
  1869. def print_variables(self):
  1870. self.have_cuda.emit()
  1871. if not self.have_cuda.value:
  1872. return
  1873. if self.use_arcadia_cuda.value and self.cuda_host_compiler.value is None:
  1874. logger.warning('$USE_ARCADIA_CUDA is set, but no $CUDA_HOST_COMPILER')
  1875. self.setup_vc_root()
  1876. self.cuda_root.emit()
  1877. self.cuda_target_root.emit()
  1878. self.cuda_version.emit()
  1879. self.cuda_architectures.emit()
  1880. self.use_arcadia_cuda.emit()
  1881. self.use_arcadia_cuda_host_compiler.emit()
  1882. self.cuda_use_clang.emit()
  1883. self.cuda_host_compiler.emit()
  1884. self.cuda_host_compiler_env.emit()
  1885. self.cuda_host_msvc_version.emit()
  1886. self.cuda_nvcc_flags.emit()
  1887. emit('NVCC', '"{}"'.format(self.build.host.exe('$CUDA_ROOT', 'bin', 'nvcc')))
  1888. emit('NVCC_OLD_UNQUOTED', self.build.host.exe('$CUDA_ROOT', 'bin', 'nvcc'))
  1889. emit('NVCC_OLD', '${quo:NVCC_OLD_UNQUOTED}')
  1890. emit('NVCC_FLAGS', self.nvcc_flags, '$CUDA_NVCC_FLAGS')
  1891. emit('NVCC_OBJ_EXT', '.o' if not self.build.target.is_windows else '.obj')
  1892. emit('NVCC_ENV', format_env({'PATH': '$CUDA_ROOT/nvvm/bin:$CUDA_ROOT/bin'}))
  1893. def print_macros(self):
  1894. mtime = ' '
  1895. custom_pid = ' '
  1896. if self.build.host_target[1].is_linux:
  1897. mtime = ' --mtime ${tool:"tools/mtime0"} '
  1898. custom_pid = '--custom-pid ${tool:"tools/custom_pid"} '
  1899. if not self.cuda_use_clang.value:
  1900. cmd = '$YMAKE_PYTHON ${input:"build/scripts/compile_cuda.py"}' + mtime + custom_pid + '$NVCC_OLD $NVCC_STD $NVCC_FLAGS -c ${input:SRC} -o ${output;suf=${OBJ_SUF}${NVCC_OBJ_EXT}:SRC} ${pre=-I:_C__INCLUDE} --cflags $C_FLAGS_PLATFORM $CXXFLAGS $NVCC_STD $NVCC_CFLAGS $SRCFLAGS ${input;hide:"build/platform/cuda/cuda_runtime_include.h"} $NVCC_ENV $CUDA_HOST_COMPILER_ENV ${kv;hide:"p CC"} ${kv;hide:"pc light-green"}' # noqa E501
  1901. else:
  1902. cmd = '$CXX_COMPILER_OLD --cuda-path=$CUDA_ROOT $C_FLAGS_PLATFORM -c ${input:SRC} -o ${output;suf=${OBJ_SUF}${NVCC_OBJ_EXT}:SRC} ${pre=-I:_C__INCLUDE} $CXXFLAGS $SRCFLAGS $TOOLCHAIN_ENV ${kv;hide:"p CU"} ${kv;hide:"pc green"}' # noqa E501
  1903. emit('_SRC_CU_CMD', cmd)
  1904. emit('_SRC_CU_PEERDIR', ' '.join(sorted(self.peerdirs)))
  1905. # This method is only used to set HAVE_CUDA=no automatically during ymake initialization
  1906. # (i.e. is used only in `auto_have_cuda` invocation)
  1907. def have_cuda_in_arcadia(self):
  1908. host, target = self.build.host_target
  1909. if not any((host.is_linux_x86_64, host.is_windows_x86_64)):
  1910. return False
  1911. if host != target:
  1912. if not (host.is_linux_x86_64 and target.is_linux_armv8):
  1913. return False
  1914. if not self.cuda_version.from_user:
  1915. return False
  1916. if self.cuda_version.value in ('11.4', '11.8', '12.1', '12.2'):
  1917. return True
  1918. elif self.cuda_version.value in ('10.2', '11.4.19') and target.is_linux_armv8:
  1919. return True
  1920. else:
  1921. raise ConfigureError('CUDA version {} is not supported in Arcadia'.format(self.cuda_version.value))
  1922. def auto_have_cuda(self):
  1923. if is_positive('MUSL'):
  1924. return False
  1925. if self.build.is_sanitized:
  1926. return False
  1927. return self.cuda_root.from_user or self.use_arcadia_cuda.value and self.have_cuda_in_arcadia()
  1928. def auto_cuda_version(self):
  1929. if self.use_arcadia_cuda.value:
  1930. return '12.2'
  1931. if not self.have_cuda.value:
  1932. return None
  1933. nvcc_exe = self.build.host.exe(os.path.expanduser(self.cuda_root.value), 'bin', 'nvcc')
  1934. def error():
  1935. raise ConfigureError('Failed to get CUDA version from {}'.format(nvcc_exe))
  1936. version_output = get_stdout([nvcc_exe, '--version']) or error()
  1937. match = re.search(r'^Cuda compilation tools, release (\d+)\.\d+,', version_output, re.MULTILINE) or error()
  1938. return match.group(1)
  1939. def convert_major_version(self, value):
  1940. if value == '11':
  1941. return '11.4'
  1942. else:
  1943. return value
  1944. def auto_cuda_architectures(self):
  1945. # empty list does not mean "no architectures"
  1946. # it means "no restriction -- any available architecture"
  1947. host, target = self.build.host_target
  1948. if not target.is_linux_x86_64:
  1949. # do not impose any restrictions, when build not for "linux 64-bit"
  1950. return ''
  1951. # Equality to CUDA 11.4 is rather strict comparison
  1952. # TODO: find out how we can relax check (e.g. to include more version of CUDA toolkit)
  1953. if self.cuda_version.value == '11.4':
  1954. # * use output of CUDA 11.4 `nvcc --help`
  1955. # * drop support for '53', '62', '72' and '87'
  1956. # (these devices run only on arm64)
  1957. # * drop support for '37'
  1958. # the single place it's used in Arcadia is https://a.yandex-team.ru/arcadia/sdg/sdc/third_party/cub/common.mk?rev=r13268523#L69
  1959. return ':'.join(
  1960. ['sm_35',
  1961. 'sm_50', 'sm_52',
  1962. 'sm_60', 'sm_61',
  1963. 'sm_70', 'sm_75',
  1964. 'sm_80', 'sm_86',
  1965. 'compute_86'])
  1966. else:
  1967. return ''
  1968. def auto_use_arcadia_cuda(self):
  1969. return not self.cuda_root.from_user
  1970. def auto_use_arcadia_cuda_host_compiler(self):
  1971. return not self.cuda_host_compiler.from_user and not self.cuda_use_clang.value
  1972. def auto_cuda_host_compiler(self):
  1973. if not self.use_arcadia_cuda_host_compiler.value:
  1974. return None
  1975. host, target = self.build.host_target
  1976. if host.is_windows_x86_64 and target.is_windows_x86_64:
  1977. return self.cuda_windows_host_compiler()
  1978. return select((
  1979. (host.is_linux_x86_64 and target.is_linux_x86_64, '$CUDA_HOST_TOOLCHAIN_RESOURCE_GLOBAL/bin/clang'),
  1980. (host.is_linux_x86_64 and target.is_linux_armv8, '$CUDA_HOST_TOOLCHAIN_RESOURCE_GLOBAL/bin/clang'),
  1981. ))
  1982. def cuda_windows_host_compiler(self):
  1983. vc_version = '14.28.29910'
  1984. env = {
  1985. 'Y_VC_Version': vc_version,
  1986. 'Y_VC_Root': '$CUDA_HOST_TOOLCHAIN_RESOURCE_GLOBAL/VC/Tools/MSVC/{}'.format(vc_version),
  1987. 'Y_SDK_Version': self.build.tc.sdk_version,
  1988. 'Y_SDK_Root': '$WINDOWS_KITS_RESOURCE_GLOBAL',
  1989. }
  1990. if not self.build.tc.ide_msvs and not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  1991. self.peerdirs.append('build/internal/platform/msvc')
  1992. self.cuda_host_compiler_env.value = format_env(env)
  1993. self.cuda_host_msvc_version.value = vc_version
  1994. return '%(Y_VC_Root)s/bin/HostX64/x64/cl.exe' % env
  1995. def setup_vc_root(self):
  1996. if not self.cuda_host_compiler.from_user:
  1997. return # Already set in cuda_windows_host_compiler()
  1998. if self.cuda_host_compiler_env.from_user:
  1999. return # We won't override user setting
  2000. def is_root(dir):
  2001. return all(os.path.isdir(os.path.join(dir, name)) for name in ('bin', 'include', 'lib'))
  2002. def get_root():
  2003. path, old_path = os.path.normpath(self.cuda_host_compiler.value), None
  2004. while path != old_path:
  2005. if is_root(path):
  2006. return path
  2007. path, old_path = os.path.dirname(path), path
  2008. vc_root = get_root()
  2009. if vc_root:
  2010. self.cuda_host_compiler_env.value = format_env({'Y_VC_Root': vc_root})
  2011. class CuDNN(object):
  2012. def __init__(self, cuda):
  2013. """
  2014. :type cuda: Cuda
  2015. """
  2016. self.cuda = cuda
  2017. self.cudnn_version = Setting('CUDNN_VERSION', auto=self.auto_cudnn_version)
  2018. def have_cudnn(self):
  2019. return self.cudnn_version.value in ('7.6.5', '8.0.5')
  2020. def auto_cudnn_version(self):
  2021. return '8.0.5'
  2022. def print_(self):
  2023. if self.cuda.have_cuda.value and self.have_cudnn():
  2024. self.cudnn_version.emit()
  2025. def customization():
  2026. if not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  2027. try:
  2028. sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'internal'))
  2029. from custom_conf import CustomConf
  2030. logger.debug('Customization extension was successfully loaded')
  2031. return CustomConf
  2032. except Exception as e:
  2033. logger.debug('Customization extension was not found; [{}]'.format(str(e)))
  2034. else:
  2035. logger.debug('Customization extension was disabled')
  2036. class DummyConf:
  2037. def __init__(self, options):
  2038. pass
  2039. def print_prologue(self):
  2040. pass
  2041. def print_epilogue(self):
  2042. pass
  2043. return DummyConf
  2044. def main():
  2045. options = opts()
  2046. CustomConfig = customization()
  2047. custom_conf = CustomConfig(options)
  2048. arcadia = Arcadia(options.arcadia_root)
  2049. ymake = YMake(arcadia)
  2050. ymake.print_core_conf()
  2051. _INTERNAL_CONF = 'internal/conf/internal.conf'
  2052. internal_conf_full_path = find_conf(_INTERNAL_CONF)
  2053. if internal_conf_full_path and not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  2054. print('@import "${{CONF_ROOT}}/{}"'.format(_INTERNAL_CONF))
  2055. custom_conf.print_prologue()
  2056. ymake.print_presets()
  2057. ymake.print_settings()
  2058. build = Build(arcadia, options.build_type, options.toolchain_params, force_ignore_local_files=not options.local_distbuild)
  2059. build.print_build()
  2060. custom_conf.print_epilogue()
  2061. if __name__ == '__main__':
  2062. main()