ymake_conf.py 98 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633
  1. #!/usr/bin/env python
  2. # coding=utf-8
  3. from __future__ import print_function
  4. import base64
  5. import itertools
  6. import json
  7. import logging
  8. import ntpath
  9. import optparse
  10. import os
  11. import posixpath
  12. import re
  13. import subprocess
  14. import sys
  15. import tempfile
  16. import six
  17. from functools import total_ordering
  18. logger = logging.getLogger(__name__ if __name__ != '__main__' else 'ymake_conf.py')
  19. class WindowsVersion(object):
  20. """
  21. Predefined values for _WIN32_WINNT macro.
  22. This macro specifies minimal Windows version required by the binary being build.
  23. A complete list of the values supported by the Windows SDK can be found at
  24. https://docs.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt
  25. """
  26. Windows07 = '0x0601'
  27. Windows08 = '0x0602'
  28. Windows10 = '0x0A00'
  29. # This is default Android API level unless `-DANDROID_API` is specified in cmdline
  30. # Android api level can be resolved to Android version using
  31. # https://android.googlesource.com/platform/bionic/+/master/docs/status.md
  32. ANDROID_API_DEFAULT = 21
  33. # This is default Linux SDK unless `-DOS_SDK` is specified in cmdline
  34. LINUX_SDK_DEFAULT = "ubuntu-14"
  35. MACOS_VERSION_MIN = "11.0"
  36. IOS_VERSION_MIN = "13.0"
  37. WINDOWS_VERSION_MIN = WindowsVersion.Windows07
  38. def init_logger(verbose):
  39. logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)
  40. def find_conf(conf_file):
  41. script_dir = os.path.dirname(__file__)
  42. full_path = os.path.join(script_dir, conf_file)
  43. if os.path.exists(full_path):
  44. return full_path
  45. return None
  46. # All paths in build graph shall be in posix format.
  47. # This ensures that UIDs of cross and native builds won't diverge
  48. # due to paths difference. This function is to be used to fix paths produced
  49. # by os.path.join on Windows.
  50. #
  51. # FIXME: upon complete switch to Python3 os.path.join may be replaced by posixpath.join
  52. # and this function will become unnecessary
  53. def win_path_fix(path):
  54. return path if sys.platform != 'win32' else path.replace('\\', '/')
  55. class DebugString(object):
  56. def __init__(self, get_string_func):
  57. self.get_string_func = get_string_func
  58. def __str__(self):
  59. return self.get_string_func()
  60. class ConfigureError(Exception):
  61. pass
  62. @total_ordering
  63. class Platform(object):
  64. def __init__(self, name, os, arch):
  65. """
  66. :type name: str
  67. :type os: str
  68. :type arch: str
  69. """
  70. self.name = name
  71. self.os = self._parse_os(os)
  72. self.arch = arch.lower()
  73. self.is_i386 = self.arch in ('i386', 'x86')
  74. self.is_i686 = self.arch == 'i686'
  75. self.is_x86 = self.is_i386 or self.is_i686
  76. self.is_x86_64 = self.arch in ('x86_64', 'amd64')
  77. self.is_intel = self.is_x86 or self.is_x86_64
  78. self.is_armv7 = self.arch in ('armv7', 'armv7a', 'armv7ahf', 'armv7a_neon', 'arm', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53')
  79. self.is_armv8 = self.arch in ('armv8', 'armv8a', 'arm64', 'aarch64', 'armv8a_cortex_a35', 'armv8a_cortex_a53')
  80. self.is_armv8m = self.arch in ('armv8m_cortex_m33', 'armv8m_cortex_m23')
  81. self.is_armv7em = self.arch in ('armv7em_cortex_m4', 'armv7em_cortex_m7')
  82. self.is_arm64 = self.arch in ('arm64',)
  83. self.is_arm = self.is_armv7 or self.is_armv8 or self.is_armv8m or self.is_armv7em
  84. self.is_armv7_neon = self.arch in ('armv7a_neon', 'armv7ahf', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53')
  85. self.is_armv7hf = self.arch in ('armv7ahf', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53')
  86. self.is_armv5te = self.arch in ('armv5te_arm968e_s')
  87. self.is_rv32imc = self.arch in ('riscv32_esp',)
  88. self.is_riscv32 = self.is_rv32imc
  89. self.is_nds32 = self.arch in ('nds32le_elf_mculib_v5f',)
  90. self.is_tc32 = self.arch in ('tc32_elf',)
  91. self.is_xtensa = self.arch in ('xtensa_hifi5',)
  92. self.armv7_float_abi = None
  93. if self.is_armv7:
  94. if self.is_armv7hf:
  95. self.armv7_float_abi = 'hard'
  96. else:
  97. self.armv7_float_abi = 'softfp'
  98. self.is_cortex_a9 = self.arch in ('armv7a_cortex_a9',)
  99. self.is_cortex_a35 = self.arch in ('armv7ahf_cortex_a35', 'armv8a_cortex_a35')
  100. self.is_cortex_a53 = self.arch in ('armv7ahf_cortex_a53', 'armv8a_cortex_a53')
  101. self.is_cortex_m33 = self.arch in ('armv8m_cortex_m33',)
  102. self.is_cortex_m23 = self.arch in ('armv8m_cortex_m23',)
  103. self.is_cortex_m4 = self.arch in ('armv7em_cortex_m4',)
  104. self.is_cortex_m7 = self.arch in ('armv7em_cortex_m7')
  105. self.is_arm968e_s = self.arch in ('armv5te_arm968e_s')
  106. self.is_power8le = self.arch == 'ppc64le'
  107. self.is_power9le = self.arch == 'power9le'
  108. self.is_powerpc = self.is_power8le or self.is_power9le
  109. self.is_wasm64 = self.arch == 'wasm64'
  110. self.is_wasm = self.is_wasm64
  111. self.is_32_bit = self.is_x86 or self.is_armv5te or self.is_armv7 or self.is_armv8m or self.is_riscv32 or self.is_nds32 or self.is_armv7em or self.is_xtensa or self.is_tc32
  112. self.is_64_bit = self.is_x86_64 or self.is_armv8 or self.is_powerpc or self.is_wasm64
  113. assert self.is_32_bit or self.is_64_bit
  114. assert not (self.is_32_bit and self.is_64_bit)
  115. self.is_linux = self.os == 'linux' or 'yocto' in self.os
  116. self.is_linux_x86_64 = self.is_linux and self.is_x86_64
  117. self.is_linux_armv8 = self.is_linux and self.is_armv8
  118. self.is_linux_armv7 = self.is_linux and self.is_armv7
  119. self.is_linux_power8le = self.is_linux and self.is_power8le
  120. self.is_linux_power9le = self.is_linux and self.is_power9le
  121. self.is_linux_powerpc = self.is_linux_power8le or self.is_linux_power9le
  122. self.is_macos = self.os == 'macos'
  123. self.is_macos_x86_64 = self.is_macos and self.is_x86_64
  124. self.is_macos_arm64 = self.is_macos and self.is_arm64
  125. self.is_iossim = self.os == 'iossim' or (self.os == 'ios' and self.is_intel)
  126. self.is_ios = self.os == 'ios' or self.is_iossim
  127. self.is_apple = self.is_macos or self.is_ios
  128. self.is_windows = self.os == 'windows'
  129. self.is_windows_x86_64 = self.is_windows and self.is_x86_64
  130. self.is_android = self.os == 'android'
  131. if self.is_android:
  132. self.android_api = int(preset('ANDROID_API', ANDROID_API_DEFAULT))
  133. self.is_cygwin = self.os == 'cygwin'
  134. self.is_yocto = self.os == 'yocto'
  135. self.is_emscripten = self.os == 'emscripten'
  136. self.is_none = self.os == 'none'
  137. self.is_posix = self.is_linux or self.is_apple or self.is_android or self.is_cygwin or self.is_yocto
  138. @staticmethod
  139. def from_json(data):
  140. name = data.get('visible_name', data['toolchain'])
  141. return Platform(name, os=data['os'], arch=data['arch'])
  142. @property
  143. def os_variables(self):
  144. # 'LINUX' variable, for backward compatibility
  145. yield self.os.upper()
  146. # 'OS_LINUX' variable
  147. yield 'OS_{}'.format(self.os.upper())
  148. # yocto is linux
  149. if 'yocto' in self.os:
  150. yield 'LINUX'
  151. yield 'OS_LINUX'
  152. if self.is_macos:
  153. yield 'DARWIN'
  154. yield 'OS_DARWIN'
  155. if self.is_iossim:
  156. yield 'IOS'
  157. yield 'OS_IOS'
  158. yield 'OS_IOSSIM'
  159. @property
  160. def arch_variables(self):
  161. return select_multiple((
  162. (self.is_i386 or self.is_i686, 'ARCH_I386'),
  163. (self.is_i686, 'ARCH_I686'),
  164. (self.is_x86_64, 'ARCH_X86_64'),
  165. (self.is_armv7, 'ARCH_ARM7'),
  166. (self.is_armv7_neon, 'ARCH_ARM7_NEON'),
  167. (self.is_armv8, 'ARCH_ARM64'),
  168. (self.is_armv8m, 'ARCH_ARM8M'),
  169. (self.is_armv7em, 'ARCH_ARM7EM'),
  170. (self.is_armv5te, 'ARCH_ARM5TE'),
  171. (self.is_arm, 'ARCH_ARM'),
  172. (self.is_linux_armv8 or self.is_macos_arm64, 'ARCH_AARCH64'),
  173. (self.is_powerpc, 'ARCH_PPC64LE'),
  174. (self.is_power8le, 'ARCH_POWER8LE'),
  175. (self.is_power9le, 'ARCH_POWER9LE'),
  176. (self.is_riscv32, 'ARCH_RISCV32'),
  177. (self.is_xtensa, 'ARCH_XTENSA'),
  178. (self.is_nds32, 'ARCH_NDS32'),
  179. (self.is_tc32, 'ARCH_TC32'),
  180. (self.is_wasm64, 'ARCH_WASM64'),
  181. (self.is_32_bit, 'ARCH_TYPE_32'),
  182. (self.is_64_bit, 'ARCH_TYPE_64'),
  183. ))
  184. @property
  185. def library_path_variables(self):
  186. return ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']
  187. def find_in_dict(self, dict_, default=None):
  188. if dict_ is None:
  189. return default
  190. for key in six.iterkeys(dict_):
  191. if self._parse_os(key) == self.os:
  192. return dict_[key]
  193. return default
  194. @property
  195. def os_compat(self):
  196. if self.is_macos:
  197. return 'DARWIN'
  198. else:
  199. return self.os.upper()
  200. @property
  201. def canonized_platform(self):
  202. os = None
  203. if self.os == 'macos':
  204. os = 'darwin'
  205. elif self.os == 'windows':
  206. os = 'win32'
  207. else:
  208. os = self.os
  209. if self.arch != 'x86_64':
  210. return '-'.join([os, self.arch])
  211. else:
  212. return os
  213. def exe(self, *paths):
  214. if self.is_windows:
  215. return ntpath.join(*itertools.chain(paths[:-1], (paths[-1] + '.exe',)))
  216. else:
  217. return posixpath.join(*paths)
  218. def __str__(self):
  219. return '{name}-{os}-{arch}'.format(name=self.name, os=self.os, arch=self.arch)
  220. def __eq__(self, other):
  221. return (self.name, self.os, self.arch) == (other.name, other.os, other.arch)
  222. def __lt__(self, other):
  223. return (self.name, self.os, self.arch) < (other.name, other.os, other.arch)
  224. def __hash__(self):
  225. return hash((self.name, self.os, self.arch))
  226. @staticmethod
  227. def _parse_os(os):
  228. os = os.lower()
  229. if os == 'darwin':
  230. return 'macos'
  231. if os in ('win', 'win32', 'win64'):
  232. return 'windows'
  233. if os.startswith('cygwin'):
  234. return 'cygwin'
  235. return os
  236. def which(prog):
  237. if os.path.exists(prog) and os.access(prog, os.X_OK):
  238. return prog
  239. # Ищем в $PATH только простые команды, без путей.
  240. if os.path.dirname(prog) != '':
  241. return None
  242. path = os.getenv('PATH', '')
  243. pathext = os.environ.get('PATHEXT')
  244. # На Windows %PATHEXT% указывает на список расширений, которые нужно проверять
  245. # при поиске команды в пути. Точное совпадение без расширения имеет приоритет.
  246. pathext = [''] if pathext is None else [''] + pathext.lower().split(os.pathsep)
  247. for dir_ in path.split(os.path.pathsep):
  248. for ext in pathext:
  249. p = os.path.join(dir_, prog + ext)
  250. if os.path.exists(p) and os.path.isfile(p) and os.access(p, os.X_OK):
  251. return win_path_fix(p)
  252. return None
  253. def get_stdout(command):
  254. stdout, code = get_stdout_and_code(command)
  255. return stdout if code == 0 else None
  256. def get_stdout_and_code(command):
  257. # noinspection PyBroadException
  258. try:
  259. process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  260. stdout, _ = process.communicate()
  261. return six.ensure_str(stdout), process.returncode
  262. except Exception as e:
  263. logger.info("While run: `%s`", e)
  264. return None, None
  265. def to_strings(o):
  266. if isinstance(o, (list, tuple)):
  267. for s in o:
  268. for ss in to_strings(s):
  269. yield ss
  270. else:
  271. if o is not None:
  272. if isinstance(o, bool):
  273. yield 'yes' if o else 'no'
  274. elif isinstance(o, (str, int)):
  275. yield str(o)
  276. else:
  277. raise ConfigureError('Unexpected value {} {}'.format(type(o), o))
  278. def emit(key, *value):
  279. print('{0}={1}'.format(key, ' '.join(to_strings(value))))
  280. def emit_with_comment(comment, key, *value):
  281. print('# {}'.format(comment))
  282. emit(key, *value)
  283. def emit_with_ignore_comment(key, *value):
  284. emit_with_comment('IGNORE YMAKE CONF CONTEXT', key, *value)
  285. def append(key, *value):
  286. print('{0}+={1}'.format(key, ' '.join(to_strings(value))))
  287. def emit_big(text):
  288. prefix = None
  289. first = True
  290. for line in text.split('\n'):
  291. if prefix is None:
  292. if not line:
  293. continue
  294. prefix = 0
  295. while prefix < len(line) and line[prefix] == ' ':
  296. prefix += 1
  297. if first: # Be pretty, prepend an empty line before the output
  298. print()
  299. first = False
  300. print(line[prefix:])
  301. class Variables(dict):
  302. def emit(self, with_ignore_comment=[]):
  303. with_ignore_comment_set = set(with_ignore_comment)
  304. for k in sorted(self.keys()):
  305. if k in with_ignore_comment_set:
  306. emit_with_ignore_comment(k, self[k])
  307. else:
  308. emit(k, self[k])
  309. def update_from_presets(self):
  310. for k in six.iterkeys(self):
  311. v = preset(k)
  312. if v is not None:
  313. self[k] = v
  314. def reset_if_any(self, value_check=None, reset_value=None):
  315. if value_check is None:
  316. def value_check(v_):
  317. return v_ is None
  318. if any(map(value_check, six.itervalues(self))):
  319. for k in six.iterkeys(self):
  320. self[k] = reset_value
  321. def format_env(env, list_separator=':'):
  322. def format_value(value):
  323. return value if isinstance(value, str) else ('\\' + list_separator).join(value)
  324. def format(kv):
  325. return '${env:"%s=%s"}' % (kv[0], format_value(kv[1]))
  326. return ' '.join(map(format, sorted(six.iteritems(env))))
  327. # TODO(somov): Проверить, используется ли это. Может быть, выпилить.
  328. def userify_presets(presets, keys):
  329. for key in keys:
  330. user_key = 'USER_{}'.format(key)
  331. values = [presets.pop(key, None), presets.get(user_key)]
  332. presets[user_key] = ' '.join(filter(None, values))
  333. def preset(key, default=None):
  334. return opts().presets.get(key, default)
  335. def is_positive(key):
  336. return is_positive_str(preset(key, ''))
  337. def is_positive_str(s):
  338. return s.lower() in ('yes', 'true', 'on', '1')
  339. def is_negative(key):
  340. return is_negative_str(preset(key, ''))
  341. def is_negative_str(s):
  342. return s.lower() in ('no', 'false', 'off', '0')
  343. def to_bool(s, default=None):
  344. if isinstance(s, six.string_types):
  345. if is_positive_str(s):
  346. return True
  347. if is_negative_str(s):
  348. return False
  349. if default is None:
  350. raise ConfigureError('{} is not a bool value'.format(s))
  351. return default
  352. def select(selectors, default=None, no_default=False):
  353. for enabled, value in selectors:
  354. if enabled:
  355. return value
  356. if no_default:
  357. raise ConfigureError()
  358. return default
  359. def select_multiple(selectors):
  360. for enabled, value in selectors:
  361. if enabled:
  362. yield value
  363. def unique(it):
  364. known = set()
  365. for i in it:
  366. if i not in known:
  367. known.add(i)
  368. yield i
  369. class Options(object):
  370. def __init__(self, argv):
  371. def parse_presets(raw_presets):
  372. presets = {}
  373. for p in raw_presets:
  374. toks = p.split('=', 1)
  375. name = toks[0]
  376. value = toks[1] if len(toks) >= 2 else ''
  377. presets[name] = value
  378. return presets
  379. parser = optparse.OptionParser(add_help_option=False)
  380. opt_group = optparse.OptionGroup(parser, 'Conf script options')
  381. opt_group.add_option('--toolchain-params', dest='toolchain_params', action='store', help='Set toolchain params via file')
  382. opt_group.add_option('-D', '--preset', dest='presets', action='append', default=[], help='set or override presets')
  383. opt_group.add_option('-l', '--local-distbuild', dest='local_distbuild', action='store_true', default=False, help='conf for local distbuild')
  384. parser.add_option_group(opt_group)
  385. self.options, self.arguments = parser.parse_args(argv)
  386. argv = self.arguments
  387. if len(argv) < 4:
  388. print('Usage: ArcRoot, --BuildType--, Verbosity, [Path to local.ymake]', file=sys.stderr)
  389. sys.exit(1)
  390. self.arcadia_root = argv[1]
  391. init_logger(argv[3] == 'verbose')
  392. # Эти переменные не надо использоваться напрямую. Их значения уже разбираются в других местах.
  393. self.build_type = argv[2].lower()
  394. self.local_distbuild = self.options.local_distbuild
  395. self.toolchain_params = self.options.toolchain_params
  396. self.presets = parse_presets(self.options.presets)
  397. userify_presets(self.presets, ('CFLAGS', 'CXXFLAGS', 'CONLYFLAGS', 'LDFLAGS', 'GO_COMPILE_FLAGS', 'GO_LINK_FLAGS', 'STD_CXX_VERSION'))
  398. Instance = None
  399. def opts():
  400. if Options.Instance is None:
  401. Options.Instance = Options(sys.argv)
  402. return Options.Instance
  403. class Profiler(object):
  404. Generic = 'generic'
  405. GProf = 'gprof'
  406. class Arcadia(object):
  407. def __init__(self, root):
  408. self.root = root
  409. class Build(object):
  410. def __init__(self, arcadia, build_type, toolchain_params, force_ignore_local_files=False):
  411. self.arcadia = arcadia
  412. self.params = self._load_json_from_base64(toolchain_params)
  413. self.build_type = build_type
  414. platform = self.params['platform']
  415. self.host = Platform.from_json(platform['host'])
  416. self.target = Platform.from_json(platform['target'])
  417. self.tc = self._get_toolchain_options()
  418. # TODO(somov): Удалить, когда перестанет использоваться.
  419. self.build_system = 'ymake'
  420. self.ignore_local_files = False
  421. dist_prefix = 'dist-'
  422. if self.build_type.startswith(dist_prefix):
  423. self.build_system = 'distbuild'
  424. self.build_type = self.build_type[len(dist_prefix):]
  425. if force_ignore_local_files:
  426. self.ignore_local_files = True
  427. if self.is_ide_build_type(self.build_type):
  428. self.ignore_local_files = True
  429. self.pic = not is_positive('FORCE_NO_PIC')
  430. @property
  431. def host_target(self):
  432. return self.host, self.target
  433. def print_build(self):
  434. self._print_build_settings()
  435. host_os = System(self.host)
  436. host_os.print_host_settings()
  437. target_os = System(self.target)
  438. target_os.print_target_settings()
  439. if self.pic:
  440. emit('PIC', 'yes')
  441. emit('COMPILER_ID', self.tc.type.upper())
  442. if self.is_valgrind:
  443. emit('WITH_VALGRIND', 'yes')
  444. if self.is_coverage:
  445. emit('_BUILD_COVERAGE', 'yes')
  446. if self.is_debug:
  447. emit('_BUILD_DEBUG', 'yes')
  448. if self.is_fast_debug:
  449. emit('_BUILD_FAST_DEBUG', 'yes')
  450. if self.is_release:
  451. emit('_BUILD_RELEASE', 'yes')
  452. if self.is_sanitized:
  453. emit('_BUILD_SANITIZED', 'yes')
  454. if self.is_size_optimized:
  455. emit('_BUILD_SIZE_OPTIMIZED', 'yes')
  456. if self.with_ndebug:
  457. emit('_BUILD_WITH_NDEBUG', 'yes')
  458. toolchain_type, compiler_type, linker_type = Compilers[self.tc.type]
  459. toolchain = toolchain_type(self.tc, self)
  460. compiler = compiler_type(self.tc, self)
  461. linker = linker_type(self.tc, self)
  462. toolchain.print_toolchain()
  463. compiler.print_compiler()
  464. linker.print_linker()
  465. self._print_other_settings(compiler)
  466. def _print_build_settings(self):
  467. emit('BUILD_TYPE', self.build_type.upper())
  468. emit('BT_' + self.build_type.upper().replace('-', '_'), 'yes')
  469. if self.build_system == 'distbuild':
  470. emit('DISTBUILD', 'yes')
  471. elif self.build_system != 'ymake':
  472. raise ConfigureError()
  473. python_bin = preset('BUILD_PYTHON_BIN', '$(PYTHON)/python')
  474. emit('YMAKE_PYTHON', python_bin)
  475. emit('YMAKE_UNPICKLER', python_bin, '$ARCADIA_ROOT/build/plugins/_unpickler.py')
  476. @property
  477. def is_release(self):
  478. # TODO(somov): Проверить, бывают ли тут суффиксы на самом деле
  479. return self.build_type in ('release', 'relwithdebinfo', 'minsizerel', 'profile', 'gprof') or self.build_type.endswith('-release')
  480. @property
  481. def is_debug(self):
  482. return self.build_type in ('debug', 'debugnoasserts', 'fastdebug') or self.build_type.endswith('-debug')
  483. @property
  484. def is_fast_debug(self):
  485. return self.build_type == 'fastdebug'
  486. @property
  487. def is_size_optimized(self):
  488. return self.build_type == 'minsizerel'
  489. @property
  490. def is_coverage(self):
  491. return self.build_type == 'coverage'
  492. @property
  493. def is_sanitized(self):
  494. sanitizer = preset('SANITIZER_TYPE')
  495. return bool(sanitizer) and not is_negative_str(sanitizer)
  496. @property
  497. def with_ndebug(self):
  498. return self.build_type in ('release', 'minsizerel', 'valgrind-release', 'profile', 'gprof', 'debugnoasserts')
  499. @property
  500. def is_valgrind(self):
  501. return self.build_type == 'valgrind' or self.build_type == 'valgrind-release'
  502. @property
  503. def is_ide(self):
  504. return self.is_ide_build_type(self.build_type)
  505. @property
  506. def profiler_type(self):
  507. if self.build_type == 'profile':
  508. return Profiler.Generic
  509. elif self.build_type == 'gprof':
  510. return Profiler.GProf
  511. else:
  512. return None
  513. @staticmethod
  514. def is_ide_build_type(build_type):
  515. return build_type == 'nobuild'
  516. def _configure_runtime_versions(self):
  517. try:
  518. res = subprocess.check_output(['xcrun', 'simctl', 'list', '--json', 'runtimes'])
  519. except subprocess.CalledProcessError as exc:
  520. logger.warning(exc)
  521. return
  522. raw_object = json.loads(res)
  523. raw_object = raw_object['runtimes']
  524. for runtime in raw_object:
  525. if runtime['isAvailable']:
  526. if "iOS" in runtime['identifier']:
  527. emit('DEFAULT_IOS_RUNTIME', '{}'.format(runtime['identifier']))
  528. def _get_toolchain_options(self):
  529. type_ = self.params['params']['type']
  530. if self.params['params'].get('local') and type_ == 'xcode':
  531. detector = CompilerDetector()
  532. detector.detect(self.params['params']['c_compiler'], self.params['params']['cxx_compiler'])
  533. emit('LOCAL_XCODE_TOOLS', 'yes')
  534. emit('XCODE', 'yes')
  535. emit('ACTOOL_PATH', self.params['params']['actool'])
  536. emit('IBTOOL_PATH', self.params['params']['ibtool'])
  537. self._configure_runtime_versions()
  538. elif type_ == 'system_cxx':
  539. c_compiler = self.params['params'].get('c_compiler')
  540. cxx_compiler = self.params['params'].get('cxx_compiler')
  541. if is_positive('USE_CLANG_CL'):
  542. c_compiler = 'clang-cl.exe' if c_compiler == 'cl.exe' else c_compiler
  543. cxx_compiler = 'clang-cl.exe' if cxx_compiler == 'cl.exe' else cxx_compiler
  544. detector = CompilerDetector()
  545. detector.detect(c_compiler, cxx_compiler)
  546. type_ = detector.type
  547. else:
  548. detector = None
  549. if type_ == 'msvc':
  550. return MSVCToolchainOptions(self, detector)
  551. else:
  552. return GnuToolchainOptions(self, detector)
  553. def _print_other_settings(self, compiler):
  554. host = self.host
  555. emit('USE_LOCAL_TOOLS', 'no' if self.ignore_local_files else 'yes')
  556. ragel = Ragel()
  557. ragel.configure_toolchain(self, compiler)
  558. ragel.print_variables()
  559. swiftc = SwiftCompiler(self)
  560. swiftc.configure()
  561. swiftc.print_compiler()
  562. if host.is_linux or host.is_macos or host.is_cygwin:
  563. if is_negative('USE_ARCADIA_PYTHON'):
  564. python = Python(self.tc)
  565. python.configure_posix()
  566. python.print_variables()
  567. cuda = Cuda(self)
  568. cuda.print_()
  569. CuDNN(cuda).print_()
  570. if self.ignore_local_files or host.is_windows or is_positive('NO_SVN_DEPENDS'):
  571. emit_with_ignore_comment('SVN_DEPENDS_CACHE__NO_UID__')
  572. else:
  573. emit_with_ignore_comment('SVN_DEPENDS_CACHE__NO_UID__', '${hide;kv:"disable_cache"}')
  574. @staticmethod
  575. def _load_json_from_base64(base64str):
  576. """
  577. :rtype: dict[str, Any]
  578. """
  579. def un_unicode(o):
  580. if isinstance(o, six.text_type):
  581. return six.ensure_str(o)
  582. if isinstance(o, list):
  583. return [un_unicode(oo) for oo in o]
  584. if isinstance(o, dict):
  585. return {un_unicode(k): un_unicode(v) for k, v in six.iteritems(o)}
  586. return o
  587. return un_unicode(json.loads(base64.b64decode(base64str)))
  588. class YMake(object):
  589. def __init__(self, arcadia):
  590. self.arcadia = arcadia
  591. def print_presets(self):
  592. presets = opts().presets
  593. if is_positive_str(presets.get('EXPORT_GRADLE', 'no')):
  594. print('@import "${CONF_ROOT}/conf/export_gradle.yes.conf"')
  595. else:
  596. print('@import "${CONF_ROOT}/conf/export_gradle.no.conf"')
  597. if presets.get('CLANG_COVERAGE', None) is None:
  598. print('@import "${CONF_ROOT}/conf/coverage_full_instrumentation.conf"')
  599. else:
  600. print('@import "${CONF_ROOT}/conf/coverage_selective_instrumentation.conf"')
  601. if presets:
  602. print('# Variables set from command line by -D options')
  603. for key in sorted(presets):
  604. emit(key, opts().presets[key])
  605. @staticmethod
  606. def _print_conf_content(path):
  607. with open(path, 'r') as fin:
  608. print(fin.read())
  609. def print_core_conf(self):
  610. print('@import "${CONF_ROOT}/ymake.core.conf"')
  611. def print_settings(self):
  612. pass
  613. class System(object):
  614. def __init__(self, platform):
  615. """
  616. :type platform: Platform
  617. """
  618. self.platform = platform
  619. def print_windows_target_const(self):
  620. # TODO(somov): Remove this variables, use generic OS/arch variables in makelists.
  621. emit('WINDOWS', 'yes')
  622. emit('WIN32', 'yes')
  623. if self.platform.is_64_bit == 64:
  624. emit('WIN64', 'yes')
  625. def print_nix_target_const(self):
  626. emit('JAVA_INCLUDE', '-I{0}/include -I{0}/include/{1}'.format('/usr/lib/jvm/default-java', self.platform.os_compat))
  627. emit('UNIX', 'yes')
  628. emit('REALPRJNAME')
  629. emit('SONAME')
  630. def print_target_settings(self):
  631. emit('TARGET_PLATFORM', self.platform.os_compat)
  632. emit('CANONIZED_TARGET_PLATFORM', self.platform.canonized_platform)
  633. emit('HARDWARE_ARCH', '32' if self.platform.is_32_bit else '64')
  634. emit('HARDWARE_TYPE', self.platform.arch)
  635. for variable in self.platform.arch_variables:
  636. emit(variable, 'yes')
  637. for variable in self.platform.os_variables:
  638. emit(variable, 'yes')
  639. if self.platform.is_armv7:
  640. emit('ARM7_FLOAT_ABI', self.platform.armv7_float_abi)
  641. if self.platform.is_android:
  642. emit('ANDROID_API', str(self.platform.android_api))
  643. if self.platform.is_posix:
  644. self.print_nix_target_const()
  645. elif self.platform.is_windows:
  646. self.print_windows_target_const()
  647. def print_host_settings(self):
  648. emit('HOST_PLATFORM', self.platform.os_compat)
  649. emit('CANONIZED_HOST_PLATFORM', self.platform.canonized_platform)
  650. for variable in itertools.chain(self.platform.os_variables, self.platform.arch_variables):
  651. emit('HOST_{var}'.format(var=variable), 'yes')
  652. class CompilerDetector(object):
  653. def __init__(self):
  654. self.type = None
  655. self.c_compiler = None
  656. self.cxx_compiler = None
  657. self.version_list = None
  658. @staticmethod
  659. def preprocess_source(compiler, source):
  660. # noinspection PyBroadException
  661. try:
  662. fd, path = tempfile.mkstemp(suffix='.cpp')
  663. try:
  664. with os.fdopen(fd, 'w') as output:
  665. output.write(source)
  666. stdout, code = get_stdout_and_code([compiler, '-E', path])
  667. except Exception as e:
  668. logger.info("While writing: `%s`", e)
  669. finally:
  670. os.remove(path)
  671. return stdout, code
  672. except Exception as e:
  673. logger.debug('Preprocessing failed: %s', e)
  674. return None, None
  675. @staticmethod
  676. def get_compiler_vars(compiler, names):
  677. prefix = '____YA_VAR_'
  678. source = '\n'.join(['{prefix}{name}={name}\n'.format(prefix=prefix, name=n) for n in names])
  679. # Некоторые препроцессоры возвращают ненулевой код возврата. Поэтому его проверять нельзя.
  680. # Мы можем только удостовериться после разбора stdout, что в нём
  681. # присутствовала хотя бы одна подставленная переменная.
  682. # TODO(somov): Исследовать, можно ли проверять ограниченный набор кодов возврата.
  683. # TODO(v-korovin): Нормально прокидывать Exception-ы, оно и так упадёт
  684. stdout, _ = CompilerDetector.preprocess_source(compiler, source)
  685. if stdout is None:
  686. return None
  687. vars_ = {}
  688. for line in six.ensure_str(stdout).split('\n'):
  689. parts = line.split('=', 1)
  690. if len(parts) == 2 and parts[0].startswith(prefix):
  691. name, value = parts[0][len(prefix):], parts[1]
  692. if value == name:
  693. continue # Preprocessor variable was not substituted
  694. vars_[name] = value
  695. return vars_
  696. def detect(self, c_compiler=None, cxx_compiler=None):
  697. c_compiler = c_compiler or os.environ.get('CC')
  698. cxx_compiler = cxx_compiler or os.environ.get('CXX') or c_compiler
  699. c_compiler = c_compiler or cxx_compiler
  700. logger.debug('e=%s', os.environ)
  701. if c_compiler is None:
  702. raise ConfigureError('Custom compiler was requested but not specified')
  703. c_compiler_path = which(c_compiler)
  704. clang_vars = ['__clang_major__', '__clang_minor__', '__clang_patchlevel__']
  705. gcc_vars = ['__GNUC__', '__GNUC_MINOR__', '__GNUC_PATCHLEVEL__']
  706. msvc_vars = ['_MSC_VER']
  707. apple_var = '__apple_build_version__'
  708. compiler_vars = self.get_compiler_vars(c_compiler_path, clang_vars + [apple_var] + gcc_vars + msvc_vars)
  709. if not compiler_vars:
  710. raise ConfigureError('Could not determine custom compiler version: {}'.format(c_compiler))
  711. def version(version_names):
  712. def iter_version():
  713. for name in version_names:
  714. yield int(compiler_vars[name])
  715. # noinspection PyBroadException
  716. try:
  717. return list(iter_version())
  718. except Exception:
  719. return None
  720. clang_version = version(clang_vars)
  721. apple_build = apple_var in compiler_vars
  722. # TODO(somov): Учитывать номера версий сборки Apple компилятора Clang.
  723. _ = apple_build
  724. gcc_version = version(gcc_vars)
  725. msvc_version = version(msvc_vars)
  726. if msvc_version:
  727. logger.debug('Detected MSVC version %s', msvc_version)
  728. self.type = 'msvc'
  729. elif clang_version:
  730. logger.debug('Detected Clang version %s', clang_version)
  731. self.type = 'clang'
  732. elif gcc_version:
  733. logger.debug('Detected GCC version %s', gcc_version)
  734. # TODO(somov): Переименовать в gcc.
  735. self.type = 'gnu'
  736. else:
  737. raise ConfigureError('Could not determine custom compiler type: {}'.format(c_compiler))
  738. self.version_list = clang_version or gcc_version or msvc_version
  739. self.c_compiler = c_compiler_path
  740. self.cxx_compiler = cxx_compiler and which(cxx_compiler) or c_compiler_path
  741. class ToolchainOptions(object):
  742. def __init__(self, build, detector):
  743. """
  744. :type build: Build
  745. """
  746. self.host = build.host
  747. self.target = build.target
  748. tc_json = build.params
  749. logger.debug('Toolchain host %s', self.host)
  750. logger.debug('Toolchain target %s', self.target)
  751. logger.debug('Toolchain json %s', DebugString(lambda: json.dumps(tc_json, indent=4, sort_keys=True)))
  752. self.params = tc_json['params']
  753. self._name = tc_json.get('name', 'theyknow')
  754. if detector:
  755. self.type = detector.type
  756. self.from_arcadia = False
  757. self.c_compiler = detector.c_compiler
  758. self.cxx_compiler = detector.cxx_compiler
  759. self.compiler_version_list = detector.version_list
  760. self.compiler_version = '.'.join(map(lambda part: six.ensure_str(str(part)), self.compiler_version_list))
  761. else:
  762. self.type = self.params['type']
  763. self.from_arcadia = True
  764. self.c_compiler = self.params['c_compiler']
  765. self.cxx_compiler = self.params['cxx_compiler']
  766. # TODO(somov): Требовать номер версии всегда.
  767. self.compiler_version = self.params.get('gcc_version') or self.params.get('version') or '0'
  768. self.compiler_version_list = list(map(int, self.compiler_version.split('.')))
  769. # 'match_root' at this point contains real name for references via toolchain
  770. self.name_marker = '$(%s)' % self.params.get('match_root', self._name.upper())
  771. self.arch_opt = self.params.get('arch_opt', [])
  772. self.triplet_opt = self.params.get('triplet_opt', {})
  773. self.target_opt = self.params.get('target_opt', [])
  774. # TODO(somov): Убрать чтение настройки из os.environ.
  775. self.werror_mode = preset('WERROR_MODE') or os.environ.get('WERROR_MODE') or self.params.get('werror_mode') or 'compiler_specific'
  776. # default C++ standard is set here, some older toolchains might need to redefine it in ya.conf.json
  777. self.cxx_std = self.params.get('cxx_std', 'c++20')
  778. self._env = tc_json.get('env', {})
  779. self.android_ndk_version = self.params.get('android_ndk_version', None)
  780. logger.debug('c_compiler=%s', self.c_compiler)
  781. logger.debug('cxx_compiler=%s', self.cxx_compiler)
  782. self.compiler_platform_projects = self.target.find_in_dict(self.params.get('platform'), [])
  783. def version_at_least(self, *args):
  784. return args <= tuple(self.compiler_version_list)
  785. def version_exactly(self, *args):
  786. if not args or len(args) > len(self.compiler_version_list):
  787. return False
  788. for left, right in zip(args, list(self.compiler_version_list)[:len(args)]):
  789. if left != right:
  790. return False
  791. return True
  792. @property
  793. def is_gcc(self):
  794. return self.type == 'gnu'
  795. @property
  796. def is_clang(self):
  797. return self.type in ('clang', 'xcode')
  798. @property
  799. def is_xcode(self):
  800. return self.type == 'xcode'
  801. @property
  802. def is_from_arcadia(self):
  803. return self.from_arcadia
  804. @property
  805. def is_system_cxx(self):
  806. return self._name == "system_cxx"
  807. def get_env(self, convert_list=None):
  808. convert_list = convert_list or (lambda x: x)
  809. r = {}
  810. for k, v in six.iteritems(self._env):
  811. if isinstance(v, str):
  812. r[k] = v
  813. elif isinstance(v, list):
  814. r[k] = convert_list(v)
  815. else:
  816. logger.debug('Unexpected values in environment: %s', self._env)
  817. raise ConfigureError('Internal error')
  818. return r
  819. class GnuToolchainOptions(ToolchainOptions):
  820. def __init__(self, build, detector):
  821. super(GnuToolchainOptions, self).__init__(build, detector)
  822. self.ar = self.params.get('ar')
  823. self.ar_plugin = self.params.get('ar_plugin')
  824. self.strip = self.params.get('strip')
  825. self.objcopy = self.params.get('objcopy')
  826. self.objdump = self.params.get('objdump')
  827. self.isystem = self.params.get('isystem')
  828. self.dwarf_tool = self.target.find_in_dict(self.params.get('dwarf_tool'))
  829. # TODO(somov): Унифицировать формат sys_lib
  830. self.sys_lib = self.params.get('sys_lib', {})
  831. if isinstance(self.sys_lib, dict):
  832. self.sys_lib = self.target.find_in_dict(self.sys_lib, [])
  833. self.os_sdk = preset('OS_SDK') or self._default_os_sdk()
  834. self.os_sdk_local = False
  835. if build.host.is_apple and build.target.is_apple and to_bool(preset('APPLE_SDK_LOCAL'), default=False):
  836. self.os_sdk_local = True
  837. if build.host.is_apple and build.target.is_ios and to_bool(preset('MAPSMOBI_BUILD_TARGET'), default=False):
  838. self.os_sdk_local = True
  839. if self.os_sdk == 'local':
  840. self.os_sdk_local = True
  841. def _default_os_sdk(self):
  842. if self.target.is_linux:
  843. if self.target.is_armv8:
  844. return 'ubuntu-16'
  845. if self.target.is_armv7 and self.target.armv7_float_abi == 'hard':
  846. return 'ubuntu-16'
  847. if self.target.is_armv7 and self.target.armv7_float_abi == 'softfp':
  848. return 'ubuntu-18'
  849. # Default OS SDK for Linux builds
  850. return LINUX_SDK_DEFAULT
  851. class Toolchain(object):
  852. def __init__(self, tc, build):
  853. """
  854. :type tc: ToolchainOptions
  855. :type build: Build
  856. """
  857. self.tc = tc
  858. self.build = build
  859. self.platform_projects = self.tc.compiler_platform_projects
  860. def print_toolchain(self):
  861. if self.platform_projects:
  862. emit('COMPILER_PLATFORM', list(unique(self.platform_projects)))
  863. class Compiler(object):
  864. def __init__(self, tc, compiler_variable):
  865. self.compiler_variable = compiler_variable
  866. self.tc = tc
  867. def print_compiler(self):
  868. # CLANG and CLANG_VER variables
  869. emit(self.compiler_variable, 'yes')
  870. cv = self.tc.compiler_version
  871. if '.' in cv:
  872. cv = cv[:cv.index('.')]
  873. emit('{}_VER'.format(self.compiler_variable), cv)
  874. if self.tc.is_xcode:
  875. emit('XCODE', 'yes')
  876. class GnuToolchain(Toolchain):
  877. def __init__(self, tc, build):
  878. """
  879. :type tc: GnuToolchainOptions
  880. :type build: Build
  881. """
  882. def get_os_sdk(target):
  883. sdk_native_version = 10.11 if not preset('EXPERIMENTAL_MACOS_M1_SUPPORT') else '11.1'
  884. if target.is_macos:
  885. return '$MACOS_SDK_RESOURCE_GLOBAL/MacOSX{}.sdk'.format(sdk_native_version)
  886. elif target.is_yocto:
  887. return '$YOCTO_SDK_RESOURCE_GLOBAL'
  888. return '$OS_SDK_ROOT_RESOURCE_GLOBAL'
  889. super(GnuToolchain, self).__init__(tc, build)
  890. self.tc = tc
  891. host = build.host
  892. target = build.target
  893. self.c_flags_platform = list(tc.target_opt)
  894. self.default_os_sdk_root = get_os_sdk(target)
  895. self.env = self.tc.get_env()
  896. self.env_go = {}
  897. if self.tc.is_clang and not self.tc.is_system_cxx:
  898. self.env_go = {'PATH': ['{}/bin'.format(self.tc.name_marker)]}
  899. if self.tc.is_gcc:
  900. self.env_go = {'PATH': ['{}/gcc/bin'.format(self.tc.name_marker)]}
  901. if 'PATH' in self.env_go:
  902. if target.is_linux:
  903. self.env_go['PATH'].append('$OS_SDK_ROOT_RESOURCE_GLOBAL/usr/bin')
  904. elif target.is_macos:
  905. self.env_go['PATH'].extend([
  906. '$MACOS_SDK_RESOURCE_GLOBAL/usr/bin',
  907. '$GO_FAKE_XCRUN_RESOURCE_GLOBAL',
  908. ])
  909. self.swift_flags_platform = []
  910. self.swift_lib_path = None
  911. if self.tc.is_from_arcadia:
  912. for lib_path in build.host.library_path_variables:
  913. self.env.setdefault(lib_path, []).append('{}/lib'.format(self.tc.name_marker))
  914. swift_target = select(default=None, selectors=[
  915. (target.is_iossim and target.is_x86_64, 'x86_64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  916. (target.is_iossim and target.is_x86, 'i386-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  917. (target.is_iossim and target.is_armv8, 'arm64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  918. (not target.is_iossim and target.is_ios and target.is_armv8, 'arm64-apple-ios9'),
  919. (not target.is_iossim and target.is_ios and target.is_armv7, 'armv7-apple-ios9'),
  920. ])
  921. if swift_target:
  922. self.swift_flags_platform += ['-target', swift_target]
  923. if self.tc.is_from_arcadia:
  924. self.swift_lib_path = select(default=None, selectors=[
  925. (host.is_macos and target.is_iossim, '$SWIFT_XCODE_TOOLCHAIN_ROOT_RESOURCE_GLOBAL/usr/lib/swift/iphonesimulator'),
  926. (host.is_macos and not target.is_iossim and target.is_ios and (target.is_armv8 or target.is_armv7), '$SWIFT_XCODE_TOOLCHAIN_ROOT_RESOURCE_GLOBAL/usr/lib/swift/iphoneos'),
  927. ])
  928. if self.tc.is_clang:
  929. target_triple = self.tc.triplet_opt.get(target.arch, None)
  930. if not target_triple:
  931. target_triple = select(default=None, selectors=[
  932. (target.is_linux and target.is_x86_64, 'x86_64-linux-gnu'),
  933. (target.is_linux and target.is_armv8, 'aarch64-linux-gnu'),
  934. (target.is_linux and target.is_armv7 and target.armv7_float_abi == 'hard', 'armv7-linux-gnueabihf'),
  935. (target.is_linux and target.is_armv7 and target.armv7_float_abi == 'softfp', 'armv7-linux-gnueabi'),
  936. (target.is_linux and target.is_powerpc, 'powerpc64le-linux-gnu'),
  937. (target.is_iossim and target.is_x86_64, 'x86_64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  938. (target.is_iossim and target.is_x86, 'i386-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  939. (target.is_iossim and target.is_armv8, 'arm64-apple-ios{}-simulator'.format(IOS_VERSION_MIN)),
  940. (not target.is_iossim and target.is_ios and target.is_armv8, 'arm64-apple-ios{}'.format(IOS_VERSION_MIN)),
  941. (not target.is_iossim and target.is_ios and target.is_armv7, 'armv7-apple-ios{}'.format(IOS_VERSION_MIN)),
  942. (target.is_apple and target.is_x86, 'i386-apple-darwin14'),
  943. (target.is_apple and target.is_x86_64, 'x86_64-apple-darwin14'),
  944. (target.is_apple and target.is_macos_arm64, 'arm64-apple-macos11'),
  945. (target.is_apple and target.is_armv7, 'armv7-apple-darwin14'),
  946. (target.is_apple and target.is_armv8, 'arm64-apple-darwin14'),
  947. (target.is_yocto and target.is_armv7, 'arm-poky-linux-gnueabi'),
  948. (target.is_android and target.is_x86, 'i686-linux-android'),
  949. (target.is_android and target.is_x86_64, 'x86_64-linux-android'),
  950. (target.is_android and target.is_armv7, 'armv7a-linux-androideabi'),
  951. (target.is_android and target.is_armv8, 'aarch64-linux-android'),
  952. (target.is_emscripten and target.is_wasm64, 'wasm64-unknown-emscripten'),
  953. ])
  954. if target.is_android:
  955. # Android NDK allows specification of API level in target triple, e.g.:
  956. # armv7a-linux-androideabi16, aarch64-linux-android21
  957. target_triple += str(target.android_api)
  958. if target_triple:
  959. self.c_flags_platform.append('--target={}'.format(target_triple))
  960. if self.tc.isystem:
  961. for root in list(self.tc.isystem):
  962. self.c_flags_platform.extend(['-isystem', root])
  963. if target.is_android:
  964. self.c_flags_platform.extend(['-isystem', '{}/sources/cxx-stl/llvm-libc++abi/include'.format(self.tc.name_marker)])
  965. if target.is_cortex_a9:
  966. self.c_flags_platform.append('-mcpu=cortex-a9')
  967. if target.is_cortex_a35:
  968. self.c_flags_platform.append('-mcpu=cortex-a35')
  969. elif target.is_cortex_a53:
  970. self.c_flags_platform.append('-mcpu=cortex-a53')
  971. elif target.is_cortex_m4:
  972. self.c_flags_platform.append('-mcpu=cortex-m4 -mfpu=fpv4-sp-d16')
  973. elif target.is_cortex_m7:
  974. self.c_flags_platform.append('-mcpu=cortex-m7 -mfpu=fpv5-sp-d16')
  975. elif target.is_cortex_m23:
  976. self.c_flags_platform.append('-mcpu=cortex-m23')
  977. elif target.is_cortex_m33:
  978. self.c_flags_platform.append('-mcpu=cortex-m33 -mfpu=fpv5-sp-d16')
  979. elif target.is_armv7_neon:
  980. self.c_flags_platform.append('-mfpu=neon')
  981. elif target.is_arm968e_s:
  982. self.c_flags_platform.append('-march=armv5te -mcpu=arm968e-s -mthumb-interwork -mlittle-endian')
  983. if (target.is_armv7 or target.is_armv8m or target.is_armv7em or target.is_armv5te) and build.is_size_optimized:
  984. # Enable ARM Thumb2 variable-length instruction encoding
  985. # to reduce code size
  986. self.c_flags_platform.append('-mthumb')
  987. if target.is_arm or target.is_powerpc:
  988. # On linux, ARM and PPC default to unsigned char
  989. # However, Arcadia requires char to be signed
  990. self.c_flags_platform.append('-fsigned-char')
  991. if target.is_rv32imc:
  992. self.c_flags_platform.append('-march=rv32imc')
  993. if self.tc.is_clang or self.tc.is_gcc and self.tc.version_at_least(8, 2):
  994. target_flags = select(default=[], selectors=[
  995. (target.is_linux and target.is_power8le, ['-mcpu=power8', '-mtune=power8', '-maltivec']),
  996. (target.is_linux and target.is_power9le, ['-mcpu=power9', '-mtune=power9', '-maltivec']),
  997. (target.is_linux and target.is_armv8, ['-march=armv8a']),
  998. (target.is_macos, ['-mmacosx-version-min={}'.format(MACOS_VERSION_MIN)]),
  999. (target.is_ios and not target.is_iossim, ['-mios-version-min={}'.format(IOS_VERSION_MIN)]),
  1000. (target.is_iossim, ['-mios-simulator-version-min={}'.format(IOS_VERSION_MIN)]),
  1001. (target.is_android and target.is_armv7, ['-march=armv7-a', '-mfloat-abi=softfp']),
  1002. (target.is_android and target.is_armv8, ['-march=armv8-a']),
  1003. (target.is_yocto and target.is_armv7, ['-march=armv7-a', '-mfpu=neon', '-mfloat-abi=hard', '-mcpu=cortex-a9', '-O1'])
  1004. ])
  1005. if target_flags:
  1006. self.c_flags_platform.extend(target_flags)
  1007. if target.is_ios:
  1008. self.c_flags_platform.append('-D__IOS__=1')
  1009. if target.is_apple:
  1010. self.setup_apple_sdk(target)
  1011. if self.tc.is_from_arcadia or self.tc.is_system_cxx:
  1012. if target.is_linux:
  1013. if not tc.os_sdk_local:
  1014. self.setup_sdk(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL')
  1015. if target.is_x86_64:
  1016. if host.is_linux and not self.tc.is_gcc:
  1017. self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu')
  1018. elif host.is_macos:
  1019. self.setup_tools(project='build/platform/binutils', var='$BINUTILS_ROOT_RESOURCE_GLOBAL', bin='x86_64-linux-gnu/bin', ldlibs=None)
  1020. elif target.is_powerpc:
  1021. self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/x86_64-linux-gnu/powerpc64le-linux-gnu/lib')
  1022. elif target.is_armv8:
  1023. self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu')
  1024. if target.is_yocto:
  1025. self.setup_sdk(project='build/platform/yocto_sdk/yocto_sdk', var='${YOCTO_SDK_ROOT_RESOURCE_GLOBAL}')
  1026. def setup_apple_sdk(self, target):
  1027. if not self.tc.os_sdk_local:
  1028. self.setup_apple_arcadia_sdk(target)
  1029. else:
  1030. self.setup_apple_local_sdk(target)
  1031. def setup_apple_arcadia_sdk(self, target):
  1032. if target.is_ios:
  1033. self.setup_xcode_sdk(project='build/internal/platform/ios_sdk', var='${IOS_SDK_ROOT_RESOURCE_GLOBAL}')
  1034. self.platform_projects.append('build/internal/platform/macos_system_stl')
  1035. if target.is_macos:
  1036. self.setup_xcode_sdk(project='build/internal/platform/macos_sdk', var='${MACOS_SDK_RESOURCE_GLOBAL}')
  1037. self.platform_projects.append('build/internal/platform/macos_system_stl')
  1038. def setup_apple_local_sdk(self, target):
  1039. def get_output(*args):
  1040. return six.ensure_str(subprocess.check_output(tuple(args))).strip()
  1041. def get_sdk_root(sdk):
  1042. root = self.env.get('SDKROOT')
  1043. if root not in (None, '', ['']):
  1044. return root
  1045. root = os.environ.get('SDKROOT')
  1046. if root:
  1047. return root
  1048. return get_output('xcrun', '-sdk', sdk, '--show-sdk-path')
  1049. if target.is_iossim:
  1050. self.env['SDKROOT'] = get_sdk_root('iphonesimulator')
  1051. elif target.is_ios:
  1052. self.env['SDKROOT'] = get_sdk_root('iphoneos')
  1053. elif target.is_macos:
  1054. self.env['SDKROOT'] = get_sdk_root('macosx')
  1055. def setup_sdk(self, project, var):
  1056. self.platform_projects.append(project)
  1057. self.c_flags_platform.append('--sysroot={}'.format(var))
  1058. self.swift_flags_platform += ['-sdk', var]
  1059. def setup_xcode_sdk(self, project, var):
  1060. self.platform_projects.append(project)
  1061. self.c_flags_platform.append('-isysroot')
  1062. self.c_flags_platform.append(var)
  1063. self.swift_flags_platform += ['-sdk', var]
  1064. # noinspection PyShadowingBuiltins
  1065. def setup_tools(self, project, var, bin, ldlibs):
  1066. self.platform_projects.append(project)
  1067. self.c_flags_platform.append('-B{}/{}'.format(var, bin))
  1068. if ldlibs:
  1069. for lib_path in self.build.host.library_path_variables:
  1070. self.env.setdefault(lib_path, []).append('{}/{}'.format(var, ldlibs))
  1071. def print_toolchain(self):
  1072. super(GnuToolchain, self).print_toolchain()
  1073. emit('TOOLCHAIN_ENV', format_env(self.env, list_separator=':'))
  1074. emit('_GO_TOOLCHAIN_ENV_PATH', format_env(self.env_go, list_separator=':'))
  1075. emit('C_FLAGS_PLATFORM', self.c_flags_platform)
  1076. emit('SWIFT_FLAGS_PLATFORM', self.swift_flags_platform)
  1077. emit('SWIFT_LD_FLAGS', '-L{}'.format(self.swift_lib_path) if self.swift_lib_path else '')
  1078. emit('PERL_SDK', preset('OS_SDK') or self.tc.os_sdk)
  1079. if preset('OS_SDK') is None:
  1080. emit('OS_SDK', self.tc.os_sdk)
  1081. emit('OS_SDK_ROOT', None if self.tc.os_sdk_local else self.default_os_sdk_root)
  1082. class GnuCompiler(Compiler):
  1083. def __init__(self, tc, build):
  1084. """
  1085. :type tc: GnuToolchainOptions
  1086. :type build: Build
  1087. """
  1088. compiler_variable = 'CLANG' if tc.is_clang else 'GCC'
  1089. super(GnuCompiler, self).__init__(tc, compiler_variable)
  1090. self.build = build
  1091. self.host = self.build.host
  1092. self.target = self.build.target
  1093. self.tc = tc
  1094. self.c_foptions = [
  1095. # Enable C++ exceptions (and allow them to be throw through pure C code)
  1096. '-fexceptions',
  1097. # Enable standard-conforming behavior and generate duplicate symbol error in case of duplicated global constants.
  1098. # See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85678#c0
  1099. '-fno-common',
  1100. # Split all functions and data into separate sections for DCE and ICF linker passes
  1101. '-ffunction-sections',
  1102. '-fdata-sections'
  1103. ]
  1104. if self.tc.is_clang and self.target.is_linux:
  1105. # Use .init_array instead of .ctors (default for old clang versions)
  1106. # See: https://maskray.me/blog/2021-11-07-init-ctors-init-array
  1107. self.c_foptions.append('-fuse-init-array')
  1108. if self.target.is_xtensa:
  1109. self.c_foptions.append('-mlongcalls')
  1110. if self.tc.is_clang:
  1111. self.c_foptions += [
  1112. # Set up output colorization
  1113. '-fcolor-diagnostics',
  1114. # Enable aligned allocation
  1115. '-faligned-allocation',
  1116. '-fdebug-default-version=4',
  1117. ]
  1118. elif self.tc.is_gcc:
  1119. if self.target.is_xtensa or self.target.is_tc32:
  1120. # Xtensa and tc32 toolchains does not support this flag
  1121. pass
  1122. else:
  1123. self.c_foptions += [
  1124. # Set up output colorization
  1125. '-fdiagnostics-color=always',
  1126. # It looks like there is no way to enable aligned allocation in gcc
  1127. ]
  1128. self.c_warnings = [
  1129. # Enable default warnings subset
  1130. '-Wall',
  1131. '-Wextra',
  1132. ]
  1133. self.cxx_warnings = [
  1134. # Issue a warning if certain overload is hidden due to inheritance
  1135. '-Woverloaded-virtual',
  1136. ]
  1137. # Disable some warnings which will fail compilation at the time
  1138. self.c_warnings += [
  1139. '-Wno-parentheses',
  1140. ]
  1141. self.c_defines = ['${hide:CPP_FAKEID}']
  1142. if self.target.is_android:
  1143. self.c_defines.append('${hide:ANDROID_FAKEID}')
  1144. self.c_defines.extend([
  1145. '-DARCADIA_ROOT=${ARCADIA_ROOT}',
  1146. '-DARCADIA_BUILD_ROOT=${ARCADIA_BUILD_ROOT}',
  1147. ])
  1148. self.c_defines.extend([
  1149. '-D_THREAD_SAFE', '-D_PTHREADS', '-D_REENTRANT',
  1150. '-D_LARGEFILE_SOURCE', '-D__STDC_CONSTANT_MACROS', '-D__STDC_FORMAT_MACROS',
  1151. ])
  1152. if not self.target.is_android:
  1153. # There is no usable _FILE_OFFSET_BITS=64 support in Androids until API 21. And it's incomplete until at least API 24.
  1154. # https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md
  1155. # Arcadia have API 16 for 32-bit Androids.
  1156. self.c_defines.append('-D_FILE_OFFSET_BITS=64')
  1157. if self.target.is_linux or self.target.is_android or self.target.is_cygwin or self.target.is_none:
  1158. self.c_defines.append('-D_GNU_SOURCE')
  1159. if self.tc.is_clang and self.target.is_linux and self.target.is_x86_64:
  1160. self.c_defines.append('-D_YNDX_LIBUNWIND_ENABLE_EXCEPTION_BACKTRACE')
  1161. self.c_flags = ['$CL_DEBUG_INFO', '$CL_DEBUG_INFO_DISABLE_CACHE__NO_UID__']
  1162. self.c_flags += self.tc.arch_opt + ['-pipe']
  1163. self.sfdl_flags = ['-E', '-C', '-x', 'c++']
  1164. if self.target.is_x86:
  1165. self.c_flags.append('-m32')
  1166. if self.target.is_x86_64:
  1167. self.c_flags.append('-m64')
  1168. if self.target.is_wasm64:
  1169. # WebAssembly-specific exception handling flags
  1170. self.c_foptions += [
  1171. '-fwasm-exceptions',
  1172. ]
  1173. self.debug_info_flags = ['-g']
  1174. if self.target.is_linux:
  1175. self.debug_info_flags.append('-ggnu-pubnames')
  1176. self.cross_suffix = '' if is_positive('FORCE_NO_PIC') else '.pic'
  1177. self.optimize = None
  1178. self.configure_build_type()
  1179. if self.tc.is_clang:
  1180. self.sfdl_flags.append('-Qunused-arguments')
  1181. self.c_warnings += [
  1182. '-Wno-implicit-const-int-float-conversion',
  1183. # For nvcc to accept the above.
  1184. '-Wno-unknown-warning-option',
  1185. ]
  1186. self.cxx_warnings += [
  1187. '-Wimport-preprocessor-directive-pedantic',
  1188. '-Wno-ambiguous-reversed-operator',
  1189. '-Wno-defaulted-function-deleted',
  1190. '-Wno-deprecated-anon-enum-enum-conversion',
  1191. '-Wno-deprecated-enum-enum-conversion',
  1192. '-Wno-deprecated-enum-float-conversion',
  1193. '-Wno-deprecated-volatile',
  1194. '-Wno-pessimizing-move',
  1195. '-Wno-return-std-move',
  1196. '-Wno-undefined-var-template',
  1197. ]
  1198. elif self.tc.is_gcc:
  1199. if self.target.is_xtensa:
  1200. # Xtensa toolchain does not support this flags
  1201. pass
  1202. else:
  1203. self.c_foptions.append('-fno-delete-null-pointer-checks')
  1204. self.c_foptions.append('-fabi-version=8')
  1205. def configure_build_type(self):
  1206. if self.build.is_valgrind:
  1207. self.c_defines.append('-DWITH_VALGRIND=1')
  1208. if self.build.is_debug:
  1209. self.c_foptions.append('$FSTACK')
  1210. if self.build.is_fast_debug:
  1211. self.c_flags.append('-Og')
  1212. if self.build.is_release:
  1213. self.c_flags.append('$OPTIMIZE')
  1214. if self.build.is_size_optimized:
  1215. # -Oz is clang's more size-aggressive version of -Os
  1216. # For ARM specifically, clang -Oz is on par with gcc -Os:
  1217. # https://github.com/android/ndk/issues/133#issuecomment-365763507
  1218. if self.tc.is_clang:
  1219. self.optimize = '-Oz'
  1220. else:
  1221. self.optimize = '-Os'
  1222. # Generate sections with address significance tables for ICF linker pass
  1223. if self.tc.is_clang:
  1224. self.c_foptions.extend(['-faddrsig'])
  1225. else:
  1226. self.optimize = '-O3'
  1227. if self.build.with_ndebug:
  1228. self.c_defines.append('-DNDEBUG')
  1229. else:
  1230. self.c_defines.append('-UNDEBUG')
  1231. if self.build.profiler_type in (Profiler.Generic, Profiler.GProf):
  1232. self.c_foptions.append('-fno-omit-frame-pointer')
  1233. if self.build.profiler_type == Profiler.GProf:
  1234. self.c_flags.append('-pg')
  1235. def print_compiler(self):
  1236. super(GnuCompiler, self).print_compiler()
  1237. emit('C_COMPILER', '"{}"'.format(self.tc.c_compiler))
  1238. emit('C_COMPILER_OLD_UNQUOTED', self.tc.c_compiler)
  1239. emit('C_COMPILER_OLD', '${quo:C_COMPILER_OLD_UNQUOTED}')
  1240. emit('OPTIMIZE', self.optimize)
  1241. emit('WERROR_MODE', self.tc.werror_mode)
  1242. emit('_C_FLAGS', self.c_flags)
  1243. emit('_C_FOPTIONS', self.c_foptions)
  1244. emit('_STD_CXX_VERSION', preset('USER_STD_CXX_VERSION') or self.tc.cxx_std)
  1245. append('C_DEFINES', self.c_defines)
  1246. append('C_WARNING_OPTS', self.c_warnings)
  1247. append('CXX_WARNING_OPTS', self.cxx_warnings)
  1248. emit('CXX_COMPILER', '"{}"'.format(self.tc.cxx_compiler))
  1249. emit('CXX_COMPILER_OLD_UNQUOTED', format(self.tc.cxx_compiler))
  1250. emit('CXX_COMPILER_OLD', '${quo:CXX_COMPILER_OLD_UNQUOTED}')
  1251. # TODO(somov): Убрать чтение настройки из os.environ
  1252. emit('USE_ARC_PROFILE', 'yes' if preset('USE_ARC_PROFILE') or os.environ.get('USE_ARC_PROFILE') else 'no')
  1253. emit('DEBUG_INFO_FLAGS', self.debug_info_flags)
  1254. if self.build.is_coverage:
  1255. emit('_IS_COVERAGE', 'yes')
  1256. if self.tc.is_clang and self.tc.version_at_least(9):
  1257. emit('_HAS_TIME_TRACE', 'yes')
  1258. print('@import "${CONF_ROOT}/conf/compilers/gnu_compiler.conf"')
  1259. class SwiftCompiler(object):
  1260. def __init__(self, build):
  1261. self.host = build.host
  1262. self.compiler = None
  1263. def configure(self):
  1264. if self.host.is_macos:
  1265. self.compiler = '$SWIFT_XCODE_TOOLCHAIN_ROOT_RESOURCE_GLOBAL/usr/bin/swiftc'
  1266. def print_compiler(self):
  1267. emit('SWIFT_COMPILER', self.compiler or '')
  1268. class Linker(object):
  1269. BFD = 'bfd'
  1270. LLD = 'lld'
  1271. GOLD = 'gold'
  1272. def __init__(self, tc, build):
  1273. """
  1274. :type tc: ToolchainOptions
  1275. :type build: Build
  1276. """
  1277. self.tc = tc
  1278. self.build = build
  1279. self.type = self._get_default_linker_type()
  1280. def _get_default_linker_type(self):
  1281. if not self.tc.is_from_arcadia or is_positive('EXPORT_CMAKE'):
  1282. # External (e.g. system) toolchain: disable linker selection logic
  1283. return None
  1284. if self.build.target.is_android:
  1285. # Android toolchain is NDK, LLD works on all supported platforms
  1286. return Linker.LLD
  1287. elif self.build.target.is_linux or self.build.target.is_macos or self.build.target.is_ios or self.build.target.is_wasm:
  1288. return Linker.LLD
  1289. # There is no linker choice on Windows (link.exe)
  1290. return None
  1291. def print_linker(self):
  1292. self._print_linker_selector()
  1293. def _print_linker_selector(self):
  1294. # if self.type is None then _DEFAULT_LINKER is set to empty string value
  1295. emit('_DEFAULT_LINKER_ID', self.type)
  1296. class LD(Linker):
  1297. def __init__(self, tc, build):
  1298. """
  1299. :type tc: GnuToolchainOptions
  1300. :type build: Build
  1301. """
  1302. super(LD, self).__init__(tc, build)
  1303. self.build = build
  1304. self.host = self.build.host
  1305. self.target = self.build.target
  1306. self.tc = tc
  1307. target = self.target
  1308. self.ar = preset('AR') or self.tc.ar
  1309. self.ar_plugin = self.tc.ar_plugin
  1310. self.strip = self.tc.strip
  1311. self.objcopy = self.tc.objcopy
  1312. self.objdump = self.tc.objdump
  1313. self.musl = Setting('MUSL', convert=to_bool)
  1314. if self.ar is None:
  1315. if self.tc.is_from_arcadia:
  1316. if self.tc.is_clang:
  1317. self.ar = '{}/bin/llvm-ar'.format(self.tc.name_marker)
  1318. if self.tc.is_gcc:
  1319. self.ar = '{}/gcc/bin/gcc-ar'.format(self.tc.name_marker)
  1320. else:
  1321. self.ar = 'ar'
  1322. self.ar_type = 'GNU_AR'
  1323. self.llvm_ar_format = 'None'
  1324. if 'libtool' in self.ar:
  1325. self.ar_type = 'LIBTOOL'
  1326. elif 'llvm-ar' in self.ar:
  1327. self.ar_type = 'LLVM_AR'
  1328. if target.is_apple:
  1329. self.llvm_ar_format = "darwin"
  1330. else:
  1331. self.llvm_ar_format = "gnu"
  1332. self.ld_flags = []
  1333. if self.musl.value:
  1334. self.ld_flags.extend(['-Wl,--no-as-needed'])
  1335. elif target.is_linux:
  1336. self.ld_flags.extend(['-ldl', '-lrt', '-Wl,--no-as-needed'])
  1337. if self.tc.is_gcc:
  1338. self.ld_flags.extend(('-Wl,-Bstatic', '-latomic', '-Wl,-Bdynamic'))
  1339. elif target.is_android:
  1340. self.ld_flags.extend(['-ldl', '-Wl,--no-as-needed'])
  1341. self.ld_sdk = select(default=None, selectors=[
  1342. # -platform_version <platform> <min_version> <sdk_version>
  1343. (target.is_macos, '-Wl,-platform_version,macos,{MACOS_VERSION_MIN},11.0'.format(MACOS_VERSION_MIN=MACOS_VERSION_MIN)),
  1344. (not target.is_iossim and target.is_ios, '-Wl,-platform_version,ios,{IOS_VERSION_MIN},13.1'.format(IOS_VERSION_MIN=IOS_VERSION_MIN)),
  1345. (target.is_iossim, '-Wl,-platform_version,ios-simulator,{IOS_VERSION_MIN},14.5'.format(IOS_VERSION_MIN=IOS_VERSION_MIN)),
  1346. ])
  1347. if self.build.profiler_type == Profiler.GProf:
  1348. self.ld_flags.append('-pg')
  1349. if self.ld_sdk:
  1350. self.ld_flags.append(self.ld_sdk)
  1351. self.sys_lib = self.tc.sys_lib
  1352. if target.is_android:
  1353. # Use toolchain defaults to link with libunwind/clang_rt.builtins
  1354. self.use_stdlib = '-nostdlib++'
  1355. def print_linker(self):
  1356. super(LD, self).print_linker()
  1357. emit('AR_TOOL', self.ar)
  1358. emit('AR_TYPE', self.ar_type)
  1359. emit('_LD_LLVM_AR_FORMAT', self.llvm_ar_format)
  1360. emit('_LD_AR_PLUGIN', self.ar_plugin or 'None')
  1361. emit('STRIP_TOOL_VENDOR', self.strip)
  1362. emit('OBJCOPY_TOOL_VENDOR', self.objcopy)
  1363. emit('OBJDUMP_TOOL_VENDOR', self.objdump)
  1364. emit('_LD_FLAGS', self.ld_flags)
  1365. emit('_LD_SYS_LIB', self.sys_lib)
  1366. emit('LD_SDK_VERSION', self.ld_sdk)
  1367. dwarf_tool = self.tc.dwarf_tool
  1368. if dwarf_tool is None and self.tc.is_clang and (self.target.is_macos or self.target.is_ios):
  1369. dsymutil = '{}/bin/{}dsymutil'.format(self.tc.name_marker, '' if self.tc.version_at_least(7) else 'llvm-')
  1370. dwarf_tool = '${YMAKE_PYTHON} ${input:"build/scripts/run_llvm_dsymutil.py"} ' + dsymutil
  1371. if self.tc.version_at_least(5, 0):
  1372. dwarf_tool += ' -flat'
  1373. if dwarf_tool is not None:
  1374. emit('DWARF_TOOL', dwarf_tool)
  1375. arch_flag = '--arch={arch}'.format(arch=self.target.os_compat)
  1376. emit('_LD_ARCH_FLAG', arch_flag)
  1377. print('@import "${CONF_ROOT}/conf/linkers/ld.conf"')
  1378. class MSVCToolchainOptions(ToolchainOptions):
  1379. def __init__(self, build, detector):
  1380. super(MSVCToolchainOptions, self).__init__(build, detector)
  1381. # C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.14.26428
  1382. self.vc_root = None
  1383. # C:\Program Files (x86)\Windows Kits\10\Include\10.0.14393.0
  1384. self.kit_includes = None
  1385. # C:\Program Files (x86)\Windows Kits\10\Lib\10.0.14393.0
  1386. self.kit_libs = None
  1387. self.under_wine_compiler = self.params.get('wine', False)
  1388. self.under_wine_tools = not build.host.is_windows
  1389. self.under_wine_link = self.under_wine_tools
  1390. self.under_wine_lib = self.under_wine_tools
  1391. self.system_msvc = self.params.get('system_msvc', False)
  1392. self.ide_msvs = self.params.get('ide_msvs', False)
  1393. self.use_clang = self.params.get('use_clang', False)
  1394. self.use_msvc_linker = is_positive('USE_MSVC_LINKER')
  1395. self.use_arcadia_toolchain = self.params.get('use_arcadia_toolchain', False)
  1396. self.sdk_version = None
  1397. if self.ide_msvs:
  1398. bindir = '$(VC_ExecutablePath_x64_x64)\\'
  1399. self.c_compiler = bindir + 'cl.exe'
  1400. self.cxx_compiler = self.c_compiler
  1401. self.link = bindir + 'link.exe'
  1402. self.lib = bindir + 'lib.exe'
  1403. self.masm_compiler = bindir + 'ml64.exe'
  1404. self.vc_root = None
  1405. sdk_dir = '$(WindowsSdkDir)'
  1406. self.sdk_version = '$(WindowsTargetPlatformVersion)'
  1407. self.kit_includes = win_path_fix(os.path.join(sdk_dir, 'Include', self.sdk_version))
  1408. self.kit_libs = win_path_fix(os.path.join(sdk_dir, 'Lib', self.sdk_version))
  1409. elif detector:
  1410. self.use_clang = is_positive('USE_CLANG_CL')
  1411. self.masm_compiler = which('ml64.exe')
  1412. self.link = which('link.exe')
  1413. self.lib = which('lib.exe')
  1414. sdk_dir = os.environ.get('WindowsSdkDir')
  1415. self.sdk_version = os.environ.get('WindowsSDKVersion').replace('\\', '')
  1416. vc_install_dir = os.environ.get('VCToolsInstallDir')
  1417. if any([x is None for x in (sdk_dir, self.sdk_version, vc_install_dir)]):
  1418. raise ConfigureError('No %WindowsSdkDir%, %WindowsSDKVersion% or %VCINSTALLDIR% present. Please, run vcvars64.bat to setup preferred environment.')
  1419. self.vc_root = os.path.normpath(vc_install_dir)
  1420. self.kit_includes = win_path_fix(os.path.normpath(os.path.join(sdk_dir, 'Include', self.sdk_version)))
  1421. self.kit_libs = win_path_fix(os.path.normpath(os.path.join(sdk_dir, 'Lib', self.sdk_version)))
  1422. # TODO(somov): Определять автоматически self.version в этом случае
  1423. else:
  1424. if self.version_at_least(2019):
  1425. self.sdk_version = '10.0.18362.0'
  1426. if is_positive('MSVC20'): # XXX: temporary flag, remove after DTCC-123 is completed
  1427. self.cxx_std = 'c++latest'
  1428. else:
  1429. self.sdk_version = '10.0.16299.0'
  1430. sdk_dir = '$WINDOWS_KITS_RESOURCE_GLOBAL'
  1431. self.vc_root = self.name_marker if not self.use_clang else '$MSVC_FOR_CLANG_RESOURCE_GLOBAL'
  1432. self.kit_includes = win_path_fix(os.path.join(sdk_dir, 'Include', self.sdk_version))
  1433. self.kit_libs = win_path_fix(os.path.join(sdk_dir, 'Lib', self.sdk_version))
  1434. bindir = win_path_fix(os.path.join(self.vc_root, 'bin', 'Hostx64'))
  1435. tools_name = select(selectors=[
  1436. (build.target.is_x86, 'x86'),
  1437. (build.target.is_x86_64, 'x64'),
  1438. (build.target.is_armv7, 'arm'),
  1439. ])
  1440. asm_name = select(selectors=[
  1441. (build.target.is_x86, 'ml.exe'),
  1442. (build.target.is_x86_64, 'ml64.exe'),
  1443. (build.target.is_armv7, 'armasm.exe'),
  1444. ])
  1445. self.masm_compiler = win_path_fix(os.path.join(bindir, tools_name, asm_name))
  1446. if self.use_clang and not self.use_msvc_linker:
  1447. self.link = self.host.exe(self.name_marker, "bin", "lld-link")
  1448. self.under_wine_link = False
  1449. else:
  1450. self.link = win_path_fix(os.path.join(bindir, tools_name, 'link.exe'))
  1451. self.under_wine_link = self.under_wine_tools
  1452. if self.use_clang:
  1453. self.lib = self.host.exe(self.name_marker, "bin", "llvm-lib")
  1454. self.under_wine_lib = False
  1455. else:
  1456. self.lib = win_path_fix(os.path.join(bindir, tools_name, 'lib.exe'))
  1457. self.under_wine_lib = self.under_wine_tools
  1458. logger.debug("link {}; lib {}".format(self.link, self.lib))
  1459. class MSVC(object):
  1460. def __init__(self, tc, build):
  1461. """
  1462. :type tc: MSVCToolchainOptions
  1463. :type build: Build
  1464. """
  1465. if not isinstance(tc, MSVCToolchainOptions):
  1466. raise TypeError('Got {} ({}) instead of an MSVCToolchainOptions'.format(tc, type(tc)))
  1467. self.build = build
  1468. self.tc = tc
  1469. class MSVCToolchain(MSVC, Toolchain):
  1470. def __init__(self, tc, build):
  1471. """
  1472. :type tc: MSVCToolchainOptions
  1473. :param build: Build
  1474. """
  1475. Toolchain.__init__(self, tc, build)
  1476. MSVC.__init__(self, tc, build)
  1477. if self.tc.from_arcadia and not self.tc.ide_msvs:
  1478. self.platform_projects.append('build/internal/platform/msvc')
  1479. if tc.under_wine_compiler or tc.under_wine_tools:
  1480. self.platform_projects.append('build/platform/wine')
  1481. def print_toolchain(self):
  1482. super(MSVCToolchain, self).print_toolchain()
  1483. emit('TOOLCHAIN_ENV', format_env(self.tc.get_env(), list_separator=';'))
  1484. if self.tc.sdk_version:
  1485. emit('WINDOWS_KITS_VERSION', self.tc.sdk_version)
  1486. if self.tc.under_wine_tools:
  1487. emit('_UNDER_WINE_TOOLS', 'yes')
  1488. if self.tc.under_wine_link:
  1489. emit('_UNDER_WINE_LINK', 'yes')
  1490. if self.tc.under_wine_lib:
  1491. emit('_UNDER_WINE_LIB', 'yes')
  1492. if self.tc.under_wine_compiler:
  1493. emit('_UNDER_WINE_COMPILER', 'yes')
  1494. if self.tc.use_clang:
  1495. emit('CLANG_CL', 'yes')
  1496. if self.tc.ide_msvs:
  1497. emit('IDE_MSVS', 'yes')
  1498. if self.tc.use_arcadia_toolchain:
  1499. emit('USE_ARCADIA_TOOLCHAIN', 'yes')
  1500. emit('_MSVC_TC_KIT_LIBS', self.tc.kit_libs)
  1501. emit('_MSVC_TC_VC_ROOT', self.tc.vc_root)
  1502. print('@import "${CONF_ROOT}/conf/toolchains/msvc_toolchain.conf"')
  1503. class MSVCCompiler(MSVC, Compiler):
  1504. def __init__(self, tc, build):
  1505. Compiler.__init__(self, tc, 'MSVC')
  1506. MSVC.__init__(self, tc, build)
  1507. def print_compiler(self):
  1508. super(MSVCCompiler, self).print_compiler()
  1509. target = self.build.target
  1510. warns_enabled = [
  1511. 4018, # 'expression' : signed/unsigned mismatch
  1512. 4265, # 'class' : class has virtual functions, but destructor is not virtual
  1513. 4296, # 'operator' : expression is always false
  1514. 4431, # missing type specifier - int assumed
  1515. ]
  1516. warns_as_error = [
  1517. 4013, # 'function' undefined; assuming extern returning int
  1518. ]
  1519. warns_disabled = [
  1520. # While this warning corresponds to enabled-by-default -Wmacro-redefinition,
  1521. # it floods clog with abundant amount of log lines,
  1522. # as yvals_core.h from Windows SDK redefines certain
  1523. # which macros logically belong to libcxx
  1524. 4005, # '__cpp_lib_*': macro redefinition.
  1525. # Ne need to recheck this, but it looks like _CRT_USE_BUILTIN_OFFSETOF still makes sense
  1526. 4117, # macro name '_CRT_USE_BUILTIN_OFFSETOF' is reserved, '#define' ignored
  1527. 4127, # conditional expression is constant
  1528. 4200, # nonstandard extension used : zero-sized array in struct/union
  1529. 4201, # nonstandard extension used : nameless struct/union
  1530. 4351, # elements of array will be default initialized
  1531. 4355, # 'this' : used in base member initializer list
  1532. 4503, # decorated name length exceeded, name was truncated
  1533. 4510, # default constructor could not be generated
  1534. 4511, # copy constructor could not be generated
  1535. 4512, # assignment operator could not be generated
  1536. 4554, # check operator precedence for possible error; use parentheses to clarify precedence
  1537. 4610, # 'object' can never be instantiated - user defined constructor required
  1538. 4706, # assignment within conditional expression
  1539. 4800, # forcing value to bool 'true' or 'false' (performance warning)
  1540. 4996, # The POSIX name for this item is deprecated
  1541. 4714, # function marked as __forceinline not inlined
  1542. 4197, # 'TAtomic' : top-level volatile in cast is ignored
  1543. 4245, # 'initializing' : conversion from 'int' to 'ui32', signed/unsigned mismatch
  1544. 4324, # 'ystd::function<void (uint8_t *)>': structure was padded due to alignment specifier
  1545. 5033, # 'register' is no longer a supported storage class
  1546. ]
  1547. defines = [
  1548. '${hide:CPP_FAKEID}',
  1549. # FIXME: This is quick fix to let catboost build from MSVS IDE
  1550. # This place is questionable overall, see YMAKE-437
  1551. '/DARCADIA_ROOT=' + ('${ARCADIA_ROOT}' if not self.tc.ide_msvs else '.'),
  1552. '/DARCADIA_BUILD_ROOT=' + ('${ARCADIA_BUILD_ROOT}' if not self.tc.ide_msvs else '.'),
  1553. '/DWIN32',
  1554. '/D_WIN32',
  1555. '/D_WINDOWS',
  1556. # Define _CRT_*_NO_WARNINGS macros to prevent ucrt from issuing a warning whenever
  1557. # a POSIX-style function is used instead of the alternative Microsoft suggests as a secure / standard replacement
  1558. # (e. g. `strncpy()` instead of `strncpy_s()`, `access()` instead of `_access()`)
  1559. # For details see:
  1560. # https://docs.microsoft.com/en-us/cpp/c-runtime-library/security-features-in-the-crt
  1561. '/D_CRT_SECURE_NO_WARNINGS',
  1562. '/D_CRT_NONSTDC_NO_WARNINGS',
  1563. # Math constants (such as M_PI, M_E, M_SQRT2) are not defined in standard C / C++
  1564. # In order to get them defined by Windows ucrt library,
  1565. # you must first define _USE_MATH_DEFINES before #including <cmath> or math.h>.
  1566. # (NB: glibc defines these macros whenever _XOPEN_SOURCE is defined)
  1567. '/D_USE_MATH_DEFINES',
  1568. '/D__STDC_CONSTANT_MACROS',
  1569. '/D__STDC_FORMAT_MACROS',
  1570. '/D_USING_V110_SDK71_',
  1571. # Below defines are covered at
  1572. # https://docs.microsoft.com/en-us/windows/win32/winprog/using-the-windows-headers#faster-builds-with-smaller-header-files
  1573. # Exclude APIs such as Cryptography, DDE, RPC, Shell, and Windows Sockets (while including <windows.h>)
  1574. '/DWIN32_LEAN_AND_MEAN',
  1575. # Define NOMINMAX to avoid min() and max() macros definition (while including <windows.h>)
  1576. '/DNOMINMAX',
  1577. ]
  1578. cxx_defines = [
  1579. # Use builtin offsetof implementation
  1580. # instead of a crutcy macro defined in ucrt/stddef.h.
  1581. # The latter can not be used in constexpr statements.
  1582. '/D_CRT_USE_BUILTIN_OFFSETOF',
  1583. ]
  1584. if target.is_x86_64:
  1585. defines.extend(('/D_WIN64', '/DWIN64'))
  1586. if target.is_armv7:
  1587. defines.extend(('/D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE', '/D__arm__'))
  1588. winapi_unicode = False
  1589. flags = [
  1590. '/nologo', '/Zm500', '/GR', '/bigobj', '/FC', '/EHs', '/errorReport:prompt', '$MSVC_INLINE_FLAG', '/utf-8',
  1591. # enable standard conforming mode
  1592. '/permissive-'
  1593. ]
  1594. flags += self.tc.arch_opt
  1595. c_warnings = ['/we{}'.format(code) for code in warns_as_error]
  1596. c_warnings += ['/w1{}'.format(code) for code in warns_enabled]
  1597. c_warnings += ['/wd{}'.format(code) for code in warns_disabled]
  1598. cxx_warnings = []
  1599. if self.tc.use_clang:
  1600. if not self.tc.is_system_cxx:
  1601. flags += [
  1602. # Allow <windows.h> to be included via <Windows.h> in case-sensitive file-systems.
  1603. '-fcase-insensitive-paths',
  1604. ]
  1605. flags += [
  1606. # At the time clang-cl identifies itself as MSVC 19.11:
  1607. # (actual value can be found in clang/lib/Driver/ToolChains/MSVC.cpp, the syntax would be like
  1608. # ```
  1609. # MSVT = VersionTuple(19, 11);
  1610. # ```
  1611. #
  1612. # We override this value to match current value of the actual MSVC being used.
  1613. '-fms-compatibility-version=19.21',
  1614. # for msvc compatibility
  1615. # https://clang.llvm.org/docs/UsersManual.html#microsoft-extensions
  1616. '-fdelayed-template-parsing',
  1617. ]
  1618. if target.is_x86:
  1619. flags.append('-m32')
  1620. elif target.is_x86_64:
  1621. flags.append('-m64')
  1622. c_warnings.extend((
  1623. '-Wno-format',
  1624. '-Wno-parentheses',
  1625. '-Wno-unknown-warning-option',
  1626. ))
  1627. cxx_warnings += [
  1628. '-Wimport-preprocessor-directive-pedantic',
  1629. '-Woverloaded-virtual',
  1630. '-Wno-ambiguous-reversed-operator',
  1631. '-Wno-defaulted-function-deleted',
  1632. '-Wno-deprecated-anon-enum-enum-conversion',
  1633. '-Wno-deprecated-enum-enum-conversion',
  1634. '-Wno-deprecated-enum-float-conversion',
  1635. '-Wno-deprecated-volatile',
  1636. '-Wno-undefined-var-template',
  1637. ]
  1638. if self.tc.ide_msvs:
  1639. cxx_warnings += [
  1640. '-Wno-unused-command-line-argument',
  1641. ]
  1642. defines.append('/D_WIN32_WINNT={0}'.format(WINDOWS_VERSION_MIN))
  1643. if winapi_unicode:
  1644. defines += ['/DUNICODE', '/D_UNICODE']
  1645. else:
  1646. defines += ['/D_MBCS']
  1647. vc_include = win_path_fix(os.path.join(self.tc.vc_root, 'include')) if not self.tc.ide_msvs else "$(VC_VC_IncludePath.Split(';')[0].Replace('\\','/'))"
  1648. if not self.tc.ide_msvs:
  1649. def include_flag(path):
  1650. return '{flag}"{path}"'.format(path=path, flag='/I ' if not self.tc.use_clang else '-imsvc')
  1651. for name in ('shared', 'ucrt', 'um', 'winrt'):
  1652. flags.append(include_flag(win_path_fix(os.path.join(self.tc.kit_includes, name))))
  1653. flags.append(include_flag(vc_include))
  1654. if self.tc.use_clang:
  1655. emit('CLANG_CL', 'yes')
  1656. if self.tc.ide_msvs:
  1657. emit('IDE_MSVS', 'yes')
  1658. if self.tc.use_arcadia_toolchain:
  1659. emit('USE_ARCADIA_TOOLCHAIN', 'yes')
  1660. emit('CXX_COMPILER_UNQUOTED', '"{}"'.format(self.tc.cxx_compiler))
  1661. emit('CXX_COMPILER_OLD_UNQUOTED', self.tc.cxx_compiler)
  1662. emit('C_COMPILER_UNQUOTED', '"{}"'.format(self.tc.c_compiler))
  1663. emit('C_COMPILER_OLD_UNQUOTED', self.tc.c_compiler)
  1664. emit('MASM_COMPILER_UNQUOTED', '"{}"'.format(self.tc.masm_compiler))
  1665. emit('MASM_COMPILER_OLD_UNQUOTED', self.tc.masm_compiler)
  1666. append('C_DEFINES', defines)
  1667. append('C_WARNING_OPTS', c_warnings)
  1668. emit('_CXX_DEFINES', cxx_defines)
  1669. append('CXX_WARNING_OPTS', cxx_warnings)
  1670. if self.build.is_release:
  1671. emit('CFLAGS_PER_TYPE', '$CFLAGS_RELEASE')
  1672. if self.build.is_debug:
  1673. emit('CFLAGS_PER_TYPE', '$CFLAGS_DEBUG')
  1674. if self.build.is_ide:
  1675. emit('CFLAGS_PER_TYPE', '@[debug|$CFLAGS_DEBUG]@[release|$CFLAGS_RELEASE]')
  1676. emit('_STD_CXX_VERSION', preset('USER_STD_CXX_VERSION') or self.tc.cxx_std)
  1677. emit('_MSVC_FLAGS', flags)
  1678. ucrt_include = win_path_fix(os.path.join(self.tc.kit_includes, 'ucrt')) if not self.tc.ide_msvs else "$(UniversalCRT_IncludePath.Split(';')[0].Replace('\\','/'))"
  1679. # clang-cl has '#include_next', and MSVC hasn't. It needs separately specified CRT and VC include directories for libc++ to include second in order standard C and C++ headers.
  1680. if not self.tc.use_clang:
  1681. emit('_CFLAGS_UCRT_VC_INCLUDES', '/DY_UCRT_INCLUDE="%s"' % ucrt_include, '/DY_MSVC_INCLUDE="%s"' % vc_include)
  1682. else:
  1683. emit('_CFLAGS_UCRT_VC_INCLUDES')
  1684. emit('WERROR_MODE', self.tc.werror_mode)
  1685. print('@import "${CONF_ROOT}/conf/compilers/msvc_compiler.conf"')
  1686. class MSVCLinker(MSVC, Linker):
  1687. def __init__(self, tc, build):
  1688. MSVC.__init__(self, tc, build)
  1689. Linker.__init__(self, tc, build)
  1690. def print_linker(self):
  1691. super(MSVCLinker, self).print_linker()
  1692. linker = self.tc.link
  1693. linker_lib = self.tc.lib
  1694. emit('_MSVC_LIB', '"{}"'.format(linker_lib))
  1695. emit('_MSVC_LIB_OLD_UNQUOTED', linker_lib)
  1696. emit('_MSVC_LIB_OLD', '${quo:_MSVC_LIB_OLD_UNQUOTED}')
  1697. emit('_MSVC_LINK', '"{}"'.format(linker))
  1698. emit('_MSVC_LINK_OLD_UNQUOTED', linker)
  1699. emit('_MSVC_LINK_OLD', '${quo:_MSVC_LINK_OLD_UNQUOTED}')
  1700. if self.build.is_release:
  1701. emit('LINK_EXE_FLAGS_PER_TYPE', '$LINK_EXE_FLAGS_RELEASE')
  1702. if self.build.is_debug:
  1703. emit('LINK_EXE_FLAGS_PER_TYPE', '$LINK_EXE_FLAGS_DEBUG')
  1704. if self.build.is_ide and self.tc.ide_msvs:
  1705. emit('LINK_EXE_FLAGS_PER_TYPE', '@[debug|$LINK_EXE_FLAGS_DEBUG]@[release|$LINK_EXE_FLAGS_RELEASE]')
  1706. print('@import "${CONF_ROOT}/conf/linkers/msvc_linker.conf"')
  1707. # TODO(somov): Rename!
  1708. Compilers = {
  1709. 'gnu': (GnuToolchain, GnuCompiler, LD),
  1710. 'clang': (GnuToolchain, GnuCompiler, LD),
  1711. 'xcode': (GnuToolchain, GnuCompiler, LD),
  1712. 'msvc': (MSVCToolchain, MSVCCompiler, MSVCLinker),
  1713. }
  1714. class Ragel(object):
  1715. def __init__(self):
  1716. self.rlgen_flags = []
  1717. self.ragel_flags = []
  1718. self.ragel6_flags = []
  1719. def configure_toolchain(self, build, compiler):
  1720. if isinstance(compiler, MSVCCompiler):
  1721. self.set_default_flags(optimized=False)
  1722. elif isinstance(compiler, GnuCompiler):
  1723. self.set_default_flags(optimized=build.is_release and not build.is_sanitized)
  1724. else:
  1725. raise ConfigureError('Unexpected compiler {}'.format(compiler))
  1726. def set_default_flags(self, optimized):
  1727. if optimized:
  1728. self.rlgen_flags.append('-G2')
  1729. self.ragel6_flags.append('-CG2')
  1730. else:
  1731. self.rlgen_flags.append('-T0')
  1732. self.ragel6_flags.append('-CT0')
  1733. def print_variables(self):
  1734. emit('RLGEN_FLAGS', self.rlgen_flags)
  1735. emit('RAGEL_FLAGS', self.ragel_flags)
  1736. emit('RAGEL6_FLAGS', self.ragel6_flags)
  1737. class Python(object):
  1738. def __init__(self, tc):
  1739. self.python = None
  1740. self.flags = None
  1741. self.ldflags = None
  1742. self.libraries = None
  1743. self.includes = None
  1744. self.tc = tc
  1745. def configure_posix(self, python=None, python_config=None):
  1746. self.check_configuration()
  1747. python = python or preset('PYTHON_BIN') or which('python')
  1748. python_config = python_config or preset('PYTHON_CONFIG') or which('python-config')
  1749. if python is None or python_config is None:
  1750. return
  1751. # python-config dumps each option on one line in the specified order
  1752. config = get_stdout([python_config, '--cflags', '--ldflags', '--includes']) or ''
  1753. config = config.split('\n')
  1754. if len(config) < 3:
  1755. return
  1756. self.python = python
  1757. self.flags = config[0]
  1758. self.ldflags = config[1]
  1759. self.includes = config[2]
  1760. # Do not split libraries from ldflags.
  1761. # They are not used separately and get overriden together, so it is safe.
  1762. # TODO(somov): Удалить эту переменную и PYTHON_LIBRARIES из makelist-ов.
  1763. self.libraries = ''
  1764. def check_configuration(self):
  1765. if preset('USE_ARCADIA_PYTHON') == 'no':
  1766. # Set USE_LOCAL_PYTHON to enable configuration
  1767. # when we still using fixed OS SDK,
  1768. # but at the same time using Python from the local system.
  1769. #
  1770. # This configuration is not guaranteed to work,
  1771. # for example, if local and fixed OS SDKs differ much.
  1772. if preset('USE_LOCAL_PYTHON') == 'yes':
  1773. return
  1774. if not preset('USE_SYSTEM_PYTHON') and not self.tc.os_sdk_local:
  1775. raise Exception("Use fixed python (see https://clubs.at.yandex-team.ru/arcadia/15392) or set OS_SDK=local flag")
  1776. def print_variables(self):
  1777. variables = Variables({
  1778. 'PYTHON_BIN': self.python,
  1779. 'PYTHON_FLAGS': self.flags,
  1780. 'PYTHON_LDFLAGS': self.ldflags,
  1781. 'PYTHON_LIBRARIES': self.libraries,
  1782. 'PYTHON_INCLUDE': self.includes
  1783. })
  1784. variables.update_from_presets()
  1785. variables.reset_if_any(reset_value='PYTHON-NOT-FOUND')
  1786. variables.emit()
  1787. class Setting(object):
  1788. def __init__(self, key, auto=None, convert=None, rewrite=False):
  1789. self.key = key
  1790. self.auto = auto
  1791. self.convert = convert
  1792. self.preset = preset(key)
  1793. self.from_user = self.preset is not None
  1794. self.rewrite = rewrite
  1795. self._value = Setting.no_value
  1796. @property
  1797. def value(self):
  1798. if self._value is Setting.no_value:
  1799. self._value = self.calculate_value()
  1800. return self._value
  1801. def calculate_value(self):
  1802. if not self.from_user:
  1803. return self.auto if not callable(self.auto) else self.auto()
  1804. else:
  1805. return self.preset if not self.convert else self.convert(self.preset)
  1806. @value.setter
  1807. def value(self, value):
  1808. if self.from_user:
  1809. raise ConfigureError("Variable {key} already set by user to {old}. Can not change it's value to {new}".format(key=self.key, old=self._value, new=value))
  1810. self._value = value
  1811. def emit(self):
  1812. if not self.from_user or self.rewrite:
  1813. emit(self.key, self.value)
  1814. no_value = object()
  1815. class Cuda(object):
  1816. def __init__(self, build):
  1817. """
  1818. :type build: Build
  1819. """
  1820. self.build = build
  1821. self.have_cuda = Setting('HAVE_CUDA', auto=self.auto_have_cuda, convert=to_bool)
  1822. self.cuda_root = Setting('CUDA_ROOT')
  1823. self.cuda_version = Setting('CUDA_VERSION', auto=self.auto_cuda_version, convert=self.convert_major_version, rewrite=True)
  1824. self.cuda_architectures = Setting('CUDA_ARCHITECTURES', auto=self.auto_cuda_architectures, rewrite=True)
  1825. self.use_arcadia_cuda = Setting('USE_ARCADIA_CUDA', auto=self.auto_use_arcadia_cuda, convert=to_bool)
  1826. self.use_arcadia_cuda_host_compiler = Setting('USE_ARCADIA_CUDA_HOST_COMPILER', auto=self.auto_use_arcadia_cuda_host_compiler, convert=to_bool)
  1827. self.cuda_use_clang = Setting('CUDA_USE_CLANG', auto=False, convert=to_bool)
  1828. self.cuda_host_compiler = Setting('CUDA_HOST_COMPILER', auto=self.auto_cuda_host_compiler)
  1829. self.cuda_host_compiler_env = Setting('CUDA_HOST_COMPILER_ENV')
  1830. self.cuda_host_msvc_version = Setting('CUDA_HOST_MSVC_VERSION')
  1831. self.cuda_nvcc_flags = Setting('CUDA_NVCC_FLAGS', auto=[])
  1832. self.peerdirs = ['build/platform/cuda']
  1833. self.nvcc_flags = [
  1834. # Compress fatbinary to reduce size of .nv_fatbin and prevent problems with linking
  1835. #
  1836. # Idea comes from many resources, one of them is https://discourse.llvm.org/t/lld-relocation-overflows-and-nv-fatbin/58889/6
  1837. # Some sources suggest using `-Xfatbin=-compress-all`, other suggest using `-Xcuda-fatbinary --compress-all`
  1838. # We will use the same flag as in nixpkgs
  1839. # (https://github.com/NixOS/nixpkgs/pull/220402/files#diff-a38e6c4e8421c03dc6c2a60c9a172ceb4059048b65798e5d4a400a7a4a5720ffR167)
  1840. "-Xfatbin=-compress-all",
  1841. # Allow __host__, __device__ annotations in lambda declaration.
  1842. "--expt-extended-lambda",
  1843. # Allow host code to invoke __device__ constexpr functions and vice versa
  1844. "--expt-relaxed-constexpr",
  1845. # Allow to use newer compilers than CUDA Toolkit officially supports
  1846. "--allow-unsupported-compiler",
  1847. ]
  1848. if not self.have_cuda.value:
  1849. return
  1850. if self.cuda_host_compiler.value:
  1851. self.nvcc_flags.append('--compiler-bindir=$CUDA_HOST_COMPILER')
  1852. if self.use_arcadia_cuda.value:
  1853. self.cuda_root.value = '$CUDA_RESOURCE_GLOBAL'
  1854. if self.build.target.is_linux_x86_64 and self.build.tc.is_clang:
  1855. # TODO(somov): Эта настройка должна приезжать сюда автоматически из другого места
  1856. self.nvcc_flags.append('-I$OS_SDK_ROOT/usr/include/x86_64-linux-gnu')
  1857. def print_(self):
  1858. self.print_variables()
  1859. self.print_macros()
  1860. print('@import "${CONF_ROOT}/conf/compilers/nvcc.conf"')
  1861. def print_variables(self):
  1862. self.have_cuda.emit()
  1863. if not self.have_cuda.value:
  1864. return
  1865. if self.use_arcadia_cuda.value and self.cuda_host_compiler.value is None:
  1866. logger.warning('$USE_ARCADIA_CUDA is set, but no $CUDA_HOST_COMPILER')
  1867. self.setup_vc_root()
  1868. self.cuda_root.emit()
  1869. self.cuda_version.emit()
  1870. self.cuda_architectures.emit()
  1871. self.use_arcadia_cuda.emit()
  1872. self.use_arcadia_cuda_host_compiler.emit()
  1873. self.cuda_use_clang.emit()
  1874. self.cuda_host_compiler.emit()
  1875. self.cuda_host_compiler_env.emit()
  1876. self.cuda_host_msvc_version.emit()
  1877. self.cuda_nvcc_flags.emit()
  1878. emit('NVCC', '"{}"'.format(self.build.host.exe('$CUDA_ROOT', 'bin', 'nvcc')))
  1879. emit('NVCC_OLD_UNQUOTED', self.build.host.exe('$CUDA_ROOT', 'bin', 'nvcc'))
  1880. emit('NVCC_OLD', '${quo:NVCC_OLD_UNQUOTED}')
  1881. emit('NVCC_FLAGS', self.nvcc_flags, '$CUDA_NVCC_FLAGS')
  1882. emit('NVCC_OBJ_EXT', '.o' if not self.build.target.is_windows else '.obj')
  1883. def print_macros(self):
  1884. mtime = ' '
  1885. if self.build.host_target[1].is_linux:
  1886. mtime = ' --mtime ${tool:"tools/mtime0"} '
  1887. if not self.cuda_use_clang.value:
  1888. cmd = '$YMAKE_PYTHON ${input:"build/scripts/compile_cuda.py"}' + mtime + '$NVCC_OLD $NVCC_STD $NVCC_FLAGS -c ${input:SRC} -o ${output;suf=${OBJ_SUF}${NVCC_OBJ_EXT}:SRC} ${pre=-I:_C__INCLUDE} --cflags $C_FLAGS_PLATFORM $CXXFLAGS $NVCC_STD $NVCC_CFLAGS $SRCFLAGS ${input;hide:"build/platform/cuda/cuda_runtime_include.h"} $CUDA_HOST_COMPILER_ENV ${kv;hide:"p CC"} ${kv;hide:"pc light-green"}' # noqa E501
  1889. else:
  1890. cmd = '$CXX_COMPILER_OLD --cuda-path=$CUDA_ROOT $C_FLAGS_PLATFORM -c ${input:SRC} -o ${output;suf=${OBJ_SUF}${NVCC_OBJ_EXT}:SRC} ${pre=-I:_C__INCLUDE} $CXXFLAGS $SRCFLAGS $TOOLCHAIN_ENV ${kv;hide:"p CU"} ${kv;hide:"pc green"}' # noqa E501
  1891. emit('_SRC_CU_CMD', cmd)
  1892. emit('_SRC_CU_PEERDIR', ' '.join(sorted(self.peerdirs)))
  1893. # This method is only used to set HAVE_CUDA=no automatically during ymake initialization
  1894. # (i.e. is used only in `auto_have_cuda` invocation)
  1895. def have_cuda_in_arcadia(self):
  1896. host, target = self.build.host_target
  1897. if not any((host.is_linux_x86_64, host.is_windows_x86_64)):
  1898. return False
  1899. if host != target:
  1900. if not (host.is_linux_x86_64 and target.is_linux_armv8):
  1901. return False
  1902. if not self.cuda_version.from_user:
  1903. return False
  1904. if self.cuda_version.value in ('11.4', '11.8', '12.1', '12.2'):
  1905. return True
  1906. else:
  1907. raise ConfigureError('CUDA version {} is not supported in Arcadia'.format(self.cuda_version.value))
  1908. def auto_have_cuda(self):
  1909. if is_positive('MUSL'):
  1910. return False
  1911. if self.build.is_sanitized:
  1912. return False
  1913. return self.cuda_root.from_user or self.use_arcadia_cuda.value and self.have_cuda_in_arcadia()
  1914. def auto_cuda_version(self):
  1915. if self.use_arcadia_cuda.value:
  1916. return '11.4'
  1917. if not self.have_cuda.value:
  1918. return None
  1919. nvcc_exe = self.build.host.exe(os.path.expanduser(self.cuda_root.value), 'bin', 'nvcc')
  1920. def error():
  1921. raise ConfigureError('Failed to get CUDA version from {}'.format(nvcc_exe))
  1922. version_output = get_stdout([nvcc_exe, '--version']) or error()
  1923. match = re.search(r'^Cuda compilation tools, release (\d+)\.\d+,', version_output, re.MULTILINE) or error()
  1924. return match.group(1)
  1925. def convert_major_version(self, value):
  1926. if value == '11':
  1927. return '11.4'
  1928. else:
  1929. return value
  1930. def auto_cuda_architectures(self):
  1931. # empty list does not mean "no architectures"
  1932. # it means "no restriction -- any available architecture"
  1933. host, target = self.build.host_target
  1934. if not target.is_linux_x86_64:
  1935. # do not impose any restrictions, when build not for "linux 64-bit"
  1936. return ''
  1937. # Equality to CUDA 11.4 is rather strict comparison
  1938. # TODO: find out how we can relax check (e.g. to include more version of CUDA toolkit)
  1939. if self.cuda_version.value == '11.4':
  1940. # * use output of CUDA 11.4 `nvcc --help`
  1941. # * drop support for '53', '62', '72' and '87'
  1942. # (these devices run only on arm64)
  1943. # * drop support for '37'
  1944. # the single place it's used in Arcadia is https://a.yandex-team.ru/arcadia/sdg/sdc/third_party/cub/common.mk?rev=r13268523#L69
  1945. return ':'.join(
  1946. ['sm_35',
  1947. 'sm_50', 'sm_52',
  1948. 'sm_60', 'sm_61',
  1949. 'sm_70', 'sm_75',
  1950. 'sm_80', 'sm_86',
  1951. 'compute_86'])
  1952. else:
  1953. return ''
  1954. def auto_use_arcadia_cuda(self):
  1955. return not self.cuda_root.from_user
  1956. def auto_use_arcadia_cuda_host_compiler(self):
  1957. return not self.cuda_host_compiler.from_user and not self.cuda_use_clang.value
  1958. def auto_cuda_host_compiler(self):
  1959. if not self.use_arcadia_cuda_host_compiler.value:
  1960. return None
  1961. host, target = self.build.host_target
  1962. if host.is_windows_x86_64 and target.is_windows_x86_64:
  1963. return self.cuda_windows_host_compiler()
  1964. return select((
  1965. (host.is_linux_x86_64 and target.is_linux_x86_64, '$CUDA_HOST_TOOLCHAIN_RESOURCE_GLOBAL/bin/clang'),
  1966. (host.is_linux_x86_64 and target.is_linux_armv8, '$CUDA_HOST_TOOLCHAIN_RESOURCE_GLOBAL/bin/clang'),
  1967. ))
  1968. def cuda_windows_host_compiler(self):
  1969. vc_version = '14.28.29910'
  1970. env = {
  1971. 'Y_VC_Version': vc_version,
  1972. 'Y_VC_Root': '$CUDA_HOST_TOOLCHAIN_RESOURCE_GLOBAL/VC/Tools/MSVC/{}'.format(vc_version),
  1973. 'Y_SDK_Version': self.build.tc.sdk_version,
  1974. 'Y_SDK_Root': '$WINDOWS_KITS_RESOURCE_GLOBAL',
  1975. }
  1976. if not self.build.tc.ide_msvs:
  1977. self.peerdirs.append('build/internal/platform/msvc')
  1978. self.cuda_host_compiler_env.value = format_env(env)
  1979. self.cuda_host_msvc_version.value = vc_version
  1980. return '%(Y_VC_Root)s/bin/HostX64/x64/cl.exe' % env
  1981. def setup_vc_root(self):
  1982. if not self.cuda_host_compiler.from_user:
  1983. return # Already set in cuda_windows_host_compiler()
  1984. if self.cuda_host_compiler_env.from_user:
  1985. return # We won't override user setting
  1986. def is_root(dir):
  1987. return all(os.path.isdir(os.path.join(dir, name)) for name in ('bin', 'include', 'lib'))
  1988. def get_root():
  1989. path, old_path = os.path.normpath(self.cuda_host_compiler.value), None
  1990. while path != old_path:
  1991. if is_root(path):
  1992. return path
  1993. path, old_path = os.path.dirname(path), path
  1994. vc_root = get_root()
  1995. if vc_root:
  1996. self.cuda_host_compiler_env.value = format_env({'Y_VC_Root': vc_root})
  1997. class CuDNN(object):
  1998. def __init__(self, cuda):
  1999. """
  2000. :type cuda: Cuda
  2001. """
  2002. self.cuda = cuda
  2003. self.cudnn_version = Setting('CUDNN_VERSION', auto=self.auto_cudnn_version)
  2004. def have_cudnn(self):
  2005. return self.cudnn_version.value in ('7.6.5', '8.0.5')
  2006. def auto_cudnn_version(self):
  2007. return '8.0.5'
  2008. def print_(self):
  2009. if self.cuda.have_cuda.value and self.have_cudnn():
  2010. self.cudnn_version.emit()
  2011. def customization():
  2012. if not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  2013. try:
  2014. sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'internal'))
  2015. from custom_conf import CustomConf
  2016. logger.debug('Customization extension was successfully loaded')
  2017. return CustomConf
  2018. except Exception as e:
  2019. logger.debug('Customization extension was not found; [{}]'.format(str(e)))
  2020. else:
  2021. logger.debug('Customization extension was disabled')
  2022. class DummyConf:
  2023. def __init__(self, options):
  2024. pass
  2025. def print_prologue(self):
  2026. pass
  2027. def print_epilogue(self):
  2028. pass
  2029. return DummyConf
  2030. def main():
  2031. options = opts()
  2032. CustomConfig = customization()
  2033. custom_conf = CustomConfig(options)
  2034. arcadia = Arcadia(options.arcadia_root)
  2035. ymake = YMake(arcadia)
  2036. ymake.print_core_conf()
  2037. _INTERNAL_CONF = 'internal/conf/internal.conf'
  2038. internal_conf_full_path = find_conf(_INTERNAL_CONF)
  2039. if internal_conf_full_path and not is_positive('DISABLE_YMAKE_CONF_CUSTOMIZATION'):
  2040. print('@import "${{CONF_ROOT}}/{}"'.format(_INTERNAL_CONF))
  2041. custom_conf.print_prologue()
  2042. ymake.print_presets()
  2043. ymake.print_settings()
  2044. build = Build(arcadia, options.build_type, options.toolchain_params, force_ignore_local_files=not options.local_distbuild)
  2045. build.print_build()
  2046. custom_conf.print_epilogue()
  2047. if __name__ == '__main__':
  2048. main()