constructor.py 69 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723
  1. # coding: utf-8
  2. import datetime
  3. import base64
  4. import binascii
  5. import sys
  6. import types
  7. import warnings
  8. from collections.abc import Hashable, MutableSequence, MutableMapping
  9. # fmt: off
  10. from ruamel.yaml.error import (MarkedYAMLError, MarkedYAMLFutureWarning,
  11. MantissaNoDotYAML1_1Warning)
  12. from ruamel.yaml.nodes import * # NOQA
  13. from ruamel.yaml.nodes import (SequenceNode, MappingNode, ScalarNode)
  14. from ruamel.yaml.compat import (builtins_module, # NOQA
  15. nprint, nprintf, version_tnf)
  16. from ruamel.yaml.compat import ordereddict
  17. from ruamel.yaml.tag import Tag
  18. from ruamel.yaml.comments import * # NOQA
  19. from ruamel.yaml.comments import (CommentedMap, CommentedOrderedMap, CommentedSet,
  20. CommentedKeySeq, CommentedSeq, TaggedScalar,
  21. CommentedKeyMap,
  22. C_KEY_PRE, C_KEY_EOL, C_KEY_POST,
  23. C_VALUE_PRE, C_VALUE_EOL, C_VALUE_POST,
  24. )
  25. from ruamel.yaml.scalarstring import (SingleQuotedScalarString, DoubleQuotedScalarString,
  26. LiteralScalarString, FoldedScalarString,
  27. PlainScalarString, ScalarString)
  28. from ruamel.yaml.scalarint import ScalarInt, BinaryInt, OctalInt, HexInt, HexCapsInt
  29. from ruamel.yaml.scalarfloat import ScalarFloat
  30. from ruamel.yaml.scalarbool import ScalarBoolean
  31. from ruamel.yaml.timestamp import TimeStamp
  32. from ruamel.yaml.util import timestamp_regexp, create_timestamp
  33. from typing import Any, Dict, List, Set, Iterator, Union, Optional # NOQA
  34. __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
  35. 'ConstructorError', 'RoundTripConstructor']
  36. # fmt: on
  37. class ConstructorError(MarkedYAMLError):
  38. pass
  39. class DuplicateKeyFutureWarning(MarkedYAMLFutureWarning):
  40. pass
  41. class DuplicateKeyError(MarkedYAMLError):
  42. pass
  43. class BaseConstructor:
  44. yaml_constructors = {} # type: Dict[Any, Any]
  45. yaml_multi_constructors = {} # type: Dict[Any, Any]
  46. def __init__(self, preserve_quotes: Optional[bool] = None, loader: Any = None) -> None:
  47. self.loader = loader
  48. if self.loader is not None and getattr(self.loader, '_constructor', None) is None:
  49. self.loader._constructor = self
  50. self.loader = loader
  51. self.yaml_base_dict_type = dict
  52. self.yaml_base_list_type = list
  53. self.constructed_objects: Dict[Any, Any] = {}
  54. self.recursive_objects: Dict[Any, Any] = {}
  55. self.state_generators: List[Any] = []
  56. self.deep_construct = False
  57. self._preserve_quotes = preserve_quotes
  58. self.allow_duplicate_keys = version_tnf((0, 15, 1), (0, 16))
  59. @property
  60. def composer(self) -> Any:
  61. if hasattr(self.loader, 'typ'):
  62. return self.loader.composer
  63. try:
  64. return self.loader._composer
  65. except AttributeError:
  66. sys.stdout.write(f'slt {type(self)}\n')
  67. sys.stdout.write(f'slc {self.loader._composer}\n')
  68. sys.stdout.write(f'{dir(self)}\n')
  69. raise
  70. @property
  71. def resolver(self) -> Any:
  72. if hasattr(self.loader, 'typ'):
  73. return self.loader.resolver
  74. return self.loader._resolver
  75. @property
  76. def scanner(self) -> Any:
  77. # needed to get to the expanded comments
  78. if hasattr(self.loader, 'typ'):
  79. return self.loader.scanner
  80. return self.loader._scanner
  81. def check_data(self) -> Any:
  82. # If there are more documents available?
  83. return self.composer.check_node()
  84. def get_data(self) -> Any:
  85. # Construct and return the next document.
  86. if self.composer.check_node():
  87. return self.construct_document(self.composer.get_node())
  88. def get_single_data(self) -> Any:
  89. # Ensure that the stream contains a single document and construct it.
  90. node = self.composer.get_single_node()
  91. if node is not None:
  92. return self.construct_document(node)
  93. return None
  94. def construct_document(self, node: Any) -> Any:
  95. data = self.construct_object(node)
  96. while bool(self.state_generators):
  97. state_generators = self.state_generators
  98. self.state_generators = []
  99. for generator in state_generators:
  100. for _dummy in generator:
  101. pass
  102. self.constructed_objects = {}
  103. self.recursive_objects = {}
  104. self.deep_construct = False
  105. return data
  106. def construct_object(self, node: Any, deep: bool = False) -> Any:
  107. """deep is True when creating an object/mapping recursively,
  108. in that case want the underlying elements available during construction
  109. """
  110. if node in self.constructed_objects:
  111. return self.constructed_objects[node]
  112. if deep:
  113. old_deep = self.deep_construct
  114. self.deep_construct = True
  115. if node in self.recursive_objects:
  116. return self.recursive_objects[node]
  117. # raise ConstructorError(
  118. # None, None, 'found unconstructable recursive node', node.start_mark
  119. # )
  120. self.recursive_objects[node] = None
  121. data = self.construct_non_recursive_object(node)
  122. self.constructed_objects[node] = data
  123. del self.recursive_objects[node]
  124. if deep:
  125. self.deep_construct = old_deep
  126. return data
  127. def construct_non_recursive_object(self, node: Any, tag: Optional[str] = None) -> Any:
  128. constructor: Any = None
  129. tag_suffix = None
  130. if tag is None:
  131. tag = node.tag
  132. if tag in self.yaml_constructors:
  133. constructor = self.yaml_constructors[tag]
  134. else:
  135. for tag_prefix in self.yaml_multi_constructors:
  136. if tag.startswith(tag_prefix):
  137. tag_suffix = tag[len(tag_prefix) :]
  138. constructor = self.yaml_multi_constructors[tag_prefix]
  139. break
  140. else:
  141. if None in self.yaml_multi_constructors:
  142. tag_suffix = tag
  143. constructor = self.yaml_multi_constructors[None]
  144. elif None in self.yaml_constructors:
  145. constructor = self.yaml_constructors[None]
  146. elif isinstance(node, ScalarNode):
  147. constructor = self.__class__.construct_scalar
  148. elif isinstance(node, SequenceNode):
  149. constructor = self.__class__.construct_sequence
  150. elif isinstance(node, MappingNode):
  151. constructor = self.__class__.construct_mapping
  152. if tag_suffix is None:
  153. data = constructor(self, node)
  154. else:
  155. data = constructor(self, tag_suffix, node)
  156. if isinstance(data, types.GeneratorType):
  157. generator = data
  158. data = next(generator)
  159. if self.deep_construct:
  160. for _dummy in generator:
  161. pass
  162. else:
  163. self.state_generators.append(generator)
  164. return data
  165. def construct_scalar(self, node: Any) -> Any:
  166. if not isinstance(node, ScalarNode):
  167. raise ConstructorError(
  168. None, None, f'expected a scalar node, but found {node.id!s}', node.start_mark,
  169. )
  170. return node.value
  171. def construct_sequence(self, node: Any, deep: bool = False) -> Any:
  172. """deep is True when creating an object/mapping recursively,
  173. in that case want the underlying elements available during construction
  174. """
  175. if not isinstance(node, SequenceNode):
  176. raise ConstructorError(
  177. None,
  178. None,
  179. f'expected a sequence node, but found {node.id!s}',
  180. node.start_mark,
  181. )
  182. return [self.construct_object(child, deep=deep) for child in node.value]
  183. def construct_mapping(self, node: Any, deep: bool = False) -> Any:
  184. """deep is True when creating an object/mapping recursively,
  185. in that case want the underlying elements available during construction
  186. """
  187. if not isinstance(node, MappingNode):
  188. raise ConstructorError(
  189. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  190. )
  191. total_mapping = self.yaml_base_dict_type()
  192. if getattr(node, 'merge', None) is not None:
  193. todo = [(node.merge, False), (node.value, False)]
  194. else:
  195. todo = [(node.value, True)]
  196. for values, check in todo:
  197. mapping: Dict[Any, Any] = self.yaml_base_dict_type()
  198. for key_node, value_node in values:
  199. # keys can be list -> deep
  200. key = self.construct_object(key_node, deep=True)
  201. # lists are not hashable, but tuples are
  202. if not isinstance(key, Hashable):
  203. if isinstance(key, list):
  204. key = tuple(key)
  205. if not isinstance(key, Hashable):
  206. raise ConstructorError(
  207. 'while constructing a mapping',
  208. node.start_mark,
  209. 'found unhashable key',
  210. key_node.start_mark,
  211. )
  212. value = self.construct_object(value_node, deep=deep)
  213. if check:
  214. if self.check_mapping_key(node, key_node, mapping, key, value):
  215. mapping[key] = value
  216. else:
  217. mapping[key] = value
  218. total_mapping.update(mapping)
  219. return total_mapping
  220. def check_mapping_key(
  221. self, node: Any, key_node: Any, mapping: Any, key: Any, value: Any,
  222. ) -> bool:
  223. """return True if key is unique"""
  224. if key in mapping:
  225. if not self.allow_duplicate_keys:
  226. mk = mapping.get(key)
  227. args = [
  228. 'while constructing a mapping',
  229. node.start_mark,
  230. f'found duplicate key "{key}" with value "{value}" '
  231. f'(original value: "{mk}")',
  232. key_node.start_mark,
  233. """
  234. To suppress this check see:
  235. http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys
  236. """,
  237. """\
  238. Duplicate keys will become an error in future releases, and are errors
  239. by default when using the new API.
  240. """,
  241. ]
  242. if self.allow_duplicate_keys is None:
  243. warnings.warn(DuplicateKeyFutureWarning(*args), stacklevel=1)
  244. else:
  245. raise DuplicateKeyError(*args)
  246. return False
  247. return True
  248. def check_set_key(self: Any, node: Any, key_node: Any, setting: Any, key: Any) -> None:
  249. if key in setting:
  250. if not self.allow_duplicate_keys:
  251. args = [
  252. 'while constructing a set',
  253. node.start_mark,
  254. f'found duplicate key "{key}"',
  255. key_node.start_mark,
  256. """
  257. To suppress this check see:
  258. http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys
  259. """,
  260. """\
  261. Duplicate keys will become an error in future releases, and are errors
  262. by default when using the new API.
  263. """,
  264. ]
  265. if self.allow_duplicate_keys is None:
  266. warnings.warn(DuplicateKeyFutureWarning(*args), stacklevel=1)
  267. else:
  268. raise DuplicateKeyError(*args)
  269. def construct_pairs(self, node: Any, deep: bool = False) -> Any:
  270. if not isinstance(node, MappingNode):
  271. raise ConstructorError(
  272. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  273. )
  274. pairs = []
  275. for key_node, value_node in node.value:
  276. key = self.construct_object(key_node, deep=deep)
  277. value = self.construct_object(value_node, deep=deep)
  278. pairs.append((key, value))
  279. return pairs
  280. # ToDo: putting stuff on the class makes it global, consider making this to work on an
  281. # instance variable once function load is dropped.
  282. @classmethod
  283. def add_constructor(cls, tag: Any, constructor: Any) -> Any:
  284. if isinstance(tag, Tag):
  285. tag = str(tag)
  286. if 'yaml_constructors' not in cls.__dict__:
  287. cls.yaml_constructors = cls.yaml_constructors.copy()
  288. ret_val = cls.yaml_constructors.get(tag, None)
  289. cls.yaml_constructors[tag] = constructor
  290. return ret_val
  291. @classmethod
  292. def add_multi_constructor(cls, tag_prefix: Any, multi_constructor: Any) -> None:
  293. if 'yaml_multi_constructors' not in cls.__dict__:
  294. cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
  295. cls.yaml_multi_constructors[tag_prefix] = multi_constructor
  296. @classmethod
  297. def add_default_constructor(
  298. cls, tag: str, method: Any = None, tag_base: str = 'tag:yaml.org,2002:',
  299. ) -> None:
  300. if not tag.startswith('tag:'):
  301. if method is None:
  302. method = 'construct_yaml_' + tag
  303. tag = tag_base + tag
  304. cls.add_constructor(tag, getattr(cls, method))
  305. class SafeConstructor(BaseConstructor):
  306. def construct_scalar(self, node: Any) -> Any:
  307. if isinstance(node, MappingNode):
  308. for key_node, value_node in node.value:
  309. if key_node.tag == 'tag:yaml.org,2002:value':
  310. return self.construct_scalar(value_node)
  311. return BaseConstructor.construct_scalar(self, node)
  312. def flatten_mapping(self, node: Any) -> Any:
  313. """
  314. This implements the merge key feature http://yaml.org/type/merge.html
  315. by inserting keys from the merge dict/list of dicts if not yet
  316. available in this node
  317. """
  318. merge: List[Any] = []
  319. index = 0
  320. while index < len(node.value):
  321. key_node, value_node = node.value[index]
  322. if key_node.tag == 'tag:yaml.org,2002:merge':
  323. if merge: # double << key
  324. if self.allow_duplicate_keys:
  325. del node.value[index]
  326. index += 1
  327. continue
  328. args = [
  329. 'while constructing a mapping',
  330. node.start_mark,
  331. f'found duplicate key "{key_node.value}"',
  332. key_node.start_mark,
  333. """
  334. To suppress this check see:
  335. http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys
  336. """,
  337. """\
  338. Duplicate keys will become an error in future releases, and are errors
  339. by default when using the new API.
  340. """,
  341. ]
  342. if self.allow_duplicate_keys is None:
  343. warnings.warn(DuplicateKeyFutureWarning(*args), stacklevel=1)
  344. else:
  345. raise DuplicateKeyError(*args)
  346. del node.value[index]
  347. if isinstance(value_node, MappingNode):
  348. self.flatten_mapping(value_node)
  349. merge.extend(value_node.value)
  350. elif isinstance(value_node, SequenceNode):
  351. submerge = []
  352. for subnode in value_node.value:
  353. if not isinstance(subnode, MappingNode):
  354. raise ConstructorError(
  355. 'while constructing a mapping',
  356. node.start_mark,
  357. f'expected a mapping for merging, but found {subnode.id!s}',
  358. subnode.start_mark,
  359. )
  360. self.flatten_mapping(subnode)
  361. submerge.append(subnode.value)
  362. submerge.reverse()
  363. for value in submerge:
  364. merge.extend(value)
  365. else:
  366. raise ConstructorError(
  367. 'while constructing a mapping',
  368. node.start_mark,
  369. 'expected a mapping or list of mappings for merging, '
  370. f'but found {value_node.id!s}',
  371. value_node.start_mark,
  372. )
  373. elif key_node.tag == 'tag:yaml.org,2002:value':
  374. key_node.tag = 'tag:yaml.org,2002:str'
  375. index += 1
  376. else:
  377. index += 1
  378. if bool(merge):
  379. node.merge = merge # separate merge keys to be able to update without duplicate
  380. node.value = merge + node.value
  381. def construct_mapping(self, node: Any, deep: bool = False) -> Any:
  382. """deep is True when creating an object/mapping recursively,
  383. in that case want the underlying elements available during construction
  384. """
  385. if isinstance(node, MappingNode):
  386. self.flatten_mapping(node)
  387. return BaseConstructor.construct_mapping(self, node, deep=deep)
  388. def construct_yaml_null(self, node: Any) -> Any:
  389. self.construct_scalar(node)
  390. return None
  391. # YAML 1.2 spec doesn't mention yes/no etc any more, 1.1 does
  392. bool_values = {
  393. 'yes': True,
  394. 'no': False,
  395. 'y': True,
  396. 'n': False,
  397. 'true': True,
  398. 'false': False,
  399. 'on': True,
  400. 'off': False,
  401. }
  402. def construct_yaml_bool(self, node: Any) -> bool:
  403. value = self.construct_scalar(node)
  404. return self.bool_values[value.lower()]
  405. def construct_yaml_int(self, node: Any) -> int:
  406. value_s = self.construct_scalar(node)
  407. value_s = value_s.replace('_', "")
  408. sign = +1
  409. if value_s[0] == '-':
  410. sign = -1
  411. if value_s[0] in '+-':
  412. value_s = value_s[1:]
  413. if value_s == '0':
  414. return 0
  415. elif value_s.startswith('0b'):
  416. return sign * int(value_s[2:], 2)
  417. elif value_s.startswith('0x'):
  418. return sign * int(value_s[2:], 16)
  419. elif value_s.startswith('0o'):
  420. return sign * int(value_s[2:], 8)
  421. elif self.resolver.processing_version == (1, 1) and value_s[0] == '0':
  422. return sign * int(value_s, 8)
  423. elif self.resolver.processing_version == (1, 1) and ':' in value_s:
  424. digits = [int(part) for part in value_s.split(':')]
  425. digits.reverse()
  426. base = 1
  427. value = 0
  428. for digit in digits:
  429. value += digit * base
  430. base *= 60
  431. return sign * value
  432. else:
  433. return sign * int(value_s)
  434. inf_value = 1e300
  435. while inf_value != inf_value * inf_value:
  436. inf_value *= inf_value
  437. nan_value = -inf_value / inf_value # Trying to make a quiet NaN (like C99).
  438. def construct_yaml_float(self, node: Any) -> float:
  439. value_so = self.construct_scalar(node)
  440. value_s = value_so.replace('_', "").lower()
  441. sign = +1
  442. if value_s[0] == '-':
  443. sign = -1
  444. if value_s[0] in '+-':
  445. value_s = value_s[1:]
  446. if value_s == '.inf':
  447. return sign * self.inf_value
  448. elif value_s == '.nan':
  449. return self.nan_value
  450. elif self.resolver.processing_version != (1, 2) and ':' in value_s:
  451. digits = [float(part) for part in value_s.split(':')]
  452. digits.reverse()
  453. base = 1
  454. value = 0.0
  455. for digit in digits:
  456. value += digit * base
  457. base *= 60
  458. return sign * value
  459. else:
  460. if self.resolver.processing_version != (1, 2) and 'e' in value_s:
  461. # value_s is lower case independent of input
  462. mantissa, exponent = value_s.split('e')
  463. if '.' not in mantissa:
  464. warnings.warn(MantissaNoDotYAML1_1Warning(node, value_so), stacklevel=1)
  465. return sign * float(value_s)
  466. def construct_yaml_binary(self, node: Any) -> Any:
  467. try:
  468. value = self.construct_scalar(node).encode('ascii')
  469. except UnicodeEncodeError as exc:
  470. raise ConstructorError(
  471. None,
  472. None,
  473. f'failed to convert base64 data into ascii: {exc!s}',
  474. node.start_mark,
  475. )
  476. try:
  477. return base64.decodebytes(value)
  478. except binascii.Error as exc:
  479. raise ConstructorError(
  480. None, None, f'failed to decode base64 data: {exc!s}', node.start_mark,
  481. )
  482. timestamp_regexp = timestamp_regexp # moved to util 0.17.17
  483. def construct_yaml_timestamp(self, node: Any, values: Any = None) -> Any:
  484. if values is None:
  485. try:
  486. match = self.timestamp_regexp.match(node.value)
  487. except TypeError:
  488. match = None
  489. if match is None:
  490. raise ConstructorError(
  491. None,
  492. None,
  493. f'failed to construct timestamp from "{node.value}"',
  494. node.start_mark,
  495. )
  496. values = match.groupdict()
  497. return create_timestamp(**values)
  498. def construct_yaml_omap(self, node: Any) -> Any:
  499. # Note: we do now check for duplicate keys
  500. omap = ordereddict()
  501. yield omap
  502. if not isinstance(node, SequenceNode):
  503. raise ConstructorError(
  504. 'while constructing an ordered map',
  505. node.start_mark,
  506. f'expected a sequence, but found {node.id!s}',
  507. node.start_mark,
  508. )
  509. for subnode in node.value:
  510. if not isinstance(subnode, MappingNode):
  511. raise ConstructorError(
  512. 'while constructing an ordered map',
  513. node.start_mark,
  514. f'expected a mapping of length 1, but found {subnode.id!s}',
  515. subnode.start_mark,
  516. )
  517. if len(subnode.value) != 1:
  518. raise ConstructorError(
  519. 'while constructing an ordered map',
  520. node.start_mark,
  521. f'expected a single mapping item, but found {len(subnode.value):d} items',
  522. subnode.start_mark,
  523. )
  524. key_node, value_node = subnode.value[0]
  525. key = self.construct_object(key_node)
  526. assert key not in omap
  527. value = self.construct_object(value_node)
  528. omap[key] = value
  529. def construct_yaml_pairs(self, node: Any) -> Any:
  530. # Note: the same code as `construct_yaml_omap`.
  531. pairs: List[Any] = []
  532. yield pairs
  533. if not isinstance(node, SequenceNode):
  534. raise ConstructorError(
  535. 'while constructing pairs',
  536. node.start_mark,
  537. f'expected a sequence, but found {node.id!s}',
  538. node.start_mark,
  539. )
  540. for subnode in node.value:
  541. if not isinstance(subnode, MappingNode):
  542. raise ConstructorError(
  543. 'while constructing pairs',
  544. node.start_mark,
  545. f'expected a mapping of length 1, but found {subnode.id!s}',
  546. subnode.start_mark,
  547. )
  548. if len(subnode.value) != 1:
  549. raise ConstructorError(
  550. 'while constructing pairs',
  551. node.start_mark,
  552. f'expected a single mapping item, but found {len(subnode.value):d} items',
  553. subnode.start_mark,
  554. )
  555. key_node, value_node = subnode.value[0]
  556. key = self.construct_object(key_node)
  557. value = self.construct_object(value_node)
  558. pairs.append((key, value))
  559. def construct_yaml_set(self, node: Any) -> Any:
  560. data: Set[Any] = set()
  561. yield data
  562. value = self.construct_mapping(node)
  563. data.update(value)
  564. def construct_yaml_str(self, node: Any) -> Any:
  565. value = self.construct_scalar(node)
  566. return value
  567. def construct_yaml_seq(self, node: Any) -> Any:
  568. data: List[Any] = self.yaml_base_list_type()
  569. yield data
  570. data.extend(self.construct_sequence(node))
  571. def construct_yaml_map(self, node: Any) -> Any:
  572. data: Dict[Any, Any] = self.yaml_base_dict_type()
  573. yield data
  574. value = self.construct_mapping(node)
  575. data.update(value)
  576. def construct_yaml_object(self, node: Any, cls: Any) -> Any:
  577. data = cls.__new__(cls)
  578. yield data
  579. if hasattr(data, '__setstate__'):
  580. state = self.construct_mapping(node, deep=True)
  581. data.__setstate__(state)
  582. else:
  583. state = self.construct_mapping(node)
  584. data.__dict__.update(state)
  585. def construct_undefined(self, node: Any) -> None:
  586. raise ConstructorError(
  587. None,
  588. None,
  589. f'could not determine a constructor for the tag {node.tag!r}',
  590. node.start_mark,
  591. )
  592. for tag in 'null bool int float binary timestamp omap pairs set str seq map'.split():
  593. SafeConstructor.add_default_constructor(tag)
  594. SafeConstructor.add_constructor(None, SafeConstructor.construct_undefined)
  595. class Constructor(SafeConstructor):
  596. def construct_python_str(self, node: Any) -> Any:
  597. return self.construct_scalar(node)
  598. def construct_python_unicode(self, node: Any) -> Any:
  599. return self.construct_scalar(node)
  600. def construct_python_bytes(self, node: Any) -> Any:
  601. try:
  602. value = self.construct_scalar(node).encode('ascii')
  603. except UnicodeEncodeError as exc:
  604. raise ConstructorError(
  605. None,
  606. None,
  607. f'failed to convert base64 data into ascii: {exc!s}',
  608. node.start_mark,
  609. )
  610. try:
  611. return base64.decodebytes(value)
  612. except binascii.Error as exc:
  613. raise ConstructorError(
  614. None, None, f'failed to decode base64 data: {exc!s}', node.start_mark,
  615. )
  616. def construct_python_long(self, node: Any) -> int:
  617. val = self.construct_yaml_int(node)
  618. return val
  619. def construct_python_complex(self, node: Any) -> Any:
  620. return complex(self.construct_scalar(node))
  621. def construct_python_tuple(self, node: Any) -> Any:
  622. return tuple(self.construct_sequence(node))
  623. def find_python_module(self, name: Any, mark: Any) -> Any:
  624. if not name:
  625. raise ConstructorError(
  626. 'while constructing a Python module',
  627. mark,
  628. 'expected non-empty name appended to the tag',
  629. mark,
  630. )
  631. try:
  632. __import__(name)
  633. except ImportError as exc:
  634. raise ConstructorError(
  635. 'while constructing a Python module',
  636. mark,
  637. f'cannot find module {name!r} ({exc!s})',
  638. mark,
  639. )
  640. return sys.modules[name]
  641. def find_python_name(self, name: Any, mark: Any) -> Any:
  642. if not name:
  643. raise ConstructorError(
  644. 'while constructing a Python object',
  645. mark,
  646. 'expected non-empty name appended to the tag',
  647. mark,
  648. )
  649. if '.' in name:
  650. lname = name.split('.')
  651. lmodule_name = lname
  652. lobject_name: List[Any] = []
  653. while len(lmodule_name) > 1:
  654. lobject_name.insert(0, lmodule_name.pop())
  655. module_name = '.'.join(lmodule_name)
  656. try:
  657. __import__(module_name)
  658. # object_name = '.'.join(object_name)
  659. break
  660. except ImportError:
  661. continue
  662. else:
  663. module_name = builtins_module
  664. lobject_name = [name]
  665. try:
  666. __import__(module_name)
  667. except ImportError as exc:
  668. raise ConstructorError(
  669. 'while constructing a Python object',
  670. mark,
  671. f'cannot find module {module_name!r} ({exc!s})',
  672. mark,
  673. )
  674. module = sys.modules[module_name]
  675. object_name = '.'.join(lobject_name)
  676. obj = module
  677. while lobject_name:
  678. if not hasattr(obj, lobject_name[0]):
  679. raise ConstructorError(
  680. 'while constructing a Python object',
  681. mark,
  682. f'cannot find {object_name!r} in the module {module.__name__!r}',
  683. mark,
  684. )
  685. obj = getattr(obj, lobject_name.pop(0))
  686. return obj
  687. def construct_python_name(self, suffix: Any, node: Any) -> Any:
  688. value = self.construct_scalar(node)
  689. if value:
  690. raise ConstructorError(
  691. 'while constructing a Python name',
  692. node.start_mark,
  693. f'expected the empty value, but found {value!r}',
  694. node.start_mark,
  695. )
  696. return self.find_python_name(suffix, node.start_mark)
  697. def construct_python_module(self, suffix: Any, node: Any) -> Any:
  698. value = self.construct_scalar(node)
  699. if value:
  700. raise ConstructorError(
  701. 'while constructing a Python module',
  702. node.start_mark,
  703. f'expected the empty value, but found {value!r}',
  704. node.start_mark,
  705. )
  706. return self.find_python_module(suffix, node.start_mark)
  707. def make_python_instance(
  708. self, suffix: Any, node: Any, args: Any = None, kwds: Any = None, newobj: bool = False,
  709. ) -> Any:
  710. if not args:
  711. args = []
  712. if not kwds:
  713. kwds = {}
  714. cls = self.find_python_name(suffix, node.start_mark)
  715. if newobj and isinstance(cls, type):
  716. return cls.__new__(cls, *args, **kwds)
  717. else:
  718. return cls(*args, **kwds)
  719. def set_python_instance_state(self, instance: Any, state: Any) -> None:
  720. if hasattr(instance, '__setstate__'):
  721. instance.__setstate__(state)
  722. else:
  723. slotstate: Dict[Any, Any] = {}
  724. if isinstance(state, tuple) and len(state) == 2:
  725. state, slotstate = state
  726. if hasattr(instance, '__dict__'):
  727. instance.__dict__.update(state)
  728. elif state:
  729. slotstate.update(state)
  730. for key, value in slotstate.items():
  731. setattr(instance, key, value)
  732. def construct_python_object(self, suffix: Any, node: Any) -> Any:
  733. # Format:
  734. # !!python/object:module.name { ... state ... }
  735. instance = self.make_python_instance(suffix, node, newobj=True)
  736. self.recursive_objects[node] = instance
  737. yield instance
  738. deep = hasattr(instance, '__setstate__')
  739. state = self.construct_mapping(node, deep=deep)
  740. self.set_python_instance_state(instance, state)
  741. def construct_python_object_apply(
  742. self, suffix: Any, node: Any, newobj: bool = False,
  743. ) -> Any:
  744. # Format:
  745. # !!python/object/apply # (or !!python/object/new)
  746. # args: [ ... arguments ... ]
  747. # kwds: { ... keywords ... }
  748. # state: ... state ...
  749. # listitems: [ ... listitems ... ]
  750. # dictitems: { ... dictitems ... }
  751. # or short format:
  752. # !!python/object/apply [ ... arguments ... ]
  753. # The difference between !!python/object/apply and !!python/object/new
  754. # is how an object is created, check make_python_instance for details.
  755. if isinstance(node, SequenceNode):
  756. args = self.construct_sequence(node, deep=True)
  757. kwds: Dict[Any, Any] = {}
  758. state: Dict[Any, Any] = {}
  759. listitems: List[Any] = []
  760. dictitems: Dict[Any, Any] = {}
  761. else:
  762. value = self.construct_mapping(node, deep=True)
  763. args = value.get('args', [])
  764. kwds = value.get('kwds', {})
  765. state = value.get('state', {})
  766. listitems = value.get('listitems', [])
  767. dictitems = value.get('dictitems', {})
  768. instance = self.make_python_instance(suffix, node, args, kwds, newobj)
  769. if bool(state):
  770. self.set_python_instance_state(instance, state)
  771. if bool(listitems):
  772. instance.extend(listitems)
  773. if bool(dictitems):
  774. for key in dictitems:
  775. instance[key] = dictitems[key]
  776. return instance
  777. def construct_python_object_new(self, suffix: Any, node: Any) -> Any:
  778. return self.construct_python_object_apply(suffix, node, newobj=True)
  779. @classmethod
  780. def add_default_constructor(
  781. cls, tag: str, method: Any = None, tag_base: str = 'tag:yaml.org,2002:python/',
  782. ) -> None:
  783. if not tag.startswith('tag:'):
  784. if method is None:
  785. method = 'construct_yaml_' + tag
  786. tag = tag_base + tag
  787. cls.add_constructor(tag, getattr(cls, method))
  788. Constructor.add_constructor('tag:yaml.org,2002:python/none', Constructor.construct_yaml_null)
  789. Constructor.add_constructor('tag:yaml.org,2002:python/bool', Constructor.construct_yaml_bool)
  790. Constructor.add_constructor('tag:yaml.org,2002:python/str', Constructor.construct_python_str)
  791. Constructor.add_constructor(
  792. 'tag:yaml.org,2002:python/unicode', Constructor.construct_python_unicode,
  793. )
  794. Constructor.add_constructor(
  795. 'tag:yaml.org,2002:python/bytes', Constructor.construct_python_bytes,
  796. )
  797. Constructor.add_constructor('tag:yaml.org,2002:python/int', Constructor.construct_yaml_int)
  798. Constructor.add_constructor('tag:yaml.org,2002:python/long', Constructor.construct_python_long)
  799. Constructor.add_constructor('tag:yaml.org,2002:python/float', Constructor.construct_yaml_float)
  800. Constructor.add_constructor(
  801. 'tag:yaml.org,2002:python/complex', Constructor.construct_python_complex,
  802. )
  803. Constructor.add_constructor('tag:yaml.org,2002:python/list', Constructor.construct_yaml_seq)
  804. Constructor.add_constructor(
  805. 'tag:yaml.org,2002:python/tuple', Constructor.construct_python_tuple,
  806. )
  807. # for tag in 'bool str unicode bytes int long float complex tuple'.split():
  808. # Constructor.add_default_constructor(tag)
  809. Constructor.add_constructor('tag:yaml.org,2002:python/dict', Constructor.construct_yaml_map)
  810. Constructor.add_multi_constructor(
  811. 'tag:yaml.org,2002:python/name:', Constructor.construct_python_name,
  812. )
  813. Constructor.add_multi_constructor(
  814. 'tag:yaml.org,2002:python/module:', Constructor.construct_python_module,
  815. )
  816. Constructor.add_multi_constructor(
  817. 'tag:yaml.org,2002:python/object:', Constructor.construct_python_object,
  818. )
  819. Constructor.add_multi_constructor(
  820. 'tag:yaml.org,2002:python/object/apply:', Constructor.construct_python_object_apply,
  821. )
  822. Constructor.add_multi_constructor(
  823. 'tag:yaml.org,2002:python/object/new:', Constructor.construct_python_object_new,
  824. )
  825. class RoundTripConstructor(SafeConstructor):
  826. """need to store the comments on the node itself,
  827. as well as on the items
  828. """
  829. def comment(self, idx: Any) -> Any:
  830. assert self.loader.comment_handling is not None
  831. x = self.scanner.comments[idx]
  832. x.set_assigned()
  833. return x
  834. def comments(self, list_of_comments: Any, idx: Optional[Any] = None) -> Any:
  835. # hand in the comment and optional pre, eol, post segment
  836. if list_of_comments is None:
  837. return []
  838. if idx is not None:
  839. if list_of_comments[idx] is None:
  840. return []
  841. list_of_comments = list_of_comments[idx]
  842. for x in list_of_comments:
  843. yield self.comment(x)
  844. def construct_scalar(self, node: Any) -> Any:
  845. if not isinstance(node, ScalarNode):
  846. raise ConstructorError(
  847. None, None, f'expected a scalar node, but found {node.id!s}', node.start_mark,
  848. )
  849. if node.style == '|' and isinstance(node.value, str):
  850. lss = LiteralScalarString(node.value, anchor=node.anchor)
  851. if self.loader and self.loader.comment_handling is None:
  852. if node.comment and node.comment[1]:
  853. lss.comment = node.comment[1][0] # type: ignore
  854. else:
  855. # NEWCMNT
  856. if node.comment is not None and node.comment[1]:
  857. # nprintf('>>>>nc1', node.comment)
  858. # EOL comment after |
  859. lss.comment = self.comment(node.comment[1][0]) # type: ignore
  860. return lss
  861. if node.style == '>' and isinstance(node.value, str):
  862. fold_positions: List[int] = []
  863. idx = -1
  864. while True:
  865. idx = node.value.find('\a', idx + 1)
  866. if idx < 0:
  867. break
  868. fold_positions.append(idx - len(fold_positions))
  869. fss = FoldedScalarString(node.value.replace('\a', ''), anchor=node.anchor)
  870. if self.loader and self.loader.comment_handling is None:
  871. if node.comment and node.comment[1]:
  872. fss.comment = node.comment[1][0] # type: ignore
  873. else:
  874. # NEWCMNT
  875. if node.comment is not None and node.comment[1]:
  876. # nprintf('>>>>nc2', node.comment)
  877. # EOL comment after >
  878. fss.comment = self.comment(node.comment[1][0]) # type: ignore
  879. if fold_positions:
  880. fss.fold_pos = fold_positions # type: ignore
  881. return fss
  882. elif bool(self._preserve_quotes) and isinstance(node.value, str):
  883. if node.style == "'":
  884. return SingleQuotedScalarString(node.value, anchor=node.anchor)
  885. if node.style == '"':
  886. return DoubleQuotedScalarString(node.value, anchor=node.anchor)
  887. # if node.ctag:
  888. # data2 = TaggedScalar()
  889. # data2.value = node.value
  890. # data2.style = node.style
  891. # data2.yaml_set_ctag(node.ctag)
  892. # if node.anchor:
  893. # from ruamel.yaml.serializer import templated_id
  894. # if not templated_id(node.anchor):
  895. # data2.yaml_set_anchor(node.anchor, always_dump=True)
  896. # return data2
  897. if node.anchor:
  898. return PlainScalarString(node.value, anchor=node.anchor)
  899. return node.value
  900. def construct_yaml_int(self, node: Any) -> Any:
  901. width: Any = None
  902. value_su = self.construct_scalar(node)
  903. try:
  904. sx = value_su.rstrip('_')
  905. underscore: Any = [len(sx) - sx.rindex('_') - 1, False, False]
  906. except ValueError:
  907. underscore = None
  908. except IndexError:
  909. underscore = None
  910. value_s = value_su.replace('_', "")
  911. sign = +1
  912. if value_s[0] == '-':
  913. sign = -1
  914. if value_s[0] in '+-':
  915. value_s = value_s[1:]
  916. if value_s == '0':
  917. return 0
  918. elif value_s.startswith('0b'):
  919. if self.resolver.processing_version > (1, 1) and value_s[2] == '0':
  920. width = len(value_s[2:])
  921. if underscore is not None:
  922. underscore[1] = value_su[2] == '_'
  923. underscore[2] = len(value_su[2:]) > 1 and value_su[-1] == '_'
  924. return BinaryInt(
  925. sign * int(value_s[2:], 2),
  926. width=width,
  927. underscore=underscore,
  928. anchor=node.anchor,
  929. )
  930. elif value_s.startswith('0x'):
  931. # default to lower-case if no a-fA-F in string
  932. if self.resolver.processing_version > (1, 1) and value_s[2] == '0':
  933. width = len(value_s[2:])
  934. hex_fun: Any = HexInt
  935. for ch in value_s[2:]:
  936. if ch in 'ABCDEF': # first non-digit is capital
  937. hex_fun = HexCapsInt
  938. break
  939. if ch in 'abcdef':
  940. break
  941. if underscore is not None:
  942. underscore[1] = value_su[2] == '_'
  943. underscore[2] = len(value_su[2:]) > 1 and value_su[-1] == '_'
  944. return hex_fun(
  945. sign * int(value_s[2:], 16),
  946. width=width,
  947. underscore=underscore,
  948. anchor=node.anchor,
  949. )
  950. elif value_s.startswith('0o'):
  951. if self.resolver.processing_version > (1, 1) and value_s[2] == '0':
  952. width = len(value_s[2:])
  953. if underscore is not None:
  954. underscore[1] = value_su[2] == '_'
  955. underscore[2] = len(value_su[2:]) > 1 and value_su[-1] == '_'
  956. return OctalInt(
  957. sign * int(value_s[2:], 8),
  958. width=width,
  959. underscore=underscore,
  960. anchor=node.anchor,
  961. )
  962. elif self.resolver.processing_version != (1, 2) and value_s[0] == '0':
  963. return OctalInt(
  964. sign * int(value_s, 8), width=width, underscore=underscore, anchor=node.anchor,
  965. )
  966. elif self.resolver.processing_version != (1, 2) and ':' in value_s:
  967. digits = [int(part) for part in value_s.split(':')]
  968. digits.reverse()
  969. base = 1
  970. value = 0
  971. for digit in digits:
  972. value += digit * base
  973. base *= 60
  974. return sign * value
  975. elif self.resolver.processing_version > (1, 1) and value_s[0] == '0':
  976. # not an octal, an integer with leading zero(s)
  977. if underscore is not None:
  978. # cannot have a leading underscore
  979. underscore[2] = len(value_su) > 1 and value_su[-1] == '_'
  980. return ScalarInt(sign * int(value_s), width=len(value_s), underscore=underscore)
  981. elif underscore:
  982. # cannot have a leading underscore
  983. underscore[2] = len(value_su) > 1 and value_su[-1] == '_'
  984. return ScalarInt(
  985. sign * int(value_s), width=None, underscore=underscore, anchor=node.anchor,
  986. )
  987. elif node.anchor:
  988. return ScalarInt(sign * int(value_s), width=None, anchor=node.anchor)
  989. else:
  990. return sign * int(value_s)
  991. def construct_yaml_float(self, node: Any) -> Any:
  992. def leading_zeros(v: Any) -> int:
  993. lead0 = 0
  994. idx = 0
  995. while idx < len(v) and v[idx] in '0.':
  996. if v[idx] == '0':
  997. lead0 += 1
  998. idx += 1
  999. return lead0
  1000. # underscore = None
  1001. m_sign: Any = False
  1002. value_so = self.construct_scalar(node)
  1003. value_s = value_so.replace('_', "").lower()
  1004. sign = +1
  1005. if value_s[0] == '-':
  1006. sign = -1
  1007. if value_s[0] in '+-':
  1008. m_sign = value_s[0]
  1009. value_s = value_s[1:]
  1010. if value_s == '.inf':
  1011. return sign * self.inf_value
  1012. if value_s == '.nan':
  1013. return self.nan_value
  1014. if self.resolver.processing_version != (1, 2) and ':' in value_s:
  1015. digits = [float(part) for part in value_s.split(':')]
  1016. digits.reverse()
  1017. base = 1
  1018. value = 0.0
  1019. for digit in digits:
  1020. value += digit * base
  1021. base *= 60
  1022. return sign * value
  1023. if 'e' in value_s:
  1024. try:
  1025. mantissa, exponent = value_so.split('e')
  1026. exp = 'e'
  1027. except ValueError:
  1028. mantissa, exponent = value_so.split('E')
  1029. exp = 'E'
  1030. if self.resolver.processing_version != (1, 2):
  1031. # value_s is lower case independent of input
  1032. if '.' not in mantissa:
  1033. warnings.warn(MantissaNoDotYAML1_1Warning(node, value_so), stacklevel=1)
  1034. lead0 = leading_zeros(mantissa)
  1035. width = len(mantissa)
  1036. prec = mantissa.find('.')
  1037. if m_sign:
  1038. width -= 1
  1039. e_width = len(exponent)
  1040. e_sign = exponent[0] in '+-'
  1041. # nprint('sf', width, prec, m_sign, exp, e_width, e_sign)
  1042. return ScalarFloat(
  1043. sign * float(value_s),
  1044. width=width,
  1045. prec=prec,
  1046. m_sign=m_sign,
  1047. m_lead0=lead0,
  1048. exp=exp,
  1049. e_width=e_width,
  1050. e_sign=e_sign,
  1051. anchor=node.anchor,
  1052. )
  1053. width = len(value_so)
  1054. # you can't use index, !!float 42 would be a float without a dot
  1055. prec = value_so.find('.')
  1056. lead0 = leading_zeros(value_so)
  1057. return ScalarFloat(
  1058. sign * float(value_s),
  1059. width=width,
  1060. prec=prec,
  1061. m_sign=m_sign,
  1062. m_lead0=lead0,
  1063. anchor=node.anchor,
  1064. )
  1065. def construct_yaml_str(self, node: Any) -> Any:
  1066. if node.ctag.handle:
  1067. value = self.construct_unknown(node)
  1068. else:
  1069. value = self.construct_scalar(node)
  1070. if isinstance(value, ScalarString):
  1071. return value
  1072. return value
  1073. def construct_rt_sequence(self, node: Any, seqtyp: Any, deep: bool = False) -> Any:
  1074. if not isinstance(node, SequenceNode):
  1075. raise ConstructorError(
  1076. None,
  1077. None,
  1078. f'expected a sequence node, but found {node.id!s}',
  1079. node.start_mark,
  1080. )
  1081. ret_val = []
  1082. if self.loader and self.loader.comment_handling is None:
  1083. if node.comment:
  1084. seqtyp._yaml_add_comment(node.comment[:2])
  1085. if len(node.comment) > 2:
  1086. # this happens e.g. if you have a sequence element that is a flow-style
  1087. # mapping and that has no EOL comment but a following commentline or
  1088. # empty line
  1089. seqtyp.yaml_end_comment_extend(node.comment[2], clear=True)
  1090. else:
  1091. # NEWCMNT
  1092. if node.comment:
  1093. nprintf('nc3', node.comment)
  1094. if node.anchor:
  1095. from ruamel.yaml.serializer import templated_id
  1096. if not templated_id(node.anchor):
  1097. seqtyp.yaml_set_anchor(node.anchor)
  1098. for idx, child in enumerate(node.value):
  1099. if child.comment:
  1100. seqtyp._yaml_add_comment(child.comment, key=idx)
  1101. child.comment = None # if moved to sequence remove from child
  1102. ret_val.append(self.construct_object(child, deep=deep))
  1103. seqtyp._yaml_set_idx_line_col(
  1104. idx, [child.start_mark.line, child.start_mark.column],
  1105. )
  1106. return ret_val
  1107. def flatten_mapping(self, node: Any) -> Any:
  1108. """
  1109. This implements the merge key feature http://yaml.org/type/merge.html
  1110. by inserting keys from the merge dict/list of dicts if not yet
  1111. available in this node
  1112. """
  1113. def constructed(value_node: Any) -> Any:
  1114. # If the contents of a merge are defined within the
  1115. # merge marker, then they won't have been constructed
  1116. # yet. But if they were already constructed, we need to use
  1117. # the existing object.
  1118. if value_node in self.constructed_objects:
  1119. value = self.constructed_objects[value_node]
  1120. else:
  1121. value = self.construct_object(value_node, deep=True)
  1122. return value
  1123. # merge = []
  1124. merge_map_list: List[Any] = []
  1125. index = 0
  1126. while index < len(node.value):
  1127. key_node, value_node = node.value[index]
  1128. if key_node.tag == 'tag:yaml.org,2002:merge':
  1129. if merge_map_list: # double << key
  1130. if self.allow_duplicate_keys:
  1131. del node.value[index]
  1132. index += 1
  1133. continue
  1134. args = [
  1135. 'while constructing a mapping',
  1136. node.start_mark,
  1137. f'found duplicate key "{key_node.value}"',
  1138. key_node.start_mark,
  1139. """
  1140. To suppress this check see:
  1141. http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys
  1142. """,
  1143. """\
  1144. Duplicate keys will become an error in future releases, and are errors
  1145. by default when using the new API.
  1146. """,
  1147. ]
  1148. if self.allow_duplicate_keys is None:
  1149. warnings.warn(DuplicateKeyFutureWarning(*args), stacklevel=1)
  1150. else:
  1151. raise DuplicateKeyError(*args)
  1152. del node.value[index]
  1153. if isinstance(value_node, MappingNode):
  1154. merge_map_list.append((index, constructed(value_node)))
  1155. # self.flatten_mapping(value_node)
  1156. # merge.extend(value_node.value)
  1157. elif isinstance(value_node, SequenceNode):
  1158. # submerge = []
  1159. for subnode in value_node.value:
  1160. if not isinstance(subnode, MappingNode):
  1161. raise ConstructorError(
  1162. 'while constructing a mapping',
  1163. node.start_mark,
  1164. f'expected a mapping for merging, but found {subnode.id!s}',
  1165. subnode.start_mark,
  1166. )
  1167. merge_map_list.append((index, constructed(subnode)))
  1168. # self.flatten_mapping(subnode)
  1169. # submerge.append(subnode.value)
  1170. # submerge.reverse()
  1171. # for value in submerge:
  1172. # merge.extend(value)
  1173. else:
  1174. raise ConstructorError(
  1175. 'while constructing a mapping',
  1176. node.start_mark,
  1177. 'expected a mapping or list of mappings for merging, '
  1178. f'but found {value_node.id!s}',
  1179. value_node.start_mark,
  1180. )
  1181. elif key_node.tag == 'tag:yaml.org,2002:value':
  1182. key_node.tag = 'tag:yaml.org,2002:str'
  1183. index += 1
  1184. else:
  1185. index += 1
  1186. return merge_map_list
  1187. # if merge:
  1188. # node.value = merge + node.value
  1189. def _sentinel(self) -> None:
  1190. pass
  1191. def construct_mapping(self, node: Any, maptyp: Any, deep: bool = False) -> Any: # type: ignore # NOQA
  1192. if not isinstance(node, MappingNode):
  1193. raise ConstructorError(
  1194. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  1195. )
  1196. merge_map = self.flatten_mapping(node)
  1197. # mapping = {}
  1198. if self.loader and self.loader.comment_handling is None:
  1199. if node.comment:
  1200. maptyp._yaml_add_comment(node.comment[:2])
  1201. if len(node.comment) > 2:
  1202. maptyp.yaml_end_comment_extend(node.comment[2], clear=True)
  1203. else:
  1204. # NEWCMNT
  1205. if node.comment:
  1206. # nprintf('nc4', node.comment, node.start_mark)
  1207. if maptyp.ca.pre is None:
  1208. maptyp.ca.pre = []
  1209. for cmnt in self.comments(node.comment, 0):
  1210. maptyp.ca.pre.append(cmnt)
  1211. if node.anchor:
  1212. from ruamel.yaml.serializer import templated_id
  1213. if not templated_id(node.anchor):
  1214. maptyp.yaml_set_anchor(node.anchor)
  1215. last_key, last_value = None, self._sentinel
  1216. for key_node, value_node in node.value:
  1217. # keys can be list -> deep
  1218. key = self.construct_object(key_node, deep=True)
  1219. # lists are not hashable, but tuples are
  1220. if not isinstance(key, Hashable):
  1221. if isinstance(key, MutableSequence):
  1222. key_s = CommentedKeySeq(key)
  1223. if key_node.flow_style is True:
  1224. key_s.fa.set_flow_style()
  1225. elif key_node.flow_style is False:
  1226. key_s.fa.set_block_style()
  1227. key_s._yaml_set_line_col(key.lc.line, key.lc.col) # type: ignore
  1228. key = key_s
  1229. elif isinstance(key, MutableMapping):
  1230. key_m = CommentedKeyMap(key)
  1231. if key_node.flow_style is True:
  1232. key_m.fa.set_flow_style()
  1233. elif key_node.flow_style is False:
  1234. key_m.fa.set_block_style()
  1235. key_m._yaml_set_line_col(key.lc.line, key.lc.col) # type: ignore
  1236. key = key_m
  1237. if not isinstance(key, Hashable):
  1238. raise ConstructorError(
  1239. 'while constructing a mapping',
  1240. node.start_mark,
  1241. 'found unhashable key',
  1242. key_node.start_mark,
  1243. )
  1244. value = self.construct_object(value_node, deep=deep)
  1245. if self.check_mapping_key(node, key_node, maptyp, key, value):
  1246. if self.loader and self.loader.comment_handling is None:
  1247. if key_node.comment and len(key_node.comment) > 4 and key_node.comment[4]:
  1248. if last_value is None:
  1249. key_node.comment[0] = key_node.comment.pop(4)
  1250. maptyp._yaml_add_comment(key_node.comment, value=last_key)
  1251. else:
  1252. key_node.comment[2] = key_node.comment.pop(4)
  1253. maptyp._yaml_add_comment(key_node.comment, key=key)
  1254. key_node.comment = None
  1255. if key_node.comment:
  1256. maptyp._yaml_add_comment(key_node.comment, key=key)
  1257. if value_node.comment:
  1258. maptyp._yaml_add_comment(value_node.comment, value=key)
  1259. else:
  1260. # NEWCMNT
  1261. if key_node.comment:
  1262. nprintf('nc5a', key, key_node.comment)
  1263. if key_node.comment[0]:
  1264. maptyp.ca.set(key, C_KEY_PRE, key_node.comment[0])
  1265. if key_node.comment[1]:
  1266. maptyp.ca.set(key, C_KEY_EOL, key_node.comment[1])
  1267. if key_node.comment[2]:
  1268. maptyp.ca.set(key, C_KEY_POST, key_node.comment[2])
  1269. if value_node.comment:
  1270. nprintf('nc5b', key, value_node.comment)
  1271. if value_node.comment[0]:
  1272. maptyp.ca.set(key, C_VALUE_PRE, value_node.comment[0])
  1273. if value_node.comment[1]:
  1274. maptyp.ca.set(key, C_VALUE_EOL, value_node.comment[1])
  1275. if value_node.comment[2]:
  1276. maptyp.ca.set(key, C_VALUE_POST, value_node.comment[2])
  1277. maptyp._yaml_set_kv_line_col(
  1278. key,
  1279. [
  1280. key_node.start_mark.line,
  1281. key_node.start_mark.column,
  1282. value_node.start_mark.line,
  1283. value_node.start_mark.column,
  1284. ],
  1285. )
  1286. maptyp[key] = value
  1287. last_key, last_value = key, value # could use indexing
  1288. # do this last, or <<: before a key will prevent insertion in instances
  1289. # of collections.OrderedDict (as they have no __contains__
  1290. if merge_map:
  1291. maptyp.add_yaml_merge(merge_map)
  1292. def construct_setting(self, node: Any, typ: Any, deep: bool = False) -> Any:
  1293. if not isinstance(node, MappingNode):
  1294. raise ConstructorError(
  1295. None, None, f'expected a mapping node, but found {node.id!s}', node.start_mark,
  1296. )
  1297. if self.loader and self.loader.comment_handling is None:
  1298. if node.comment:
  1299. typ._yaml_add_comment(node.comment[:2])
  1300. if len(node.comment) > 2:
  1301. typ.yaml_end_comment_extend(node.comment[2], clear=True)
  1302. else:
  1303. # NEWCMNT
  1304. if node.comment:
  1305. nprintf('nc6', node.comment)
  1306. if node.anchor:
  1307. from ruamel.yaml.serializer import templated_id
  1308. if not templated_id(node.anchor):
  1309. typ.yaml_set_anchor(node.anchor)
  1310. for key_node, value_node in node.value:
  1311. # keys can be list -> deep
  1312. key = self.construct_object(key_node, deep=True)
  1313. # lists are not hashable, but tuples are
  1314. if not isinstance(key, Hashable):
  1315. if isinstance(key, list):
  1316. key = tuple(key)
  1317. if not isinstance(key, Hashable):
  1318. raise ConstructorError(
  1319. 'while constructing a mapping',
  1320. node.start_mark,
  1321. 'found unhashable key',
  1322. key_node.start_mark,
  1323. )
  1324. # construct but should be null
  1325. value = self.construct_object(value_node, deep=deep) # NOQA
  1326. self.check_set_key(node, key_node, typ, key)
  1327. if self.loader and self.loader.comment_handling is None:
  1328. if key_node.comment:
  1329. typ._yaml_add_comment(key_node.comment, key=key)
  1330. if value_node.comment:
  1331. typ._yaml_add_comment(value_node.comment, value=key)
  1332. else:
  1333. # NEWCMNT
  1334. if key_node.comment:
  1335. nprintf('nc7a', key_node.comment)
  1336. if value_node.comment:
  1337. nprintf('nc7b', value_node.comment)
  1338. typ.add(key)
  1339. def construct_yaml_seq(self, node: Any) -> Iterator[CommentedSeq]:
  1340. data = CommentedSeq()
  1341. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1342. # if node.comment:
  1343. # data._yaml_add_comment(node.comment)
  1344. yield data
  1345. data.extend(self.construct_rt_sequence(node, data))
  1346. self.set_collection_style(data, node)
  1347. def construct_yaml_map(self, node: Any) -> Iterator[CommentedMap]:
  1348. data = CommentedMap()
  1349. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1350. yield data
  1351. self.construct_mapping(node, data, deep=True)
  1352. self.set_collection_style(data, node)
  1353. def set_collection_style(self, data: Any, node: Any) -> None:
  1354. if len(data) == 0:
  1355. return
  1356. if node.flow_style is True:
  1357. data.fa.set_flow_style()
  1358. elif node.flow_style is False:
  1359. data.fa.set_block_style()
  1360. def construct_yaml_object(self, node: Any, cls: Any) -> Any:
  1361. from dataclasses import is_dataclass, InitVar, MISSING
  1362. data = cls.__new__(cls)
  1363. yield data
  1364. if hasattr(data, '__setstate__'):
  1365. state = SafeConstructor.construct_mapping(self, node, deep=True)
  1366. data.__setstate__(state)
  1367. elif is_dataclass(data):
  1368. mapping = SafeConstructor.construct_mapping(self, node)
  1369. init_var_defaults = {}
  1370. for field in data.__dataclass_fields__.values():
  1371. # nprintf('field', field, field.default is MISSING,
  1372. # isinstance(field.type, InitVar))
  1373. # in 3.7, InitVar is a singleton
  1374. if (
  1375. isinstance(field.type, InitVar) or field.type is InitVar
  1376. ) and field.default is not MISSING:
  1377. init_var_defaults[field.name] = field.default
  1378. for attr, value in mapping.items():
  1379. if attr not in init_var_defaults:
  1380. setattr(data, attr, value)
  1381. post_init = getattr(data, '__post_init__', None)
  1382. if post_init is not None:
  1383. kw = {}
  1384. for name, default in init_var_defaults.items():
  1385. kw[name] = mapping.get(name, default)
  1386. post_init(**kw)
  1387. else:
  1388. state = SafeConstructor.construct_mapping(self, node)
  1389. if hasattr(data, '__attrs_attrs__'): # issue 394
  1390. data.__init__(**state)
  1391. else:
  1392. data.__dict__.update(state)
  1393. if node.anchor:
  1394. from ruamel.yaml.serializer import templated_id
  1395. from ruamel.yaml.anchor import Anchor
  1396. if not templated_id(node.anchor):
  1397. if not hasattr(data, Anchor.attrib):
  1398. a = Anchor()
  1399. setattr(data, Anchor.attrib, a)
  1400. else:
  1401. a = getattr(data, Anchor.attrib)
  1402. a.value = node.anchor
  1403. def construct_yaml_omap(self, node: Any) -> Iterator[CommentedOrderedMap]:
  1404. # Note: we do now check for duplicate keys
  1405. omap = CommentedOrderedMap()
  1406. omap._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1407. if node.flow_style is True:
  1408. omap.fa.set_flow_style()
  1409. elif node.flow_style is False:
  1410. omap.fa.set_block_style()
  1411. yield omap
  1412. if self.loader and self.loader.comment_handling is None:
  1413. if node.comment:
  1414. omap._yaml_add_comment(node.comment[:2])
  1415. if len(node.comment) > 2:
  1416. omap.yaml_end_comment_extend(node.comment[2], clear=True)
  1417. else:
  1418. # NEWCMNT
  1419. if node.comment:
  1420. nprintf('nc8', node.comment)
  1421. if not isinstance(node, SequenceNode):
  1422. raise ConstructorError(
  1423. 'while constructing an ordered map',
  1424. node.start_mark,
  1425. f'expected a sequence, but found {node.id!s}',
  1426. node.start_mark,
  1427. )
  1428. for subnode in node.value:
  1429. if not isinstance(subnode, MappingNode):
  1430. raise ConstructorError(
  1431. 'while constructing an ordered map',
  1432. node.start_mark,
  1433. f'expected a mapping of length 1, but found {subnode.id!s}',
  1434. subnode.start_mark,
  1435. )
  1436. if len(subnode.value) != 1:
  1437. raise ConstructorError(
  1438. 'while constructing an ordered map',
  1439. node.start_mark,
  1440. f'expected a single mapping item, but found {len(subnode.value):d} items',
  1441. subnode.start_mark,
  1442. )
  1443. key_node, value_node = subnode.value[0]
  1444. key = self.construct_object(key_node)
  1445. assert key not in omap
  1446. value = self.construct_object(value_node)
  1447. if self.loader and self.loader.comment_handling is None:
  1448. if key_node.comment:
  1449. omap._yaml_add_comment(key_node.comment, key=key)
  1450. if subnode.comment:
  1451. omap._yaml_add_comment(subnode.comment, key=key)
  1452. if value_node.comment:
  1453. omap._yaml_add_comment(value_node.comment, value=key)
  1454. else:
  1455. # NEWCMNT
  1456. if key_node.comment:
  1457. nprintf('nc9a', key_node.comment)
  1458. if subnode.comment:
  1459. nprintf('nc9b', subnode.comment)
  1460. if value_node.comment:
  1461. nprintf('nc9c', value_node.comment)
  1462. omap[key] = value
  1463. def construct_yaml_set(self, node: Any) -> Iterator[CommentedSet]:
  1464. data = CommentedSet()
  1465. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1466. if node.flow_style is True:
  1467. data.fa.set_flow_style()
  1468. elif node.flow_style is False:
  1469. data.fa.set_block_style()
  1470. yield data
  1471. self.construct_setting(node, data)
  1472. def construct_unknown(
  1473. self, node: Any,
  1474. ) -> Iterator[Union[CommentedMap, TaggedScalar, CommentedSeq]]:
  1475. try:
  1476. if isinstance(node, MappingNode):
  1477. data = CommentedMap()
  1478. data._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1479. if node.flow_style is True:
  1480. data.fa.set_flow_style()
  1481. elif node.flow_style is False:
  1482. data.fa.set_block_style()
  1483. data.yaml_set_ctag(node.ctag)
  1484. yield data
  1485. if node.anchor:
  1486. from ruamel.yaml.serializer import templated_id
  1487. if not templated_id(node.anchor):
  1488. data.yaml_set_anchor(node.anchor)
  1489. self.construct_mapping(node, data)
  1490. return
  1491. elif isinstance(node, ScalarNode):
  1492. data2 = TaggedScalar()
  1493. data2.value = self.construct_scalar(node)
  1494. data2.style = node.style
  1495. data2.yaml_set_ctag(node.ctag)
  1496. yield data2
  1497. if node.anchor:
  1498. from ruamel.yaml.serializer import templated_id
  1499. if not templated_id(node.anchor):
  1500. data2.yaml_set_anchor(node.anchor, always_dump=True)
  1501. return
  1502. elif isinstance(node, SequenceNode):
  1503. data3 = CommentedSeq()
  1504. data3._yaml_set_line_col(node.start_mark.line, node.start_mark.column)
  1505. if node.flow_style is True:
  1506. data3.fa.set_flow_style()
  1507. elif node.flow_style is False:
  1508. data3.fa.set_block_style()
  1509. data3.yaml_set_ctag(node.ctag)
  1510. yield data3
  1511. if node.anchor:
  1512. from ruamel.yaml.serializer import templated_id
  1513. if not templated_id(node.anchor):
  1514. data3.yaml_set_anchor(node.anchor)
  1515. data3.extend(self.construct_sequence(node))
  1516. return
  1517. except: # NOQA
  1518. pass
  1519. raise ConstructorError(
  1520. None,
  1521. None,
  1522. f'could not determine a constructor for the tag {node.tag!r}',
  1523. node.start_mark,
  1524. )
  1525. def construct_yaml_timestamp(
  1526. self, node: Any, values: Any = None,
  1527. ) -> Union[datetime.date, datetime.datetime, TimeStamp]:
  1528. try:
  1529. match = self.timestamp_regexp.match(node.value)
  1530. except TypeError:
  1531. match = None
  1532. if match is None:
  1533. raise ConstructorError(
  1534. None,
  1535. None,
  1536. f'failed to construct timestamp from "{node.value}"',
  1537. node.start_mark,
  1538. )
  1539. values = match.groupdict()
  1540. if not values['hour']:
  1541. return create_timestamp(**values)
  1542. # return SafeConstructor.construct_yaml_timestamp(self, node, values)
  1543. for part in ['t', 'tz_sign', 'tz_hour', 'tz_minute']:
  1544. if values[part]:
  1545. break
  1546. else:
  1547. return create_timestamp(**values)
  1548. # return SafeConstructor.construct_yaml_timestamp(self, node, values)
  1549. dd = create_timestamp(**values) # this has delta applied
  1550. delta = None
  1551. if values['tz_sign']:
  1552. tz_hour = int(values['tz_hour'])
  1553. minutes = values['tz_minute']
  1554. tz_minute = int(minutes) if minutes else 0
  1555. delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
  1556. if values['tz_sign'] == '-':
  1557. delta = -delta
  1558. # should check for None and solve issue 366 should be tzinfo=delta)
  1559. # isinstance(datetime.datetime.now, datetime.date) is true)
  1560. if isinstance(dd, datetime.datetime):
  1561. data = TimeStamp(
  1562. dd.year, dd.month, dd.day, dd.hour, dd.minute, dd.second, dd.microsecond,
  1563. )
  1564. else:
  1565. # ToDo: make this into a DateStamp?
  1566. data = TimeStamp(dd.year, dd.month, dd.day, 0, 0, 0, 0)
  1567. return data
  1568. if delta:
  1569. data._yaml['delta'] = delta
  1570. tz = values['tz_sign'] + values['tz_hour']
  1571. if values['tz_minute']:
  1572. tz += ':' + values['tz_minute']
  1573. data._yaml['tz'] = tz
  1574. else:
  1575. if values['tz']: # no delta
  1576. data._yaml['tz'] = values['tz']
  1577. if values['t']:
  1578. data._yaml['t'] = True
  1579. return data
  1580. def construct_yaml_sbool(self, node: Any) -> Union[bool, ScalarBoolean]:
  1581. b = SafeConstructor.construct_yaml_bool(self, node)
  1582. if node.anchor:
  1583. return ScalarBoolean(b, anchor=node.anchor)
  1584. return b
  1585. RoundTripConstructor.add_default_constructor('bool', method='construct_yaml_sbool')
  1586. for tag in 'null int float binary timestamp omap pairs set str seq map'.split():
  1587. RoundTripConstructor.add_default_constructor(tag)
  1588. RoundTripConstructor.add_constructor(None, RoundTripConstructor.construct_unknown)