tokens.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. # # header
  2. # coding: utf-8
  3. from __future__ import unicode_literals
  4. if False: # MYPY
  5. from typing import Text, Any, Dict, Optional, List # NOQA
  6. from .error import StreamMark # NOQA
  7. SHOWLINES = True
  8. class Token(object):
  9. __slots__ = 'start_mark', 'end_mark', '_comment'
  10. def __init__(self, start_mark, end_mark):
  11. # type: (StreamMark, StreamMark) -> None
  12. self.start_mark = start_mark
  13. self.end_mark = end_mark
  14. def __repr__(self):
  15. # type: () -> Any
  16. # attributes = [key for key in self.__slots__ if not key.endswith('_mark') and
  17. # hasattr('self', key)]
  18. attributes = [key for key in self.__slots__ if not key.endswith('_mark')]
  19. attributes.sort()
  20. arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) for key in attributes])
  21. if SHOWLINES:
  22. try:
  23. arguments += ', line: ' + str(self.start_mark.line)
  24. except: # NOQA
  25. pass
  26. try:
  27. arguments += ', comment: ' + str(self._comment)
  28. except: # NOQA
  29. pass
  30. return '{}({})'.format(self.__class__.__name__, arguments)
  31. def add_post_comment(self, comment):
  32. # type: (Any) -> None
  33. if not hasattr(self, '_comment'):
  34. self._comment = [None, None]
  35. self._comment[0] = comment
  36. def add_pre_comments(self, comments):
  37. # type: (Any) -> None
  38. if not hasattr(self, '_comment'):
  39. self._comment = [None, None]
  40. assert self._comment[1] is None
  41. self._comment[1] = comments
  42. def get_comment(self):
  43. # type: () -> Any
  44. return getattr(self, '_comment', None)
  45. @property
  46. def comment(self):
  47. # type: () -> Any
  48. return getattr(self, '_comment', None)
  49. def move_comment(self, target, empty=False):
  50. # type: (Any, bool) -> Any
  51. """move a comment from this token to target (normally next token)
  52. used to combine e.g. comments before a BlockEntryToken to the
  53. ScalarToken that follows it
  54. empty is a special for empty values -> comment after key
  55. """
  56. c = self.comment
  57. if c is None:
  58. return
  59. # don't push beyond last element
  60. if isinstance(target, (StreamEndToken, DocumentStartToken)):
  61. return
  62. delattr(self, '_comment')
  63. tc = target.comment
  64. if not tc: # target comment, just insert
  65. # special for empty value in key: value issue 25
  66. if empty:
  67. c = [c[0], c[1], None, None, c[0]]
  68. target._comment = c
  69. # nprint('mco2:', self, target, target.comment, empty)
  70. return self
  71. if c[0] and tc[0] or c[1] and tc[1]:
  72. raise NotImplementedError('overlap in comment %r %r' % (c, tc))
  73. if c[0]:
  74. tc[0] = c[0]
  75. if c[1]:
  76. tc[1] = c[1]
  77. return self
  78. def split_comment(self):
  79. # type: () -> Any
  80. """ split the post part of a comment, and return it
  81. as comment to be added. Delete second part if [None, None]
  82. abc: # this goes to sequence
  83. # this goes to first element
  84. - first element
  85. """
  86. comment = self.comment
  87. if comment is None or comment[0] is None:
  88. return None # nothing to do
  89. ret_val = [comment[0], None]
  90. if comment[1] is None:
  91. delattr(self, '_comment')
  92. return ret_val
  93. # class BOMToken(Token):
  94. # id = '<byte order mark>'
  95. class DirectiveToken(Token):
  96. __slots__ = 'name', 'value'
  97. id = '<directive>'
  98. def __init__(self, name, value, start_mark, end_mark):
  99. # type: (Any, Any, Any, Any) -> None
  100. Token.__init__(self, start_mark, end_mark)
  101. self.name = name
  102. self.value = value
  103. class DocumentStartToken(Token):
  104. __slots__ = ()
  105. id = '<document start>'
  106. class DocumentEndToken(Token):
  107. __slots__ = ()
  108. id = '<document end>'
  109. class StreamStartToken(Token):
  110. __slots__ = ('encoding',)
  111. id = '<stream start>'
  112. def __init__(self, start_mark=None, end_mark=None, encoding=None):
  113. # type: (Any, Any, Any) -> None
  114. Token.__init__(self, start_mark, end_mark)
  115. self.encoding = encoding
  116. class StreamEndToken(Token):
  117. __slots__ = ()
  118. id = '<stream end>'
  119. class BlockSequenceStartToken(Token):
  120. __slots__ = ()
  121. id = '<block sequence start>'
  122. class BlockMappingStartToken(Token):
  123. __slots__ = ()
  124. id = '<block mapping start>'
  125. class BlockEndToken(Token):
  126. __slots__ = ()
  127. id = '<block end>'
  128. class FlowSequenceStartToken(Token):
  129. __slots__ = ()
  130. id = '['
  131. class FlowMappingStartToken(Token):
  132. __slots__ = ()
  133. id = '{'
  134. class FlowSequenceEndToken(Token):
  135. __slots__ = ()
  136. id = ']'
  137. class FlowMappingEndToken(Token):
  138. __slots__ = ()
  139. id = '}'
  140. class KeyToken(Token):
  141. __slots__ = ()
  142. id = '?'
  143. # def x__repr__(self):
  144. # return 'KeyToken({})'.format(
  145. # self.start_mark.buffer[self.start_mark.index:].split(None, 1)[0])
  146. class ValueToken(Token):
  147. __slots__ = ()
  148. id = ':'
  149. class BlockEntryToken(Token):
  150. __slots__ = ()
  151. id = '-'
  152. class FlowEntryToken(Token):
  153. __slots__ = ()
  154. id = ','
  155. class AliasToken(Token):
  156. __slots__ = ('value',)
  157. id = '<alias>'
  158. def __init__(self, value, start_mark, end_mark):
  159. # type: (Any, Any, Any) -> None
  160. Token.__init__(self, start_mark, end_mark)
  161. self.value = value
  162. class AnchorToken(Token):
  163. __slots__ = ('value',)
  164. id = '<anchor>'
  165. def __init__(self, value, start_mark, end_mark):
  166. # type: (Any, Any, Any) -> None
  167. Token.__init__(self, start_mark, end_mark)
  168. self.value = value
  169. class TagToken(Token):
  170. __slots__ = ('value',)
  171. id = '<tag>'
  172. def __init__(self, value, start_mark, end_mark):
  173. # type: (Any, Any, Any) -> None
  174. Token.__init__(self, start_mark, end_mark)
  175. self.value = value
  176. class ScalarToken(Token):
  177. __slots__ = 'value', 'plain', 'style'
  178. id = '<scalar>'
  179. def __init__(self, value, plain, start_mark, end_mark, style=None):
  180. # type: (Any, Any, Any, Any, Any) -> None
  181. Token.__init__(self, start_mark, end_mark)
  182. self.value = value
  183. self.plain = plain
  184. self.style = style
  185. class CommentToken(Token):
  186. __slots__ = 'value', 'pre_done'
  187. id = '<comment>'
  188. def __init__(self, value, start_mark, end_mark):
  189. # type: (Any, Any, Any) -> None
  190. Token.__init__(self, start_mark, end_mark)
  191. self.value = value
  192. def reset(self):
  193. # type: () -> None
  194. if hasattr(self, 'pre_done'):
  195. delattr(self, 'pre_done')
  196. def __repr__(self):
  197. # type: () -> Any
  198. v = '{!r}'.format(self.value)
  199. if SHOWLINES:
  200. try:
  201. v += ', line: ' + str(self.start_mark.line)
  202. v += ', col: ' + str(self.start_mark.column)
  203. except: # NOQA
  204. pass
  205. return 'CommentToken({})'.format(v)
  206. def __eq__(self, other):
  207. # type: (Any) -> bool
  208. if self.start_mark != other.start_mark:
  209. return False
  210. if self.end_mark != other.end_mark:
  211. return False
  212. if self.value != other.value:
  213. return False
  214. return True
  215. def __ne__(self, other):
  216. # type: (Any) -> bool
  217. return not self.__eq__(other)