_token.py 2.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150
  1. """
  2. FIXME:https://github.com/twisted/twisted/issues/3843
  3. This can be removed once t.persisted.aot is removed.
  4. New code should not make use of this.
  5. Token constants.
  6. vendored from https://github.com/python/cpython/blob/6b825c1b8a14460641ca6f1647d83005c68199aa/Lib/token.py
  7. Licence: https://docs.python.org/3/license.html
  8. """
  9. # Auto-generated by Tools/scripts/generate_token.py
  10. __all__ = ["tok_name", "ISTERMINAL", "ISNONTERMINAL", "ISEOF"]
  11. ENDMARKER = 0
  12. NAME = 1
  13. NUMBER = 2
  14. STRING = 3
  15. NEWLINE = 4
  16. INDENT = 5
  17. DEDENT = 6
  18. LPAR = 7
  19. RPAR = 8
  20. LSQB = 9
  21. RSQB = 10
  22. COLON = 11
  23. COMMA = 12
  24. SEMI = 13
  25. PLUS = 14
  26. MINUS = 15
  27. STAR = 16
  28. SLASH = 17
  29. VBAR = 18
  30. AMPER = 19
  31. LESS = 20
  32. GREATER = 21
  33. EQUAL = 22
  34. DOT = 23
  35. PERCENT = 24
  36. LBRACE = 25
  37. RBRACE = 26
  38. EQEQUAL = 27
  39. NOTEQUAL = 28
  40. LESSEQUAL = 29
  41. GREATEREQUAL = 30
  42. TILDE = 31
  43. CIRCUMFLEX = 32
  44. LEFTSHIFT = 33
  45. RIGHTSHIFT = 34
  46. DOUBLESTAR = 35
  47. PLUSEQUAL = 36
  48. MINEQUAL = 37
  49. STAREQUAL = 38
  50. SLASHEQUAL = 39
  51. PERCENTEQUAL = 40
  52. AMPEREQUAL = 41
  53. VBAREQUAL = 42
  54. CIRCUMFLEXEQUAL = 43
  55. LEFTSHIFTEQUAL = 44
  56. RIGHTSHIFTEQUAL = 45
  57. DOUBLESTAREQUAL = 46
  58. DOUBLESLASH = 47
  59. DOUBLESLASHEQUAL = 48
  60. AT = 49
  61. ATEQUAL = 50
  62. RARROW = 51
  63. ELLIPSIS = 52
  64. COLONEQUAL = 53
  65. OP = 54
  66. AWAIT = 55
  67. ASYNC = 56
  68. TYPE_IGNORE = 57
  69. TYPE_COMMENT = 58
  70. SOFT_KEYWORD = 59
  71. # These aren't used by the C tokenizer but are needed for tokenize.py
  72. ERRORTOKEN = 60
  73. COMMENT = 61
  74. NL = 62
  75. ENCODING = 63
  76. N_TOKENS = 64
  77. # Special definitions for cooperation with parser
  78. NT_OFFSET = 256
  79. tok_name = {
  80. value: name
  81. for name, value in globals().items()
  82. if isinstance(value, int) and not name.startswith("_")
  83. }
  84. __all__.extend(tok_name.values())
  85. EXACT_TOKEN_TYPES = {
  86. "!=": NOTEQUAL,
  87. "%": PERCENT,
  88. "%=": PERCENTEQUAL,
  89. "&": AMPER,
  90. "&=": AMPEREQUAL,
  91. "(": LPAR,
  92. ")": RPAR,
  93. "*": STAR,
  94. "**": DOUBLESTAR,
  95. "**=": DOUBLESTAREQUAL,
  96. "*=": STAREQUAL,
  97. "+": PLUS,
  98. "+=": PLUSEQUAL,
  99. ",": COMMA,
  100. "-": MINUS,
  101. "-=": MINEQUAL,
  102. "->": RARROW,
  103. ".": DOT,
  104. "...": ELLIPSIS,
  105. "/": SLASH,
  106. "//": DOUBLESLASH,
  107. "//=": DOUBLESLASHEQUAL,
  108. "/=": SLASHEQUAL,
  109. ":": COLON,
  110. ":=": COLONEQUAL,
  111. ";": SEMI,
  112. "<": LESS,
  113. "<<": LEFTSHIFT,
  114. "<<=": LEFTSHIFTEQUAL,
  115. "<=": LESSEQUAL,
  116. "=": EQUAL,
  117. "==": EQEQUAL,
  118. ">": GREATER,
  119. ">=": GREATEREQUAL,
  120. ">>": RIGHTSHIFT,
  121. ">>=": RIGHTSHIFTEQUAL,
  122. "@": AT,
  123. "@=": ATEQUAL,
  124. "[": LSQB,
  125. "]": RSQB,
  126. "^": CIRCUMFLEX,
  127. "^=": CIRCUMFLEXEQUAL,
  128. "{": LBRACE,
  129. "|": VBAR,
  130. "|=": VBAREQUAL,
  131. "}": RBRACE,
  132. "~": TILDE,
  133. }
  134. def ISTERMINAL(x):
  135. return x < NT_OFFSET
  136. def ISNONTERMINAL(x):
  137. return x >= NT_OFFSET
  138. def ISEOF(x):
  139. return x == ENDMARKER