123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150 |
- """
- FIXME:https://github.com/twisted/twisted/issues/3843
- This can be removed once t.persisted.aot is removed.
- New code should not make use of this.
- Token constants.
- vendored from https://github.com/python/cpython/blob/6b825c1b8a14460641ca6f1647d83005c68199aa/Lib/token.py
- Licence: https://docs.python.org/3/license.html
- """
- # Auto-generated by Tools/scripts/generate_token.py
- __all__ = ["tok_name", "ISTERMINAL", "ISNONTERMINAL", "ISEOF"]
- ENDMARKER = 0
- NAME = 1
- NUMBER = 2
- STRING = 3
- NEWLINE = 4
- INDENT = 5
- DEDENT = 6
- LPAR = 7
- RPAR = 8
- LSQB = 9
- RSQB = 10
- COLON = 11
- COMMA = 12
- SEMI = 13
- PLUS = 14
- MINUS = 15
- STAR = 16
- SLASH = 17
- VBAR = 18
- AMPER = 19
- LESS = 20
- GREATER = 21
- EQUAL = 22
- DOT = 23
- PERCENT = 24
- LBRACE = 25
- RBRACE = 26
- EQEQUAL = 27
- NOTEQUAL = 28
- LESSEQUAL = 29
- GREATEREQUAL = 30
- TILDE = 31
- CIRCUMFLEX = 32
- LEFTSHIFT = 33
- RIGHTSHIFT = 34
- DOUBLESTAR = 35
- PLUSEQUAL = 36
- MINEQUAL = 37
- STAREQUAL = 38
- SLASHEQUAL = 39
- PERCENTEQUAL = 40
- AMPEREQUAL = 41
- VBAREQUAL = 42
- CIRCUMFLEXEQUAL = 43
- LEFTSHIFTEQUAL = 44
- RIGHTSHIFTEQUAL = 45
- DOUBLESTAREQUAL = 46
- DOUBLESLASH = 47
- DOUBLESLASHEQUAL = 48
- AT = 49
- ATEQUAL = 50
- RARROW = 51
- ELLIPSIS = 52
- COLONEQUAL = 53
- OP = 54
- AWAIT = 55
- ASYNC = 56
- TYPE_IGNORE = 57
- TYPE_COMMENT = 58
- SOFT_KEYWORD = 59
- # These aren't used by the C tokenizer but are needed for tokenize.py
- ERRORTOKEN = 60
- COMMENT = 61
- NL = 62
- ENCODING = 63
- N_TOKENS = 64
- # Special definitions for cooperation with parser
- NT_OFFSET = 256
- tok_name = {
- value: name
- for name, value in globals().items()
- if isinstance(value, int) and not name.startswith("_")
- }
- __all__.extend(tok_name.values())
- EXACT_TOKEN_TYPES = {
- "!=": NOTEQUAL,
- "%": PERCENT,
- "%=": PERCENTEQUAL,
- "&": AMPER,
- "&=": AMPEREQUAL,
- "(": LPAR,
- ")": RPAR,
- "*": STAR,
- "**": DOUBLESTAR,
- "**=": DOUBLESTAREQUAL,
- "*=": STAREQUAL,
- "+": PLUS,
- "+=": PLUSEQUAL,
- ",": COMMA,
- "-": MINUS,
- "-=": MINEQUAL,
- "->": RARROW,
- ".": DOT,
- "...": ELLIPSIS,
- "/": SLASH,
- "//": DOUBLESLASH,
- "//=": DOUBLESLASHEQUAL,
- "/=": SLASHEQUAL,
- ":": COLON,
- ":=": COLONEQUAL,
- ";": SEMI,
- "<": LESS,
- "<<": LEFTSHIFT,
- "<<=": LEFTSHIFTEQUAL,
- "<=": LESSEQUAL,
- "=": EQUAL,
- "==": EQEQUAL,
- ">": GREATER,
- ">=": GREATEREQUAL,
- ">>": RIGHTSHIFT,
- ">>=": RIGHTSHIFTEQUAL,
- "@": AT,
- "@=": ATEQUAL,
- "[": LSQB,
- "]": RSQB,
- "^": CIRCUMFLEX,
- "^=": CIRCUMFLEXEQUAL,
- "{": LBRACE,
- "|": VBAR,
- "|=": VBAREQUAL,
- "}": RBRACE,
- "~": TILDE,
- }
- def ISTERMINAL(x):
- return x < NT_OFFSET
- def ISNONTERMINAL(x):
- return x >= NT_OFFSET
- def ISEOF(x):
- return x == ENDMARKER
|