123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154 |
- #ifndef Py_TOKENIZER_H
- #define Py_TOKENIZER_H
- #ifdef __cplusplus
- extern "C" {
- #endif
- #include "object.h"
- #include "pycore_token.h"
- #define MAXINDENT 100
- #define MAXLEVEL 200
- #define MAXFSTRINGLEVEL 150
- enum decoding_state {
- STATE_INIT,
- STATE_SEEK_CODING,
- STATE_NORMAL
- };
- enum interactive_underflow_t {
-
- IUNDERFLOW_NORMAL,
-
- IUNDERFLOW_STOP,
- };
- struct token {
- int level;
- int lineno, col_offset, end_lineno, end_col_offset;
- const char *start, *end;
- PyObject *metadata;
- };
- enum tokenizer_mode_kind_t {
- TOK_REGULAR_MODE,
- TOK_FSTRING_MODE,
- };
- #define MAX_EXPR_NESTING 3
- typedef struct _tokenizer_mode {
- enum tokenizer_mode_kind_t kind;
- int curly_bracket_depth;
- int curly_bracket_expr_start_depth;
- char f_string_quote;
- int f_string_quote_size;
- int f_string_raw;
- const char* f_string_start;
- const char* f_string_multi_line_start;
- int f_string_line_start;
- Py_ssize_t f_string_start_offset;
- Py_ssize_t f_string_multi_line_start_offset;
- Py_ssize_t last_expr_size;
- Py_ssize_t last_expr_end;
- char* last_expr_buffer;
- int f_string_debug;
- } tokenizer_mode;
- struct tok_state {
-
-
- char *buf;
- char *cur;
- char *inp;
- int fp_interactive;
- char *interactive_src_start;
- char *interactive_src_end;
- const char *end;
- const char *start;
- int done;
-
- FILE *fp;
- int tabsize;
- int indent;
- int indstack[MAXINDENT];
- int atbol;
- int pendin;
- const char *prompt, *nextprompt;
- int lineno;
- int first_lineno;
- int starting_col_offset;
- int col_offset;
- int level;
-
- char parenstack[MAXLEVEL];
- int parenlinenostack[MAXLEVEL];
- int parencolstack[MAXLEVEL];
- PyObject *filename;
-
- int altindstack[MAXINDENT];
-
- enum decoding_state decoding_state;
- int decoding_erred;
- char *encoding;
- int cont_line;
- const char* line_start;
- const char* multi_line_start;
- PyObject *decoding_readline;
- PyObject *decoding_buffer;
- PyObject *readline;
- const char* enc;
- char* str;
- char* input;
- int type_comments;
-
- int async_hacks;
- int async_def;
- int async_def_indent;
- int async_def_nl;
-
- enum interactive_underflow_t interactive_underflow;
- int report_warnings;
-
- tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
- int tok_mode_stack_index;
- int tok_extra_tokens;
- int comment_newline;
- int implicit_newline;
- #ifdef Py_DEBUG
- int debug;
- #endif
- };
- extern struct tok_state *_PyTokenizer_FromString(const char *, int, int);
- extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
- extern struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int);
- extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
- const char *, const char *);
- extern void _PyTokenizer_Free(struct tok_state *);
- extern void _PyToken_Free(struct token *);
- extern void _PyToken_Init(struct token *);
- extern int _PyTokenizer_Get(struct tok_state *, struct token *);
- #define tok_dump _Py_tok_dump
- #ifdef __cplusplus
- }
- #endif
- #endif
|