ydb/library/cpp/yson/lexer_detail.h

#pragma once

#include "detail.h"
#include "token.h"

namespace NYson {
    ////////////////////////////////////////////////////////////////////////////////

    namespace NDetail {
        /*! \internal */
        ////////////////////////////////////////////////////////////////////////////////

        // EReadStartCase tree representation:
        // Root                                =     xb
        //     BinaryStringOrOtherSpecialToken =    x0b
        //         BinaryString                =    00b
        //         OtherSpecialToken           =    10b
        //     Other                           =    x1b
        //         BinaryScalar                =  xx01b
        //             BinaryInt64             =  0001b
        //             BinaryDouble            =  0101b
        //             BinaryFalse             =  1001b
        //             BinaryTrue              =  1101b
        //         Other                       = xxx11b
        //             Quote                   = 00011b
        //             DigitOrMinus            = 00111b
        //             String                  = 01011b
        //             Space                   = 01111b
        //             Plus                    = 10011b
        //             None                    = 10111b
        //             Percent                 = 11011b

        enum EReadStartCase : unsigned {
            BinaryString = 0,      // =    00b
            OtherSpecialToken = 2, // =    10b

            BinaryInt64 = 1,   // =   001b
            BinaryDouble = 5,  // =   101b
            BinaryFalse = 9,   // =  1001b
            BinaryTrue = 13,   // =  1101b
            BinaryUint64 = 17, // = 10001b

            Quote = 3,        // = 00011b
            DigitOrMinus = 7, // = 00111b
            String = 11,      // = 01011b
            Space = 15,       // = 01111b
            Plus = 19,        // = 10011b
            None = 23,        // = 10111b
            Percent = 27      // = 11011b
        };

        template <class TBlockStream, bool EnableLinePositionInfo>
        class TLexer
           : public TLexerBase<TBlockStream, EnableLinePositionInfo> {
        private:
            using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>;

            static EReadStartCase GetStartState(char ch) {
#define NN EReadStartCase::None
#define BS EReadStartCase::BinaryString
#define BI EReadStartCase::BinaryInt64
#define BD EReadStartCase::BinaryDouble
#define BF EReadStartCase::BinaryFalse
#define BT EReadStartCase::BinaryTrue
#define BU EReadStartCase::BinaryUint64
#define SP NN // EReadStartCase::Space
#define DM EReadStartCase::DigitOrMinus
#define ST EReadStartCase::String
#define PL EReadStartCase::Plus
#define QU EReadStartCase::Quote
#define PC EReadStartCase::Percent
#define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken)

                static const ui8 lookupTable[] =
                    {
                        NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,

                        // 32
                        SP,                   // ' '
                        NN,                   // '!'
                        QU,                   // '"'
                        TT(Hash),             // '#'
                        NN,                   // '$'
                        PC,                   // '%'
                        NN,                   // '&'
                        NN,                   // "'"
                        TT(LeftParenthesis),  // '('
                        TT(RightParenthesis), // ')'
                        NN,                   // '*'
                        PL,                   // '+'
                        TT(Comma),            // ','
                        DM,                   // '-'
                        NN,                   // '.'
                        NN,                   // '/'

                        // 48
                        DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9'
                        TT(Colon),                              // ':'
                        TT(Semicolon),                          // ';'
                        TT(LeftAngle),                          // '<'
                        TT(Equals),                             // '='
                        TT(RightAngle),                         // '>'
                        NN,                                     // '?'

                        // 64
                        NN,                                                 // '@'
                        ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M'
                        ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'N' - 'Z'
                        TT(LeftBracket),                                    // '['
                        NN,                                                 // '\'
                        TT(RightBracket),                                   // ']'
                        NN,                                                 // '^'
                        ST,                                                 // '_'

                        // 96
                        NN, // '`'

                        ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm'
                        ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z'
                        TT(LeftBrace),                                      // '{'
                        NN,                                                 // '|'
                        TT(RightBrace),                                     // '}'
                        NN,                                                 // '~'
                        NN,                                                 // '^?' non-printable
                        // 128
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,

                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN,
                        NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN};

#undef NN
#undef BS
#undef BI
#undef BD
#undef SP
#undef DM
#undef ST
#undef PL
#undef QU
#undef TT
                return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]);
            }

        public:
            TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit)
                : TBase(blockStream, memoryLimit)
            {
            }

            void GetToken(TToken* token) {
                char ch1 = TBase::SkipSpaceAndGetChar();
                auto state = GetStartState(ch1);
                auto stateBits = static_cast<unsigned>(state);

                if (ch1 == '\0') {
                    *token = TToken::EndOfStream;
                    return;
                }

                if (stateBits & 1) {          // Other = x1b
                    if (stateBits & 1 << 1) { // Other = xxx11b
                        if (state == EReadStartCase::Quote) {
                            TStringBuf value;
                            TBase::Advance(1);
                            TBase::ReadQuotedString(&value);
                            *token = TToken(value);
                        } else if (state == EReadStartCase::DigitOrMinus) {
                            ReadNumeric<true>(token);
                        } else if (state == EReadStartCase::Plus) {
                            TBase::Advance(1);

                            char ch2 = TBase::template GetChar<true>();

                            if (!isdigit(ch2)) {
                                *token = TToken(ETokenType::Plus);
                            } else {
                                ReadNumeric<true>(token);
                            }
                        } else if (state == EReadStartCase::String) {
                            TStringBuf value;
                            TBase::template ReadUnquotedString<true>(&value);
                            *token = TToken(value);
                        } else if (state == EReadStartCase::Percent) {
                            TBase::Advance(1);
                            char ch3 = TBase::template GetChar<true>();
                            if (ch3 == 't' || ch3 == 'f') {
                                *token = TToken(TBase::template ReadBoolean<true>());
                            } else {
                                *token = TToken(TBase::template ReadNanOrInf<true>());
                            }
                        } else { // None
                            Y_ASSERT(state == EReadStartCase::None);
                            ythrow TYsonException() << "Unexpected " << ch1;
                        }
                    } else { // BinaryScalar = x01b
                        TBase::Advance(1);
                        if (state == EReadStartCase::BinaryDouble) {
                            double value;
                            TBase::ReadBinaryDouble(&value);
                            *token = TToken(value);
                        } else if (state == EReadStartCase::BinaryInt64) {
                            i64 value;
                            TBase::ReadBinaryInt64(&value);
                            *token = TToken(value);
                        } else if (state == EReadStartCase::BinaryUint64) {
                            ui64 value;
                            TBase::ReadBinaryUint64(&value);
                            *token = TToken(value);
                        } else if (state == EReadStartCase::BinaryFalse) {
                            *token = TToken(false);
                        } else if (state == EReadStartCase::BinaryTrue) {
                            *token = TToken(true);
                        } else {
                            Y_ABORT("unreachable");
                        }
                    }
                } else { // BinaryStringOrOtherSpecialToken = x0b
                    TBase::Advance(1);
                    if (stateBits & 1 << 1) { // OtherSpecialToken = 10b
                        Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken));
                        *token = TToken(ETokenType(stateBits >> 2));
                    } else { // BinaryString = 00b
                        Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::BinaryString));
                        TStringBuf value;
                        TBase::ReadBinaryString(&value);
                        *token = TToken(value);
                    }
                }
            }

            template <bool AllowFinish>
            void ReadNumeric(TToken* token) {
                TStringBuf valueBuffer;
                ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer);

                if (numericResult == ENumericResult::Double) {
                    try {
                        *token = TToken(FromString<double>(valueBuffer));
                    } catch (yexception&) {
                        ythrow TYsonException() << "Error parsing double literal " << valueBuffer;
                    }
                } else if (numericResult == ENumericResult::Int64) {
                    try {
                        *token = TToken(FromString<i64>(valueBuffer));
                    } catch (yexception&) {
                        ythrow TYsonException() << "Error parsing int64 literal " << valueBuffer;
                    }
                } else if (numericResult == ENumericResult::Uint64) {
                    try {
                        *token = TToken(FromString<ui64>(valueBuffer.SubStr(0, valueBuffer.size() - 1)));
                    } catch (yexception&) {
                        ythrow TYsonException() << "Error parsing uint64 literal " << valueBuffer;
                    }
                }
            }
        };
        ////////////////////////////////////////////////////////////////////////////////
        /*! \endinternal */
    }

    class TStatelessYsonLexerImplBase {
    public:
        virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0;

        virtual ~TStatelessYsonLexerImplBase() {
        }
    };

    template <bool EnableLinePositionInfo>
    class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase {
    private:
        using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>;
        TLexer Lexer;

    public:
        TStatelesYsonLexerImpl()
            : Lexer(TStringReader(), Nothing())
        {
        }

        size_t GetToken(const TStringBuf& data, TToken* token) override {
            Lexer.SetBuffer(data.begin(), data.end());
            Lexer.GetToken(token);
            return Lexer.Begin() - data.begin();
        }
    };

    ////////////////////////////////////////////////////////////////////////////////

} // namespace NYson