Tokenizer.hpp

Idź do dokumentacji tego pliku.

#if defined(_MSC_VER) && (_MSC_VER >= 1200)
#pragma once
#endif
#ifndef COMMON_TOKENIZER_H_
#define COMMON_TOKENIZER_H_

#include "Error.hpp"

namespace common
{

class Stream;


class TokenizerError : public Error
{
private:
        size_t m_Char, m_Row, m_Col;
public:
        TokenizerError(size_t Char, size_t Row, size_t Col, const tstring &Msg = _T(""), const tstring &File = _T(""), int a_Line = 0);

        size_t GetCharNum() { return m_Char; }
        size_t GetRowNum() { return m_Row; }
        size_t GetColNum() { return m_Col; }
};

class Tokenizer_pimpl;

class Tokenizer
{
private:
        scoped_ptr<Tokenizer_pimpl> pimpl;

public:
        static const uint FLAG_TOKEN_EOL         = 0x01;
        static const uint FLAG_MULTILINE_STRINGS = 0x02;

        enum TOKEN
        {
                TOKEN_EOF,        
                TOKEN_EOL,        
                TOKEN_SYMBOL,     
                TOKEN_INTEGER,    
                TOKEN_FLOAT,      
                TOKEN_CHAR,       
                TOKEN_IDENTIFIER, 
                TOKEN_KEYWORD,    
                TOKEN_STRING,     
        };

        Tokenizer(const tchar *Input, size_t NumChars, uint Flags);
        Tokenizer(const tstring *Input, uint Flags);
        Tokenizer(Stream *Input, uint Flags);

        ~Tokenizer();

        void RegisterKeyword(uint Id, const tstring &Keyword);

        void RegisterKeywords(const tchar **Keywords, size_t KeywordCount);


        void Next();

        TOKEN GetToken();

        size_t GetCharNum();
        size_t GetRowNum();
        size_t GetColNum();

        const tstring & GetString();
        tchar GetChar();
        uint GetId();
        bool GetUint1(uint1 *Out);
        bool GetUint2(uint2 *Out);
        bool GetUint4(uint4 *Out);
        bool GetUint8(uint8 *Out);
        bool GetInt1(int1 *Out);
        bool GetInt2(int2 *Out);
        bool GetInt4(int4 *Out);
        bool GetInt8(int8 *Out);
        uint1 MustGetUint1();
        uint2 MustGetUint2();
        uint4 MustGetUint4();
        uint8 MustGetUint8();
        int1 MustGetInt1();
        int2 MustGetInt2();
        int4 MustGetInt4();
        int8 MustGetInt8();
        bool GetFloat(float *Out);
        bool GetDouble(double *Out);
        float MustGetFloat();
        double MustGetDouble();


        void CreateError();
        void CreateError(const tstring &Msg);

        bool QueryToken(TOKEN Token);
        bool QueryToken(TOKEN Token1, TOKEN Token2);
        bool QueryEOF();
        bool QueryEOL();
        bool QuerySymbol(tchar Symbol);
        bool QueryIdentifier(const tstring &Identifier);
        bool QueryKeyword(uint KeywordId);
        bool QueryKeyword(const tstring &Keyword);


        void AssertToken(TOKEN Token);
        void AssertToken(TOKEN Token1, TOKEN Token2);
        void AssertEOF();
        void AssertEOL();
        void AssertSymbol(tchar Symbol);
        void AssertIdentifier(const tstring &Identifier);
        void AssertKeyword(uint KeywordId);
        void AssertKeyword(const tstring &Keyword);

        static const tchar * GetTokenName(TOKEN Token);

        static const uint ESCAPE_EOL   = 0x01; 
        static const uint ESCAPE_OTHER = 0x02; 


        static void Escape(tstring *Out, const tstring &In, uint EscapeFlags);
};

// code_tokenizer

} // namespace common

#endif

---