GCSx/include/gcsx_tokenize.h

/* GCSx
** TOKENIZE.H
**
** Script tokenization (to feed to compiler)
*/

/*****************************************************************************
** Copyright (C) 2003-2006 Janson
**
** This program is free software; you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation; either version 2 of the License, or
** (at your option) any later version.
** 
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU General Public License for more details.
**
** You should have received a copy of the GNU General Public License
** along with this program; if not, write to the Free Software
** Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.
*****************************************************************************/

#ifndef __GCSx_TOKENIZE_H_
#define __GCSx_TOKENIZE_H_

class Tokenizer {
private:
    const std::list<std::string>* source;
    std::list<std::string>::const_iterator row;
    int rowNum;
    int col;
    int rowLen;
    int atNewLine;
    int nextCloseBrace;
    
    int errorCount;
    int warningCount;
    int silent;
    
    int errRow;
    int errCol;
    char* errBuffer;

    struct Token {
        int type;
        std::string* text;
        int rowN;
        int colN;
    };

    // Used for both peeks and bookmarks
    int nextBookmarkName;
    int cacheRecord;
    int bookmarkNew;
    std::list<Token> cached;
    std::list<Token>::iterator cacheReplay;
    std::map<int, std::list<Token>::iterator> bookmarks;
    void deallocRange(std::list<Token>::iterator start, std::list<Token>::iterator end);

    struct TokenStr {
        int type;
        const char* text;
    };
    static const TokenStr tokenStrings[];
    static const char* debugText[];
    static std::map<std::string, int>* tokenLookup;
    static void initTokenLookups();

    // All other functions assert not at EOF
    // Can never reach EOF until after reading an EOL, however
    int atEOF();
    char getCharacter(); // Returns '\0' for EOL
    void moveNext();
    void nextLine(); // Assumes there IS a next line
    
    // If next char is boundary, will return blank
    // If reaches EOL, throws 1 and does not move position
    // Otherwise, boundary character is next to be retrieved
    std::string grabUntil(const char* boundaries) throw_int;
    // If next char is not in charset, will return blank
    // If reached EOL, ends there; EOL should then be retrieved via getCharacter()
    std::string grabWhile(const char* charset); 
    std::string grabRestOfLine();

public:
    Tokenizer(const std::list<std::string>* src);
    ~Tokenizer();

    static void destroyGlobals();
    
    // Returns true if a new token was retrieved
    // False = EOF (always a ENDLINE before this, though)
    // Discards comments
    // Lowercases everything but strings
    // Automatically ensures EOL before a }, and NOT before a { or after a { or }
    int nextToken(int& type, std::string& token);

    // Look at next token without retrieving it
    int peekToken(int& type, std::string& token);
    void skipToken();

    // Bookmark position within token stream; return to it or cancel bookmark
    // Reserved bookmarks:
    //  1- when checking an identifier for label status
    void bookmarkStore(int name = 0);
    void bookmarkReturn(int name = 0);
    void bookmarkCancel(int name = 0);
    // Creates a unique bookmark name over 1000
    int getBookmarkName();
    
    // For error outputting with char/line- outputs at location
    // of start of token last retrieved with nextToken OR peekToken
    void outputError(const char* text, ...);
    void outputWarning(const char* text, ...);
    void silentErrors(int newSilent = 1);
    void resetErrors();
    int numErrors() { return errorCount; }
    int numWarnings() { return warningCount; }
};

#endif