Tokenizer.h

#pragma once


#include <libutl/Array.h>
#include <libutl/List.h>
#include <libutl/Token.h>


UTL_NS_BEGIN;


class Stream;
class TokenizerTokens;

// Tokenizer ///////////////////////////////////////////////////////////////////////////////////////


class Tokenizer : public Object
{
    UTL_CLASS_DECL(Tokenizer, Object);
    UTL_CLASS_NO_COMPARE;

public:
    virtual void copy(const Object& rhs);

    void clear();

    void addTerminal(uint_t id, const String& regex);

    void scan(TokenizerTokens& tokens, Stream* is, bool owner = true) const;

protected:
    Array _terminals;
    bool _reverse;

private:
    void
    init()
    {
        _reverse = false;
    }
    void
    deInit()
    {
    }
};

// StringTokenizer /////////////////////////////////////////////////////////////////////////////////


class StringTokenizer : public Tokenizer
{
    UTL_CLASS_DECL(StringTokenizer, Tokenizer);
    UTL_CLASS_DEFID;

public:
    void scan(TokenizerTokens& tokens, const String* s) const;
};

// TokenizerTokens /////////////////////////////////////////////////////////////////////////////////


class TokenizerTokens : public Object, protected FlagsMI
{
    UTL_CLASS_DECL(TokenizerTokens, Object);
    UTL_CLASS_DEFID;

public:
    void
    clear()
    {
        _tokens.clear();
    }

    void
    add(String* tk)
    {
        _tokens.pushBack(tk);
    }

    bool
    empty() const
    {
        return _tokens.empty();
    }

    Token peek() const;

    Token next();

    void skip(size_t numTokens);

    bool
    isReverse() const
    {
        return getFlag(flg_reverse);
    }

    void
    setReverse(bool reverse)
    {
        setFlag(flg_reverse, reverse);
    }

private:
    enum flg_t
    {
        flg_reverse
    };

private:
    List _tokens;
};


UTL_NS_END;