Lexer.h

// WARNING: Changes to this file must be contributed back to Sawyer or else they will
//          be clobbered by the next update from Sawyer.  The Sawyer repository is at
//          https://github.com/matzke1/sawyer.
 
 
 
 
#ifndef Sawyer_Lexer_H
#define Sawyer_Lexer_H
 
#include <Sawyer/AllocatingBuffer.h>
#include <Sawyer/LineVector.h>
#include <Sawyer/Optional.h>
#include <Sawyer/Sawyer.h>
 
#include <boost/filesystem.hpp>
 
namespace Sawyer {
 
namespace Lexer {
 
template<typename T>
class SAWYER_EXPORT Token {
public:
    typedef T TokenEnum;
 
private:
    Optional<TokenEnum> type_;                          // empty means EOF
    size_t begin_;                                      // beginning character position in the input
    size_t end_;                                        // ending (exclusive) character position in the input
 
public:
    Token(): begin_(0), end_(0) {}
 
    Token(TokenEnum type, size_t begin, size_t end)
        : type_(type), begin_(begin), end_(end) {
        ASSERT_require(end >= begin);
    }
 
    bool isEof() const {
        if (type_)
            return false;
        return true;
    }
 
    TokenEnum type() const {
        return *type_;
    }
 
    size_t begin() const {
        return begin_;
    }
 
    size_t end() const {
        return end_;
    }
};
 
template<class T>
class SAWYER_EXPORT TokenStream {
public:
    typedef T Token;
 
private:
    std::string name_;                                  // name of stream (e.g., a file name)
    Container::LineVector content_;                     // line-oriented character contents of the stream
    size_t at_;                                         // cursor position in stream
    std::vector<Token> tokens_;                         // current token at [0] plus lookahead
 
public:
    virtual ~TokenStream() {}
 
    explicit TokenStream(const boost::filesystem::path &fileName)
        : name_(fileName.string()), content_(fileName.string()), at_(0) {}
 
    explicit TokenStream(const std::string &inputString)
        : name_("string"), content_(Container::AllocatingBuffer<size_t, char>::instance(inputString)), at_(0) {}
 
    explicit TokenStream(const Container::Buffer<size_t, char>::Ptr &buffer)
        : name_("string"), content_(buffer), at_(0) {}
 
    const std::string& name() const {
        return name_;
    }
 
    const Token& current() {
        return (*this)[0];
    }
 
    bool atEof() {
        return current().isEof();
    }
 
    const Token& operator[](size_t lookahead) {
        static const Token eof_;
        while (lookahead >= tokens_.size()) {
            if (!tokens_.empty() && tokens_.back().isEof())
                return eof_;
            tokens_.push_back(scanNextToken(content_, at_/*in,out*/));
        }
        return tokens_[lookahead];
    }
 
    void consume(size_t n = 1) {
        const Token &t = current();
        if (t.isEof()) {
            // void
        } else if (n >= tokens_.size()) {
            tokens_.clear();
        } else {
            tokens_.erase(tokens_.begin(), tokens_.begin() + n);
        }
    }
 
    std::string lexeme(const Token &t) {
        if (const char *s = content_.characters(t.begin())) {
            return std::string(s, t.end() - t.begin());
        } else {
            return "";
        }
    }
    std::string lexeme() {
        return lexeme(current());
    }
    bool isa(const Token &t, typename Token::TokenEnum type) {
        return !t.isEof() && t.type() == type;
    }
 
    bool isa(typename Token::TokenEnum type) {
        return isa(current(), type);
    }
    bool match(const Token &t, const char *s) {
        ASSERT_not_null(s);
        size_t n1 = t.end() - t.begin();
        size_t n2 = strlen(s);
        if (n1 != n2)
            return false;
        const char *lexeme = content_.characters(t.begin());
        return 0 == strncmp(lexeme, s, n1);
    }
    bool match(const char *s) {
        return match(current(), s);
    }
    std::pair<size_t, size_t> location(size_t position) {
        return content_.location(position);
    }
 
    std::pair<size_t, size_t> locationEof() {
        size_t nChars = content_.nCharacters();
        return nChars > 0 ? content_.location(nChars-1) : content_.location(0);
    }
    
    std::string lineString(size_t lineIdx) {
        return content_.lineString(lineIdx);
    }
 
    virtual Token scanNextToken(const Container::LineVector &content, size_t &at /*in,out*/) = 0;
};
 
} // namespace
} // namespace
 
#endif