11 #include <Sawyer/AllocatingBuffer.h>
12 #include <Sawyer/LineVector.h>
13 #include <Sawyer/Optional.h>
14 #include <Sawyer/Sawyer.h>
16 #include <boost/filesystem.hpp>
47 Token(TokenEnum type,
size_t begin,
size_t end)
48 : type_(type), begin_(begin), end_(end) {
49 ASSERT_require(end >= begin);
99 std::vector<Token> tokens_;
106 : name_(fileName.string()), content_(fileName.string()), at_(0) {}
113 : name_(
"string"), content_(Container::AllocatingBuffer<size_t, char>::instance(inputString)), at_(0) {}
119 : name_(
"string"), content_(buffer), at_(0) {}
122 const std::string&
name()
const {
137 return current().isEof();
145 static const Token eof_;
146 while (lookahead >= tokens_.size()) {
147 if (!tokens_.empty() && tokens_.back().isEof())
149 tokens_.push_back(scanNextToken(content_, at_));
151 return tokens_[lookahead];
159 const Token &t = current();
162 }
else if (n >= tokens_.size()) {
165 tokens_.erase(tokens_.begin(), tokens_.begin() + n);
183 if (
const char *s = content_.
characters(t.begin())) {
184 return std::string(s, t.end() - t.begin());
190 return lexeme(current());
200 bool isa(
const Token &t,
typename Token::TokenEnum type) {
201 return !t.isEof() && t.type() == type;
204 bool isa(
typename Token::TokenEnum type) {
205 return isa(current(), type);
217 bool match(
const Token &t,
const char *s) {
219 size_t n1 = t.end() - t.begin();
220 size_t n2 = strlen(s);
223 const char *lexeme = content_.
characters(t.begin());
224 return 0 == strncmp(lexeme, s, n1);
227 return match(current(), s);
235 std::pair<size_t, size_t>
location(
size_t position) {
bool isEof() const
Whether this is an EOF token.
Represents one token of input.
size_t nCharacters() const
Number of characters.
TokenStream(const Container::Buffer< size_t, char >::Ptr &buffer)
Create a token stream from a buffer.
size_t end() const
Token lexeme ending position.
std::string lexeme()
Return the lexeme for a token.
A buffer of characters indexed by line number.
const char * characters(size_t charIdx) const
Characters at file offset.
std::string lineString(size_t lineIdx) const
Line as a string.
bool atEof()
Returns true if the stream is at the end.
bool isa(typename Token::TokenEnum type)
Determine whether token is a specific type.
An ordered list of tokens scanned from input.
void consume(size_t n=1)
Consume some tokens.
TokenEnum type() const
Returns the token.
std::string lineString(size_t lineIdx)
Return the entire string for some line index.
bool match(const char *s)
Determine whether a token matches a string.
const Token & operator[](size_t lookahead)
Return the current or future token.
std::string lexeme(const Token &t)
Return the lexeme for a token.
const Token & current()
Return the current token.
Reference-counting intrusive smart pointer.
Name space for the entire library.
bool isa(const Token &t, typename Token::TokenEnum type)
Determine whether token is a specific type.
const std::string & name() const
Property: Name of stream.
std::pair< size_t, size_t > location(size_t position)
Return the line number and offset for an input position.
size_t begin() const
Token lexeme starting position.
TokenStream(const std::string &inputString)
Create a token stream from a string.
TokenStream(const boost::filesystem::path &fileName)
Create a token stream from the contents of a file.
Token(TokenEnum type, size_t begin, size_t end)
Construct a token.
Token()
Construct an EOF token.
std::pair< size_t, size_t > locationEof()
Returns the last line index and character offset.
std::pair< size_t, size_t > location(size_t charIndex) const
Convert a character index to a line and column index.
bool match(const Token &t, const char *s)
Determine whether a token matches a string.