Line data    Source code

       1             : /* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
       2             :  * Use of this file is governed by the BSD 3-clause license that
       3             :  * can be found in the LICENSE.txt file in the project root.
       4             :  */
       5             : 
       6             : #pragma once
       7             : 
       8             : #include "TokenStream.h"
       9             : 
      10             : namespace antlr4 {
      11             : 
      12             :   /**
      13             :    * This implementation of {@link TokenStream} loads tokens from a
      14             :    * {@link TokenSource} on-demand, and places the tokens in a buffer to provide
      15             :    * access to any previous token by index.
      16             :    *
      17             :    * <p>
      18             :    * This token stream ignores the value of {@link Token#getChannel}. If your
      19             :    * parser requires the token stream filter tokens to only those on a particular
      20             :    * channel, such as {@link Token#DEFAULT_CHANNEL} or
      21             :    * {@link Token#HIDDEN_CHANNEL}, use a filtering token stream such a
      22             :    * {@link CommonTokenStream}.</p>
      23             :    */
      24           0 :   class ANTLR4CPP_PUBLIC BufferedTokenStream : public TokenStream {
      25             :   public:
      26             :     BufferedTokenStream(TokenSource *tokenSource);
      27             :     BufferedTokenStream(const BufferedTokenStream& other) = delete;
      28             : 
      29             :     BufferedTokenStream& operator = (const BufferedTokenStream& other) = delete;
      30             : 
      31             :     virtual TokenSource* getTokenSource() const override;
      32             :     virtual size_t index() override;
      33             :     virtual ssize_t mark() override;
      34             : 
      35             :     virtual void release(ssize_t marker) override;
      36             :     virtual void reset();
      37             :     virtual void seek(size_t index) override;
      38             : 
      39             :     virtual size_t size() override;
      40             :     virtual void consume() override;
      41             : 
      42             :     virtual Token* get(size_t i) const override;
      43             : 
      44             :     /// Get all tokens from start..stop inclusively.
      45             :     virtual std::vector<Token *> get(size_t start, size_t stop);
      46             : 
      47             :     virtual size_t LA(ssize_t i) override;
      48             :     virtual Token* LT(ssize_t k) override;
      49             : 
      50             :     /// Reset this token stream by setting its token source.
      51             :     virtual void setTokenSource(TokenSource *tokenSource);
      52             :     virtual std::vector<Token *> getTokens();
      53             :     virtual std::vector<Token *> getTokens(size_t start, size_t stop);
      54             : 
      55             :     /// <summary>
      56             :     /// Given a start and stop index, return a List of all tokens in
      57             :     ///  the token type BitSet.  Return null if no tokens were found.  This
      58             :     ///  method looks at both on and off channel tokens.
      59             :     /// </summary>
      60             :     virtual std::vector<Token *> getTokens(size_t start, size_t stop, const std::vector<size_t> &types);
      61             :     virtual std::vector<Token *> getTokens(size_t start, size_t stop, size_t ttype);
      62             : 
      63             :     /// Collect all tokens on specified channel to the right of
      64             :     ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
      65             :     ///  EOF. If channel is -1, find any non default channel token.
      66             :     virtual std::vector<Token *> getHiddenTokensToRight(size_t tokenIndex, ssize_t channel);
      67             : 
      68             :     /// <summary>
      69             :     /// Collect all hidden tokens (any off-default channel) to the right of
      70             :     ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL
      71             :     ///  or EOF.
      72             :     /// </summary>
      73             :     virtual std::vector<Token *> getHiddenTokensToRight(size_t tokenIndex);
      74             : 
      75             :     /// <summary>
      76             :     /// Collect all tokens on specified channel to the left of
      77             :     ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
      78             :     ///  If channel is -1, find any non default channel token.
      79             :     /// </summary>
      80             :     virtual std::vector<Token *> getHiddenTokensToLeft(size_t tokenIndex, ssize_t channel);
      81             : 
      82             :     /// <summary>
      83             :     /// Collect all hidden tokens (any off-default channel) to the left of
      84             :     ///  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
      85             :     /// </summary>
      86             :     virtual std::vector<Token *> getHiddenTokensToLeft(size_t tokenIndex);
      87             : 
      88             :     virtual std::string getSourceName() const override;
      89             :     virtual std::string getText() override;
      90             :     virtual std::string getText(const misc::Interval &interval) override;
      91             :     virtual std::string getText(RuleContext *ctx) override;
      92             :     virtual std::string getText(Token *start, Token *stop) override;
      93             : 
      94             :     /// Get all tokens from lexer until EOF.
      95             :     virtual void fill();
      96             : 
      97             :   protected:
      98             :     /**
      99             :      * The {@link TokenSource} from which tokens for this stream are fetched.
     100             :      */
     101             :     TokenSource *_tokenSource;
     102             : 
     103             :     /**
     104             :      * A collection of all tokens fetched from the token source. The list is
     105             :      * considered a complete view of the input once {@link #fetchedEOF} is set
     106             :      * to {@code true}.
     107             :      */
     108             :     std::vector<std::unique_ptr<Token>> _tokens;
     109             : 
     110             :     /**
     111             :      * The index into {@link #tokens} of the current token (next token to
     112             :      * {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be
     113             :      * {@link #LT LT(1)}.
     114             :      *
     115             :      * <p>This field is set to -1 when the stream is first constructed or when
     116             :      * {@link #setTokenSource} is called, indicating that the first token has
     117             :      * not yet been fetched from the token source. For additional information,
     118             :      * see the documentation of {@link IntStream} for a description of
     119             :      * Initializing Methods.</p>
     120             :      */
     121             :     // ml: since -1 requires to make this member signed for just this single aspect we use a member _needSetup instead.
     122             :     //     Use bool isInitialized() to find out if this stream has started reading.
     123             :     size_t _p;
     124             : 
     125             :     /**
     126             :      * Indicates whether the {@link Token#EOF} token has been fetched from
     127             :      * {@link #tokenSource} and added to {@link #tokens}. This field improves
     128             :      * performance for the following cases:
     129             :      *
     130             :      * <ul>
     131             :      * <li>{@link #consume}: The lookahead check in {@link #consume} to prevent
     132             :      * consuming the EOF symbol is optimized by checking the values of
     133             :      * {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.</li>
     134             :      * <li>{@link #fetch}: The check to prevent adding multiple EOF symbols into
     135             :      * {@link #tokens} is trivial with this field.</li>
     136             :      * <ul>
     137             :      */
     138             :     bool _fetchedEOF;
     139             : 
     140             :     /// <summary>
     141             :     /// Make sure index {@code i} in tokens has a token.
     142             :     /// </summary>
     143             :     /// <returns> {@code true} if a token is located at index {@code i}, otherwise
     144             :     ///    {@code false}. </returns>
     145             :     /// <seealso cref= #get(int i) </seealso>
     146             :     virtual bool sync(size_t i);
     147             : 
     148             :     /// <summary>
     149             :     /// Add {@code n} elements to buffer.
     150             :     /// </summary>
     151             :     /// <returns> The actual number of elements added to the buffer. </returns>
     152             :     virtual size_t fetch(size_t n);
     153             : 
     154             :     virtual Token* LB(size_t k);
     155             : 
     156             :     /// Allowed derived classes to modify the behavior of operations which change
     157             :     /// the current stream position by adjusting the target token index of a seek
     158             :     /// operation. The default implementation simply returns {@code i}. If an
     159             :     /// exception is thrown in this method, the current stream index should not be
     160             :     /// changed.
     161             :     /// <p/>
     162             :     /// For example, <seealso cref="CommonTokenStream"/> overrides this method to ensure that
     163             :     /// the seek target is always an on-channel token.
     164             :     ///
     165             :     /// <param name="i"> The target token index. </param>
     166             :     /// <returns> The adjusted target token index. </returns>
     167             :     virtual ssize_t adjustSeekIndex(size_t i);
     168             :     void lazyInit();
     169             :     virtual void setup();
     170             : 
     171             :     /**
     172             :      * Given a starting index, return the index of the next token on channel.
     173             :      * Return {@code i} if {@code tokens[i]} is on channel. Return the index of
     174             :      * the EOF token if there are no tokens on channel between {@code i} and
     175             :      * EOF.
     176             :      */
     177             :     virtual ssize_t nextTokenOnChannel(size_t i, size_t channel);
     178             : 
     179             :     /**
     180             :      * Given a starting index, return the index of the previous token on
     181             :      * channel. Return {@code i} if {@code tokens[i]} is on channel. Return -1
     182             :      * if there are no tokens on channel between {@code i} and 0.
     183             :      *
     184             :      * <p>
     185             :      * If {@code i} specifies an index at or after the EOF token, the EOF token
     186             :      * index is returned. This is due to the fact that the EOF token is treated
     187             :      * as though it were on every channel.</p>
     188             :      */
     189             :     virtual ssize_t previousTokenOnChannel(size_t i, size_t channel);
     190             : 
     191             :     virtual std::vector<Token *> filterForChannel(size_t from, size_t to, ssize_t channel);
     192             : 
     193             :     bool isInitialized() const;
     194             : 
     195             :   private:
     196             :     bool _needSetup;
     197             :     void InitializeInstanceFields();
     198             :   };
     199             : 
     200             : } // namespace antlr4