Default implementation of CommonTokenStream. More...
#include <antlr3tokenstream.h>
Default implementation of CommonTokenStream.
static ANTLR3_UINT32 _LA | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_INT32 | i | |||
) | [static] |
References ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_COMMON_TOKEN_struct::getType, and ANTLR3_INT_STREAM_struct::super.
Referenced by dbgLA().
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenDebugStreamSourceNew | ( | ANTLR3_UINT32 | hint, | |
pANTLR3_TOKEN_SOURCE | source, | |||
pANTLR3_DEBUG_EVENT_LISTENER | debugger | |||
) |
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, antlr3CommonTokenStreamSourceNew(), ANTLR3_INT_STREAM_struct::consume, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, ANTLR3_INT_STREAM_struct::seek, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamNew | ( | ANTLR3_UINT32 | hint | ) |
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_MALLOC, antlr3IntStreamNew(), antlr3TokenStreamNew(), antlr3VectorNew(), ANTLR3_INT_STREAM_struct::consume, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannelToks, ANTLR3_COMMON_TOKEN_STREAM_struct::discardTokenType, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::getSourceName, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokenRange, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokens, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensList, ANTLR3_TOKEN_STREAM_struct::getTokenSource, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensType, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::release, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, ANTLR3_INT_STREAM_struct::seek, ANTLR3_TOKEN_STREAM_struct::setDebugListener, ANTLR3_TOKEN_STREAM_struct::setTokenSource, ANTLR3_COMMON_TOKEN_STREAM_struct::setTokenTypeChannel, ANTLR3_INT_STREAM_struct::size, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, ANTLR3_TOKEN_STREAM_struct::toString, ANTLR3_TOKEN_STREAM_struct::toStringSS, ANTLR3_TOKEN_STREAM_struct::toStringTT, ANTLR3_COMMON_TOKEN_STREAM_struct::tstream, and ANTLR3_INT_STREAM_struct::type.
Referenced by antlr3CommonTokenStreamSourceNew().
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamSourceNew | ( | ANTLR3_UINT32 | hint, | |
pANTLR3_TOKEN_SOURCE | source | |||
) |
References antlr3CommonTokenStreamNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_COMMON_TOKEN_STREAM_struct::free, ANTLR3_TOKEN_STREAM_struct::setTokenSource, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
Referenced by antlr3CommonTokenDebugStreamSourceNew().
static void antlr3CTSFree | ( | pANTLR3_COMMON_TOKEN_STREAM | stream | ) | [static] |
References ANTLR3_FREE, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_LIST_struct::free, ANTLR3_TOKEN_STREAM_struct::free, ANTLR3_INT_STREAM_struct::free, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
static void antlr3TokenStreamFree | ( | pANTLR3_TOKEN_STREAM | stream | ) | [static] |
References ANTLR3_FREE.
ANTLR3_API pANTLR3_TOKEN_STREAM antlr3TokenStreamNew | ( | ) |
References ANTLR3_MALLOC, and ANTLR3_TOKEN_STREAM_struct::free.
Referenced by antlr3CommonTokenStreamNew().
static void consume | ( | pANTLR3_INT_STREAM | is | ) | [static] |
Move the input pointer to the next incoming token.
The stream must become active with LT(1) available. consume() simply moves the input pointer so that LT(1) points at the next input symbol. Consume at least one token.
Walk past any token not on the channel the parser is listening to.
References ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_VECTOR_struct::size, skipOffTokenChannels(), ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_INT_STREAM_struct::super, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by dbgConsume().
static void consumeInitialHiddenTokens | ( | pANTLR3_INT_STREAM | is | ) | [static] |
Debug only method to flag consumption of initial off-channel tokens in the input stream.
References ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeHiddenToken, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::initialStreamState, and ANTLR3_INT_STREAM_struct::super.
Referenced by dbgConsume(), dbgLA(), and dbgTokLT().
static void dbgConsume | ( | pANTLR3_INT_STREAM | is | ) | [static] |
As per ordinary consume but notifies the debugger about hidden tokens and so on.
References ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_TRUE, consume(), ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeHiddenToken, consumeInitialHiddenTokens(), ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeToken, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::initialStreamState, and ANTLR3_INT_STREAM_struct::super.
static ANTLR3_UINT32 dbgLA | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_INT32 | i | |||
) | [static] |
As per _LA() but for debug mode.
References _LA(), ANTLR3_TRUE, consumeInitialHiddenTokens(), ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_DEBUG_EVENT_LISTENER_struct::LT, ANTLR3_INT_STREAM_struct::super, and tokLT().
static ANTLR3_MARKER dbgMark | ( | pANTLR3_INT_STREAM | is | ) | [static] |
As per mark() but with a call to tell the debugger we are doing this.
References ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_INT_STREAM_struct::index, ANTLR3_INT_STREAM_struct::lastMarker, ANTLR3_DEBUG_EVENT_LISTENER_struct::mark, and ANTLR3_INT_STREAM_struct::super.
static void dbgRewindLast | ( | pANTLR3_INT_STREAM | is | ) | [static] |
static void dbgRewindStream | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_MARKER | marker | |||
) | [static] |
static void dbgSeek | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_MARKER | index | |||
) | [static] |
static pANTLR3_COMMON_TOKEN dbgTokLT | ( | pANTLR3_TOKEN_STREAM | ts, | |
ANTLR3_INT32 | k | |||
) | [static] |
As per the normal tokLT but sends information to the debugger.
References ANTLR3_TRUE, consumeInitialHiddenTokens(), ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, and tokLT().
static void discardOffChannel | ( | pANTLR3_COMMON_TOKEN_STREAM | cts, | |
ANTLR3_BOOLEAN | discard | |||
) | [static] |
static void discardTokenType | ( | pANTLR3_COMMON_TOKEN_STREAM | cts, | |
ANTLR3_INT32 | ttype | |||
) | [static] |
References ANTLR3_FUNC_PTR, antlr3ListNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, and ANTLR3_LIST_struct::put.
static void fillBuffer | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream | ) | [static] |
References ANTLR3_VECTOR_struct::add, ANTLR3_FALSE, ANTLR3_TOKEN_EOF, ANTLR3_TRUE, ANTLR3_UINT32_CAST, ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_VECTOR_struct::count, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_LIST_struct::get, ANTLR3_COMMON_TOKEN_struct::getChannel, ANTLR3_COMMON_TOKEN_struct::getType, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_TOKEN_SOURCE_struct::nextToken, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_COMMON_TOKEN_struct::setChannel, ANTLR3_COMMON_TOKEN_struct::setTokenIndex, skipOffTokenChannels(), ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, ANTLR3_TOKEN_STREAM_struct::tokenSource, ANTLR3_COMMON_TOKEN_STREAM_struct::tstream, and ANTLR3_COMMON_TOKEN_struct::type.
Referenced by fillBufferExt(), getTokens(), getTokensSet(), LB(), tokLT(), toString(), and toStringSS().
ANTLR3_API void fillBufferExt | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream | ) |
static pANTLR3_COMMON_TOKEN get | ( | pANTLR3_TOKEN_STREAM | ts, | |
ANTLR3_UINT32 | i | |||
) | [static] |
static pANTLR3_STRING getSourceName | ( | pANTLR3_INT_STREAM | is | ) | [static] |
Return a string that represents the name assoicated with the input source.
/param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
/returns /implements ANTLR3_INT_STREAM_struct::getSourceName()
References ANTLR3_INT_STREAM_struct::super.
static pANTLR3_LIST getTokenRange | ( | pANTLR3_COMMON_TOKEN_STREAM | cts, | |
ANTLR3_UINT32 | start, | |||
ANTLR3_UINT32 | stop | |||
) | [static] |
References ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet.
static pANTLR3_VECTOR getTokens | ( | pANTLR3_COMMON_TOKEN_STREAM | cts | ) | [static] |
References fillBuffer(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
static pANTLR3_LIST getTokensList | ( | pANTLR3_COMMON_TOKEN_STREAM | cts, | |
ANTLR3_UINT32 | start, | |||
ANTLR3_UINT32 | stop, | |||
pANTLR3_LIST | list | |||
) | [static] |
References antlr3BitsetList(), ANTLR3_BITSET_struct::free, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, and ANTLR3_LIST_struct::table.
static pANTLR3_TOKEN_SOURCE getTokenSource | ( | pANTLR3_TOKEN_STREAM | ts | ) | [static] |
References ANTLR3_TOKEN_STREAM_struct::tokenSource.
static pANTLR3_LIST getTokensSet | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream, | |
ANTLR3_UINT32 | start, | |||
ANTLR3_UINT32 | stop, | |||
pANTLR3_BITSET | types | |||
) | [static] |
Given a start and stop index, return a List of all tokens in the token type BitSet.
Return null if no tokens were found. This method looks at both on and off channel tokens.
References antlr3ListNew(), fillBuffer(), ANTLR3_LIST_struct::free, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_COMMON_TOKEN_struct::getType, ANTLR3_BITSET_struct::isMember, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_LIST_struct::put, ANTLR3_LIST_struct::size, ANTLR3_INT_STREAM_struct::size, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
static pANTLR3_LIST getTokensType | ( | pANTLR3_COMMON_TOKEN_STREAM | cts, | |
ANTLR3_UINT32 | start, | |||
ANTLR3_UINT32 | stop, | |||
ANTLR3_UINT32 | type | |||
) | [static] |
References antlr3BitsetOf(), ANTLR3_BITSET_struct::free, and ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet.
static pANTLR3_COMMON_TOKEN LB | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream, | |
ANTLR3_INT32 | i | |||
) | [static] |
References ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, fillBuffer(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, skipOffTokenChannelsReverse(), and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by tokLT().
static ANTLR3_MARKER mark | ( | pANTLR3_INT_STREAM | is | ) | [static] |
References ANTLR3_INT_STREAM_struct::index, and ANTLR3_INT_STREAM_struct::lastMarker.
static void release | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_MARKER | mark | |||
) | [static] |
static void rewindLast | ( | pANTLR3_INT_STREAM | is | ) | [static] |
References ANTLR3_INT_STREAM_struct::lastMarker, and ANTLR3_INT_STREAM_struct::rewind.
static void rewindStream | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_MARKER | marker | |||
) | [static] |
References ANTLR3_INT_STREAM_struct::seek.
static void seek | ( | pANTLR3_INT_STREAM | is, | |
ANTLR3_MARKER | index | |||
) | [static] |
References ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_INT_STREAM_struct::super.
Referenced by dbgSeek().
static void setDebugListener | ( | pANTLR3_TOKEN_STREAM | ts, | |
pANTLR3_DEBUG_EVENT_LISTENER | debugger | |||
) | [static] |
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_INT_STREAM_struct::consume, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, and ANTLR3_INT_STREAM_struct::seek.
static void setTokenSource | ( | pANTLR3_TOKEN_STREAM | ts, | |
pANTLR3_TOKEN_SOURCE | tokenSource | |||
) | [static] |
References ANTLR3_TOKEN_STREAM_struct::tokenSource.
static void setTokenTypeChannel | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream, | |
ANTLR3_UINT32 | ttype, | |||
ANTLR3_UINT32 | channel | |||
) | [static] |
A simple filter mechanism whereby you can tell this token stream to force all tokens of type ttype to be on channel.
For example, when interpreting, we cannot execute actions so we need to tell the stream to force all WS and NEWLINE to be a different, ignored, channel.
References ANTLR3_FUNC_PTR, antlr3ListNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, and ANTLR3_LIST_struct::put.
static ANTLR3_UINT32 size | ( | pANTLR3_INT_STREAM | is | ) | [static] |
static ANTLR3_UINT32 skipOffTokenChannels | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream, | |
ANTLR3_INT32 | i | |||
) | [static] |
Given a starting index, return the index of the first on-channel token.
References ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_struct::channel, ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
Referenced by consume(), fillBuffer(), and tokLT().
static ANTLR3_UINT32 skipOffTokenChannelsReverse | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream, | |
ANTLR3_INT32 | i | |||
) | [static] |
References ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_struct::channel, ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by LB().
static ANTLR3_MARKER tindex | ( | pANTLR3_INT_STREAM | is | ) | [static] |
static pANTLR3_COMMON_TOKEN tokLT | ( | pANTLR3_TOKEN_STREAM | ts, | |
ANTLR3_INT32 | k | |||
) | [static] |
Get the ith token from the current position 1..n where k=1 is the first symbol of lookahead.
References ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, ANTLR3_TOKEN_SOURCE_struct::eofToken, fillBuffer(), ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::istream, LB(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_COMMON_TOKEN_struct::setStartIndex, ANTLR3_COMMON_TOKEN_struct::setStopIndex, skipOffTokenChannels(), ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_TOKEN_STREAM_struct::tokenSource.
Referenced by dbgLA(), and dbgTokLT().
static pANTLR3_STRING toString | ( | pANTLR3_TOKEN_STREAM | ts | ) | [static] |
References fillBuffer(), ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::size, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_TOKEN_STREAM_struct::toStringSS.
static pANTLR3_STRING toStringSS | ( | pANTLR3_TOKEN_STREAM | ts, | |
ANTLR3_UINT32 | start, | |||
ANTLR3_UINT32 | stop | |||
) | [static] |
References fillBuffer(), ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_COMMON_TOKEN_struct::getText, ANTLR3_TOKEN_STREAM_struct::getTokenSource, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_STRING_FACTORY_struct::newRaw, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::size, ANTLR3_TOKEN_SOURCE_struct::strFactory, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
static pANTLR3_STRING toStringTT | ( | pANTLR3_TOKEN_STREAM | ts, | |
pANTLR3_COMMON_TOKEN | start, | |||
pANTLR3_COMMON_TOKEN | stop | |||
) | [static] |