Package org.mozilla.javascript
Class TokenStream
- java.lang.Object
-
- org.mozilla.javascript.TokenStream
-
class TokenStream extends java.lang.Object
This class implements the JavaScript scanner.It is based on the C source files jsscan.c and jsscan.h in the jsref package.
- See Also:
Parser
-
-
Field Summary
Fields Modifier and Type Field Description private ObjToIntMap
allStrings
private java.math.BigInteger
bigInt
private static char
BYTE_ORDER_MARK
private int
commentCursor
private java.lang.String
commentPrefix
(package private) Token.CommentType
commentType
(package private) int
cursor
private boolean
dirtyLine
private static int
EOF_CHAR
private boolean
hitEOF
private boolean
isBinary
private boolean
isHex
private boolean
isOctal
private boolean
isOldOctal
private int
lineEndChar
(package private) int
lineno
private int
lineStart
private double
number
private static char
NUMERIC_SEPARATOR
private Parser
parser
private int
quoteChar
private java.lang.StringBuilder
rawString
(package private) java.lang.String
regExpFlags
private static int
REPORT_NUMBER_FORMAT_ERROR
private char[]
sourceBuffer
(package private) int
sourceCursor
private int
sourceEnd
private java.io.Reader
sourceReader
private java.lang.String
sourceString
private java.lang.String
string
private char[]
stringBuffer
private int
stringBufferTop
(package private) int
tokenBeg
(package private) int
tokenEnd
private int[]
ungetBuffer
private int
ungetCursor
private boolean
xmlIsAttribute
private boolean
xmlIsTagContent
private int
xmlOpenTagsCount
-
Constructor Summary
Constructors Constructor Description TokenStream(Parser parser, java.io.Reader sourceReader, java.lang.String sourceString, int lineno)
-
Method Summary
All Methods Static Methods Instance Methods Concrete Methods Modifier and Type Method Description private void
addToString(int c)
private boolean
canUngetChar()
private int
charAt(int index)
private static java.lang.String
convertLastCharToHex(java.lang.String str)
(package private) boolean
eof()
private boolean
fillSourceBuffer()
(package private) java.lang.String
getAndResetCurrentComment()
(package private) java.math.BigInteger
getBigInt()
private int
getChar()
private int
getChar(boolean skipFormattingChars)
private int
getChar(boolean skipFormattingChars, boolean ignoreLineEnd)
private int
getCharIgnoreLineEnd()
private int
getCharIgnoreLineEnd(boolean skipFormattingChars)
Token.CommentType
getCommentType()
Return the type of the last scanned comment.int
getCursor()
Return the current position of the scanner cursor.(package private) int
getFirstXMLToken()
(package private) java.lang.String
getLine()
(package private) java.lang.String
getLine(int position, int[] linep)
(package private) int
getLineno()
(package private) int
getNextXMLToken()
(package private) double
getNumber()
(package private) int
getOffset()
Returns the offset into the current line.(package private) char
getQuoteChar()
(package private) java.lang.String
getRawString()
(package private) java.lang.String
getSourceString()
(package private) java.lang.String
getString()
private java.lang.String
getStringFromBuffer()
private int
getTemplateLiteralChar()
(package private) int
getToken()
int
getTokenBeg()
Return the absolute source offset of the last scanned token.int
getTokenEnd()
Return the absolute source end-offset of the last scanned token.int
getTokenLength()
Return tokenEnd - tokenBegprivate static boolean
isAlpha(int c)
private static boolean
isDigit(int c)
private static boolean
isDigit(int base, int c)
private static boolean
isDualDigit(int c)
private static boolean
isHexDigit(int c)
private static boolean
isJSFormatChar(int c)
private static boolean
isJSSpace(int c)
(package private) static boolean
isKeyword(java.lang.String s, int version, boolean isStrict)
private boolean
isMarkingComment()
(package private) boolean
isNumericBinary()
(package private) boolean
isNumericHex()
(package private) boolean
isNumericOctal()
(package private) boolean
isNumericOldOctal()
private static boolean
isOctalDigit(int c)
(package private) boolean
isXMLAttribute()
private void
markCommentStart()
private void
markCommentStart(java.lang.String prefix)
private boolean
matchChar(int test)
private boolean
matchTemplateLiteralChar(int test)
private int
peekChar()
private int
peekTemplateLiteralChar()
(package private) java.lang.String
readAndClearRegExpFlags()
private boolean
readCDATA()
private int
readDigits(int base, int c)
private boolean
readEntity()
private boolean
readPI()
private boolean
readQuotedString(int quote)
(package private) void
readRegExp(int startToken)
Parser calls the method when it gets / or /= in literal context.(package private) int
readTemplateLiteral(boolean isTaggedLiteral)
private boolean
readXmlComment()
private void
skipLine()
private static int
stringToKeyword(java.lang.String name, int version, boolean isStrict)
private static int
stringToKeywordForES(java.lang.String name, boolean isStrict)
ECMAScript 6.private static int
stringToKeywordForJS(java.lang.String name)
JavaScript 1.8 and earlierprivate java.lang.String
substring(int beginIndex, int endIndex)
(package private) java.lang.String
tokenToString(int token)
private void
ungetChar(int c)
private void
ungetCharIgnoreLineEnd(int c)
private void
ungetTemplateLiteralChar(int c)
-
-
-
Field Detail
-
EOF_CHAR
private static final int EOF_CHAR
- See Also:
- Constant Field Values
-
REPORT_NUMBER_FORMAT_ERROR
private static final int REPORT_NUMBER_FORMAT_ERROR
- See Also:
- Constant Field Values
-
BYTE_ORDER_MARK
private static final char BYTE_ORDER_MARK
- See Also:
- Constant Field Values
-
NUMERIC_SEPARATOR
private static final char NUMERIC_SEPARATOR
- See Also:
- Constant Field Values
-
rawString
private java.lang.StringBuilder rawString
-
dirtyLine
private boolean dirtyLine
-
regExpFlags
java.lang.String regExpFlags
-
string
private java.lang.String string
-
number
private double number
-
bigInt
private java.math.BigInteger bigInt
-
isBinary
private boolean isBinary
-
isOldOctal
private boolean isOldOctal
-
isOctal
private boolean isOctal
-
isHex
private boolean isHex
-
quoteChar
private int quoteChar
-
stringBuffer
private char[] stringBuffer
-
stringBufferTop
private int stringBufferTop
-
allStrings
private ObjToIntMap allStrings
-
ungetBuffer
private final int[] ungetBuffer
-
ungetCursor
private int ungetCursor
-
hitEOF
private boolean hitEOF
-
lineStart
private int lineStart
-
lineEndChar
private int lineEndChar
-
lineno
int lineno
-
sourceString
private java.lang.String sourceString
-
sourceReader
private java.io.Reader sourceReader
-
sourceBuffer
private char[] sourceBuffer
-
sourceEnd
private int sourceEnd
-
sourceCursor
int sourceCursor
-
cursor
int cursor
-
tokenBeg
int tokenBeg
-
tokenEnd
int tokenEnd
-
commentType
Token.CommentType commentType
-
xmlIsAttribute
private boolean xmlIsAttribute
-
xmlIsTagContent
private boolean xmlIsTagContent
-
xmlOpenTagsCount
private int xmlOpenTagsCount
-
parser
private Parser parser
-
commentPrefix
private java.lang.String commentPrefix
-
commentCursor
private int commentCursor
-
-
Constructor Detail
-
TokenStream
TokenStream(Parser parser, java.io.Reader sourceReader, java.lang.String sourceString, int lineno)
-
-
Method Detail
-
tokenToString
java.lang.String tokenToString(int token)
-
isKeyword
static boolean isKeyword(java.lang.String s, int version, boolean isStrict)
-
stringToKeyword
private static int stringToKeyword(java.lang.String name, int version, boolean isStrict)
-
stringToKeywordForJS
private static int stringToKeywordForJS(java.lang.String name)
JavaScript 1.8 and earlier
-
stringToKeywordForES
private static int stringToKeywordForES(java.lang.String name, boolean isStrict)
ECMAScript 6.
-
getSourceString
final java.lang.String getSourceString()
-
getLineno
final int getLineno()
-
getString
final java.lang.String getString()
-
getQuoteChar
final char getQuoteChar()
-
getNumber
final double getNumber()
-
getBigInt
final java.math.BigInteger getBigInt()
-
isNumericBinary
final boolean isNumericBinary()
-
isNumericOldOctal
final boolean isNumericOldOctal()
-
isNumericOctal
final boolean isNumericOctal()
-
isNumericHex
final boolean isNumericHex()
-
eof
final boolean eof()
-
getToken
final int getToken() throws java.io.IOException
- Throws:
java.io.IOException
-
readDigits
private int readDigits(int base, int c) throws java.io.IOException
- Throws:
java.io.IOException
-
isAlpha
private static boolean isAlpha(int c)
-
isDigit
private static boolean isDigit(int base, int c)
-
isDualDigit
private static boolean isDualDigit(int c)
-
isOctalDigit
private static boolean isOctalDigit(int c)
-
isDigit
private static boolean isDigit(int c)
-
isHexDigit
private static boolean isHexDigit(int c)
-
isJSSpace
private static boolean isJSSpace(int c)
-
isJSFormatChar
private static boolean isJSFormatChar(int c)
-
readRegExp
void readRegExp(int startToken) throws java.io.IOException
Parser calls the method when it gets / or /= in literal context.- Throws:
java.io.IOException
-
readAndClearRegExpFlags
java.lang.String readAndClearRegExpFlags()
-
getRawString
java.lang.String getRawString()
-
getTemplateLiteralChar
private int getTemplateLiteralChar() throws java.io.IOException
- Throws:
java.io.IOException
-
ungetTemplateLiteralChar
private void ungetTemplateLiteralChar(int c)
-
matchTemplateLiteralChar
private boolean matchTemplateLiteralChar(int test) throws java.io.IOException
- Throws:
java.io.IOException
-
peekTemplateLiteralChar
private int peekTemplateLiteralChar() throws java.io.IOException
- Throws:
java.io.IOException
-
readTemplateLiteral
int readTemplateLiteral(boolean isTaggedLiteral) throws java.io.IOException
- Throws:
java.io.IOException
-
isXMLAttribute
boolean isXMLAttribute()
-
getFirstXMLToken
int getFirstXMLToken() throws java.io.IOException
- Throws:
java.io.IOException
-
getNextXMLToken
int getNextXMLToken() throws java.io.IOException
- Throws:
java.io.IOException
-
readQuotedString
private boolean readQuotedString(int quote) throws java.io.IOException
- Throws:
java.io.IOException
-
readXmlComment
private boolean readXmlComment() throws java.io.IOException
- Throws:
java.io.IOException
-
readCDATA
private boolean readCDATA() throws java.io.IOException
- Throws:
java.io.IOException
-
readEntity
private boolean readEntity() throws java.io.IOException
- Throws:
java.io.IOException
-
readPI
private boolean readPI() throws java.io.IOException
- Throws:
java.io.IOException
-
getStringFromBuffer
private java.lang.String getStringFromBuffer()
-
addToString
private void addToString(int c)
-
canUngetChar
private boolean canUngetChar()
-
ungetChar
private void ungetChar(int c)
-
matchChar
private boolean matchChar(int test) throws java.io.IOException
- Throws:
java.io.IOException
-
peekChar
private int peekChar() throws java.io.IOException
- Throws:
java.io.IOException
-
getChar
private int getChar() throws java.io.IOException
- Throws:
java.io.IOException
-
getChar
private int getChar(boolean skipFormattingChars) throws java.io.IOException
- Throws:
java.io.IOException
-
getChar
private int getChar(boolean skipFormattingChars, boolean ignoreLineEnd) throws java.io.IOException
- Throws:
java.io.IOException
-
getCharIgnoreLineEnd
private int getCharIgnoreLineEnd() throws java.io.IOException
- Throws:
java.io.IOException
-
getCharIgnoreLineEnd
private int getCharIgnoreLineEnd(boolean skipFormattingChars) throws java.io.IOException
- Throws:
java.io.IOException
-
ungetCharIgnoreLineEnd
private void ungetCharIgnoreLineEnd(int c)
-
skipLine
private void skipLine() throws java.io.IOException
- Throws:
java.io.IOException
-
getOffset
final int getOffset()
Returns the offset into the current line.
-
charAt
private final int charAt(int index)
-
substring
private final java.lang.String substring(int beginIndex, int endIndex)
-
getLine
final java.lang.String getLine()
-
getLine
final java.lang.String getLine(int position, int[] linep)
-
fillSourceBuffer
private boolean fillSourceBuffer() throws java.io.IOException
- Throws:
java.io.IOException
-
getCursor
public int getCursor()
Return the current position of the scanner cursor.
-
getTokenBeg
public int getTokenBeg()
Return the absolute source offset of the last scanned token.
-
getTokenEnd
public int getTokenEnd()
Return the absolute source end-offset of the last scanned token.
-
getTokenLength
public int getTokenLength()
Return tokenEnd - tokenBeg
-
getCommentType
public Token.CommentType getCommentType()
Return the type of the last scanned comment.- Returns:
- type of last scanned comment, or 0 if none have been scanned.
-
markCommentStart
private void markCommentStart()
-
markCommentStart
private void markCommentStart(java.lang.String prefix)
-
isMarkingComment
private boolean isMarkingComment()
-
getAndResetCurrentComment
final java.lang.String getAndResetCurrentComment()
-
convertLastCharToHex
private static java.lang.String convertLastCharToHex(java.lang.String str)
-
-