Package org.mozilla.javascript

Class TokenStream

java.lang.Object
org.mozilla.javascript.TokenStream
class TokenStream extends Object
This class implements the JavaScript scanner.

It is based on the C source files jsscan.c and jsscan.h in the jsref package.

See Also:

  • Field Details

    • EOF_CHAR

      private static final int EOF_CHAR
      See Also:

    • REPORT_NUMBER_FORMAT_ERROR

      private static final int REPORT_NUMBER_FORMAT_ERROR
      See Also:

    • BYTE_ORDER_MARK

      private static final char BYTE_ORDER_MARK
      See Also:

    • NUMERIC_SEPARATOR

      private static final char NUMERIC_SEPARATOR
      See Also:

    • rawString

      private StringBuilder rawString

    • dirtyLine

      private boolean dirtyLine

    • regExpFlags

      String regExpFlags

    • string

      private String string

    • number

      private double number

    • bigInt

      private BigInteger bigInt

    • isBinary

      private boolean isBinary

    • isOldOctal

      private boolean isOldOctal

    • isOctal

      private boolean isOctal

    • isHex

      private boolean isHex

    • quoteChar

      private int quoteChar

    • stringBuffer

      private char[] stringBuffer

    • stringBufferTop

      private int stringBufferTop

    • allStrings

      private ObjToIntMap allStrings

    • ungetBuffer

      private final int[] ungetBuffer

    • ungetCursor

      private int ungetCursor

    • hitEOF

      private boolean hitEOF

    • lineStart

      private int lineStart

    • lineEndChar

      private int lineEndChar

    • lineno

      int lineno

    • sourceString

      private String sourceString

    • sourceReader

      private Reader sourceReader

    • sourceBuffer

      private char[] sourceBuffer

    • sourceEnd

      private int sourceEnd

    • sourceCursor

      int sourceCursor

    • cursor

      int cursor

    • tokenBeg

      int tokenBeg

    • tokenEnd

      int tokenEnd

    • commentType

      Token.CommentType commentType

    • xmlIsAttribute

      private boolean xmlIsAttribute

    • xmlIsTagContent

      private boolean xmlIsTagContent

    • xmlOpenTagsCount

      private int xmlOpenTagsCount

    • parser

      private Parser parser

    • commentPrefix

      private String commentPrefix

    • commentCursor

      private int commentCursor

  • Constructor Details

    • TokenStream

      TokenStream(Parser parser, Reader sourceReader, String sourceString, int lineno)

  • Method Details

    • tokenToString

      String tokenToString(int token)

    • isKeyword

      static boolean isKeyword(String s, int version, boolean isStrict)

    • stringToKeyword

      private static int stringToKeyword(String name, int version, boolean isStrict)

    • stringToKeywordForJS

      private static int stringToKeywordForJS(String name)
      JavaScript 1.8 and earlier

    • stringToKeywordForES

      private static int stringToKeywordForES(String name, boolean isStrict)
      ECMAScript 6.

    • getSourceString

      final String getSourceString()

    • getLineno

      final int getLineno()

    • getString

      final String getString()

    • getQuoteChar

      final char getQuoteChar()

    • getNumber

      final double getNumber()

    • getBigInt

      final BigInteger getBigInt()

    • isNumericBinary

      final boolean isNumericBinary()

    • isNumericOldOctal

      final boolean isNumericOldOctal()

    • isNumericOctal

      final boolean isNumericOctal()

    • isNumericHex

      final boolean isNumericHex()

    • eof

      final boolean eof()

    • getToken

      final int getToken() throws IOException
      Throws:
      IOException

    • readDigits

      private int readDigits(int base, int c) throws IOException
      Throws:
      IOException

    • isAlpha

      private static boolean isAlpha(int c)

    • isDigit

      private static boolean isDigit(int base, int c)

    • isDualDigit

      private static boolean isDualDigit(int c)

    • isOctalDigit

      private static boolean isOctalDigit(int c)

    • isDigit

      private static boolean isDigit(int c)

    • isHexDigit

      private static boolean isHexDigit(int c)

    • isJSSpace

      private static boolean isJSSpace(int c)

    • isJSFormatChar

      private static boolean isJSFormatChar(int c)

    • readRegExp

      void readRegExp(int startToken) throws IOException
      Parser calls the method when it gets / or /= in literal context.
      Throws:
      IOException

    • readAndClearRegExpFlags

      String readAndClearRegExpFlags()

    • getRawString

      String getRawString()

    • getTemplateLiteralChar

      private int getTemplateLiteralChar() throws IOException
      Throws:
      IOException

    • ungetTemplateLiteralChar

      private void ungetTemplateLiteralChar(int c)

    • matchTemplateLiteralChar

      private boolean matchTemplateLiteralChar(int test) throws IOException
      Throws:
      IOException

    • peekTemplateLiteralChar

      private int peekTemplateLiteralChar() throws IOException
      Throws:
      IOException

    • readTemplateLiteral

      int readTemplateLiteral(boolean isTaggedLiteral) throws IOException
      Throws:
      IOException

    • isXMLAttribute

      boolean isXMLAttribute()

    • getFirstXMLToken

      int getFirstXMLToken() throws IOException
      Throws:
      IOException

    • getNextXMLToken

      int getNextXMLToken() throws IOException
      Throws:
      IOException

    • readQuotedString

      private boolean readQuotedString(int quote) throws IOException
      Throws:
      IOException

    • readXmlComment

      private boolean readXmlComment() throws IOException
      Throws:
      IOException

    • readCDATA

      private boolean readCDATA() throws IOException
      Throws:
      IOException

    • readEntity

      private boolean readEntity() throws IOException
      Throws:
      IOException

    • readPI

      private boolean readPI() throws IOException
      Throws:
      IOException

    • getStringFromBuffer

      private String getStringFromBuffer()

    • addToString

      private void addToString(int c)

    • canUngetChar

      private boolean canUngetChar()

    • ungetChar

      private void ungetChar(int c)

    • matchChar

      private boolean matchChar(int test) throws IOException
      Throws:
      IOException

    • peekChar

      private int peekChar() throws IOException
      Throws:
      IOException

    • getChar

      private int getChar() throws IOException
      Throws:
      IOException

    • getChar

      private int getChar(boolean skipFormattingChars) throws IOException
      Throws:
      IOException

    • getChar

      private int getChar(boolean skipFormattingChars, boolean ignoreLineEnd) throws IOException
      Throws:
      IOException

    • getCharIgnoreLineEnd

      private int getCharIgnoreLineEnd() throws IOException
      Throws:
      IOException

    • getCharIgnoreLineEnd

      private int getCharIgnoreLineEnd(boolean skipFormattingChars) throws IOException
      Throws:
      IOException

    • ungetCharIgnoreLineEnd

      private void ungetCharIgnoreLineEnd(int c)

    • skipLine

      private void skipLine() throws IOException
      Throws:
      IOException

    • getOffset

      final int getOffset()
      Returns the offset into the current line.

    • charAt

      private final int charAt(int index)

    • substring

      private final String substring(int beginIndex, int endIndex)

    • getLine

      final String getLine()

    • getLine

      final String getLine(int position, int[] linep)

    • fillSourceBuffer

      private boolean fillSourceBuffer() throws IOException
      Throws:
      IOException

    • getCursor

      public int getCursor()
      Return the current position of the scanner cursor.

    • getTokenBeg

      public int getTokenBeg()
      Return the absolute source offset of the last scanned token.

    • getTokenEnd

      public int getTokenEnd()
      Return the absolute source end-offset of the last scanned token.

    • getTokenLength

      public int getTokenLength()
      Return tokenEnd - tokenBeg

    • getCommentType

      public Token.CommentType getCommentType()
      Return the type of the last scanned comment.
      Returns:
      type of last scanned comment, or 0 if none have been scanned.

    • markCommentStart

      private void markCommentStart()

    • markCommentStart

      private void markCommentStart(String prefix)

    • isMarkingComment

      private boolean isMarkingComment()

    • getAndResetCurrentComment

      final String getAndResetCurrentComment()

    • convertLastCharToHex

      private static String convertLastCharToHex(String str)