net.morimekta.providence.reflect.parser

Class ThriftTokenizer

java.lang.Object

java.io.Reader

net.morimekta.util.io.LineBufferedReader

net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

net.morimekta.providence.reflect.parser.ThriftTokenizer

All Implemented Interfaces:

Closeable, AutoCloseable, Readable, net.morimekta.util.lexer.Tokenizer<ThriftTokenType,ThriftToken>

public class ThriftTokenizer
extends net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

Specialization of the 'pretty' tokenizer to make it handle some special cases only applicable when parsing thrift files, but not allowed in pretty format or config files.

Field Summary

Fields

Modifier and Type	Field and Description
static String	kConst
static String	kEnum
static String	kException
static String	kExtends
static String	kImplements
static String	kInclude
static String	kInterface
static String	kNamespace
static String	kNull
static String	kOf
static String	kOneway
static String	kOptional
static String	kRequired
static String	kService
static String	kStruct
static String	kThrows
static String	kTypedef
static String	kUnion
static String	kVoid

Fields inherited from class net.morimekta.util.lexer.TokenizerBase

DEFAULT_BUFFER_SIZE

Fields inherited from class net.morimekta.util.io.LineBufferedReader

buffer, bufferLimit, bufferLineEnd, bufferOffset, lastChar, lineNo, linePos, preLoaded, reader

Fields inherited from class java.io.Reader

lock

Constructor Summary

Constructors

Constructor and Description
ThriftTokenizer(InputStream in)
ThriftTokenizer(Reader reader)

Method Summary

Modifier and Type	Method and Description
protected boolean	allowIdentifier(int last)
protected ThriftToken	genericToken(char[] buffer, int offset, int len, ThriftTokenType type, int lineNo, int linePos)
protected ThriftToken	identifierToken(char[] buffer, int offset, int len, int lineNo, int linePos)
protected ThriftToken	nextSymbol()
protected ThriftToken	numberToken(char[] buffer, int offset, int len, int lineNo, int linePos)
ThriftToken	parseNextToken()
protected boolean	startString()
protected ThriftToken	stringToken(char[] buffer, int offset, int len, int lineNo, int linePos)
protected ThriftToken	symbolToken(char[] buffer, int offset, int len, int lineNo, int linePos)

Methods inherited from class net.morimekta.util.lexer.TokenizerBase

currentLine, currentLineNo, currentLinePos, eofFailure, failure, identifierSeparator, isWhitespace, readUntil, startIdentifier, startNumber, startSymbol, toString

Methods inherited from class net.morimekta.util.io.LineBufferedReader

close, getLine, getLineNo, getLinePos, getRemainingLines, getRestOfLine, maybeConsolidateBuffer, read, read, readNextChar

Methods inherited from class java.io.Reader

mark, markSupported, read, read, ready, reset, skip

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

kNamespace

public static final String kNamespace

See Also:

Constant Field Values

kInclude

public static final String kInclude

See Also:

Constant Field Values

kTypedef

public static final String kTypedef

See Also:

Constant Field Values

kEnum

public static final String kEnum

See Also:

Constant Field Values

kStruct

public static final String kStruct

See Also:

Constant Field Values

kUnion

public static final String kUnion

See Also:

Constant Field Values

kException

public static final String kException

See Also:

Constant Field Values

kInterface

public static final String kInterface

See Also:

Constant Field Values

kConst

public static final String kConst

See Also:

Constant Field Values

kService

public static final String kService

See Also:

Constant Field Values

kExtends

public static final String kExtends

See Also:

Constant Field Values

kVoid

public static final String kVoid

See Also:

Constant Field Values

kOneway

public static final String kOneway

See Also:

Constant Field Values

kThrows

public static final String kThrows

See Also:

Constant Field Values

kRequired

public static final String kRequired

See Also:

Constant Field Values

kOptional

public static final String kOptional

See Also:

Constant Field Values

kImplements

public static final String kImplements

See Also:

Constant Field Values

kOf

public static final String kOf

See Also:

Constant Field Values

kNull

public static final String kNull

See Also:

Constant Field Values

Constructor Detail

ThriftTokenizer

public ThriftTokenizer(InputStream in)

ThriftTokenizer

public ThriftTokenizer(Reader reader)

Method Detail

genericToken

protected ThriftToken genericToken(char[] buffer,
                                   int offset,
                                   int len,
                                   @Nonnull
                                   ThriftTokenType type,
                                   int lineNo,
                                   int linePos)

Specified by:

genericToken in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

parseNextToken

@Nullable
public ThriftToken parseNextToken()
                                     throws IOException

Specified by:

parseNextToken in interface net.morimekta.util.lexer.Tokenizer<ThriftTokenType,ThriftToken>

Overrides:

parseNextToken in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

Throws:

IOException

identifierToken

protected ThriftToken identifierToken(char[] buffer,
                                      int offset,
                                      int len,
                                      int lineNo,
                                      int linePos)

Specified by:

identifierToken in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

stringToken

protected ThriftToken stringToken(char[] buffer,
                                  int offset,
                                  int len,
                                  int lineNo,
                                  int linePos)

Specified by:

stringToken in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

numberToken

protected ThriftToken numberToken(char[] buffer,
                                  int offset,
                                  int len,
                                  int lineNo,
                                  int linePos)

Specified by:

numberToken in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

symbolToken

protected ThriftToken symbolToken(char[] buffer,
                                  int offset,
                                  int len,
                                  int lineNo,
                                  int linePos)

Specified by:

symbolToken in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

startString

protected boolean startString()

Overrides:

startString in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

nextSymbol

@Nonnull
protected ThriftToken nextSymbol()
                                   throws IOException

Overrides:

nextSymbol in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>

Throws:

IOException

allowIdentifier

protected boolean allowIdentifier(int last)

Overrides:

allowIdentifier in class net.morimekta.util.lexer.TokenizerBase<ThriftTokenType,ThriftToken>