Sample Context-sensitive Lexer in ANTLR v3

parser grammar T;

prog: stat+ EOF ;

stat: kreturn e ';'
	| id '=' e ';'
	| id ':'
	;

e	:	integer | id ;

id	:	letter+ ;

kreturn : 'r' 'e' 't' 'u' 'r' 'n' ;

integer : digit+ ;

digit
	:	'0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
	;
	
letter
	:	'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j'
	|	'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't'
	|	'u' | 'v' | 'w' | 'x' | 'y' | 'z'
	;

import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Token;
import org.antlr.runtime.TokenSource;
import org.antlr.tool.Grammar;

import java.util.LinkedHashMap;
import java.util.Map;

public class CharsAsTokens implements TokenSource {
	CharStream input;
	String[] tokenNames;
	int line=1;
	int charPosInLine;
	Map<Integer, Integer> charToTokenType = new LinkedHashMap<Integer, Integer>();

	public CharsAsTokens(CharStream input, String[] tokenNames) {
		this.input = input;
		this.tokenNames = tokenNames;
		int ttype = 0;
		for (String tname : tokenNames) {
			if ( tname.charAt(0)=='\'' ) {
				charToTokenType.put(Grammar.getCharValueFromGrammarCharLiteral(tname), ttype);
			}
			ttype++;
		}
		System.out.println(charToTokenType);
	}

	public Token nextToken() {
		Token t = null;
		consumeUnknown();
		int c = input.LA(1);
		int i = input.index();
		if ( c == CharStream.EOF ) {
			t = Token.EOF_TOKEN;
		}
		else {
			Integer	ttypeI = charToTokenType.get(c);
			t = new CommonToken(input, ttypeI, Token.DEFAULT_CHANNEL, i,  i);
		}
		t.setLine(line);
		t.setCharPositionInLine(charPosInLine);
		consume();
		return t;
	}

	protected void consumeUnknown() {
		int c = input.LA(1);
		Integer ttypeI = charToTokenType.get(c);
		while ( ttypeI==null && c != CharStream.EOF ) {
			System.err.println("no token type for char '"+(char)c+"'");
			c = consume();
			ttypeI = charToTokenType.get(c);
		}
	}

	protected int consume() {
		int c = input.LA(1);
		input.consume();
		if ( c == '\n' ) { charPosInLine = 0; line++; }
		charPosInLine++;
		return input.LA(1);
	}

	public String getSourceName() {
		return null;
	}
}

import org.antlr.runtime.*;
public class Test {
	public static void main(String[] args) throws Exception {
		CharStream in = null;
		if ( args.length>0 ) in = new ANTLRFileStream(args[0]);
		else in = new ANTLRInputStream(System.in);

		CharsAsTokens fauxLexer = new CharsAsTokens(in, T.tokenNames);
		CommonTokenStream tokens = new CommonTokenStream(fauxLexer);
		tokens.fill();
		System.out.println(tokens.getTokens());
		T parser = new T(tokens);
		parser.prog();
	}
}