All you have to do is override Lexer.emit() and define your own token class:
grammar s; @lexer::members { public class MyToken extends CommonToken { int x; // custom field public MyToken(CharStream input, int type, int channel, int start, int stop) { super(input, type, channel, start, stop); x = 0; } public String toString() { return super.toString()+",x="+x; } } // override standard token emission public Token emit() { MyToken t = new MyToken(input, type, channel, tokenStartCharIndex, getCharIndex()-1); t.setLine(tokenStartLine); t.setText(text); t.setCharPositionInLine(tokenStartCharPositionInLine); t.x = 1; emit(t); return t; } } a : 'int' ID (',' ID)* ';' ; ID : 'a'..'z'+ ;INT : '0'..'9'+; WS : (' '|'\n') {$channel=HIDDEN;} ;
import org.antlr.runtime.*; public class Test { public static void main(String[] args) throws Exception { CharStream input = new ANTLRFileStream(args[0]); sLexer lex = new sLexer(input); CommonTokenStream tokens = new CommonTokenStream(lex); sParser parser = new sParser(tokens); parser.a(); System.out.println("Tokens: "); for (int i = 0; i<tokens.size(); i++) { Token t = tokens.get(i); System.out.println(t); i++; } } }
Sample run:
$ java org.antlr.Tool s.g ANTLR Parser Generator Version 3.0 1989-2007 $ javac Test.java $ cat input int a,b; $ java Test input Tokens: [@0,0:2='int',<7>,1:0],x=1 [@2,4:4='a',<4>,1:4],x=1 [@4,6:6='b',<4>,1:6],x=1 [@6,8:8='\n',<6>,channel=99,1:8],x=1