StreamTokenizer
In this chapter you will learn:
- How to use StreamTokenizer
- nval and nval represent the next value
- Set comment char for reading
- Get line number
Use StreamTokenizer
StreamTokenizer breaks up the input stream into tokens.
We create a StreamTokenizer from a Reader implementation and read input stream
token by token.
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StreamTokenizer;
/*from ja v a 2 s . co m*/
public class Main {
public static void main(String[] av) throws IOException {
StreamTokenizer tf = new StreamTokenizer(new BufferedReader(new InputStreamReader(System.in)));
int i;
while ((i = tf.nextToken()) != StreamTokenizer.TT_EOF) {
switch (i) {
case StreamTokenizer.TT_EOF:
System.out.println("End of file");
break;
case StreamTokenizer.TT_EOL:
System.out.println("End of line");
break;
case StreamTokenizer.TT_NUMBER:
System.out.println("Number " + tf.nval);
break;
case StreamTokenizer.TT_WORD:
System.out.println("Word, length " + tf.sval.length() + "->" + tf.sval);
break;
default:
System.out.println("What is it? i = " + i);
}
}
}
}
nval and nval represent the next value
We can get the value out from nval and sval.
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StreamTokenizer;
//from j a va 2 s. c o m
public class MainClass {
public static void main(String args[]) {
try {
// Create a file reader
FileReader fr = new FileReader(args[0]);
// Create a buffered reader
BufferedReader br = new BufferedReader(fr);
// Create a stream tokenizer
StreamTokenizer st = new StreamTokenizer(br);
// Process tokens
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case StreamTokenizer.TT_WORD:
System.out.println(st.lineno() + ") " + st.sval);
break;
case StreamTokenizer.TT_NUMBER:
System.out.println(st.lineno() + ") " + st.nval);
break;
default:
System.out.println(st.lineno() + ") " + (char) st.ttype);
}
}
// Close file reader
fr.close();
} catch (Exception e) {
System.out.println("Exception: " + e);
}
}
}
Set comment char for reading
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StreamTokenizer;
/*j a v a 2 s.c om*/
public class Main {
public static void main(String args[]) throws IOException {
BufferedReader inData = new BufferedReader(new InputStreamReader(System.in));
StreamTokenizer inStream = new StreamTokenizer(inData);
inStream.commentChar('#');
boolean eof = false;
do {
int token = inStream.nextToken();
switch (token) {
case StreamTokenizer.TT_EOF:
System.out.println("EOF encountered.");
eof = true;
break;
case StreamTokenizer.TT_EOL:
System.out.println("EOL encountered.");
break;
case StreamTokenizer.TT_WORD:
System.out.println("Word: " + inStream.sval);
break;
case StreamTokenizer.TT_NUMBER:
System.out.println("Number: " + inStream.nval);
break;
default:
System.out.println((char) token + " encountered.");
if (token == '!')
eof = true;
}
} while (!eof);
}
}
Get line number
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.StreamTokenizer;
// j av a 2s . co m
public class MainClass {
public static void main(String args[]) {
try {
FileReader fr = new FileReader(args[0]);
BufferedReader br = new BufferedReader(fr);
StreamTokenizer st = new StreamTokenizer(br);
st.ordinaryChar('.');
st.wordChars('\'', '\'');
while (st.nextToken() != StreamTokenizer.TT_EOF) {
switch (st.ttype) {
case StreamTokenizer.TT_WORD:
System.out.println(st.lineno() + ") " + st.sval);
break;
case StreamTokenizer.TT_NUMBER:
System.out.println(st.lineno() + ") " + st.nval);
break;
default:
System.out.println(st.lineno() + ") " + (char) st.ttype);
}
}
fr.close();
} catch (Exception e) {
System.out.println("Exception: " + e);
}
}
}
Next chapter...
What you will learn in the next chapter: