Example usage for java.io StreamTokenizer resetSyntax

Introduction

In this page you can find the example usage for java.io StreamTokenizer resetSyntax.

Prototype

public void resetSyntax() 

Source Link

Document

Resets this tokenizer's syntax table so that all characters are "ordinary."

Usage

From source file:com.indeed.imhotep.index.builder.util.SmartArgs.java

/**
 * parse a String into an argument list, in the same way java parses
 * arguments passed to main()/*  w w w.  j  a  va 2 s.co m*/
 */
public static String[] parseArgParams(String line) {
    StreamTokenizer tok = new StreamTokenizer(new StringReader(line));
    tok.resetSyntax();
    tok.wordChars('\u0000', '\uFFFF');
    tok.whitespaceChars(' ', ' ');
    tok.quoteChar('\"');
    ArrayList<String> output = new ArrayList<String>();
    try {
        while (tok.nextToken() != StreamTokenizer.TT_EOF) {
            output.add(tok.sval);
        }
    } catch (IOException e) {
    }
    return output.toArray(new String[output.size()]);
}

---

From source file:com.xpn.xwiki.util.Util.java

/**
 * Create a Map from a string holding a space separated list of key=value pairs. If keys or values must contain
 * spaces, they can be placed inside quotes, like <code>"this key"="a larger value"</code>. To use a quote as part
 * of a key/value, use <code>%_Q_%</code>.
 * /*from  w ww. jav  a  2s.c o  m*/
 * @param mapString The string that must be parsed.
 * @return A Map containing the keys and values. If a key is defined more than once, the last value is used.
 */
public static Hashtable<String, String> keyValueToHashtable(String mapString) throws IOException {
    Hashtable<String, String> result = new Hashtable<String, String>();
    StreamTokenizer st = new StreamTokenizer(new BufferedReader(new StringReader(mapString)));
    st.resetSyntax();
    st.quoteChar('"');
    st.wordChars('a', 'z');
    st.wordChars('A', 'Z');
    st.whitespaceChars(' ', ' ');
    st.whitespaceChars('=', '=');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        String key = st.sval;
        st.nextToken();
        String value = (st.sval != null) ? st.sval : "";
        result.put(key, restoreValue(value));
    }
    return result;
}

---

From source file:com.zimbra.common.calendar.ZoneInfo2iCalendar.java

private static void readExtraData(Reader reader) throws IOException, ParseException {
    char dquote = '"';
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars(32, 126);//from   www .  ja va  2 s. c om
    tokenizer.whitespaceChars(' ', ' ');
    tokenizer.whitespaceChars('\t', '\t');
    tokenizer.whitespaceChars(0, 20);
    tokenizer.commentChar('#');
    tokenizer.quoteChar(dquote);
    tokenizer.eolIsSignificant(true);

    List<String> tokenList = new ArrayList<String>();
    LineType lineType = LineType.UNKNOWN;
    boolean atLineStart = true;

    int ttype;
    int prevTtype = StreamTokenizer.TT_EOL; // used for empty line detection
    while ((ttype = tokenizer.nextToken()) != StreamTokenizer.TT_EOF) {
        int lineNum = tokenizer.lineno();
        if (ttype == StreamTokenizer.TT_WORD || ttype == dquote) {
            String token = tokenizer.sval;
            if (atLineStart) {
                lineType = LineType.lookUp(token);
                if (LineType.UNKNOWN.equals(lineType))
                    throw new ParseException("Invalid line type", lineNum);
            } else {
                tokenList.add(token);
            }
            atLineStart = false;
        } else if (ttype == StreamTokenizer.TT_EOL) {
            if (prevTtype == StreamTokenizer.TT_EOL) {
                prevTtype = ttype;
                continue;
            }
            atLineStart = true;
            switch (lineType) {
            case PRIMARYZONE:
                if (tokenList.size() < 1)
                    throw new ParseException("Not enough fields in a PrimaryZone line", lineNum);
                String primaryTZID = tokenList.get(0);
                sPrimaryTZIDs.add(primaryTZID);
                break;
            case ZONEMATCHSCORE:
                if (tokenList.size() < 2)
                    throw new ParseException("Not enough fields in a ZoneMatchScore line", lineNum);
                String zoneName = tokenList.get(0);
                String zoneMatchScoreStr = tokenList.get(1);
                int zoneMatchScore = 0;
                try {
                    zoneMatchScore = Integer.parseInt(zoneMatchScoreStr);
                } catch (NumberFormatException e) {
                    throw new ParseException("Zone match score must be an integer: " + zoneMatchScoreStr,
                            lineNum);
                }
                sMatchScores.put(zoneName, zoneMatchScore);
                break;
            }
            if (atLineStart) {
                tokenList.clear();
                lineType = LineType.UNKNOWN;
            }
        } else if (ttype == StreamTokenizer.TT_NUMBER) {
            // shouldn't happen
            throw new ParseException("Invalid parser state: TT_NUMBER found", lineNum);
        }
        prevTtype = ttype;
    }
}

---

From source file:Matrix.java

/**
 * Read a matrix from a stream. The format is the same the print method, so
 * printed matrices can be read back in (provided they were printed using US
 * Locale). Elements are separated by whitespace, all the elements for each
 * row appear on a single line, the last row is followed by a blank line.
 * /*from  ww  w .ja  v  a2 s  .c om*/
 * @param input
 *            the input stream.
 */

public static Matrix read(BufferedReader input) throws java.io.IOException {
    StreamTokenizer tokenizer = new StreamTokenizer(input);

    // Although StreamTokenizer will parse numbers, it doesn't recognize
    // scientific notation (E or D); however, Double.valueOf does.
    // The strategy here is to disable StreamTokenizer's number parsing.
    // We'll only get whitespace delimited words, EOL's and EOF's.
    // These words should all be numbers, for Double.valueOf to parse.

    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.eolIsSignificant(true);
    java.util.Vector v = new java.util.Vector();

    // Ignore initial empty lines
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL)
        ;
    if (tokenizer.ttype == StreamTokenizer.TT_EOF)
        throw new java.io.IOException("Unexpected EOF on matrix read.");
    do {
        v.addElement(Double.valueOf(tokenizer.sval)); // Read & store 1st
        // row.
    } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);

    int n = v.size(); // Now we've got the number of columns!
    double row[] = new double[n];
    for (int j = 0; j < n; j++)
        // extract the elements of the 1st row.
        row[j] = ((Double) v.elementAt(j)).doubleValue();
    v.removeAllElements();
    v.addElement(row); // Start storing rows instead of columns.
    while (tokenizer.nextToken() == StreamTokenizer.TT_WORD) {
        // While non-empty lines
        v.addElement(row = new double[n]);
        int j = 0;
        do {
            if (j >= n)
                throw new java.io.IOException("Row " + v.size() + " is too long.");
            row[j++] = Double.valueOf(tokenizer.sval).doubleValue();
        } while (tokenizer.nextToken() == StreamTokenizer.TT_WORD);
        if (j < n)
            throw new java.io.IOException("Row " + v.size() + " is too short.");
    }
    int m = v.size(); // Now we've got the number of rows.
    double[][] A = new double[m][];
    v.copyInto(A); // copy the rows out of the vector
    return new Matrix(A);
}

---

From source file:org.apache.openjpa.jdbc.kernel.SQLStoreQuery.java

/**
 * Utility method to substitute '?num' for parameters in the given SQL
 * statement, and fill-in the order of the parameter tokens
 *//* w w  w.j  a va  2 s .  co  m*/
public static String substituteParams(String sql, List<Integer> paramOrder) throws IOException {
    // if there's no "?" parameter marker, then we don't need to
    // perform the parsing process
    if (sql.indexOf("?") == -1)
        return sql;

    paramOrder.clear();
    StreamTokenizer tok = new StreamTokenizer(new StringReader(sql));
    tok.resetSyntax();
    tok.quoteChar('\'');
    tok.wordChars('0', '9');
    tok.wordChars('?', '?');

    StringBuilder buf = new StringBuilder(sql.length());
    for (int ttype; (ttype = tok.nextToken()) != StreamTokenizer.TT_EOF;) {
        switch (ttype) {
        case StreamTokenizer.TT_WORD:
            // a token is a positional parameter if it starts with
            // a "?" and the rest of the token are all numbers
            if (tok.sval.startsWith("?")) {
                buf.append("?");
                String pIndex = tok.sval.substring(1);
                if (pIndex.length() > 0) {
                    paramOrder.add(Integer.valueOf(pIndex));
                } else { // or nothing
                    paramOrder.add(paramOrder.size() + 1);
                }
            } else
                buf.append(tok.sval);
            break;
        case '\'':
            buf.append('\'');
            if (tok.sval != null) {
                buf.append(tok.sval);
                buf.append('\'');
            }
            break;
        default:
            buf.append((char) ttype);
        }
    }
    return buf.toString();
}

---

From source file:org.yestech.jmlnitrate.transformation.inbound.BaseInboundTransformation.java

/**
 *  Parses a Parameter Request String and return a array of the tokens in the
 *  string. The default delimeters are ^ and ;
 *
 * @param  rawRequest the Parameter Request
 * @return  Array of the tokens//from   w  w  w  .j  a v a 2 s  .  c om
 * @throws  Exception if an error happens
 */
private Object[] parseParameterRequest(String rawRequest) throws Exception {
    StreamTokenizer tokenStream = new StreamTokenizer(
            new BufferedReader(new StringReader(rawRequest), BUFFER_SIZE));
    //reset the stream
    tokenStream.resetSyntax();
    //configure tokens
    tokenStream.lowerCaseMode(false);
    tokenStream.eolIsSignificant(false);

    //add word chars
    tokenStream.wordChars(32, 58);
    tokenStream.wordChars(60, 93);
    tokenStream.wordChars(95, 126);

    //add <CR>\r <LF>\n
    tokenStream.wordChars(10, 10);
    tokenStream.wordChars(13, 13);

    //set ^ AND ; as delimeters to string tokens
    tokenStream.quoteChar(94);

    //removed ;
    //tokenStream.quoteChar(59);

    //set up the temp arraylist
    ArrayList tokens = new ArrayList();

    int result = tokenStream.ttype;
    while (result != StreamTokenizer.TT_EOF) {
        result = tokenStream.nextToken();
        switch (result) {
        case StreamTokenizer.TT_EOF:
            break;
        default:
            tokens.add(tokenStream.sval);
            break;
        }
    }
    return tokens.toArray();
}

---