Example usage for java.io StreamTokenizer TT

Introduction

In this page you can find the example usage for java.io StreamTokenizer TT_WORD.

Prototype

int TT_WORD

To view the source code for java.io StreamTokenizer TT_WORD.

Click Source Link

Document

A constant indicating that a word token has been read.

Usage

From source file:org.apache.wiki.plugin.DefaultPluginManager.java

/**
 *  Parses plugin arguments.  Handles quotes and all other kewl stuff.
 *
 *  <h3>Special parameters</h3>
 *  The plugin body is put into a special parameter defined by {@link #PARAM_BODY};
 *  the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE};
 *  and the bounds of the plugin within the wiki page text by a parameter defined
 *  by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array,
 *  i.e., <tt>[start,end]</tt>.
 *
 * @param argstring The argument string to the plugin.  This is
 *  typically a list of key-value pairs, using "'" to escape
 *  spaces in strings, followed by an empty line and then the
 *  plugin body.  In case the parameter is null, will return an
 *  empty parameter list.//w  w w. ja v  a  2s. co m
 *
 * @return A parsed list of parameters.
 *
 * @throws IOException If the parsing fails.
 */
public Map<String, String> parseArgs(String argstring) throws IOException {
    Map<String, String> arglist = new HashMap<String, String>();

    //
    //  Protection against funny users.
    //
    if (argstring == null)
        return arglist;

    arglist.put(PARAM_CMDLINE, argstring);

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString((int) tok.nval);
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;

                arglist.put(param, value);

                // log.debug("ARG: "+param+"="+value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

From source file:org.eclipse.mylyn.internal.phabricator.core.client.TracWebClient.java

/**
 * Parses the JavaScript code from the query page to extract repository configuration.
 *//*from  w ww  . j a va  2  s.c om*/
private void parseAttributesTokenizer(String text) throws IOException {
    StreamTokenizer t = new StreamTokenizer(new StringReader(text));
    t.quoteChar('"');

    TracConfiguration configuration = new TracConfiguration(data);
    AttributeFactory attributeFactory = null;
    String attributeType = null;

    AttributeState state = AttributeState.INIT;
    int tokenType;
    while ((tokenType = t.nextToken()) != StreamTokenizer.TT_EOF) {
        switch (tokenType) {
        case StreamTokenizer.TT_WORD:
        case '"':
            if (state == AttributeState.IN_LIST) {
                attributeFactory = configuration.getFactoryByField(t.sval);
                if (attributeFactory != null) {
                    attributeFactory.initialize();
                }
            } else if (state == AttributeState.IN_ATTRIBUTE_KEY) {
                attributeType = t.sval;
            } else if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST && "options".equals(attributeType)) { //$NON-NLS-1$
                if (attributeFactory != null) {
                    attributeFactory.addAttribute(t.sval);
                }
            }
            break;
        case ':':
            if (state == AttributeState.IN_ATTRIBUTE_KEY) {
                state = AttributeState.IN_ATTRIBUTE_VALUE;
            }
            break;
        case ',':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_ATTRIBUTE_KEY;
            }
            break;
        case '[':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_ATTRIBUTE_VALUE_LIST;
            }
            break;
        case ']':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST) {
                state = AttributeState.IN_ATTRIBUTE_VALUE;
            }
            break;
        case '{':
            if (state == AttributeState.INIT) {
                state = AttributeState.IN_LIST;
            } else if (state == AttributeState.IN_LIST) {
                state = AttributeState.IN_ATTRIBUTE_KEY;
            } else {
                throw new IOException("Error parsing attributes: unexpected token '{'"); //$NON-NLS-1$
            }
            break;
        case '}':
            if (state == AttributeState.IN_ATTRIBUTE_KEY || state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_LIST;
            } else if (state == AttributeState.IN_LIST) {
                state = AttributeState.INIT;
            } else {
                throw new IOException("Error parsing attributes: unexpected token '}'"); //$NON-NLS-1$
            }
            break;
        }
    }
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

private boolean isNumberNext() throws IOException {
    int type = tokenizer.nextToken();
    tokenizer.pushBack();//from  www.j  av  a 2 s .  c  o  m
    return type == StreamTokenizer.TT_WORD;
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Parses the next number in the stream. Numbers with exponents are handled.
 * //from   w ww .j  av  a2  s .  c om
 * @param tokenizer tokenizer over a stream of text in Well-known Text
 *        format. The next token must be a number.
 * @return the next number in the stream
 * @throws ParseException if the next token is not a valid number
 * @throws IOException if an I/O error occurs
 */
private double getNextNumber() throws IOException, ParseException {
    int type = tokenizer.nextToken();
    switch (type) {
    case StreamTokenizer.TT_WORD: {
        try {
            return Double.parseDouble(tokenizer.sval);
        } catch (NumberFormatException ex) {
            throw new ParseException("Invalid number: " + tokenizer.sval);
        }
    }
    }
    parseError("number");
    return 0.0;
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Returns the next word in the stream.//from w  w  w  . j a v a2 s.co  m
 * 
 * @param tokenizer tokenizer over a stream of text in Well-known Text
 *        format. The next token must be a word.
 * @return the next word in the stream as uppercase text
 * @throws ParseException if the next token is not a word
 * @throws IOException if an I/O error occurs
 */
private String getNextWord() throws IOException, ParseException {
    int type = tokenizer.nextToken();
    switch (type) {
    case StreamTokenizer.TT_WORD:

        String word = tokenizer.sval;
        if (word.equalsIgnoreCase(EMPTY))
            return EMPTY;
        return word;

    case '(':
        return L_PAREN;
    case ')':
        return R_PAREN;
    case ',':
        return COMMA;
    case '=':
        return EQUALS;
    case ';':
        return SEMICOLON;
    }
    parseError("word");
    return null;
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Gets a description of the current token
 * /*from ww w  .  j  av a2s . c  om*/
 * @return a description of the current token
 */
private String tokenString() {
    switch (tokenizer.ttype) {
    case StreamTokenizer.TT_NUMBER:
        return "<NUMBER>";
    case StreamTokenizer.TT_EOL:
        return "End-of-Line";
    case StreamTokenizer.TT_EOF:
        return "End-of-Stream";
    case StreamTokenizer.TT_WORD:
        return "'" + tokenizer.sval + "'";
    }
    return "'" + (char) tokenizer.ttype + "'";
}

From source file:org.jdesigner.platform.web.converter.AbstractArrayConverter.java

/**
 * <p>//from  ww  w  .j  a  v a2  s .co  m
 * Parse an incoming String of the form similar to an array initializer in
 * the Java language into a <code>List</code> individual Strings for each
 * element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around
 * the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes.
 * Within a quoted elements, the normal Java escape sequences are valid.</li>
 * </ul>
 * 
 * @param svalue
 *            String value to be parsed
 * @return The parsed list of String values
 * 
 * @exception ConversionException
 *                if the syntax of <code>svalue</code> is not syntactically
 *                valid
 * @exception NullPointerException
 *                if <code>svalue</code> is <code>null</code>
 */
protected List parseElements(String svalue) {

    // Validate the passed argument
    if (svalue == null) {
        throw new NullPointerException();
    }

    // Trim any matching '{' and '}' delimiters
    svalue = svalue.trim();
    if (svalue.startsWith("{") && svalue.endsWith("}")) {
        svalue = svalue.substring(1, svalue.length() - 1);
    }

    try {

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(svalue));
        st.whitespaceChars(',', ','); // Commas are delimiters
        st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
        st.ordinaryChars('.', '.');
        st.ordinaryChars('-', '-');
        st.wordChars('0', '9'); // Needed to make part of tokens
        st.wordChars('.', '.');
        st.wordChars('-', '-');

        // Split comma-delimited tokens into a List
        ArrayList list = new ArrayList();
        while (true) {
            int ttype = st.nextToken();
            if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
                list.add(st.sval);
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype);
            }
        }

        // Return the completed list
        return (list);

    } catch (IOException e) {

        throw new ConversionException(e);

    }

}

From source file:org.jdesigner.platform.web.converter.ArrayConverter.java

/**
 * <p>//w ww  .j  a  va2  s .com
 * Parse an incoming String of the form similar to an array initializer in
 * the Java language into a <code>List</code> individual Strings for each
 * element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around
 * the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes.
 * Within a quoted elements, the normal Java escape sequences are valid.</li>
 * </ul>
 * 
 * @param type
 *            The type to convert the value to
 * @param value
 *            String value to be parsed
 * @return List of parsed elements.
 * 
 * @throws ConversionException
 *             if the syntax of <code>svalue</code> is not syntactically
 *             valid
 * @throws NullPointerException
 *             if <code>svalue</code> is <code>null</code>
 */
private List parseElements(Class type, String value) {

    if (log().isDebugEnabled()) {
        log().debug("Parsing elements, delimiter=[" + delimiter + "], value=[" + value + "]");
    }

    // Trim any matching '{' and '}' delimiters
    value = value.trim();
    if (value.startsWith("{") && value.endsWith("}")) {
        value = value.substring(1, value.length() - 1);
    }

    try {

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(value));
        st.whitespaceChars(delimiter, delimiter); // Set the delimiters
        st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
        st.wordChars('0', '9'); // Needed to make part of tokens
        for (int i = 0; i < allowedChars.length; i++) {
            st.ordinaryChars(allowedChars[i], allowedChars[i]);
            st.wordChars(allowedChars[i], allowedChars[i]);
        }

        // Split comma-delimited tokens into a List
        List list = null;
        while (true) {
            int ttype = st.nextToken();
            if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
                if (st.sval != null) {
                    if (list == null) {
                        list = new ArrayList();
                    }
                    list.add(st.sval);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException(
                        "Encountered token of type " + ttype + " parsing elements to '" + toString(type) + ".");
            }
        }

        if (list == null) {
            list = Collections.EMPTY_LIST;
        }
        if (log().isDebugEnabled()) {
            log().debug(list.size() + " elements parsed");
        }

        // Return the completed list
        return (list);

    } catch (IOException e) {

        throw new ConversionException(
                "Error converting from String to '" + toString(type) + "': " + e.getMessage(), e);

    }

}

From source file:org.jrman.parser.Parser.java

public void parse(String filename) throws Exception {
    if (currentDirectory != null) {
        String fullFileName = (String) fullFileNames.get(filename);
        if (fullFileName == null) {
            fullFileName = currentDirectory + File.separator + filename;
            fullFileNames.put(filename, fullFileName);
        }/*from w  w  w .  j av a 2  s .c o  m*/
        filename = fullFileName;
    }
    FileReader fr = new FileReader(filename);
    Tokenizer st = new Tokenizer(new BufferedReader(fr));
    // st.commentChar('#');
    int tk;
    while ((tk = st.nextToken()) != StreamTokenizer.TT_EOF) {
        try {
            if (tk != StreamTokenizer.TT_WORD)
                throw new Exception("Expected keyword at line " + st.lineno());
            String keyword = st.sval;
            KeywordParser kp = getKeyWordParser(keyword);
            if (!kp.getValidStates().contains(state))
                throw new IllegalStateException(
                        "Keyword" + kp + " is not valid in state " + state + ", at line " + st.lineno());
            kp.parse(st);
        } catch (Exception pe) {
            System.err.println("Error: " + pe);
            pe.printStackTrace();
        }
    }
    fr.close();
}

From source file:org.mindswap.pellet.KRSSLoader.java

private ATermAppl parseExpr() throws IOException {
    ATermAppl a = null;//from ww  w .  ja v a  2  s.com

    int token = in.nextToken();
    String s = in.sval;
    if (token == ':') {
        s = nextString();
        if (s.equalsIgnoreCase("TOP"))
            a = ATermUtils.TOP;
        else if (s.equalsIgnoreCase("BOTTOM"))
            a = ATermUtils.BOTTOM;
        else
            throw new RuntimeException("Parse exception after ':' " + s);
    } else if (token == '(') {
        token = in.nextToken();
        ATermUtils.assertTrue(token == StreamTokenizer.TT_WORD);

        s = in.sval;
        if (s.equalsIgnoreCase("NOT")) {
            ATermAppl c = parseExpr();
            a = ATermUtils.makeNot(c);

            if (ATermUtils.isPrimitive(c))
                kb.addClass(c);
        } else if (s.equalsIgnoreCase("AND")) {
            ATermList list = ATermUtils.EMPTY_LIST;

            while (!peekNext(')')) {
                ATermAppl c = parseExpr();

                if (ATermUtils.isPrimitive(c))
                    kb.addClass(c);
                list = list.insert(c);
            }
            a = ATermUtils.makeAnd(list);
        } else if (s.equalsIgnoreCase("OR")) {
            ATermList list = ATermUtils.EMPTY_LIST;

            while (!peekNext(')')) {
                ATermAppl c = parseExpr();

                if (ATermUtils.isPrimitive(c))
                    kb.addClass(c);
                list = list.insert(c);
            }
            a = ATermUtils.makeOr(list);
        } else if (s.equalsIgnoreCase("ONE-OF")) {
            ATermList list = ATermUtils.EMPTY_LIST;

            while (!peekNext(')')) {
                ATermAppl c = parseExpr();

                kb.addIndividual(c);
                list = list.insert(ATermUtils.makeValue(c));
            }
            a = ATermUtils.makeOr(list);
        } else if (s.equalsIgnoreCase("ALL")) {
            ATermAppl r = parseExpr();
            kb.addObjectProperty(r);
            ATermAppl c = parseExpr();
            if (ATermUtils.isPrimitive(c))
                kb.addClass(c);

            a = ATermUtils.makeAllValues(r, c);
        } else if (s.equalsIgnoreCase("SOME")) {
            ATermAppl r = parseExpr();
            kb.addObjectProperty(r);
            ATermAppl c = parseExpr();
            if (ATermUtils.isPrimitive(c))
                kb.addClass(c);
            a = ATermUtils.makeSomeValues(r, c);
        } else if (s.equalsIgnoreCase("AT-LEAST") || s.equalsIgnoreCase("ATLEAST")) {
            int n = nextInt();
            ATermAppl r = parseExpr();
            kb.addObjectProperty(r);

            a = ATermUtils.makeMin(r, n);
        } else if (s.equalsIgnoreCase("AT-MOST") || s.equalsIgnoreCase("ATMOST")) {
            int n = nextInt();
            ATermAppl r = parseExpr();
            kb.addObjectProperty(r);
            a = ATermUtils.makeMax(r, n);
        } else if (s.equalsIgnoreCase("A")) {
            ATermAppl r = nextTerm();
            // TODO what does term 'A' stand for
            kb.addProperty(r);
            kb.addFunctionalProperty(r);
            a = ATermUtils.makeMin(r, 1);
        } else if (s.equalsIgnoreCase("MIN") || s.equals(">=")) {
            ATermAppl r = nextTerm();
            kb.addDatatypeProperty(r);
            Object val = nextNumber();
            DatatypeReasoner dtReasoner = kb.getDatatypeReasoner();
            Datatype dt = xsdInteger.restrictMinInclusive(val);
            String dtName = dtReasoner.defineDatatype(dt);
            ATermAppl datatype = ATermUtils.makeTermAppl(dtName);
            a = ATermUtils.makeAllValues(r, datatype);
        } else if (s.equalsIgnoreCase("MAX") || s.equals("<=")) {
            ATermAppl r = nextTerm();
            kb.addDatatypeProperty(r);
            Object val = nextNumber();
            DatatypeReasoner dtReasoner = kb.getDatatypeReasoner();
            Datatype dt = xsdInteger.restrictMaxInclusive(val);
            String dtName = dtReasoner.defineDatatype(dt);
            ATermAppl datatype = ATermUtils.makeTermAppl(dtName);
            a = ATermUtils.makeAllValues(r, datatype);
        } else if (s.equals("=")) {
            ATermAppl r = nextTerm();
            kb.addDatatypeProperty(r);
            Object val = nextNumber();
            DatatypeReasoner dtReasoner = kb.getDatatypeReasoner();
            Datatype dt = xsdInteger.singleton(val);
            String dtName = dtReasoner.defineDatatype(dt);
            ATermAppl datatype = ATermUtils.makeTermAppl(dtName);
            a = ATermUtils.makeAllValues(r, datatype);
        } else if (s.equalsIgnoreCase("EXACTLY")) {
            int n = nextInt();
            ATermAppl r = parseExpr();
            kb.addObjectProperty(r);
            a = ATermUtils.makeAnd(ATermUtils.makeMax(r, n), ATermUtils.makeMin(r, n));
        } else if (s.equalsIgnoreCase("INV")) {
            ATermAppl r = parseExpr();
            kb.addObjectProperty(r);
            a = kb.getProperty(r).getInverse().getName();
        } else {
            throw new RuntimeException("Unknown expression " + s);
        }

        if (in.nextToken() != ')') {
            if (s.equalsIgnoreCase("AT-LEAST") || s.equalsIgnoreCase("AT-MOST") || s.equalsIgnoreCase("ATLEAST")
                    || s.equalsIgnoreCase("ATMOST")) {
                s = nextString();
                if (s.equalsIgnoreCase("TOP") || s.equalsIgnoreCase("*TOP*") || s.equalsIgnoreCase(":TOP"))
                    skipNext(')');
                else
                    throw new UnsupportedFeatureException("Qualified cardinality restrictions");
            } else
                throw new RuntimeException("Parse exception at term " + s);
        }
    } else if (token == '#') {
        int n = nextInt();
        if (peekNext('#')) {
            skipNext();
            a = (ATermAppl) terms.get(n);
            if (a == null)
                throw new RuntimeException("Parse exception: #" + n + "# is not defined");
        } else {
            skipNext("=");
            a = parseExpr();

            while (terms.size() <= n)
                terms.add(null);

            terms.set(n, a);
        }
    } else if (token == StreamTokenizer.TT_EOF)
        a = null;
    else if (s.equalsIgnoreCase("TOP") || s.equalsIgnoreCase("*TOP*") || s.equalsIgnoreCase(":TOP"))
        a = ATermUtils.TOP;
    else if (s.equalsIgnoreCase("BOTTOM") || s.equalsIgnoreCase("*BOTTOM*"))
        a = ATermUtils.BOTTOM;
    else {
        if (forceUppercase)
            s = s.toUpperCase();
        a = ATermUtils.makeTermAppl(s);
    }

    return a;
}

Example usage for java.io StreamTokenizer TT_WORD

Introduction

Prototype

Document

Usage