Example usage for java.io StreamTokenizer StreamTokenizer

List of usage examples for java.io StreamTokenizer StreamTokenizer

  1. HOME
  2. Java
  3. java
  4. java.io.*
  5. StreamTokenizer
  6. StreamTokenizer

Introduction

In this page you can find the example usage for java.io StreamTokenizer StreamTokenizer.

Prototype

public StreamTokenizer(Reader r)

Source Link

Document

Create a tokenizer that parses the given character stream.

Usage

From source file:org.apache.wiki.plugin.DefaultPluginManager.java

/**
 *  Parses plugin arguments.  Handles quotes and all other kewl stuff.
 *
 *  <h3>Special parameters</h3>
 *  The plugin body is put into a special parameter defined by {@link #PARAM_BODY};
 *  the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE};
 *  and the bounds of the plugin within the wiki page text by a parameter defined
 *  by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array,
 *  i.e., <tt>[start,end]</tt>.
 *
 * @param argstring The argument string to the plugin.  This is
 *  typically a list of key-value pairs, using "'" to escape
 *  spaces in strings, followed by an empty line and then the
 *  plugin body.  In case the parameter is null, will return an
 *  empty parameter list.//w w  w .  j a  v a 2s. co m
 *
 * @return A parsed list of parameters.
 *
 * @throws IOException If the parsing fails.
 */
public Map<String, String> parseArgs(String argstring) throws IOException {
    Map<String, String> arglist = new HashMap<String, String>();

    //
    //  Protection against funny users.
    //
    if (argstring == null)
        return arglist;

    arglist.put(PARAM_CMDLINE, argstring);

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString((int) tok.nval);
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;

                arglist.put(param, value);

                // log.debug("ARG: "+param+"="+value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

From source file:org.eclipse.mylyn.internal.phabricator.core.client.TracWebClient.java

/**
 * Parses the JavaScript code from the query page to extract repository configuration.
 *//*w w w  .j av a2  s .  c  o m*/
private void parseAttributesTokenizer(String text) throws IOException {
    StreamTokenizer t = new StreamTokenizer(new StringReader(text));
    t.quoteChar('"');

    TracConfiguration configuration = new TracConfiguration(data);
    AttributeFactory attributeFactory = null;
    String attributeType = null;

    AttributeState state = AttributeState.INIT;
    int tokenType;
    while ((tokenType = t.nextToken()) != StreamTokenizer.TT_EOF) {
        switch (tokenType) {
        case StreamTokenizer.TT_WORD:
        case '"':
            if (state == AttributeState.IN_LIST) {
                attributeFactory = configuration.getFactoryByField(t.sval);
                if (attributeFactory != null) {
                    attributeFactory.initialize();
                }
            } else if (state == AttributeState.IN_ATTRIBUTE_KEY) {
                attributeType = t.sval;
            } else if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST && "options".equals(attributeType)) { //$NON-NLS-1$
                if (attributeFactory != null) {
                    attributeFactory.addAttribute(t.sval);
                }
            }
            break;
        case ':':
            if (state == AttributeState.IN_ATTRIBUTE_KEY) {
                state = AttributeState.IN_ATTRIBUTE_VALUE;
            }
            break;
        case ',':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_ATTRIBUTE_KEY;
            }
            break;
        case '[':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_ATTRIBUTE_VALUE_LIST;
            }
            break;
        case ']':
            if (state == AttributeState.IN_ATTRIBUTE_VALUE_LIST) {
                state = AttributeState.IN_ATTRIBUTE_VALUE;
            }
            break;
        case '{':
            if (state == AttributeState.INIT) {
                state = AttributeState.IN_LIST;
            } else if (state == AttributeState.IN_LIST) {
                state = AttributeState.IN_ATTRIBUTE_KEY;
            } else {
                throw new IOException("Error parsing attributes: unexpected token '{'"); //$NON-NLS-1$
            }
            break;
        case '}':
            if (state == AttributeState.IN_ATTRIBUTE_KEY || state == AttributeState.IN_ATTRIBUTE_VALUE) {
                state = AttributeState.IN_LIST;
            } else if (state == AttributeState.IN_LIST) {
                state = AttributeState.INIT;
            } else {
                throw new IOException("Error parsing attributes: unexpected token '}'"); //$NON-NLS-1$
            }
            break;
        }
    }
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Reads a Well-Known Text representation of a {@link Geometry} from a
 * {@link java.io.Reader}.//ww w .j a  v  a 2s .co m
 * 
 * @param reader a Reader which will return a <Geometry Tagged Text> string
 *        (see the OpenGIS Simple Features Specification)
 * @return a <code>Geometry</code> read from <code>reader</code>
 * @throws ParseException if a parsing problem occurs
 */
private Geometry read_internal(Reader reader) throws ParseException {

    try {

        synchronized (this) {
            if (this.tokenizer != null) {
                throw new RuntimeException("EWKT-Reader is already in use.");
            }
            tokenizer = new StreamTokenizer(reader);
        }

        // set tokenizer to NOT parse numbers
        tokenizer.resetSyntax();
        tokenizer.wordChars('a', 'z');
        tokenizer.wordChars('A', 'Z');
        tokenizer.wordChars(128 + 32, 255);
        tokenizer.wordChars('0', '9');
        tokenizer.wordChars('-', '-');
        tokenizer.wordChars('+', '+');
        tokenizer.wordChars('.', '.');
        tokenizer.whitespaceChars(0, ' ');
        tokenizer.commentChar('#');

        this.hasM = null;
        this.dimension = -1;

        return readGeometryTaggedText();

    } catch (IOException e) {
        throw new ParseException(e.toString());
    } finally {
        this.tokenizer = null;
    }
}

From source file:org.jdesigner.platform.web.converter.AbstractArrayConverter.java

/**
 * <p>//from  w w w  .j  a va  2 s. com
 * Parse an incoming String of the form similar to an array initializer in
 * the Java language into a <code>List</code> individual Strings for each
 * element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around
 * the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes.
 * Within a quoted elements, the normal Java escape sequences are valid.</li>
 * </ul>
 * 
 * @param svalue
 *            String value to be parsed
 * @return The parsed list of String values
 * 
 * @exception ConversionException
 *                if the syntax of <code>svalue</code> is not syntactically
 *                valid
 * @exception NullPointerException
 *                if <code>svalue</code> is <code>null</code>
 */
protected List parseElements(String svalue) {

    // Validate the passed argument
    if (svalue == null) {
        throw new NullPointerException();
    }

    // Trim any matching '{' and '}' delimiters
    svalue = svalue.trim();
    if (svalue.startsWith("{") && svalue.endsWith("}")) {
        svalue = svalue.substring(1, svalue.length() - 1);
    }

    try {

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(svalue));
        st.whitespaceChars(',', ','); // Commas are delimiters
        st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
        st.ordinaryChars('.', '.');
        st.ordinaryChars('-', '-');
        st.wordChars('0', '9'); // Needed to make part of tokens
        st.wordChars('.', '.');
        st.wordChars('-', '-');

        // Split comma-delimited tokens into a List
        ArrayList list = new ArrayList();
        while (true) {
            int ttype = st.nextToken();
            if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
                list.add(st.sval);
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype);
            }
        }

        // Return the completed list
        return (list);

    } catch (IOException e) {

        throw new ConversionException(e);

    }

}

From source file:org.jdesigner.platform.web.converter.ArrayConverter.java

/**
 * <p>//  w ww  .  ja  v  a 2 s.c o  m
 * Parse an incoming String of the form similar to an array initializer in
 * the Java language into a <code>List</code> individual Strings for each
 * element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around
 * the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes.
 * Within a quoted elements, the normal Java escape sequences are valid.</li>
 * </ul>
 * 
 * @param type
 *            The type to convert the value to
 * @param value
 *            String value to be parsed
 * @return List of parsed elements.
 * 
 * @throws ConversionException
 *             if the syntax of <code>svalue</code> is not syntactically
 *             valid
 * @throws NullPointerException
 *             if <code>svalue</code> is <code>null</code>
 */
private List parseElements(Class type, String value) {

    if (log().isDebugEnabled()) {
        log().debug("Parsing elements, delimiter=[" + delimiter + "], value=[" + value + "]");
    }

    // Trim any matching '{' and '}' delimiters
    value = value.trim();
    if (value.startsWith("{") && value.endsWith("}")) {
        value = value.substring(1, value.length() - 1);
    }

    try {

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(value));
        st.whitespaceChars(delimiter, delimiter); // Set the delimiters
        st.ordinaryChars('0', '9'); // Needed to turn off numeric flag
        st.wordChars('0', '9'); // Needed to make part of tokens
        for (int i = 0; i < allowedChars.length; i++) {
            st.ordinaryChars(allowedChars[i], allowedChars[i]);
            st.wordChars(allowedChars[i], allowedChars[i]);
        }

        // Split comma-delimited tokens into a List
        List list = null;
        while (true) {
            int ttype = st.nextToken();
            if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
                if (st.sval != null) {
                    if (list == null) {
                        list = new ArrayList();
                    }
                    list.add(st.sval);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException(
                        "Encountered token of type " + ttype + " parsing elements to '" + toString(type) + ".");
            }
        }

        if (list == null) {
            list = Collections.EMPTY_LIST;
        }
        if (log().isDebugEnabled()) {
            log().debug(list.size() + " elements parsed");
        }

        // Return the completed list
        return (list);

    } catch (IOException e) {

        throw new ConversionException(
                "Error converting from String to '" + toString(type) + "': " + e.getMessage(), e);

    }

}

From source file:org.mindswap.pellet.KRSSLoader.java

private void initTokenizer(Reader reader) {
    in = new StreamTokenizer(reader);
    in.lowerCaseMode(false);/*from ww w  .jav  a  2  s  .  c om*/
    in.commentChar(';');
    in.wordChars('/', '/');
    in.wordChars('_', '_');
    in.wordChars('*', '*');
    in.wordChars('?', '?');
    in.wordChars('%', '%');
    in.wordChars('>', '>');
    in.wordChars('<', '<');
    in.wordChars('=', '=');
    in.quoteChar('|');
}

From source file:org.opencms.main.CmsShell.java

/**
 * Executes all commands read from the given input stream.<p>
 * /* ww w .ja v  a2s  .  c  o  m*/
 * @param fileInputStream a file input stream from which the commands are read
 */
private void executeCommands(FileInputStream fileInputStream) {

    try {
        LineNumberReader lnr = new LineNumberReader(new InputStreamReader(fileInputStream));
        while (!m_exitCalled) {
            printPrompt();
            String line = lnr.readLine();
            if (line == null) {
                // if null the file has been read to the end
                try {
                    Thread.sleep(500);
                } catch (Throwable t) {
                    // noop
                }
                break;
            }
            if (line.trim().startsWith("#")) {
                System.out.println(line);
                continue;
            }
            StringReader reader = new StringReader(line);
            StreamTokenizer st = new StreamTokenizer(reader);
            st.eolIsSignificant(true);

            // put all tokens into a List
            List<String> parameters = new ArrayList<String>();
            while (st.nextToken() != StreamTokenizer.TT_EOF) {
                if (st.ttype == StreamTokenizer.TT_NUMBER) {
                    parameters.add(Integer.toString(new Double(st.nval).intValue()));
                } else {
                    parameters.add(st.sval);
                }
            }
            reader.close();

            // extract command and arguments
            if (parameters.size() == 0) {
                if (m_echo) {
                    System.out.println();
                }
                continue;
            }
            String command = parameters.get(0);
            parameters = parameters.subList(1, parameters.size());

            // execute the command
            executeCommand(command, parameters);
        }
    } catch (Throwable t) {
        t.printStackTrace(System.err);
    }
}

From source file:org.openhab.io.caldav.internal.util.ExecuteCommandJob.java

/**
 * Parses a <code>command</code>. Utilizes the {@link StreamTokenizer} which
 * takes care of quoted Strings as well.
 * /*ww  w . j  av a  2  s  .com*/
 * @param command the command to parse 
 * @return the tokenized command which can be processed by the 
 * <code>ConsoleInterpreter</code>
 * 
 * @see org.openhab.io.console.ConsoleInterpreter
 */
protected String[] parseCommand(String command) {
    logger.trace("going to parse command '{}'", command);

    // if the command starts with '>' it contains a script which needs no
    // further handling here ...
    if (command.startsWith(">")) {
        return new String[] { ">", command.substring(1).trim() };
    }

    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(command));
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('.', '.');

    List<String> tokens = new ArrayList<String>();
    try {
        int tokenType = 0;
        while (tokenType != StreamTokenizer.TT_EOF && tokenType != StreamTokenizer.TT_EOL) {
            tokenType = tokenizer.nextToken();
            String token = "";
            switch (tokenType) {
            case StreamTokenizer.TT_WORD:
            case 34 /* quoted String */:
                token = tokenizer.sval;
                break;
            case StreamTokenizer.TT_NUMBER:
                token = String.valueOf(tokenizer.nval);
                break;
            }
            tokens.add(token);
            logger.trace("read value {} from the given command", token);
        }
    } catch (IOException ioe) {
    }

    return tokens.toArray(new String[0]);
}

From source file:org.openmrs.cohort.CohortSearchHistory.java

public PatientSearch createCompositionFilter(String description) {
    Set<String> andWords = new HashSet<String>();
    Set<String> orWords = new HashSet<String>();
    Set<String> notWords = new HashSet<String>();
    andWords.add("and");
    andWords.add("intersection");
    andWords.add("*");
    orWords.add("or");
    orWords.add("union");
    orWords.add("+");
    notWords.add("not");
    notWords.add("!");

    List<Object> currentLine = new ArrayList<Object>();

    try {//from w w  w .ja  va2s . c  o  m
        StreamTokenizer st = new StreamTokenizer(new StringReader(description));
        st.ordinaryChar('(');
        st.ordinaryChar(')');
        Stack<List<Object>> stack = new Stack<List<Object>>();
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                Integer thisInt = new Integer((int) st.nval);
                if (thisInt < 1 || thisInt > searchHistory.size()) {
                    log.error("number < 1 or > search history size");
                    return null;
                }
                currentLine.add(thisInt);
            } else if (st.ttype == '(') {
                stack.push(currentLine);
                currentLine = new ArrayList<Object>();
            } else if (st.ttype == ')') {
                List<Object> l = stack.pop();
                l.add(currentLine);
                currentLine = l;
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                String str = st.sval.toLowerCase();
                if (andWords.contains(str))
                    currentLine.add(PatientSetService.BooleanOperator.AND);
                else if (orWords.contains(str))
                    currentLine.add(PatientSetService.BooleanOperator.OR);
                else if (notWords.contains(str))
                    currentLine.add(PatientSetService.BooleanOperator.NOT);
                else
                    throw new IllegalArgumentException("Don't recognize " + st.sval);
            }
        }
    } catch (Exception ex) {
        log.error("Error in description string: " + description, ex);
        return null;
    }

    if (!testCompositionList(currentLine)) {
        log.error("Description string failed test: " + description);
        return null;
    }

    //return toPatientFilter(currentLine);
    PatientSearch ret = new PatientSearch();
    ret.setParsedComposition(currentLine);
    return ret;
}

From source file:org.openmrs.module.reporting.cohort.definition.util.CohortExpressionParser.java

/**
 * Elements in this list can be: an Integer, indicating a 1-based index into a search history a
 * BooleanOperator (AND, OR, NOT) a CohortDefinition a PatientSearch another List of the same form,
 * which indicates a parenthetical expression
 *///  w  w  w .  j a  v a  2 s  . c  o m
public static List<Object> parseIntoTokens(String expression) {

    List<Object> tokens = new ArrayList<Object>();
    try {
        StreamTokenizer st = new StreamTokenizer(new StringReader(expression));
        for (Character c : characterWords) {
            st.ordinaryChar(c);
        }
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                Integer thisInt = new Integer((int) st.nval);
                if (thisInt < 1) {
                    log.error("number < 1");
                    return null;
                }
                tokens.add(thisInt);
            } else if (openParenthesesWords.contains(Character.valueOf((char) st.ttype))) {
                tokens.add("(");
            } else if (closeParenthesesWords.contains(Character.valueOf((char) st.ttype))) {
                tokens.add(")");
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                tokens.add(st.sval);
            }
        }
        return parseIntoTokens(tokens);
    } catch (Exception ex) {
        log.error("Error in description string: " + expression, ex);
        return null;
    }
}