Example usage for java.io StreamTokenizer nextToken

Introduction

In this page you can find the example usage for java.io StreamTokenizer nextToken.

Prototype

public int nextToken() throws IOException 

Source Link

Document

Parses the next token from the input stream of this tokenizer.

Usage

From source file:org.openmrs.cohort.CohortSearchHistory.java

public PatientSearch createCompositionFilter(String description) {
    Set<String> andWords = new HashSet<String>();
    Set<String> orWords = new HashSet<String>();
    Set<String> notWords = new HashSet<String>();
    andWords.add("and");
    andWords.add("intersection");
    andWords.add("*");
    orWords.add("or");
    orWords.add("union");
    orWords.add("+");
    notWords.add("not");
    notWords.add("!");

    List<Object> currentLine = new ArrayList<Object>();

    try {//from w ww.j a  va2  s. c  om
        StreamTokenizer st = new StreamTokenizer(new StringReader(description));
        st.ordinaryChar('(');
        st.ordinaryChar(')');
        Stack<List<Object>> stack = new Stack<List<Object>>();
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                Integer thisInt = new Integer((int) st.nval);
                if (thisInt < 1 || thisInt > searchHistory.size()) {
                    log.error("number < 1 or > search history size");
                    return null;
                }
                currentLine.add(thisInt);
            } else if (st.ttype == '(') {
                stack.push(currentLine);
                currentLine = new ArrayList<Object>();
            } else if (st.ttype == ')') {
                List<Object> l = stack.pop();
                l.add(currentLine);
                currentLine = l;
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                String str = st.sval.toLowerCase();
                if (andWords.contains(str))
                    currentLine.add(PatientSetService.BooleanOperator.AND);
                else if (orWords.contains(str))
                    currentLine.add(PatientSetService.BooleanOperator.OR);
                else if (notWords.contains(str))
                    currentLine.add(PatientSetService.BooleanOperator.NOT);
                else
                    throw new IllegalArgumentException("Don't recognize " + st.sval);
            }
        }
    } catch (Exception ex) {
        log.error("Error in description string: " + description, ex);
        return null;
    }

    if (!testCompositionList(currentLine)) {
        log.error("Description string failed test: " + description);
        return null;
    }

    //return toPatientFilter(currentLine);
    PatientSearch ret = new PatientSearch();
    ret.setParsedComposition(currentLine);
    return ret;
}

---

From source file:org.openmrs.module.reporting.cohort.definition.util.CohortExpressionParser.java

/**
 * Elements in this list can be: an Integer, indicating a 1-based index into a search history a
 * BooleanOperator (AND, OR, NOT) a CohortDefinition a PatientSearch another List of the same form,
 * which indicates a parenthetical expression
 *///from w  w  w. j a  v a  2 s . com
public static List<Object> parseIntoTokens(String expression) {

    List<Object> tokens = new ArrayList<Object>();
    try {
        StreamTokenizer st = new StreamTokenizer(new StringReader(expression));
        for (Character c : characterWords) {
            st.ordinaryChar(c);
        }
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                Integer thisInt = new Integer((int) st.nval);
                if (thisInt < 1) {
                    log.error("number < 1");
                    return null;
                }
                tokens.add(thisInt);
            } else if (openParenthesesWords.contains(Character.valueOf((char) st.ttype))) {
                tokens.add("(");
            } else if (closeParenthesesWords.contains(Character.valueOf((char) st.ttype))) {
                tokens.add(")");
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                tokens.add(st.sval);
            }
        }
        return parseIntoTokens(tokens);
    } catch (Exception ex) {
        log.error("Error in description string: " + expression, ex);
        return null;
    }
}

---

From source file:org.openmrs.module.reporting.query.evaluator.CompositionQueryEvaluator.java

/**
 * Elements in this list can be: an Integer, indicating a 1-based index into a search history a
 * BooleanOperator (AND, OR, NOT) a Query, another List of the same form, which indicates a parenthetical expression
 *///from  w w  w .j  a v  a 2  s  . c  o m
public List<Object> parseIntoTokens(String expression) throws EvaluationException {

    List<Object> tokens = new ArrayList<Object>();
    try {
        StreamTokenizer st = new StreamTokenizer(new StringReader(expression));
        for (Character c : CHARACTER_WORDS) {
            st.ordinaryChar(c);
        }
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                Integer thisInt = (int) st.nval;
                if (thisInt < 1) {
                    throw new IllegalArgumentException("Invalid number < 1 found");
                }
                tokens.add(thisInt);
            } else if (OPEN_PARENTHESES_WORDS.contains(Character.valueOf((char) st.ttype))) {
                tokens.add("(");
            } else if (CLOSE_PARENTHESES_WORDS.contains(Character.valueOf((char) st.ttype))) {
                tokens.add(")");
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                tokens.add(st.sval);
            }
        }
        return parseIntoTokens(tokens);
    } catch (Exception e) {
        throw new EvaluationException("Unable to parse expression <" + expression + "> into tokens", e);
    }
}

---

From source file:org.openmrs.reporting.PatientSearch.java

public static PatientSearch createCompositionSearch(String description) {
    // TODO This is a rewrite of the code in CohortSearchHistory.createCompositionFilter(String). That method should probably delegate to this one in some way.
    // TODO use open/closeParenthesesWords declared above
    List<Object> tokens = new ArrayList<Object>();
    try {/*from   w ww . j  av a  2 s  . c  om*/
        StreamTokenizer st = new StreamTokenizer(new StringReader(description));
        st.ordinaryChar('(');
        st.ordinaryChar(')');
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                Integer thisInt = new Integer((int) st.nval);
                if (thisInt < 1) {
                    log.error("number < 1");
                    return null;
                }
                tokens.add(thisInt);
            } else if (st.ttype == '(') {
                tokens.add("(");
            } else if (st.ttype == ')') {
                tokens.add(")");
            } else if (st.ttype == StreamTokenizer.TT_WORD) {
                String str = st.sval.toLowerCase();
                tokens.add(str);
            }
        }
        return createCompositionSearch(tokens);
    } catch (Exception ex) {
        log.error("Error in description string: " + description, ex);
        return null;
    }
}

---

From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java

/**
 * Parse a string into arguments as if it were provided on the command line.
 *
 * @param commandLineString/*  w w w  .j  a  va 2  s .  c  om*/
 *          A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus"
 * @param variableSpace
 *          Context for resolving variable names. If {@code null}, no variable resolution we happen.
 * @param ignoreSqoopCommand
 *          If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export".
 * @return List of parsed arguments
 * @throws IOException
 *           when the command line could not be parsed
 */
public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace,
        boolean ignoreSqoopCommand) throws IOException {
    List<String> args = new ArrayList<String>();
    StringReader reader = new StringReader(commandLineString);
    try {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        // Treat a dash as an ordinary character so it gets included in the token
        tokenizer.ordinaryChar('-');
        tokenizer.ordinaryChar('.');
        tokenizer.ordinaryChars('0', '9');
        // Treat all characters as word characters so nothing is parsed out
        tokenizer.wordChars('\u0000', '\uFFFF');

        // Re-add whitespace characters
        tokenizer.whitespaceChars(0, ' ');

        // Use " and ' as quote characters
        tokenizer.quoteChar('"');
        tokenizer.quoteChar('\'');

        // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments,
        // e.g. "sqoop <tool>")
        boolean skipToken = false;
        // Add all non-null string values tokenized from the string to the argument list
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.sval != null) {
                String s = tokenizer.sval;
                if (variableSpace != null) {
                    s = variableSpace.environmentSubstitute(s);
                }
                if (ignoreSqoopCommand && args.isEmpty()) {
                    // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of
                    // arguments directly
                    // from a working command line
                    if ("sqoop".equals(s)) {
                        skipToken = true;
                        continue; // skip this one and the next
                    } else if (skipToken) {
                        ignoreSqoopCommand = false; // Don't attempt to ignore any more commands
                        // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing
                        skipToken = false;
                        continue;
                    }
                }

                if (s.startsWith(ARG_D)) {
                    handleCustomOption(args, s, tokenizer, variableSpace);
                    continue;
                }
                args.add(escapeEscapeSequences(s));
            }
        }
    } finally {
        reader.close();
    }
    return args;
}

---

From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java

private static void handleCustomOption(List<String> args, String option, StreamTokenizer tokenizer,
        VariableSpace variableSpace) throws IOException {
    String key = null;//w  ww.  ja  v  a2 s .  c  om
    String value = null;

    args.add(ARG_D);
    if (ARG_D.equals(option)) {
        tokenizer.nextToken();
        key = tokenizer.sval;
    } else {
        key = option.substring(ARG_D.length());
    }

    if (key.contains(EQUALS)) {
        if (key.endsWith(EQUALS)) {
            key = key.substring(0, key.length() - 1);
            tokenizer.nextToken();
            value = tokenizer.sval;
        } else {
            String[] split = key.split(EQUALS);
            key = split[0];
            value = split[1];
        }
    } else {
        tokenizer.nextToken();
        value = tokenizer.sval;
    }
    if (variableSpace != null) {
        key = variableSpace.environmentSubstitute(key);
        value = variableSpace.environmentSubstitute(value);
    }
    args.add(key + EQUALS + escapeEscapeSequences(value));
}

---

From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java

/**
 * Parse a string into arguments as if it were provided on the command line.
 *
 * @param commandLineString A command line string.
 * @return List of parsed arguments/*from  w  ww. j av a  2s . co m*/
 * @throws IOException when the command line could not be parsed
 */
public List<String> parseCommandLine(String commandLineString) throws IOException {
    List<String> args = new ArrayList<String>();
    StringReader reader = new StringReader(commandLineString);
    try {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        // Treat a dash as an ordinary character so it gets included in the token
        tokenizer.ordinaryChar('-');
        tokenizer.ordinaryChar('.');
        tokenizer.ordinaryChars('0', '9');
        // Treat all characters as word characters so nothing is parsed out
        tokenizer.wordChars('\u0000', '\uFFFF');

        // Re-add whitespace characters
        tokenizer.whitespaceChars(0, ' ');

        // Use " and ' as quote characters
        tokenizer.quoteChar('"');
        tokenizer.quoteChar('\'');

        // Add all non-null string values tokenized from the string to the argument list
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.sval != null) {
                String s = tokenizer.sval;
                s = environmentSubstitute(s);
                args.add(s);
            }
        }
    } finally {
        reader.close();
    }

    return args;
}

---

From source file:org.structr.core.function.Functions.java

public static int nextToken(final StreamTokenizer tokenizer) {

    try {/*from  w  ww.  j  a v a  2 s  .  c  o m*/

        return tokenizer.nextToken();

    } catch (IOException ioex) {
    }

    return StreamTokenizer.TT_EOF;
}

---

From source file:org.xwiki.extension.internal.ExtensionUtils.java

/**
 * @param str the String to parse/*from w ww .  ja va  2  s .c o m*/
 * @param trim true if the passed String should be trimmed
 * @return the collection of Strings extracted from the passed String
 * @since 8.3M1
 */
public static List<String> importPropertyStringList(String str, boolean trim) {
    try {
        String cleanedString = str;

        // Trim
        if (trim) {
            cleanedString = cleanedString.trim();
        }

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString));

        // Everything is word
        st.ordinaryChars(0, 255);
        st.wordChars(0, 255);

        // Except quote chars
        st.quoteChar('"');
        st.quoteChar('\'');

        // And delimiters
        st.whitespaceChars(',', ',');
        st.whitespaceChars(' ', ' ');
        st.whitespaceChars('\t', '\t');
        st.whitespaceChars('\n', '\n');
        st.whitespaceChars('\r', '\r');

        // Split comma-delimited tokens into a List
        List<String> collection = new ArrayList<>();
        while (true) {
            int ttype = st.nextToken();
            if (ttype == StreamTokenizer.TT_WORD || ttype > 0) {
                if (st.sval != null) {
                    collection.add(st.sval);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype + " parsing elements.");
            }
        }

        // Return the completed list
        return collection;
    } catch (Exception e) {
        // Log ?
    }

    return Collections.emptyList();
}

---

From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java

/**
 * <p>/*  w ww.j a  v  a2 s.co  m*/
 * Parse an incoming String of the form similar to an array initializer in the Java language into a
 * <code>List</code> individual Strings for each element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes. Within a quoted elements, the normal Java
 * escape sequences are valid.</li>
 * </ul>
 * 
 * @param value String value to be parsed
 * @param genericType the generic type
 * @return List of parsed elements.
 * @throws ConversionException if the syntax of <code>value</code> is not syntactically valid
 * @throws NullPointerException if <code>value</code> is <code>null</code>
 */
protected Collection parseElements(String value, Type genericType) {
    String cleanedValue = cleanValue(value);

    try {
        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = createStreamTokenizer(cleanedValue);

        // Split comma-delimited tokens into a List
        Collection collection = newCollection();
        while (true) {
            int ttype = st.nextToken();
            if (ttype == StreamTokenizer.TT_WORD || ttype > 0) {
                if (st.sval != null) {
                    Object objValue = st.sval;
                    if (genericType != null && genericType != String.class) {
                        objValue = this.converterManager.convert(genericType, objValue);
                    }

                    collection.add(objValue);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype + " parsing elements.");
            }
        }

        // Return the completed list
        return collection;
    } catch (IOException e) {
        throw new ConversionException("Error converting from String: " + e.getMessage(), e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »