Example usage for java.io StreamTokenizer TT

Introduction

In this page you can find the example usage for java.io StreamTokenizer TT_EOF.

Prototype

int TT_EOF

To view the source code for java.io StreamTokenizer TT_EOF.

Click Source Link

Document

A constant indicating that the end of the stream has been read.

Usage

From source file:org.osaf.cosmo.calendar.ICalValueParser.java

/**
 * Parses the text value.//from   www. j a va2  s .com
 */
public void parse() throws ParseException {
    int nextToken = nextToken();
    // log.debug("starting token: " + tokenizer);
    if (nextToken != ';')
        return;

    nextToken = nextToken();
    while (nextToken != ':' && nextToken != StreamTokenizer.TT_EOF) {
        // log.debug("param name token: " + tokenizer);
        if (nextToken != StreamTokenizer.TT_WORD)
            throw new ParseException("expected word, read " + tokenizer.ttype, 1);
        String name = tokenizer.sval;

        nextToken = nextToken();
        // log.debug("param = token: " + tokenizer);
        if (nextToken != '=')
            throw new ParseException("expected =, read " + tokenizer.ttype, 1);

        nextToken = nextToken();
        // log.debug("param val token: " + tokenizer);
        if (!(nextToken == StreamTokenizer.TT_WORD || nextToken == '"'))
            throw new ParseException("expected word, read " + tokenizer.ttype, 1);
        String value = tokenizer.sval;

        // log.debug("parameter " + name + ": " + value);

        params.put(name, value);

        nextToken = nextToken();
        // log.debug("post param token: " + tokenizer);

        if (nextToken == ':')
            break;
        else if (nextToken == ';')
            nextToken = nextToken();
        else
            throw new ParseException("expected either : or ;, read " + tokenizer.ttype, 1);
    }

    nextToken = nextToken();
    // log.debug("prop val token: " + tokenizer);
    if (nextToken != StreamTokenizer.TT_WORD)
        throw new ParseException("expected " + StreamTokenizer.TT_WORD + ", read " + tokenizer.ttype, 1);
    value = tokenizer.sval;

    // log.debug("property: " + value + ", params: " + params);
}

From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java

/**
 * Parse a string into arguments as if it were provided on the command line.
 *
 * @param commandLineString/*from w w  w .  j  av a 2  s .  c  o  m*/
 *          A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus"
 * @param variableSpace
 *          Context for resolving variable names. If {@code null}, no variable resolution we happen.
 * @param ignoreSqoopCommand
 *          If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export".
 * @return List of parsed arguments
 * @throws IOException
 *           when the command line could not be parsed
 */
public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace,
        boolean ignoreSqoopCommand) throws IOException {
    List<String> args = new ArrayList<String>();
    StringReader reader = new StringReader(commandLineString);
    try {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        // Treat a dash as an ordinary character so it gets included in the token
        tokenizer.ordinaryChar('-');
        tokenizer.ordinaryChar('.');
        tokenizer.ordinaryChars('0', '9');
        // Treat all characters as word characters so nothing is parsed out
        tokenizer.wordChars('\u0000', '\uFFFF');

        // Re-add whitespace characters
        tokenizer.whitespaceChars(0, ' ');

        // Use " and ' as quote characters
        tokenizer.quoteChar('"');
        tokenizer.quoteChar('\'');

        // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments,
        // e.g. "sqoop <tool>")
        boolean skipToken = false;
        // Add all non-null string values tokenized from the string to the argument list
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.sval != null) {
                String s = tokenizer.sval;
                if (variableSpace != null) {
                    s = variableSpace.environmentSubstitute(s);
                }
                if (ignoreSqoopCommand && args.isEmpty()) {
                    // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of
                    // arguments directly
                    // from a working command line
                    if ("sqoop".equals(s)) {
                        skipToken = true;
                        continue; // skip this one and the next
                    } else if (skipToken) {
                        ignoreSqoopCommand = false; // Don't attempt to ignore any more commands
                        // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing
                        skipToken = false;
                        continue;
                    }
                }

                if (s.startsWith(ARG_D)) {
                    handleCustomOption(args, s, tokenizer, variableSpace);
                    continue;
                }
                args.add(escapeEscapeSequences(s));
            }
        }
    } finally {
        reader.close();
    }
    return args;
}

From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java

/**
 * Parse a string into arguments as if it were provided on the command line.
 *
 * @param commandLineString A command line string.
 * @return List of parsed arguments/*from w ww  .  j av  a  2s. c  o m*/
 * @throws IOException when the command line could not be parsed
 */
public List<String> parseCommandLine(String commandLineString) throws IOException {
    List<String> args = new ArrayList<String>();
    StringReader reader = new StringReader(commandLineString);
    try {
        StreamTokenizer tokenizer = new StreamTokenizer(reader);
        // Treat a dash as an ordinary character so it gets included in the token
        tokenizer.ordinaryChar('-');
        tokenizer.ordinaryChar('.');
        tokenizer.ordinaryChars('0', '9');
        // Treat all characters as word characters so nothing is parsed out
        tokenizer.wordChars('\u0000', '\uFFFF');

        // Re-add whitespace characters
        tokenizer.whitespaceChars(0, ' ');

        // Use " and ' as quote characters
        tokenizer.quoteChar('"');
        tokenizer.quoteChar('\'');

        // Add all non-null string values tokenized from the string to the argument list
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.sval != null) {
                String s = tokenizer.sval;
                s = environmentSubstitute(s);
                args.add(s);
            }
        }
    } finally {
        reader.close();
    }

    return args;
}

From source file:org.structr.core.function.Functions.java

public static Object evaluate(final ActionContext actionContext, final GraphObject entity,
        final String expression) throws FrameworkException {

    final String expressionWithoutNewlines = expression.replace('\n', ' ');
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(expressionWithoutNewlines));
    tokenizer.eolIsSignificant(true);/*from   w ww  .  j  a va2 s  . c o m*/
    tokenizer.ordinaryChar('.');
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('.', '.');
    tokenizer.wordChars('!', '!');

    Expression root = new RootExpression();
    Expression current = root;
    Expression next = null;
    String lastToken = null;
    int token = 0;
    int level = 0;

    while (token != StreamTokenizer.TT_EOF) {

        token = nextToken(tokenizer);

        switch (token) {

        case StreamTokenizer.TT_EOF:
            break;

        case StreamTokenizer.TT_EOL:
            break;

        case StreamTokenizer.TT_NUMBER:
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before NUMBER");
            }
            next = new ConstantExpression(tokenizer.nval);
            current.add(next);
            lastToken += "NUMBER";
            break;

        case StreamTokenizer.TT_WORD:
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before " + tokenizer.sval);
            }
            next = checkReservedWords(tokenizer.sval);
            Expression previousExpression = current.getPrevious();
            if (tokenizer.sval.startsWith(".") && previousExpression != null
                    && previousExpression instanceof FunctionExpression && next instanceof ValueExpression) {

                final FunctionExpression previousFunctionExpression = (FunctionExpression) previousExpression;
                final ValueExpression valueExpression = (ValueExpression) next;

                current.replacePrevious(
                        new FunctionValueExpression(previousFunctionExpression, valueExpression));
            } else {
                current.add(next);
            }
            lastToken = tokenizer.sval;
            break;

        case '(':
            if (((current == null || current instanceof RootExpression) && next == null) || current == next) {

                // an additional bracket without a new function,
                // this can only be an execution group.
                next = new GroupExpression();
                current.add(next);
            }

            current = next;
            lastToken += "(";
            level++;
            break;

        case ')':
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before " + lastToken);
            }
            current = current.getParent();
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched closing bracket after " + lastToken);
            }
            lastToken += ")";
            level--;
            break;

        case '[':
            // bind directly to the previous expression
            next = new ArrayExpression();
            current.add(next);
            current = next;
            lastToken += "[";
            level++;
            break;

        case ']':
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched closing bracket before " + lastToken);
            }
            current = current.getParent();
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched closing bracket after " + lastToken);
            }
            lastToken += "]";
            level--;
            break;

        case ';':
            next = null;
            lastToken += ";";
            break;

        case ',':
            next = current;
            lastToken += ",";
            break;

        default:
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before " + tokenizer.sval);
            }
            current.add(new ConstantExpression(tokenizer.sval));
            lastToken = tokenizer.sval;

        }
    }

    if (level > 0) {
        throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken);
    }

    return root.evaluate(actionContext, entity);
}

From source file:org.structr.core.function.Functions.java

public static int nextToken(final StreamTokenizer tokenizer) {

    try {// w  ww  . j  a  v  a 2 s.c  o  m

        return tokenizer.nextToken();

    } catch (IOException ioex) {
    }

    return StreamTokenizer.TT_EOF;
}

From source file:org.unitedinternet.cosmo.calendar.ICalValueParser.java

/**
 * Parses the text value.//from w ww  .  ja  v  a  2s  . c o  m
 * @throws ParseException - if something is wrong this exception is thrown.
 */
public void parse() throws ParseException {
    int nextToken = nextToken();
    // log.debug("starting token: " + tokenizer);
    if (nextToken != ';') {
        return;
    }

    nextToken = nextToken();
    while (nextToken != ':' && nextToken != StreamTokenizer.TT_EOF) {
        // log.debug("param name token: " + tokenizer);
        if (nextToken != StreamTokenizer.TT_WORD) {
            throw new ParseException("expected word, read " + tokenizer.ttype, 1);
        }
        String name = tokenizer.sval;

        nextToken = nextToken();
        // log.debug("param = token: " + tokenizer);
        if (nextToken != '=') {
            throw new ParseException("expected =, read " + tokenizer.ttype, 1);
        }

        nextToken = nextToken();
        // log.debug("param val token: " + tokenizer);
        if (!(nextToken == StreamTokenizer.TT_WORD || nextToken == '"')) {
            throw new ParseException("expected word, read " + tokenizer.ttype, 1);
        }
        String value = tokenizer.sval;

        // log.debug("parameter " + name + ": " + value);

        params.put(name, value);

        nextToken = nextToken();
        // log.debug("post param token: " + tokenizer);

        if (nextToken == ':') {
            break;
        } else if (nextToken == ';') {
            nextToken = nextToken();
        } else {
            throw new ParseException("expected either : or ;, read " + tokenizer.ttype, 1);
        }
    }

    nextToken = nextToken();
    // log.debug("prop val token: " + tokenizer);
    if (nextToken != StreamTokenizer.TT_WORD) {
        throw new ParseException("expected " + StreamTokenizer.TT_WORD + ", read " + tokenizer.ttype, 1);
    }
    value = tokenizer.sval;

    // log.debug("property: " + value + ", params: " + params);
}

From source file:org.wso2.andes.server.security.access.config.PlainConfiguration.java

@Override
public RuleSet load() throws ConfigurationException {
    RuleSet ruleSet = super.load();

    try {//  ww  w  .j av  a 2s . c o  m
        _st = new StreamTokenizer(new BufferedReader(new FileReader(_file)));
        _st.resetSyntax(); // setup the tokenizer

        _st.commentChar(COMMENT); // single line comments
        _st.eolIsSignificant(true); // return EOL as a token
        _st.lowerCaseMode(true); // case insensitive tokens
        _st.ordinaryChar('='); // equals is a token
        _st.ordinaryChar(CONTINUATION); // continuation character (when followed by EOL)
        _st.quoteChar('"'); // double quote
        _st.quoteChar('\''); // single quote
        _st.whitespaceChars('\u0000', '\u0020'); // whitespace (to be ignored) TODO properly
        _st.wordChars('a', 'z'); // unquoted token characters [a-z]
        _st.wordChars('A', 'Z'); // [A-Z]
        _st.wordChars('0', '9'); // [0-9]
        _st.wordChars('_', '_'); // underscore
        _st.wordChars('-', '-'); // dash
        _st.wordChars('.', '.'); // dot
        _st.wordChars('*', '*'); // star
        _st.wordChars('@', '@'); // at
        _st.wordChars(':', ':'); // colon

        // parse the acl file lines
        Stack<String> stack = new Stack<String>();
        int current;
        do {
            current = _st.nextToken();
            switch (current) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                if (stack.isEmpty()) {
                    break; // blank line
                }

                // pull out the first token from the bottom of the stack and check arguments exist
                String first = stack.firstElement();
                stack.removeElementAt(0);
                if (stack.isEmpty()) {
                    throw new ConfigurationException(String.format(NOT_ENOUGH_TOKENS_MSG, getLine()));
                }

                // check for and parse optional initial number for ACL lines
                Integer number = null;
                if (StringUtils.isNumeric(first)) {
                    // set the acl number and get the next element
                    number = Integer.valueOf(first);
                    first = stack.firstElement();
                    stack.removeElementAt(0);
                }

                if (StringUtils.equalsIgnoreCase(ACL, first)) {
                    parseAcl(number, stack);
                } else if (number == null) {
                    if (StringUtils.equalsIgnoreCase(GROUP, first)) {
                        parseGroup(stack);
                    } else if (StringUtils.equalsIgnoreCase(CONFIG, first)) {
                        parseConfig(stack);
                    } else {
                        throw new ConfigurationException(
                                String.format(UNRECOGNISED_INITIAL_MSG, first, getLine()));
                    }
                } else {
                    throw new ConfigurationException(String.format(NUMBER_NOT_ALLOWED_MSG, first, getLine()));
                }

                // reset stack, start next line
                stack.clear();
                break;
            case StreamTokenizer.TT_NUMBER:
                stack.push(Integer.toString(Double.valueOf(_st.nval).intValue()));
                break;
            case StreamTokenizer.TT_WORD:
                stack.push(_st.sval); // token
                break;
            default:
                if (_st.ttype == CONTINUATION) {
                    int next = _st.nextToken();
                    if (next == StreamTokenizer.TT_EOL) {
                        break; // continue reading next line
                    }

                    // invalid location for continuation character (add one to line beacuse we ate the EOL)
                    throw new ConfigurationException(String.format(PREMATURE_CONTINUATION_MSG, getLine() + 1));
                } else if (_st.ttype == '\'' || _st.ttype == '"') {
                    stack.push(_st.sval); // quoted token
                } else {
                    stack.push(Character.toString((char) _st.ttype)); // single character
                }
            }
        } while (current != StreamTokenizer.TT_EOF);

        if (!stack.isEmpty()) {
            throw new ConfigurationException(String.format(PREMATURE_EOF_MSG, getLine()));
        }
    } catch (IllegalArgumentException iae) {
        throw new ConfigurationException(String.format(PARSE_TOKEN_FAILED_MSG, getLine()), iae);
    } catch (FileNotFoundException fnfe) {
        throw new ConfigurationException(String.format(CONFIG_NOT_FOUND_MSG, getFile().getName()), fnfe);
    } catch (IOException ioe) {
        throw new ConfigurationException(String.format(CANNOT_LOAD_MSG, getFile().getName()), ioe);
    }

    return ruleSet;
}

From source file:org.xwiki.extension.internal.ExtensionUtils.java

/**
 * @param str the String to parse// w w  w.j  av a  2s  . com
 * @param trim true if the passed String should be trimmed
 * @return the collection of Strings extracted from the passed String
 * @since 8.3M1
 */
public static List<String> importPropertyStringList(String str, boolean trim) {
    try {
        String cleanedString = str;

        // Trim
        if (trim) {
            cleanedString = cleanedString.trim();
        }

        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString));

        // Everything is word
        st.ordinaryChars(0, 255);
        st.wordChars(0, 255);

        // Except quote chars
        st.quoteChar('"');
        st.quoteChar('\'');

        // And delimiters
        st.whitespaceChars(',', ',');
        st.whitespaceChars(' ', ' ');
        st.whitespaceChars('\t', '\t');
        st.whitespaceChars('\n', '\n');
        st.whitespaceChars('\r', '\r');

        // Split comma-delimited tokens into a List
        List<String> collection = new ArrayList<>();
        while (true) {
            int ttype = st.nextToken();
            if (ttype == StreamTokenizer.TT_WORD || ttype > 0) {
                if (st.sval != null) {
                    collection.add(st.sval);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype + " parsing elements.");
            }
        }

        // Return the completed list
        return collection;
    } catch (Exception e) {
        // Log ?
    }

    return Collections.emptyList();
}

From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java

/**
 * <p>/*w  ww. ja  v  a 2 s.  c o m*/
 * Parse an incoming String of the form similar to an array initializer in the Java language into a
 * <code>List</code> individual Strings for each element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes. Within a quoted elements, the normal Java
 * escape sequences are valid.</li>
 * </ul>
 * 
 * @param value String value to be parsed
 * @param genericType the generic type
 * @return List of parsed elements.
 * @throws ConversionException if the syntax of <code>value</code> is not syntactically valid
 * @throws NullPointerException if <code>value</code> is <code>null</code>
 */
protected Collection parseElements(String value, Type genericType) {
    String cleanedValue = cleanValue(value);

    try {
        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = createStreamTokenizer(cleanedValue);

        // Split comma-delimited tokens into a List
        Collection collection = newCollection();
        while (true) {
            int ttype = st.nextToken();
            if (ttype == StreamTokenizer.TT_WORD || ttype > 0) {
                if (st.sval != null) {
                    Object objValue = st.sval;
                    if (genericType != null && genericType != String.class) {
                        objValue = this.converterManager.convert(genericType, objValue);
                    }

                    collection.add(objValue);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype + " parsing elements.");
            }
        }

        // Return the completed list
        return collection;
    } catch (IOException e) {
        throw new ConversionException("Error converting from String: " + e.getMessage(), e);
    }
}

From source file:org.xwiki.properties.converter.collection.AbstractCollectionConverter.java

/**
 * <p>//from w ww.  j a  v  a2 s. c  o  m
 * Parse an incoming String of the form similar to an array initializer in the Java language into a
 * <code>List</code> individual Strings for each element, according to the following rules.
 * </p>
 * <ul>
 * <li>The string is expected to be a comma-separated list of values.</li>
 * <li>The string may optionally have matching '{' and '}' delimiters around the list.</li>
 * <li>Whitespace before and after each element is stripped.</li>
 * <li>Elements in the list may be delimited by single or double quotes. Within a quoted elements, the normal Java
 * escape sequences are valid.</li>
 * </ul>
 *
 * @param <G> the type in which the provided value has to be converted
 * @param targetType Data type to which this value should be converted.
 * @param value String value to be parsed
 * @param elementType the generic type
 * @return List of parsed elements.
 * @throws ConversionException if the syntax of <code>value</code> is not syntactically valid
 * @throws NullPointerException if <code>value</code> is <code>null</code>
 */
protected <G extends T> G parseElements(Type targetType, String value, Type elementType) {
    String cleanedValue = cleanValue(value);

    try {
        // Set up a StreamTokenizer on the characters in this String
        StreamTokenizer st = createStreamTokenizer(cleanedValue);

        // Split comma-delimited tokens into a List
        T collection = newCollection(targetType);
        while (true) {
            int ttype = st.nextToken();
            if (ttype == StreamTokenizer.TT_WORD || ttype > 0) {
                if (st.sval != null) {
                    Object objValue = st.sval;
                    if (elementType != null && elementType != String.class) {
                        objValue = this.converterManager.convert(elementType, objValue);
                    }

                    collection.add(objValue);
                }
            } else if (ttype == StreamTokenizer.TT_EOF) {
                break;
            } else {
                throw new ConversionException("Encountered token of type " + ttype + " parsing elements.");
            }
        }

        // Return the completed list
        return (G) collection;
    } catch (IOException e) {
        throw new ConversionException("Error converting from String: " + e.getMessage(), e);
    }
}

Example usage for java.io StreamTokenizer TT_EOF

Introduction

Prototype

Document

Usage