Example usage for java.io StreamTokenizer TT

Introduction

In this page you can find the example usage for java.io StreamTokenizer TT_EOL.

Prototype

int TT_EOL

To view the source code for java.io StreamTokenizer TT_EOL.

Click Source Link

Document

A constant indicating that the end of the line has been read.

Usage

From source file:com.rapidminer.tools.Tools.java

/** Delivers the next token and skip empty lines. */
public static void getFirstToken(StreamTokenizer tokenizer) throws IOException {
    // skip empty lines
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
    }/*from ww w .j a v  a2  s. c  o m*/
    ;

    if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
        tokenizer.ttype = '?';
    }
}

From source file:com.rapidminer.tools.Tools.java

/** Delivers the next token and checks if its the end of line. */
public static void getLastToken(StreamTokenizer tokenizer, boolean endOfFileOk) throws IOException {
    if (tokenizer.nextToken() != StreamTokenizer.TT_EOL
            && (tokenizer.ttype != StreamTokenizer.TT_EOF || !endOfFileOk)) {
        throw new IOException("expected the end of the line " + tokenizer.lineno());
    }//from  w w w. j a v a2  s. c o  m
}

From source file:com.rapidminer.tools.Tools.java

/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
    if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
        throw new IOException("unexpected end of line " + tokenizer.lineno());
    }//  w w  w.  j  a  v  a2 s . co m

    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
        throw new IOException("unexpected end of file in line " + tokenizer.lineno());
    } else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
        tokenizer.ttype = '?';
    }
}

From source file:com.rapidminer.tools.Tools.java

/** Skips all tokens before next end of line (EOL). */
public static void waitForEOL(StreamTokenizer tokenizer) throws IOException {
    // skip everything until EOL
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
    }//from   w w  w.j a  v a2s.co m
    ;
    tokenizer.pushBack();
}

From source file:org.apache.qpid.server.security.access.config.PlainConfiguration.java

@Override
public RuleSet load() {
    RuleSet ruleSet = super.load();

    File file = getFile();/*from   w  ww  . ja  v a 2s  .  c  om*/
    FileReader fileReader = null;

    try {
        if (_logger.isDebugEnabled()) {
            _logger.debug("About to load ACL file " + file);
        }

        fileReader = new FileReader(file);
        _st = new StreamTokenizer(new BufferedReader(fileReader));
        _st.resetSyntax(); // setup the tokenizer

        _st.commentChar(COMMENT); // single line comments
        _st.eolIsSignificant(true); // return EOL as a token
        _st.ordinaryChar('='); // equals is a token
        _st.ordinaryChar(CONTINUATION); // continuation character (when followed by EOL)
        _st.quoteChar('"'); // double quote
        _st.quoteChar('\''); // single quote
        _st.whitespaceChars('\u0000', '\u0020'); // whitespace (to be ignored) TODO properly
        _st.wordChars('a', 'z'); // unquoted token characters [a-z]
        _st.wordChars('A', 'Z'); // [A-Z]
        _st.wordChars('0', '9'); // [0-9]
        _st.wordChars('_', '_'); // underscore
        _st.wordChars('-', '-'); // dash
        _st.wordChars('.', '.'); // dot
        _st.wordChars('*', '*'); // star
        _st.wordChars('@', '@'); // at
        _st.wordChars(':', ':'); // colon

        // parse the acl file lines
        Stack<String> stack = new Stack<String>();
        int current;
        do {
            current = _st.nextToken();
            switch (current) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                if (stack.isEmpty()) {
                    break; // blank line
                }

                // pull out the first token from the bottom of the stack and check arguments exist
                String first = stack.firstElement();
                stack.removeElementAt(0);
                if (stack.isEmpty()) {
                    throw new IllegalConfigurationException(String.format(NOT_ENOUGH_TOKENS_MSG, getLine()));
                }

                // check for and parse optional initial number for ACL lines
                Integer number = null;
                if (StringUtils.isNumeric(first)) {
                    // set the acl number and get the next element
                    number = Integer.valueOf(first);
                    first = stack.firstElement();
                    stack.removeElementAt(0);
                }

                if (StringUtils.equalsIgnoreCase(ACL, first)) {
                    parseAcl(number, stack);
                } else if (number == null) {
                    if (StringUtils.equalsIgnoreCase("GROUP", first)) {
                        throw new IllegalConfigurationException(String.format(
                                "GROUP keyword not supported. Groups should defined via a Group Provider, not in the ACL file.",
                                getLine()));
                    } else if (StringUtils.equalsIgnoreCase(CONFIG, first)) {
                        parseConfig(stack);
                    } else {
                        throw new IllegalConfigurationException(
                                String.format(UNRECOGNISED_INITIAL_MSG, first, getLine()));
                    }
                } else {
                    throw new IllegalConfigurationException(
                            String.format(NUMBER_NOT_ALLOWED_MSG, first, getLine()));
                }

                // reset stack, start next line
                stack.clear();
                break;
            case StreamTokenizer.TT_NUMBER:
                stack.push(Integer.toString(Double.valueOf(_st.nval).intValue()));
                break;
            case StreamTokenizer.TT_WORD:
                stack.push(_st.sval); // token
                break;
            default:
                if (_st.ttype == CONTINUATION) {
                    int next = _st.nextToken();
                    if (next == StreamTokenizer.TT_EOL) {
                        break; // continue reading next line
                    }

                    // invalid location for continuation character (add one to line beacuse we ate the EOL)
                    throw new IllegalConfigurationException(
                            String.format(PREMATURE_CONTINUATION_MSG, getLine() + 1));
                } else if (_st.ttype == '\'' || _st.ttype == '"') {
                    stack.push(_st.sval); // quoted token
                } else {
                    stack.push(Character.toString((char) _st.ttype)); // single character
                }
            }
        } while (current != StreamTokenizer.TT_EOF);

        if (!stack.isEmpty()) {
            throw new IllegalConfigurationException(String.format(PREMATURE_EOF_MSG, getLine()));
        }
    } catch (IllegalArgumentException iae) {
        throw new IllegalConfigurationException(String.format(PARSE_TOKEN_FAILED_MSG, getLine()), iae);
    } catch (FileNotFoundException fnfe) {
        throw new IllegalConfigurationException(String.format(CONFIG_NOT_FOUND_MSG, file.getName()), fnfe);
    } catch (IOException ioe) {
        throw new IllegalConfigurationException(String.format(CANNOT_LOAD_MSG, file.getName()), ioe);
    } finally {
        if (fileReader != null) {
            try {
                fileReader.close();
            } catch (IOException e) {
                throw new IllegalConfigurationException(String.format(CANNOT_CLOSE_MSG, file.getName()), e);
            }
        }
    }

    return ruleSet;
}

From source file:org.apache.wiki.plugin.DefaultPluginManager.java

/**
 *  Parses plugin arguments.  Handles quotes and all other kewl stuff.
 *
 *  <h3>Special parameters</h3>
 *  The plugin body is put into a special parameter defined by {@link #PARAM_BODY};
 *  the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE};
 *  and the bounds of the plugin within the wiki page text by a parameter defined
 *  by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array,
 *  i.e., <tt>[start,end]</tt>.
 *
 * @param argstring The argument string to the plugin.  This is
 *  typically a list of key-value pairs, using "'" to escape
 *  spaces in strings, followed by an empty line and then the
 *  plugin body.  In case the parameter is null, will return an
 *  empty parameter list./*from  w w w  .  j  a v a 2  s  .  co  m*/
 *
 * @return A parsed list of parameters.
 *
 * @throws IOException If the parsing fails.
 */
public Map<String, String> parseArgs(String argstring) throws IOException {
    Map<String, String> arglist = new HashMap<String, String>();

    //
    //  Protection against funny users.
    //
    if (argstring == null)
        return arglist;

    arglist.put(PARAM_CMDLINE, argstring);

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString((int) tok.nval);
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;

                arglist.put(param, value);

                // log.debug("ARG: "+param+"="+value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Throws a formatted ParseException for the current token.
 * /*from w ww  .j  a va  2 s. co m*/
 * @param expected a description of what was expected
 * @throws ParseException
 * @throws com.vividsolutions.jts.util.AssertionFailedException if an
 *         invalid token is encountered
 */
private void parseError(String expected) throws ParseException {
    // throws Asserts for tokens that should never be seen
    if (tokenizer.ttype == StreamTokenizer.TT_NUMBER)
        Assert.shouldNeverReachHere("Unexpected NUMBER token");
    if (tokenizer.ttype == StreamTokenizer.TT_EOL)
        Assert.shouldNeverReachHere("Unexpected EOL token");

    String tokenStr = tokenString();
    throw new ParseException("Expected " + expected + " but found " + tokenStr);
}

From source file:org.gvnix.jpa.geo.hibernatespatial.util.EWKTReader.java

/**
 * Gets a description of the current token
 * /*ww w .j av a  2s  .  co m*/
 * @return a description of the current token
 */
private String tokenString() {
    switch (tokenizer.ttype) {
    case StreamTokenizer.TT_NUMBER:
        return "<NUMBER>";
    case StreamTokenizer.TT_EOL:
        return "End-of-Line";
    case StreamTokenizer.TT_EOF:
        return "End-of-Stream";
    case StreamTokenizer.TT_WORD:
        return "'" + tokenizer.sval + "'";
    }
    return "'" + (char) tokenizer.ttype + "'";
}

From source file:org.openhab.io.caldav.internal.util.ExecuteCommandJob.java

/**
 * Parses a <code>command</code>. Utilizes the {@link StreamTokenizer} which
 * takes care of quoted Strings as well.
 * /* w w  w .  j  a  v  a  2s .  c  o m*/
 * @param command the command to parse 
 * @return the tokenized command which can be processed by the 
 * <code>ConsoleInterpreter</code>
 * 
 * @see org.openhab.io.console.ConsoleInterpreter
 */
protected String[] parseCommand(String command) {
    logger.trace("going to parse command '{}'", command);

    // if the command starts with '>' it contains a script which needs no
    // further handling here ...
    if (command.startsWith(">")) {
        return new String[] { ">", command.substring(1).trim() };
    }

    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(command));
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('.', '.');

    List<String> tokens = new ArrayList<String>();
    try {
        int tokenType = 0;
        while (tokenType != StreamTokenizer.TT_EOF && tokenType != StreamTokenizer.TT_EOL) {
            tokenType = tokenizer.nextToken();
            String token = "";
            switch (tokenType) {
            case StreamTokenizer.TT_WORD:
            case 34 /* quoted String */:
                token = tokenizer.sval;
                break;
            case StreamTokenizer.TT_NUMBER:
                token = String.valueOf(tokenizer.nval);
                break;
            }
            tokens.add(token);
            logger.trace("read value {} from the given command", token);
        }
    } catch (IOException ioe) {
    }

    return tokens.toArray(new String[0]);
}

From source file:org.structr.core.function.Functions.java

public static Object evaluate(final ActionContext actionContext, final GraphObject entity,
        final String expression) throws FrameworkException {

    final String expressionWithoutNewlines = expression.replace('\n', ' ');
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(expressionWithoutNewlines));
    tokenizer.eolIsSignificant(true);/*from w  w w  .  java 2s  . c  o  m*/
    tokenizer.ordinaryChar('.');
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('.', '.');
    tokenizer.wordChars('!', '!');

    Expression root = new RootExpression();
    Expression current = root;
    Expression next = null;
    String lastToken = null;
    int token = 0;
    int level = 0;

    while (token != StreamTokenizer.TT_EOF) {

        token = nextToken(tokenizer);

        switch (token) {

        case StreamTokenizer.TT_EOF:
            break;

        case StreamTokenizer.TT_EOL:
            break;

        case StreamTokenizer.TT_NUMBER:
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before NUMBER");
            }
            next = new ConstantExpression(tokenizer.nval);
            current.add(next);
            lastToken += "NUMBER";
            break;

        case StreamTokenizer.TT_WORD:
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before " + tokenizer.sval);
            }
            next = checkReservedWords(tokenizer.sval);
            Expression previousExpression = current.getPrevious();
            if (tokenizer.sval.startsWith(".") && previousExpression != null
                    && previousExpression instanceof FunctionExpression && next instanceof ValueExpression) {

                final FunctionExpression previousFunctionExpression = (FunctionExpression) previousExpression;
                final ValueExpression valueExpression = (ValueExpression) next;

                current.replacePrevious(
                        new FunctionValueExpression(previousFunctionExpression, valueExpression));
            } else {
                current.add(next);
            }
            lastToken = tokenizer.sval;
            break;

        case '(':
            if (((current == null || current instanceof RootExpression) && next == null) || current == next) {

                // an additional bracket without a new function,
                // this can only be an execution group.
                next = new GroupExpression();
                current.add(next);
            }

            current = next;
            lastToken += "(";
            level++;
            break;

        case ')':
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before " + lastToken);
            }
            current = current.getParent();
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched closing bracket after " + lastToken);
            }
            lastToken += ")";
            level--;
            break;

        case '[':
            // bind directly to the previous expression
            next = new ArrayExpression();
            current.add(next);
            current = next;
            lastToken += "[";
            level++;
            break;

        case ']':
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched closing bracket before " + lastToken);
            }
            current = current.getParent();
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched closing bracket after " + lastToken);
            }
            lastToken += "]";
            level--;
            break;

        case ';':
            next = null;
            lastToken += ";";
            break;

        case ',':
            next = current;
            lastToken += ",";
            break;

        default:
            if (current == null) {
                throw new FrameworkException(422,
                        "Invalid expression: mismatched opening bracket before " + tokenizer.sval);
            }
            current.add(new ConstantExpression(tokenizer.sval));
            lastToken = tokenizer.sval;

        }
    }

    if (level > 0) {
        throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken);
    }

    return root.evaluate(actionContext, entity);
}

Example usage for java.io StreamTokenizer TT_EOL

Introduction

Prototype

Document

Usage