Example usage for java.io StreamTokenizer TT_EOF

Introduction

In this page you can find the example usage for java.io StreamTokenizer TT_EOF.

Prototype

int TT_EOF

To view the source code for java.io StreamTokenizer TT_EOF.

Click Source Link

Document

A constant indicating that the end of the stream has been read.

Usage

From source file:com.indeed.imhotep.index.builder.util.SmartArgs.java

/**
 * parse a String into an argument list, in the same way java parses
 * arguments passed to main()/*from   w ww .j  a v a2s.c o m*/
 */
public static String[] parseArgParams(String line) {
    StreamTokenizer tok = new StreamTokenizer(new StringReader(line));
    tok.resetSyntax();
    tok.wordChars('\u0000', '\uFFFF');
    tok.whitespaceChars(' ', ' ');
    tok.quoteChar('\"');
    ArrayList<String> output = new ArrayList<String>();
    try {
        while (tok.nextToken() != StreamTokenizer.TT_EOF) {
            output.add(tok.sval);
        }
    } catch (IOException e) {
    }
    return output.toArray(new String[output.size()]);
}

---

From source file:keel.Algorithms.ImbalancedClassification.CSMethods.C45CS.C45CS.java

/** Function to read the options from the execution file and assign the values to the parameters.
 *
 * @param options       The StreamTokenizer that reads the parameters file.
 *
 * @throws Exception   If the format of the file is not correct.
 *///w  w  w .  j a va  2 s  .  c  o  m
protected void setOptions(StreamTokenizer options) throws Exception {
    options.nextToken();

    /* Checks that the file starts with the token algorithm */
    if (options.sval.equalsIgnoreCase("algorithm")) {
        options.nextToken();
        options.nextToken();

        //if (!options.sval.equalsIgnoreCase( "C4.5" ) )
        //   throw new Exception( "The name of the algorithm is not correct." );

        options.nextToken();
        options.nextToken();
        options.nextToken();
        options.nextToken();

        /* Reads the names of the input files*/
        if (options.sval.equalsIgnoreCase("inputData")) {
            options.nextToken();
            options.nextToken();
            modelFileName = options.sval;

            if (options.nextToken() != StreamTokenizer.TT_EOL) {
                trainFileName = options.sval;
                options.nextToken();
                testFileName = options.sval;
                if (options.nextToken() != StreamTokenizer.TT_EOL) {
                    trainFileName = modelFileName;
                    options.nextToken();
                }
            }

        } else {
            throw new Exception("No file test provided.");
        }

        /* Reads the names of the output files*/
        while (true) {
            if (options.nextToken() == StreamTokenizer.TT_EOF) {
                throw new Exception("No output file provided.");
            }

            if (options.sval == null) {
                continue;
            } else if (options.sval.equalsIgnoreCase("outputData")) {
                break;
            }
        }

        options.nextToken();
        options.nextToken();
        trainOutputFileName = options.sval;
        options.nextToken();
        testOutputFileName = options.sval;
        options.nextToken();
        resultFileName = options.sval;

        if (!getNextToken(options)) {
            return;
        }

        while (options.ttype != StreamTokenizer.TT_EOF) {
            /* Reads the prune parameter */
            if (options.sval.equalsIgnoreCase("pruned")) {
                options.nextToken();
                options.nextToken();

                if (options.sval.equalsIgnoreCase("TRUE")) {
                    prune = true;
                } else {
                    //prune = false;
                    prune = true;
                }
            }

            /* Reads the confidence parameter */
            if (options.sval.equalsIgnoreCase("confidence")) {
                if (!prune) {
                    throw new Exception("Doesn't make sense to change confidence for prune " + "tree!");
                }

                options.nextToken();
                options.nextToken();

                /* Checks that the confidence threshold is between 0 and 1. */
                float cf = Float.parseFloat(options.sval);

                if (cf <= 1 || cf >= 0) {
                    confidence = Float.parseFloat(options.sval);
                }
            }

            /* Reads the itemsets per leaf parameter */
            if (options.sval.equalsIgnoreCase("itemsetsPerLeaf")) {
                options.nextToken();
                options.nextToken();

                if (Integer.parseInt(options.sval) > 0) {
                    minItemsets = Integer.parseInt(options.sval);
                }
            }

            /* Reads the minimum expected cost parameter */
            if (options.sval.equalsIgnoreCase("minimumExpectedCost")) {
                options.nextToken();
                options.nextToken();

                if (options.sval.equalsIgnoreCase("TRUE")) {
                    minimumExpectedCost = true;
                } else {
                    minimumExpectedCost = false;
                }
            }

            getNextToken(options);
        }
    }
}

---

From source file:com.enonic.cms.business.portal.datasource.expressionfunctions.ExpressionFunctions.java

/**
 * This method will take a freetext search string and create a valid query that can be used in the getContent* methods.  The search
 * string is spilt into tokens.  Using the operator, it may be specified whether the field must contain all or any of the words in the
 * search string.//www .  j  av  a2  s  .  c  om
 *
 * @param fieldName    The name of the field to search for the words in the search string.
 * @param searchString The words to search for.
 * @param operator     Must be either AND or OR.  Case doesn't matter.
 * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care,
 *         it may also be merged with other queries using AND or OR.
 * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR.
 */
public String buildFreetextQuery(String fieldName, String searchString, String operator) {
    if (searchString == null || searchString.trim().equals("")) {
        return "";
    }
    if (fieldName == null || fieldName.trim().equals("")) {
        throw new IllegalArgumentException("fieldName can not be empty.");
    }

    String op = "";
    if (operator != null) {
        op = operator.trim().toUpperCase();
    }
    if (!(op.equals("AND") || op.equals("OR"))) {
        throw new IllegalArgumentException("Illegal operator: " + operator);
    }

    boolean first = true;
    StringBuffer queryTokens = new StringBuffer();
    Reader searchStringReader = new StringReader(searchString);
    StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader);
    searchStringTokens.slashSlashComments(false);
    searchStringTokens.slashStarComments(false);
    searchStringTokens.eolIsSignificant(false);
    searchStringTokens.ordinaryChar('!');
    searchStringTokens.ordinaryChars('#', '}');
    searchStringTokens.wordChars('!', '!');
    searchStringTokens.wordChars('#', '}');

    try {
        while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) {
            String token = searchStringTokens.sval;
            addQueryPart(queryTokens, first, fieldName, token, op);
            if (first) {
                first = false;
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!");
    }

    return queryTokens.toString();
}

---

From source file:com.enonic.cms.core.portal.datasource.el.ExpressionFunctions.java

/**
 * This method will take a freetext search string and create a valid query that can be used in the getContent* methods.  The search
 * string is spilt into tokens.  Using the operator, it may be specified whether the field must contain all or any of the words in the
 * search string./*w w w  .j ava  2  s .  c om*/
 *
 * @param fieldName    The name of the field to search for the words in the search string.
 * @param searchString The words to search for.
 * @param operator     Must be either AND or OR.  Case doesn't matter.
 * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care,
 *         it may also be merged with other queries using AND or OR.
 * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR.
 */
public String buildFreetextQuery(String fieldName, String searchString, String operator) {
    if (searchString == null || searchString.trim().equals("")) {
        return "";
    }
    if (fieldName == null || fieldName.trim().equals("")) {
        throw new IllegalArgumentException("fieldName can not be empty.");
    }

    String op = "";
    if (operator != null) {
        op = operator.trim().toUpperCase();
    }
    if (!(op.equals("AND") || op.equals("OR"))) {
        throw new IllegalArgumentException("Illegal operator: " + operator);
    }

    boolean first = true;
    StringBuilder queryTokens = new StringBuilder();
    Reader searchStringReader = new StringReader(searchString);
    StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader);
    searchStringTokens.slashSlashComments(false);
    searchStringTokens.slashStarComments(false);
    searchStringTokens.eolIsSignificant(false);
    searchStringTokens.ordinaryChar('!');
    searchStringTokens.ordinaryChars('#', '}');
    searchStringTokens.wordChars('!', '!');
    searchStringTokens.wordChars('#', '}');

    try {
        while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) {
            String token = searchStringTokens.sval;
            addQueryPart(queryTokens, first, fieldName, token, op);
            if (first) {
                first = false;
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!");
    }

    return queryTokens.toString();
}

---

From source file:keel.Algorithms.Genetic_Rule_Learning.PART.C45.java

/** Function to read the options from the execution file and assign the values to the parameters.
 *
 * @param options The StreamTokenizer that reads the parameters file.
 *
 * @throws Exception If the format of the file is not correct.
 *//*w w  w  .  jav a2 s  .c  om*/
protected void setOptions(StreamTokenizer options) throws Exception {
    options.nextToken();

    /* Checks that the file starts with the token algorithm */
    if (options.sval.equalsIgnoreCase("algorithm")) {
        options.nextToken();
        options.nextToken();

        //if (!options.sval.equalsIgnoreCase( "C4.5" ) )
        //   throw new Exception( "The name of the algorithm is not correct." );

        options.nextToken();
        System.out.println(options.sval + "\n");
        options.nextToken();
        System.out.println(options.sval + "\n");
        //options.nextToken();
        //System.out.println(options.sval+"\n");
        //options.nextToken();
        //System.out.println(options.sval+"\n");

        /* Reads the names of the input files*/
        if (options.sval.equalsIgnoreCase("inputData")) {
            options.nextToken();
            options.nextToken();
            modelFileName = options.sval;
            System.out.println("Hay inputs\n");

            if (options.nextToken() != StreamTokenizer.TT_EOL) {
                trainFileName = options.sval;
                options.nextToken();
                testFileName = options.sval;
                if (options.nextToken() != StreamTokenizer.TT_EOL) {
                    trainFileName = modelFileName;
                    options.nextToken();
                }
                System.out.println(trainFileName + "\n");
                System.out.println(testFileName + "\n");
            }

        } else
            throw new Exception("No file test provided.");

        /* Reads the names of the output files*/
        while (true) {
            if (options.nextToken() == StreamTokenizer.TT_EOF)
                throw new Exception("No output file provided.");

            if (options.sval == null)
                continue;
            else if (options.sval.equalsIgnoreCase("outputData"))
                break;
        }

        options.nextToken();
        options.nextToken();
        trainOutputFileName = options.sval;
        options.nextToken();
        testOutputFileName = options.sval;
        options.nextToken();
        resultFileName = options.sval;

        System.out.println(trainOutputFileName + "\n");
        System.out.println(testOutputFileName + "\n");
        System.out.println(resultFileName + "\n");

        if (!getNextToken(options))
            return;

        while (options.ttype != StreamTokenizer.TT_EOF) {
            /* Reads the prune parameter */
            if (options.sval.equalsIgnoreCase("pruned")) {
                options.nextToken();
                options.nextToken();

                if (options.sval.equalsIgnoreCase("TRUE"))
                    prune = true;
                else
                    prune = false;
                //prune = true;
            }

            /* Reads the confidence parameter */
            if (options.sval.equalsIgnoreCase("confidence")) {
                if (!prune)
                    throw new Exception("Doesn't make sense to change confidence for prune tree!");

                options.nextToken();
                options.nextToken();

                /* Checks that the confidence threshold is between 0 and 1. */
                float cf = Float.parseFloat(options.sval);

                if (cf <= 1 || cf >= 0)
                    confidence = Float.parseFloat(options.sval);
            }

            /* Reads the itemsets per leaf parameter */
            if (options.sval.equalsIgnoreCase("itemsetsPerLeaf")) {
                options.nextToken();
                options.nextToken();

                if (Integer.parseInt(options.sval) > 0)
                    minItemsets = Integer.parseInt(options.sval);
            }

            getNextToken(options);
        }
    }
}

---

From source file:edu.buffalo.cse.pigout.parser.PigOutMacro.java

void validate() throws IOException {
    if (rets.isEmpty()) {
        return;//w w  w  . j  a  va 2  s .c o  m
    }

    HashSet<String> testSet = new HashSet<String>();

    StreamTokenizer st = new StreamTokenizer(new StringReader(body));

    st.wordChars('.', '.');
    st.wordChars('0', '9');
    st.wordChars('_', '_');
    st.wordChars('$', '$');
    st.lowerCaseMode(false);
    st.ordinaryChar('/');
    st.slashStarComments(true);

    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        if (matchWord(st, "define", false) && matchDollarAlias(st, true)) {
            testSet.add(st.sval.substring(1));
        } else if (matchDollarAlias(st, false)) {
            String prevWord = st.sval;
            if (matchWord(st, "if", true) || matchWord(st, "otherwise", true)) {
                testSet.add(prevWord.substring(1));
            } else if (matchChar(st, '=', true) && !matchChar(st, '=', true)) {
                testSet.add(prevWord.substring(1));
            } else if (matchChar(st, ',', true)) {
                // possible mult-alias inlining of a macro
                ArrayList<String> mlist = new ArrayList<String>();
                mlist.add(prevWord);
                if (isMultiValueReturn(st, mlist, true)) {
                    for (String s : mlist) {
                        testSet.add(s.substring(1));
                    }
                }
            }
        } else if (matchChar(st, '-', false) && matchChar(st, '-', true)) {
            skipSingleLineComment(st);
        }
    }

    for (String s : rets) {
        if (!testSet.contains(s)) {
            throw new IOException("Macro '" + name + "' missing return alias: " + s);
        }
    }
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private File runScript(String filename) throws Exception {
    File file = new File(filename);
    StreamTokenizer tokenizer = openScript(file);
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
        if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
            runCommand(tokenizer, file, file.getName(), new ExecutionContext());
        }//from   ww w  .  j a  v  a 2  s  .  c  o  m
    }
    return file;
}

---

From source file:edu.buffalo.cse.pigout.parser.PigOutMacro.java

private static void skipSingleLineComment(StreamTokenizer st) throws IOException {
    int lineNo = st.lineno();
    int lookahead = st.nextToken();
    while (lookahead != StreamTokenizer.TT_EOF && lookahead != '\n') {
        if (st.lineno() > lineNo)
            break;
        lookahead = st.nextToken();//from w ww. j  ava 2 s .c  o  m
    }
    st.pushBack();
}

---

From source file:com.ecyrd.jspwiki.plugin.PluginManager.java

/**
 *  Parses plugin arguments.  Handles quotes and all other kewl stuff.
 *
 *  <h3>Special parameters</h3>
 *  The plugin body is put into a special parameter defined by {@link #PARAM_BODY};
 *  the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE};
 *  and the bounds of the plugin within the wiki page text by a parameter defined
 *  by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array,
 *  i.e., <tt>[start,end]</tt>.
 *
 * @param argstring The argument string to the plugin.  This is
 *  typically a list of key-value pairs, using "'" to escape
 *  spaces in strings, followed by an empty line and then the
 *  plugin body.  In case the parameter is null, will return an
 *  empty parameter list.//from ww w.  j  a v a 2s  .c om
 *
 * @return A parsed list of parameters.
 *
 * @throws IOException If the parsing fails.
 */
public Map parseArgs(String argstring) throws IOException {
    HashMap<String, Object> arglist = new HashMap<String, Object>();

    //
    //  Protection against funny users.
    //
    if (argstring == null)
        return arglist;

    arglist.put(PARAM_CMDLINE, argstring);

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString((int) tok.nval);
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;

                arglist.put(param, value);

                // log.debug("ARG: "+param+"="+value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private boolean executeFunction(String name, File file, StreamTokenizer parent, ExecutionContext script)
        throws Exception {
    ExecutionContext context = functions.get(name);
    if (null != context) {
        // If this context has parameters then gather argument values
        if (null != parent && null != script) {
            List<String> params = context.getParams();
            if (null != params && params.size() > 0) {
                context.clearArgs();//from w  w w  .j  a  va  2 s .  co m
                int count = params.size();
                while (count-- > 0) {
                    // get argument
                    parent.nextToken();
                    System.out.print(' ');
                    switch (parent.ttype) {
                    case StreamTokenizer.TT_NUMBER:
                        System.out.print(parent.nval);
                        context.addArg(parent.nval);
                        break;
                    default:
                        System.out.print(parent.sval);
                        context.addArg(script.getExpandedString(parent));
                        break;
                    }
                }
            }
            System.out.println();
        }

        // Get function body and execute it
        String code = context.getBody();
        if (null != code) {
            StreamTokenizer tokenizer = openString(code);
            while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
                if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
                    runCommand(tokenizer, file, name, context);
                }
            }
            return true;
        }
    }
    return false;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »