Example usage for java.io StreamTokenizer eolIsSignificant

Introduction

In this page you can find the example usage for java.io StreamTokenizer eolIsSignificant.

Prototype

public void eolIsSignificant(boolean flag) 

Source Link

Document

Determines whether or not ends of line are treated as tokens.

Usage

From source file:Main.java

public static void main(String args[]) throws Exception {
    FileReader fr = new FileReader(args[0]);
    BufferedReader br = new BufferedReader(fr);
    StreamTokenizer st = new StreamTokenizer(br);
    st.eolIsSignificant(true);
    int lines = 1;

    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        switch (st.ttype) {
        case StreamTokenizer.TT_EOL:
            ++lines;//from w w w  . ja va2s.  c  om
        }
    }
    System.out.println("There are " + lines + " lines");
    fr.close();
}

---

From source file:Main.java

public static void main(String[] args) throws Exception {
    BufferedReader in = null;/*  w w w .j  a  va  2 s .  c o  m*/
    in = new BufferedReader(new FileReader("fileeditor.txt"));
    StreamTokenizer st = new StreamTokenizer(in);
    st.eolIsSignificant(false);
    // remove comment handling
    st.slashSlashComments(false);
    st.slashStarComments(false);

    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        if (st.ttype == StreamTokenizer.TT_NUMBER) {
            // the default is to treat numbers differently than words
            // also the numbers are doubles
            System.out.println((int) st.nval);
        } else {
            System.out.println(st.sval);
        }
    }
}

---

From source file:MainClass.java

public static void main(String args[]) {

    try {/*from   w  w w.j  a  v  a  2s  .c om*/
        FileReader fr = new FileReader(args[0]);

        BufferedReader br = new BufferedReader(fr);

        StreamTokenizer st = new StreamTokenizer(br);

        // Consider end-of-line as a token
        st.eolIsSignificant(true);

        // Declare variable to count lines
        int lines = 1;

        // Process tokens
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            switch (st.ttype) {
            case StreamTokenizer.TT_EOL:
                ++lines;
            }
        }

        System.out.println("There are " + lines + " lines");

        fr.close();
    } catch (Exception e) {
        System.out.println("Exception: " + e);
    }
}

---

From source file:Main.java

public static void main(String[] argv) throws Exception {
    FileReader rd = new FileReader("filename.java");
    StreamTokenizer st = new StreamTokenizer(rd);

    st.parseNumbers();//from ww  w.  j  av  a 2s . c o m
    st.wordChars('_', '_');
    st.eolIsSignificant(true);
    st.ordinaryChars(0, ' ');
    st.slashSlashComments(true);
    st.slashStarComments(true);

    int token = st.nextToken();
    while (token != StreamTokenizer.TT_EOF) {
        token = st.nextToken();
        switch (token) {
        case StreamTokenizer.TT_NUMBER:
            double num = st.nval;
            System.out.println(num);
            break;
        case StreamTokenizer.TT_WORD:
            String word = st.sval;
            System.out.println(word);
            break;
        case '"':
            String dquoteVal = st.sval;
            System.out.println(dquoteVal);
            break;
        case '\'':
            String squoteVal = st.sval;
            System.out.println(squoteVal);
            break;
        case StreamTokenizer.TT_EOL:
            break;
        case StreamTokenizer.TT_EOF:
            break;
        default:
            char ch = (char) st.ttype;
            System.out.println(ch);
            break;
        }
    }
    rd.close();
}

---

From source file:SumLine.java

static void sumLines(String filename) throws IOException {
    LineNumberReader lnr = new LineNumberReader(new FileReader(filename));
    lnr.setLineNumber(1);//w ww. j a va 2s  . c o m
    StreamTokenizer stok = new StreamTokenizer(lnr);
    stok.parseNumbers();
    stok.eolIsSignificant(true);
    stok.nextToken();
    while (stok.ttype != StreamTokenizer.TT_EOF) {
        int lineno = lnr.getLineNumber();
        double sum = 0;
        while (stok.ttype != StreamTokenizer.TT_EOL) {
            if (stok.ttype == StreamTokenizer.TT_NUMBER)
                sum += stok.nval;
            stok.nextToken();
        }
        System.out.println("Sum of line " + lineno + " is " + sum);
        stok.nextToken();
    }
}

---

From source file:com.enonic.cms.business.portal.datasource.expressionfunctions.ExpressionFunctions.java

/**
 * This method will take a freetext search string and create a valid query that can be used in the getContent* methods.  The search
 * string is spilt into tokens.  Using the operator, it may be specified whether the field must contain all or any of the words in the
 * search string.// w  w  w . java2s .c o m
 *
 * @param fieldName    The name of the field to search for the words in the search string.
 * @param searchString The words to search for.
 * @param operator     Must be either AND or OR.  Case doesn't matter.
 * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care,
 *         it may also be merged with other queries using AND or OR.
 * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR.
 */
public String buildFreetextQuery(String fieldName, String searchString, String operator) {
    if (searchString == null || searchString.trim().equals("")) {
        return "";
    }
    if (fieldName == null || fieldName.trim().equals("")) {
        throw new IllegalArgumentException("fieldName can not be empty.");
    }

    String op = "";
    if (operator != null) {
        op = operator.trim().toUpperCase();
    }
    if (!(op.equals("AND") || op.equals("OR"))) {
        throw new IllegalArgumentException("Illegal operator: " + operator);
    }

    boolean first = true;
    StringBuffer queryTokens = new StringBuffer();
    Reader searchStringReader = new StringReader(searchString);
    StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader);
    searchStringTokens.slashSlashComments(false);
    searchStringTokens.slashStarComments(false);
    searchStringTokens.eolIsSignificant(false);
    searchStringTokens.ordinaryChar('!');
    searchStringTokens.ordinaryChars('#', '}');
    searchStringTokens.wordChars('!', '!');
    searchStringTokens.wordChars('#', '}');

    try {
        while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) {
            String token = searchStringTokens.sval;
            addQueryPart(queryTokens, first, fieldName, token, op);
            if (first) {
                first = false;
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!");
    }

    return queryTokens.toString();
}

---

From source file:net.duckling.ddl.service.render.dml.ParseHtmlImg.java

public Map parseArgs(String argstring) throws IOException {
    HashMap<String, String> arglist = new HashMap<String, String>();

    //// w w w .  j  ava  2  s . c  om
    //  Protection against funny users.
    //
    if (argstring == null) {
        return arglist;
    }

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString(new Double(tok.nval).intValue());
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;
                arglist.put(param, value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

---

From source file:com.enonic.cms.core.portal.datasource.el.ExpressionFunctions.java

/**
 * This method will take a freetext search string and create a valid query that can be used in the getContent* methods.  The search
 * string is spilt into tokens.  Using the operator, it may be specified whether the field must contain all or any of the words in the
 * search string.// ww  w  .j  av  a 2s .com
 *
 * @param fieldName    The name of the field to search for the words in the search string.
 * @param searchString The words to search for.
 * @param operator     Must be either AND or OR.  Case doesn't matter.
 * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care,
 *         it may also be merged with other queries using AND or OR.
 * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR.
 */
public String buildFreetextQuery(String fieldName, String searchString, String operator) {
    if (searchString == null || searchString.trim().equals("")) {
        return "";
    }
    if (fieldName == null || fieldName.trim().equals("")) {
        throw new IllegalArgumentException("fieldName can not be empty.");
    }

    String op = "";
    if (operator != null) {
        op = operator.trim().toUpperCase();
    }
    if (!(op.equals("AND") || op.equals("OR"))) {
        throw new IllegalArgumentException("Illegal operator: " + operator);
    }

    boolean first = true;
    StringBuilder queryTokens = new StringBuilder();
    Reader searchStringReader = new StringReader(searchString);
    StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader);
    searchStringTokens.slashSlashComments(false);
    searchStringTokens.slashStarComments(false);
    searchStringTokens.eolIsSignificant(false);
    searchStringTokens.ordinaryChar('!');
    searchStringTokens.ordinaryChars('#', '}');
    searchStringTokens.wordChars('!', '!');
    searchStringTokens.wordChars('#', '}');

    try {
        while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) {
            String token = searchStringTokens.sval;
            addQueryPart(queryTokens, first, fieldName, token, op);
            if (first) {
                first = false;
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!");
    }

    return queryTokens.toString();
}

---

From source file:com.zimbra.common.calendar.ZoneInfo2iCalendar.java

private static void readExtraData(Reader reader) throws IOException, ParseException {
    char dquote = '"';
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();/*from   w ww . j a  v a 2 s . c  om*/
    tokenizer.wordChars(32, 126);
    tokenizer.whitespaceChars(' ', ' ');
    tokenizer.whitespaceChars('\t', '\t');
    tokenizer.whitespaceChars(0, 20);
    tokenizer.commentChar('#');
    tokenizer.quoteChar(dquote);
    tokenizer.eolIsSignificant(true);

    List<String> tokenList = new ArrayList<String>();
    LineType lineType = LineType.UNKNOWN;
    boolean atLineStart = true;

    int ttype;
    int prevTtype = StreamTokenizer.TT_EOL; // used for empty line detection
    while ((ttype = tokenizer.nextToken()) != StreamTokenizer.TT_EOF) {
        int lineNum = tokenizer.lineno();
        if (ttype == StreamTokenizer.TT_WORD || ttype == dquote) {
            String token = tokenizer.sval;
            if (atLineStart) {
                lineType = LineType.lookUp(token);
                if (LineType.UNKNOWN.equals(lineType))
                    throw new ParseException("Invalid line type", lineNum);
            } else {
                tokenList.add(token);
            }
            atLineStart = false;
        } else if (ttype == StreamTokenizer.TT_EOL) {
            if (prevTtype == StreamTokenizer.TT_EOL) {
                prevTtype = ttype;
                continue;
            }
            atLineStart = true;
            switch (lineType) {
            case PRIMARYZONE:
                if (tokenList.size() < 1)
                    throw new ParseException("Not enough fields in a PrimaryZone line", lineNum);
                String primaryTZID = tokenList.get(0);
                sPrimaryTZIDs.add(primaryTZID);
                break;
            case ZONEMATCHSCORE:
                if (tokenList.size() < 2)
                    throw new ParseException("Not enough fields in a ZoneMatchScore line", lineNum);
                String zoneName = tokenList.get(0);
                String zoneMatchScoreStr = tokenList.get(1);
                int zoneMatchScore = 0;
                try {
                    zoneMatchScore = Integer.parseInt(zoneMatchScoreStr);
                } catch (NumberFormatException e) {
                    throw new ParseException("Zone match score must be an integer: " + zoneMatchScoreStr,
                            lineNum);
                }
                sMatchScores.put(zoneName, zoneMatchScore);
                break;
            }
            if (atLineStart) {
                tokenList.clear();
                lineType = LineType.UNKNOWN;
            }
        } else if (ttype == StreamTokenizer.TT_NUMBER) {
            // shouldn't happen
            throw new ParseException("Invalid parser state: TT_NUMBER found", lineNum);
        }
        prevTtype = ttype;
    }
}

---

From source file:com.ecyrd.jspwiki.plugin.PluginManager.java

/**
 *  Parses plugin arguments.  Handles quotes and all other kewl stuff.
 *
 *  <h3>Special parameters</h3>
 *  The plugin body is put into a special parameter defined by {@link #PARAM_BODY};
 *  the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE};
 *  and the bounds of the plugin within the wiki page text by a parameter defined
 *  by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array,
 *  i.e., <tt>[start,end]</tt>.
 *
 * @param argstring The argument string to the plugin.  This is
 *  typically a list of key-value pairs, using "'" to escape
 *  spaces in strings, followed by an empty line and then the
 *  plugin body.  In case the parameter is null, will return an
 *  empty parameter list./*from ww  w  . ja  v a2s.c  o m*/
 *
 * @return A parsed list of parameters.
 *
 * @throws IOException If the parsing fails.
 */
public Map parseArgs(String argstring) throws IOException {
    HashMap<String, Object> arglist = new HashMap<String, Object>();

    //
    //  Protection against funny users.
    //
    if (argstring == null)
        return arglist;

    arglist.put(PARAM_CMDLINE, argstring);

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString((int) tok.nval);
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;

                arglist.put(param, value);

                // log.debug("ARG: "+param+"="+value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

---

• «
• 1
• 2
• »