Example usage for java.io StreamTokenizer slashStarComments

Introduction

In this page you can find the example usage for java.io StreamTokenizer slashStarComments.

Prototype

public void slashStarComments(boolean flag) 

Source Link

Document

Determines whether or not the tokenizer recognizes C-style comments.

Usage

From source file:Main.java

public static void main(String[] args) throws Exception {
    BufferedReader in = null;/*  w ww  .  j  a v  a  2 s.c o  m*/
    in = new BufferedReader(new FileReader("fileeditor.txt"));
    StreamTokenizer st = new StreamTokenizer(in);
    st.eolIsSignificant(false);
    // remove comment handling
    st.slashSlashComments(false);
    st.slashStarComments(false);

    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        if (st.ttype == StreamTokenizer.TT_NUMBER) {
            // the default is to treat numbers differently than words
            // also the numbers are doubles
            System.out.println((int) st.nval);
        } else {
            System.out.println(st.sval);
        }
    }
}

---

From source file:Main.java

public static void main(String[] argv) throws Exception {
    FileReader rd = new FileReader("filename.java");
    StreamTokenizer st = new StreamTokenizer(rd);

    st.parseNumbers();//from  w w  w . j  a va  2 s .c o  m
    st.wordChars('_', '_');
    st.eolIsSignificant(true);
    st.ordinaryChars(0, ' ');
    st.slashSlashComments(true);
    st.slashStarComments(true);

    int token = st.nextToken();
    while (token != StreamTokenizer.TT_EOF) {
        token = st.nextToken();
        switch (token) {
        case StreamTokenizer.TT_NUMBER:
            double num = st.nval;
            System.out.println(num);
            break;
        case StreamTokenizer.TT_WORD:
            String word = st.sval;
            System.out.println(word);
            break;
        case '"':
            String dquoteVal = st.sval;
            System.out.println(dquoteVal);
            break;
        case '\'':
            String squoteVal = st.sval;
            System.out.println(squoteVal);
            break;
        case StreamTokenizer.TT_EOL:
            break;
        case StreamTokenizer.TT_EOF:
            break;
        default:
            char ch = (char) st.ttype;
            System.out.println(ch);
            break;
        }
    }
    rd.close();
}

---

From source file:com.denimgroup.threadfix.framework.filefilter.ClassAnnotationBasedFileFilter.java

@Override
public boolean accept(@Nullable File file) {
    boolean returnValue = false;
    boolean hasArroba = false;

    if (file != null && file.exists() && file.isFile() && file.getName().endsWith(".java")) {
        Reader reader = null;/* w ww .ja v a  2s  .c  o  m*/

        try {

            reader = new InputStreamReader(new FileInputStream(file), "UTF-8");

            StreamTokenizer tokenizer = new StreamTokenizer(reader);
            tokenizer.slashSlashComments(true);
            tokenizer.slashStarComments(true);

            while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
                if (hasArroba && tokenizer.sval != null && getClassAnnotations().contains(tokenizer.sval)) {
                    returnValue = true;
                    break;
                } else if (tokenizer.sval != null && tokenizer.sval.equals("class")) {
                    // we've gone too far
                    break;
                }

                hasArroba = tokenizer.ttype == '@';
            }
        } catch (IOException e) {
            log.warn("Encountered IOException while tokenizing file.", e);
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                    log.error("Encountered IOException while attempting to close file.", e);
                }
            }
        }
    }

    return returnValue;
}

---

From source file:com.enonic.cms.business.portal.datasource.expressionfunctions.ExpressionFunctions.java

/**
 * This method will take a freetext search string and create a valid query that can be used in the getContent* methods.  The search
 * string is spilt into tokens.  Using the operator, it may be specified whether the field must contain all or any of the words in the
 * search string.//from w ww .  ja  v  a2  s . c  o  m
 *
 * @param fieldName    The name of the field to search for the words in the search string.
 * @param searchString The words to search for.
 * @param operator     Must be either AND or OR.  Case doesn't matter.
 * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care,
 *         it may also be merged with other queries using AND or OR.
 * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR.
 */
public String buildFreetextQuery(String fieldName, String searchString, String operator) {
    if (searchString == null || searchString.trim().equals("")) {
        return "";
    }
    if (fieldName == null || fieldName.trim().equals("")) {
        throw new IllegalArgumentException("fieldName can not be empty.");
    }

    String op = "";
    if (operator != null) {
        op = operator.trim().toUpperCase();
    }
    if (!(op.equals("AND") || op.equals("OR"))) {
        throw new IllegalArgumentException("Illegal operator: " + operator);
    }

    boolean first = true;
    StringBuffer queryTokens = new StringBuffer();
    Reader searchStringReader = new StringReader(searchString);
    StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader);
    searchStringTokens.slashSlashComments(false);
    searchStringTokens.slashStarComments(false);
    searchStringTokens.eolIsSignificant(false);
    searchStringTokens.ordinaryChar('!');
    searchStringTokens.ordinaryChars('#', '}');
    searchStringTokens.wordChars('!', '!');
    searchStringTokens.wordChars('#', '}');

    try {
        while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) {
            String token = searchStringTokens.sval;
            addQueryPart(queryTokens, first, fieldName, token, op);
            if (first) {
                first = false;
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!");
    }

    return queryTokens.toString();
}

---

From source file:com.enonic.cms.core.portal.datasource.el.ExpressionFunctions.java

/**
 * This method will take a freetext search string and create a valid query that can be used in the getContent* methods.  The search
 * string is spilt into tokens.  Using the operator, it may be specified whether the field must contain all or any of the words in the
 * search string./*from   www .j  a  va2  s  .c  om*/
 *
 * @param fieldName    The name of the field to search for the words in the search string.
 * @param searchString The words to search for.
 * @param operator     Must be either AND or OR.  Case doesn't matter.
 * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care,
 *         it may also be merged with other queries using AND or OR.
 * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR.
 */
public String buildFreetextQuery(String fieldName, String searchString, String operator) {
    if (searchString == null || searchString.trim().equals("")) {
        return "";
    }
    if (fieldName == null || fieldName.trim().equals("")) {
        throw new IllegalArgumentException("fieldName can not be empty.");
    }

    String op = "";
    if (operator != null) {
        op = operator.trim().toUpperCase();
    }
    if (!(op.equals("AND") || op.equals("OR"))) {
        throw new IllegalArgumentException("Illegal operator: " + operator);
    }

    boolean first = true;
    StringBuilder queryTokens = new StringBuilder();
    Reader searchStringReader = new StringReader(searchString);
    StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader);
    searchStringTokens.slashSlashComments(false);
    searchStringTokens.slashStarComments(false);
    searchStringTokens.eolIsSignificant(false);
    searchStringTokens.ordinaryChar('!');
    searchStringTokens.ordinaryChars('#', '}');
    searchStringTokens.wordChars('!', '!');
    searchStringTokens.wordChars('#', '}');

    try {
        while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) {
            String token = searchStringTokens.sval;
            addQueryPart(queryTokens, first, fieldName, token, op);
            if (first) {
                first = false;
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!");
    }

    return queryTokens.toString();
}

---

From source file:edu.buffalo.cse.pigout.parser.PigOutMacro.java

void validate() throws IOException {
    if (rets.isEmpty()) {
        return;/*from   ww  w  .ja  v a2 s  . c  om*/
    }

    HashSet<String> testSet = new HashSet<String>();

    StreamTokenizer st = new StreamTokenizer(new StringReader(body));

    st.wordChars('.', '.');
    st.wordChars('0', '9');
    st.wordChars('_', '_');
    st.wordChars('$', '$');
    st.lowerCaseMode(false);
    st.ordinaryChar('/');
    st.slashStarComments(true);

    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        if (matchWord(st, "define", false) && matchDollarAlias(st, true)) {
            testSet.add(st.sval.substring(1));
        } else if (matchDollarAlias(st, false)) {
            String prevWord = st.sval;
            if (matchWord(st, "if", true) || matchWord(st, "otherwise", true)) {
                testSet.add(prevWord.substring(1));
            } else if (matchChar(st, '=', true) && !matchChar(st, '=', true)) {
                testSet.add(prevWord.substring(1));
            } else if (matchChar(st, ',', true)) {
                // possible mult-alias inlining of a macro
                ArrayList<String> mlist = new ArrayList<String>();
                mlist.add(prevWord);
                if (isMultiValueReturn(st, mlist, true)) {
                    for (String s : mlist) {
                        testSet.add(s.substring(1));
                    }
                }
            }
        } else if (matchChar(st, '-', false) && matchChar(st, '-', true)) {
            skipSingleLineComment(st);
        }
    }

    for (String s : rets) {
        if (!testSet.contains(s)) {
            throw new IOException("Macro '" + name + "' missing return alias: " + s);
        }
    }
}

---

From source file:com.feilong.core.bean.ConvertUtilTest.java

/**
 * TestConvertUtilTest.//  w ww .ja  v a  2 s. co  m
 * 
 * @throws IOException
 */
@Test
public void testConvertUtilTest5() throws IOException {
    StreamTokenizer streamTokenizer = new StreamTokenizer(new StringReader("abaBc^babac^cb//ab/*test*/"));
    streamTokenizer.whitespaceChars('^', '^'); // Set the delimiters
    streamTokenizer.lowerCaseMode(true);

    streamTokenizer.slashSlashComments(false);
    streamTokenizer.slashStarComments(false);
    // Split comma-delimited tokens into a List
    List<String> list = new ArrayList<String>();
    while (true) {
        int ttype = streamTokenizer.nextToken();
        if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
            if (streamTokenizer.sval != null) {
                list.add(streamTokenizer.sval);
            }
        } else if (ttype == StreamTokenizer.TT_EOF) {
            break;
        }
    }

    LOGGER.debug(JsonUtil.format(list));
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private void initTokenizer(StreamTokenizer tokenizer) {
    tokenizer.quoteChar('"');
    tokenizer.slashStarComments(true);
    tokenizer.slashSlashComments(true);/*from   w  w  w.j ava 2 s . c  om*/
    tokenizer.whitespaceChars(' ', ' ');
    tokenizer.whitespaceChars(0x09, 0x09);
    tokenizer.wordChars('$', '$'); // treat $ as part of word
    tokenizer.wordChars('#', '#'); // treat # as part of word
    tokenizer.wordChars('_', '_'); // treat $# as part of word
}

---