Java String Tokenize getTokens(String vbt)

Description

get Tokens

License

BSD License

Declaration

public static Vector getTokens(String vbt) 

Method Source Code

//package com.java2s;
/*L// w w  w  .  j a v  a  2  s. c o m
 * Copyright Northrop Grumman Information Technology.
 *
 * Distributed under the OSI-approved BSD 3-Clause License.
 * See http://ncip.github.com/nci-mapping-tool/LICENSE.txt for details.
 */

import java.util.*;

public class Main {
    private static Vector filler_vec = null;

    public static Vector getTokens(String vbt) {
        String t = removeFillers(vbt);
        Vector v = new Vector();
        StringTokenizer st = new StringTokenizer(t, " ");
        int knt = 0;
        String retstr = "";
        while (st.hasMoreTokens()) {
            String val = st.nextToken();
            val = val.toLowerCase();
            v.add(val);
        }

        return v;
    }

    public static String removeFillers(String str) {
        String delim = getDelimiters();
        return removeFillers(str, delim);
    }

    public static String removeFillers(String str, String delim) {
        Vector fillers = getFillers();
        StringTokenizer st = new StringTokenizer(str, delim);
        int knt = 0;
        //String retstr = "";
        StringBuffer buf = new StringBuffer();
        while (st.hasMoreTokens()) {
            String val = st.nextToken();
            if (!fillers.contains(val)) {
                //retstr = retstr + " " + val;
                buf.append(" " + val);
            }
        }
        String retstr = buf.toString();

        retstr = retstr.trim();
        return retstr;
    }

    public static String getDelimiters() {

        return "(-)";
    }

    public static Vector getFillers() {
        return filler_vec;
    }
}

Related

1. getTokens(String str, String delimiter)
2. getTokens(String string)
3. getTokens(String string)
4. getTokens(String string, String tokenSeparator)
5. getTokens(String value)
6. getTokenTypes(CommonTree tree)
7. getUnitType(String typeToken)
8. hasSuffix(String _token)
9. isPOSTag(String token)