Java String Tokenize tokenize(String s, String delimiters)

Description

Tokenize given string for specified delimiter(s).

License

Open Source License

Parameter

Parameter	Description
s	String to be tokenized
delimiters	Delimiter character(s)

Return

Array of token strings

Declaration

public static String[] tokenize(String s, String delimiters) 

Method Source Code

//package com.java2s;

import java.util.*;

public class Main {
    /**/*  ww  w .jav a  2s . com*/
     * Tokenize given string for specified delimiter(s).
     *
     * @param s          String to be tokenized
     * @param delimiters Delimiter character(s)
     * @return Array of token strings
     */
    public static String[] tokenize(String s, String delimiters) {
        if (s == null) {
            return new String[] {};
        }

        StringTokenizer tokenizer = new StringTokenizer(s, delimiters);
        String result[] = new String[tokenizer.countTokens()];
        int index = 0;
        while (tokenizer.hasMoreTokens()) {
            result[index++] = tokenizer.nextToken();
        }

        return result;
    }

    public static String[] tokenize(String s, String delimiters, boolean trimTokens, boolean allowEmptyTokens) {
        if (s == null) {
            return new String[] {};
        }

        StringTokenizer tokenizer = new StringTokenizer(s, delimiters, true);
        List<String> tokenList = new LinkedList<String>();
        boolean lastWasSeparator = false;
        while (tokenizer.hasMoreTokens()) {
            String token = tokenizer.nextToken();
            boolean isSeparator = delimiters.indexOf(token) >= 0;
            if (trimTokens) {
                token = token.trim();
            }
            if (isSeparator) {
                if (lastWasSeparator && allowEmptyTokens) {
                    tokenList.add("");
                }
            } else if (!"".equals(token) || allowEmptyTokens) {
                tokenList.add(token);
            }
            lastWasSeparator = isSeparator;
        }
        if (lastWasSeparator && allowEmptyTokens) {
            tokenList.add("");
        }

        return tokenList.toArray(new String[tokenList.size()]);
    }
}

Related

1. tokenize(String s)
2. tokenize(String s)
3. tokenize(String s, char separator)
4. tokenize(String s, char separator, int maxTokens)
5. tokenize(String s, char separator, int maxTokens)
6. tokenize(String s1)
7. Tokenize(String sent)
8. tokenize(String sentence)
9. tokenize(String source, char separator)