Java String Tokenize tokenize(final String s)

Description

tokenize

License

Apache License

Declaration

public static List<String> tokenize(final String s) 

Method Source Code

//package com.java2s;
/*/*from w w  w. j a va  2s  .c o  m*/
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You should have received a copy of  the license along with this library.
 * You may also obtain a copy of the License at
 *         http://www.apache.org/licenses/LICENSE-2.0.
 */

import java.util.ArrayList;
import java.util.List;

public class Main {
    public static List<String> tokenize(final String s) {
        return tokenize(" \t\n\r\u0085\u2028", s);
    }

    public static List<String> tokenize(final String delims, final String s) {
        final List<String> ret = new ArrayList<String>();
        final int len = s.length();
        int start = -1;
        for (int i = 0; i < len; ++i) {
            final char c = s.charAt(i);
            if (delims.indexOf(c) >= 0) {
                if (start >= 0) {
                    ret.add(s.substring(start, i));
                    start = -1;
                }
                continue;
            }
            if (start < 0) {
                start = i;
            }
        }
        if (start >= 0) {
            ret.add(s.substring(start, len));
        }
        return ret;
    }
}

Related

1. removeStopWords(String[] tokens, Set stopWords)
2. stringTokenizer(String in)
3. substringsBetween(final String str, final String open, final String close, boolean tokenReservedFlag)
4. tokenArray(StringTokenizer st)
5. tokenize(final String aInput, final String aDelimiters)
6. tokenize(String array, String delimiter)
7. tokenize(String formula)
8. tokenize(String in)
9. tokenize(String input)