Here you can find the source of splitOnTokens(String text)
static String[] splitOnTokens(String text)
//package com.java2s; import java.util.ArrayList; public class Main { static String[] splitOnTokens(String text) { char[] array = text.toCharArray(); if (text.indexOf("?") == -1 && text.indexOf("*") == -1) { return new String[] { text }; }/*from w w w.j av a2 s . c o m*/ ArrayList list = new ArrayList(); StringBuffer buffer = new StringBuffer(); for (int i = 0; i < array.length; i++) { if (array[i] == '?' || array[i] == '*') { if (buffer.length() != 0) { list.add(buffer.toString()); buffer.setLength(0); } list.add(new String(new char[] { array[i] })); } else { buffer.append(array[i]); } } if (buffer.length() != 0) { list.add(buffer.toString()); } return (String[]) list.toArray(new String[0]); } }