Here you can find the source of conservativeTokenize(String text)
public static List<String> conservativeTokenize(String text)
//package com.java2s; //License from project: Open Source License import java.util.Arrays; import java.util.List; public class Main { /** Conservatively normalize a string while tokenizing it */ public static List<String> conservativeTokenize(String text) { String[] token_arr = text.toLowerCase() .split("[ \t~`@#$%^&\\*\\(\\)_\\+-=\\{\\}\\[\\]:\";'<>\\?,./\\|\\\\]+"); return Arrays.asList(token_arr); }/*from w w w .j a v a2s . c o m*/ }