Here you can find the source of normalizeTokens(String[] tokens, boolean removeEmpty)
public static String[] normalizeTokens(String[] tokens, boolean removeEmpty)
//package com.java2s; //License from project: Apache License import java.util.ArrayList; import java.util.Arrays; public class Main { private static final char[] CHARACTER_REPLACE_MAPPING = new char[256]; public static String[] normalizeTokens(String[] tokens, boolean removeEmpty) { for (int i = 0; i < tokens.length; i++) { tokens[i] = normalizeString(tokens[i]); }/*from w ww. java2s . co m*/ if (removeEmpty) { tokens = removeEmpty(tokens); } return tokens; } public static String normalizeString(String token) { char[] charArray = token.toCharArray(); char[] toReturn = new char[charArray.length]; int index = 0; for (int i = 0; i < charArray.length; i++) { char x = charArray[i]; if (x < CHARACTER_REPLACE_MAPPING.length) { if (CHARACTER_REPLACE_MAPPING[x] > 0) { toReturn[index++] = CHARACTER_REPLACE_MAPPING[x]; } } } return String.valueOf(Arrays.copyOf(toReturn, index)); } /** * Removes empty tokens from given array. The empty slots will be filled with * the follow-up tokens. */ public static String[] removeEmpty(String[] arr) { ArrayList<String> list = new ArrayList<>(); for (String s : arr) { if (s != null && !s.isEmpty()) list.add(s); } return list.toArray(new String[list.size()]); } }