Java tutorial
/* * Copyright (c) 1994, 2004, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. Oracle designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. */ package java.util; import java.lang.*; /** * The string tokenizer class allows an application to break a * string into tokens. The tokenization method is much simpler than * the one used by the {@code StreamTokenizer} class. The * {@code StringTokenizer} methods do not distinguish among * identifiers, numbers, and quoted strings, nor do they recognize * and skip comments. * <p> * The set of delimiters (the characters that separate tokens) may * be specified either at creation time or on a per-token basis. * <p> * An instance of {@code StringTokenizer} behaves in one of two * ways, depending on whether it was created with the * {@code returnDelims} flag having the value {@code true} * or {@code false}: * <ul> * <li>If the flag is {@code false}, delimiter characters serve to * separate tokens. A token is a maximal sequence of consecutive * characters that are not delimiters. * <li>If the flag is {@code true}, delimiter characters are themselves * considered to be tokens. A token is thus either one delimiter * character, or a maximal sequence of consecutive characters that are * not delimiters. * </ul><p> * A {@code StringTokenizer} object internally maintains a current * position within the string to be tokenized. Some operations advance this * current position past the characters processed.<p> * A token is returned by taking a substring of the string that was used to * create the {@code StringTokenizer} object. * <p> * The following is one example of the use of the tokenizer. The code: * <blockquote><pre> * StringTokenizer st = new StringTokenizer("this is a test"); * while (st.hasMoreTokens()) { * System.out.println(st.nextToken()); * } * </pre></blockquote> * <p> * prints the following output: * <blockquote><pre> * this * is * a * test * </pre></blockquote> * * <p> * {@code StringTokenizer} is a legacy class that is retained for * compatibility reasons although its use is discouraged in new code. It is * recommended that anyone seeking this functionality use the {@code split} * method of {@code String} or the java.util.regex package instead. * <p> * The following example illustrates how the {@code String.split} * method can be used to break up a string into its basic tokens: * <blockquote><pre> * String[] result = "this is a test".split("\\s"); * for (int x=0; x<result.length; x++) * System.out.println(result[x]); * </pre></blockquote> * <p> * prints the following output: * <blockquote><pre> * this * is * a * test * </pre></blockquote> * * @author unascribed * @see java.io.StreamTokenizer * @since 1.0 */ public class StringTokenizer implements Enumeration<Object> { private int currentPosition; private int newPosition; private int maxPosition; private String str; private String delimiters; private boolean retDelims; private boolean delimsChanged; /** * maxDelimCodePoint stores the value of the delimiter character with the * highest value. It is used to optimize the detection of delimiter * characters. * * It is unlikely to provide any optimization benefit in the * hasSurrogates case because most string characters will be * smaller than the limit, but we keep it so that the two code * paths remain similar. */ private int maxDelimCodePoint; /** * If delimiters include any surrogates (including surrogate * pairs), hasSurrogates is true and the tokenizer uses the * different code path. This is because String.indexOf(int) * doesn't handle unpaired surrogates as a single character. */ private boolean hasSurrogates = false; /** * When hasSurrogates is true, delimiters are converted to code * points and isDelimiter(int) is used to determine if the given * codepoint is a delimiter. */ private int[] delimiterCodePoints; /** * Set maxDelimCodePoint to the highest char in the delimiter set. */ private void setMaxDelimCodePoint() { if (delimiters == null) { maxDelimCodePoint = 0; return; } int m = 0; int c; int count = 0; for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) { c = delimiters.charAt(i); if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) { c = delimiters.codePointAt(i); hasSurrogates = true; } if (m < c) m = c; count++; } maxDelimCodePoint = m; if (hasSurrogates) { delimiterCodePoints = new int[count]; for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) { c = delimiters.codePointAt(j); delimiterCodePoints[i] = c; } } } /** * Constructs a string tokenizer for the specified string. All * characters in the {@code delim} argument are the delimiters * for separating tokens. * <p> * If the {@code returnDelims} flag is {@code true}, then * the delimiter characters are also returned as tokens. Each * delimiter is returned as a string of length one. If the flag is * {@code false}, the delimiter characters are skipped and only * serve as separators between tokens. * <p> * Note that if {@code delim} is {@code null}, this constructor does * not throw an exception. However, trying to invoke other methods on the * resulting {@code StringTokenizer} may result in a * {@code NullPointerException}. * * @param str a string to be parsed. * @param delim the delimiters. * @param returnDelims flag indicating whether to return the delimiters * as tokens. * @exception NullPointerException if str is {@code null} */ public StringTokenizer(String str, String delim, boolean returnDelims) { currentPosition = 0; newPosition = -1; delimsChanged = false; this.str = str; maxPosition = str.length(); delimiters = delim; retDelims = returnDelims; setMaxDelimCodePoint(); } /** * Constructs a string tokenizer for the specified string. The * characters in the {@code delim} argument are the delimiters * for separating tokens. Delimiter characters themselves will not * be treated as tokens. * <p> * Note that if {@code delim} is {@code null}, this constructor does * not throw an exception. However, trying to invoke other methods on the * resulting {@code StringTokenizer} may result in a * {@code NullPointerException}. * * @param str a string to be parsed. * @param delim the delimiters. * @exception NullPointerException if str is {@code null} */ public StringTokenizer(String str, String delim) { this(str, delim, false); } /** * Constructs a string tokenizer for the specified string. The * tokenizer uses the default delimiter set, which is * <code>" \t\n\r\f"</code>: the space character, * the tab character, the newline character, the carriage-return character, * and the form-feed character. Delimiter characters themselves will * not be treated as tokens. * * @param str a string to be parsed. * @exception NullPointerException if str is {@code null} */ public StringTokenizer(String str) { this(str, " \t\n\r\f", false); } /** * Skips delimiters starting from the specified position. If retDelims * is false, returns the index of the first non-delimiter character at or * after startPos. If retDelims is true, startPos is returned. */ private int skipDelimiters(int startPos) { if (delimiters == null) throw new NullPointerException(); int position = startPos; while (!retDelims && position < maxPosition) { if (!hasSurrogates) { char c = str.charAt(position); if ((c > maxDelimCodePoint) || (delimiters.indexOf(c) < 0)) break; position++; } else { int c = str.codePointAt(position); if ((c > maxDelimCodePoint) || !isDelimiter(c)) { break; } position += Character.charCount(c); } } return position; } /** * Skips ahead from startPos and returns the index of the next delimiter * character encountered, or maxPosition if no such delimiter is found. */ private int scanToken(int startPos) { int position = startPos; while (position < maxPosition) { if (!hasSurrogates) { char c = str.charAt(position); if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0)) break; position++; } else { int c = str.codePointAt(position); if ((c <= maxDelimCodePoint) && isDelimiter(c)) break; position += Character.charCount(c); } } if (retDelims && (startPos == position)) { if (!hasSurrogates) { char c = str.charAt(position); if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0)) position++; } else { int c = str.codePointAt(position); if ((c <= maxDelimCodePoint) && isDelimiter(c)) position += Character.charCount(c); } } return position; } private boolean isDelimiter(int codePoint) { for (int delimiterCodePoint : delimiterCodePoints) { if (delimiterCodePoint == codePoint) { return true; } } return false; } /** * Tests if there are more tokens available from this tokenizer's string. * If this method returns {@code true}, then a subsequent call to * {@code nextToken} with no argument will successfully return a token. * * @return {@code true} if and only if there is at least one token * in the string after the current position; {@code false} * otherwise. */ public boolean hasMoreTokens() { /* * Temporarily store this position and use it in the following * nextToken() method only if the delimiters haven't been changed in * that nextToken() invocation. */ newPosition = skipDelimiters(currentPosition); return (newPosition < maxPosition); } /** * Returns the next token from this string tokenizer. * * @return the next token from this string tokenizer. * @exception NoSuchElementException if there are no more tokens in this * tokenizer's string. */ public String nextToken() { /* * If next position already computed in hasMoreElements() and * delimiters have changed between the computation and this invocation, * then use the computed value. */ currentPosition = (newPosition >= 0 && !delimsChanged) ? newPosition : skipDelimiters(currentPosition); /* Reset these anyway */ delimsChanged = false; newPosition = -1; if (currentPosition >= maxPosition) throw new NoSuchElementException(); int start = currentPosition; currentPosition = scanToken(currentPosition); return str.substring(start, currentPosition); } /** * Returns the next token in this string tokenizer's string. First, * the set of characters considered to be delimiters by this * {@code StringTokenizer} object is changed to be the characters in * the string {@code delim}. Then the next token in the string * after the current position is returned. The current position is * advanced beyond the recognized token. The new delimiter set * remains the default after this call. * * @param delim the new delimiters. * @return the next token, after switching to the new delimiter set. * @exception NoSuchElementException if there are no more tokens in this * tokenizer's string. * @exception NullPointerException if delim is {@code null} */ public String nextToken(String delim) { delimiters = delim; /* delimiter string specified, so set the appropriate flag. */ delimsChanged = true; setMaxDelimCodePoint(); return nextToken(); } /** * Returns the same value as the {@code hasMoreTokens} * method. It exists so that this class can implement the * {@code Enumeration} interface. * * @return {@code true} if there are more tokens; * {@code false} otherwise. * @see java.util.Enumeration * @see java.util.StringTokenizer#hasMoreTokens() */ public boolean hasMoreElements() { return hasMoreTokens(); } /** * Returns the same value as the {@code nextToken} method, * except that its declared return value is {@code Object} rather than * {@code String}. It exists so that this class can implement the * {@code Enumeration} interface. * * @return the next token in the string. * @exception NoSuchElementException if there are no more tokens in this * tokenizer's string. * @see java.util.Enumeration * @see java.util.StringTokenizer#nextToken() */ public Object nextElement() { return nextToken(); } /** * Calculates the number of times that this tokenizer's * {@code nextToken} method can be called before it generates an * exception. The current position is not advanced. * * @return the number of tokens remaining in the string using the current * delimiter set. * @see java.util.StringTokenizer#nextToken() */ public int countTokens() { int count = 0; int currpos = currentPosition; while (currpos < maxPosition) { currpos = skipDelimiters(currpos); if (currpos >= maxPosition) break; currpos = scanToken(currpos); count++; } return count; } }