List of usage examples for java.io StreamTokenizer ordinaryChars
public void ordinaryChars(int low, int hi)
low <= c <= high are "ordinary" in this tokenizer. From source file:Main.java
public static void main(String[] argv) throws Exception { FileReader rd = new FileReader("filename.java"); StreamTokenizer st = new StreamTokenizer(rd); st.parseNumbers();/*www . j ava 2s .c om*/ st.wordChars('_', '_'); st.eolIsSignificant(true); st.ordinaryChars(0, ' '); st.slashSlashComments(true); st.slashStarComments(true); int token = st.nextToken(); while (token != StreamTokenizer.TT_EOF) { token = st.nextToken(); switch (token) { case StreamTokenizer.TT_NUMBER: double num = st.nval; System.out.println(num); break; case StreamTokenizer.TT_WORD: String word = st.sval; System.out.println(word); break; case '"': String dquoteVal = st.sval; System.out.println(dquoteVal); break; case '\'': String squoteVal = st.sval; System.out.println(squoteVal); break; case StreamTokenizer.TT_EOL: break; case StreamTokenizer.TT_EOF: break; default: char ch = (char) st.ttype; System.out.println(ch); break; } } rd.close(); }
From source file:com.enonic.cms.business.portal.datasource.expressionfunctions.ExpressionFunctions.java
/** * This method will take a freetext search string and create a valid query that can be used in the getContent* methods. The search * string is spilt into tokens. Using the operator, it may be specified whether the field must contain all or any of the words in the * search string./*from w w w . ja va 2 s. c o m*/ * * @param fieldName The name of the field to search for the words in the search string. * @param searchString The words to search for. * @param operator Must be either AND or OR. Case doesn't matter. * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care, * it may also be merged with other queries using AND or OR. * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR. */ public String buildFreetextQuery(String fieldName, String searchString, String operator) { if (searchString == null || searchString.trim().equals("")) { return ""; } if (fieldName == null || fieldName.trim().equals("")) { throw new IllegalArgumentException("fieldName can not be empty."); } String op = ""; if (operator != null) { op = operator.trim().toUpperCase(); } if (!(op.equals("AND") || op.equals("OR"))) { throw new IllegalArgumentException("Illegal operator: " + operator); } boolean first = true; StringBuffer queryTokens = new StringBuffer(); Reader searchStringReader = new StringReader(searchString); StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader); searchStringTokens.slashSlashComments(false); searchStringTokens.slashStarComments(false); searchStringTokens.eolIsSignificant(false); searchStringTokens.ordinaryChar('!'); searchStringTokens.ordinaryChars('#', '}'); searchStringTokens.wordChars('!', '!'); searchStringTokens.wordChars('#', '}'); try { while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) { String token = searchStringTokens.sval; addQueryPart(queryTokens, first, fieldName, token, op); if (first) { first = false; } } } catch (IOException e) { throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!"); } return queryTokens.toString(); }
From source file:com.enonic.cms.core.portal.datasource.el.ExpressionFunctions.java
/** * This method will take a freetext search string and create a valid query that can be used in the getContent* methods. The search * string is spilt into tokens. Using the operator, it may be specified whether the field must contain all or any of the words in the * search string.// w w w . ja v a 2 s .co m * * @param fieldName The name of the field to search for the words in the search string. * @param searchString The words to search for. * @param operator Must be either AND or OR. Case doesn't matter. * @return A syntactically correct search that may be used as the query parameter in getContent* methods on the data source. With care, * it may also be merged with other queries using AND or OR. * @throws IllegalArgumentException If any of the parameters are empty or the operator is not AND or OR. */ public String buildFreetextQuery(String fieldName, String searchString, String operator) { if (searchString == null || searchString.trim().equals("")) { return ""; } if (fieldName == null || fieldName.trim().equals("")) { throw new IllegalArgumentException("fieldName can not be empty."); } String op = ""; if (operator != null) { op = operator.trim().toUpperCase(); } if (!(op.equals("AND") || op.equals("OR"))) { throw new IllegalArgumentException("Illegal operator: " + operator); } boolean first = true; StringBuilder queryTokens = new StringBuilder(); Reader searchStringReader = new StringReader(searchString); StreamTokenizer searchStringTokens = new StreamTokenizer(searchStringReader); searchStringTokens.slashSlashComments(false); searchStringTokens.slashStarComments(false); searchStringTokens.eolIsSignificant(false); searchStringTokens.ordinaryChar('!'); searchStringTokens.ordinaryChars('#', '}'); searchStringTokens.wordChars('!', '!'); searchStringTokens.wordChars('#', '}'); try { while (searchStringTokens.nextToken() != StreamTokenizer.TT_EOF) { String token = searchStringTokens.sval; addQueryPart(queryTokens, first, fieldName, token, op); if (first) { first = false; } } } catch (IOException e) { throw new IllegalStateException("This should never happen, since the IO class is wrapping a string!"); } return queryTokens.toString(); }
From source file:org.jdesigner.platform.web.converter.AbstractArrayConverter.java
/** * <p>//from www . ja va 2 s . com * Parse an incoming String of the form similar to an array initializer in * the Java language into a <code>List</code> individual Strings for each * element, according to the following rules. * </p> * <ul> * <li>The string is expected to be a comma-separated list of values.</li> * <li>The string may optionally have matching '{' and '}' delimiters around * the list.</li> * <li>Whitespace before and after each element is stripped.</li> * <li>Elements in the list may be delimited by single or double quotes. * Within a quoted elements, the normal Java escape sequences are valid.</li> * </ul> * * @param svalue * String value to be parsed * @return The parsed list of String values * * @exception ConversionException * if the syntax of <code>svalue</code> is not syntactically * valid * @exception NullPointerException * if <code>svalue</code> is <code>null</code> */ protected List parseElements(String svalue) { // Validate the passed argument if (svalue == null) { throw new NullPointerException(); } // Trim any matching '{' and '}' delimiters svalue = svalue.trim(); if (svalue.startsWith("{") && svalue.endsWith("}")) { svalue = svalue.substring(1, svalue.length() - 1); } try { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(svalue)); st.whitespaceChars(',', ','); // Commas are delimiters st.ordinaryChars('0', '9'); // Needed to turn off numeric flag st.ordinaryChars('.', '.'); st.ordinaryChars('-', '-'); st.wordChars('0', '9'); // Needed to make part of tokens st.wordChars('.', '.'); st.wordChars('-', '-'); // Split comma-delimited tokens into a List ArrayList list = new ArrayList(); while (true) { int ttype = st.nextToken(); if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) { list.add(st.sval); } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype); } } // Return the completed list return (list); } catch (IOException e) { throw new ConversionException(e); } }
From source file:org.jdesigner.platform.web.converter.ArrayConverter.java
/** * <p>/*w w w . j av a 2 s .c om*/ * Parse an incoming String of the form similar to an array initializer in * the Java language into a <code>List</code> individual Strings for each * element, according to the following rules. * </p> * <ul> * <li>The string is expected to be a comma-separated list of values.</li> * <li>The string may optionally have matching '{' and '}' delimiters around * the list.</li> * <li>Whitespace before and after each element is stripped.</li> * <li>Elements in the list may be delimited by single or double quotes. * Within a quoted elements, the normal Java escape sequences are valid.</li> * </ul> * * @param type * The type to convert the value to * @param value * String value to be parsed * @return List of parsed elements. * * @throws ConversionException * if the syntax of <code>svalue</code> is not syntactically * valid * @throws NullPointerException * if <code>svalue</code> is <code>null</code> */ private List parseElements(Class type, String value) { if (log().isDebugEnabled()) { log().debug("Parsing elements, delimiter=[" + delimiter + "], value=[" + value + "]"); } // Trim any matching '{' and '}' delimiters value = value.trim(); if (value.startsWith("{") && value.endsWith("}")) { value = value.substring(1, value.length() - 1); } try { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); st.whitespaceChars(delimiter, delimiter); // Set the delimiters st.ordinaryChars('0', '9'); // Needed to turn off numeric flag st.wordChars('0', '9'); // Needed to make part of tokens for (int i = 0; i < allowedChars.length; i++) { st.ordinaryChars(allowedChars[i], allowedChars[i]); st.wordChars(allowedChars[i], allowedChars[i]); } // Split comma-delimited tokens into a List List list = null; while (true) { int ttype = st.nextToken(); if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) { if (st.sval != null) { if (list == null) { list = new ArrayList(); } list.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException( "Encountered token of type " + ttype + " parsing elements to '" + toString(type) + "."); } } if (list == null) { list = Collections.EMPTY_LIST; } if (log().isDebugEnabled()) { log().debug(list.size() + " elements parsed"); } // Return the completed list return (list); } catch (IOException e) { throw new ConversionException( "Error converting from String to '" + toString(type) + "': " + e.getMessage(), e); } }
From source file:org.pentaho.big.data.kettle.plugins.sqoop.SqoopUtils.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString//from ww w .j a v a 2 s . co m * A command line string, e.g. "sqoop import --table test --connect jdbc:mysql://bogus/bogus" * @param variableSpace * Context for resolving variable names. If {@code null}, no variable resolution we happen. * @param ignoreSqoopCommand * If set, the first "sqoop <tool>" arguments will be ignored, e.g. "sqoop import" or "sqoop export". * @return List of parsed arguments * @throws IOException * when the command line could not be parsed */ public static List<String> parseCommandLine(String commandLineString, VariableSpace variableSpace, boolean ignoreSqoopCommand) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Flag to indicate if the next token needs to be skipped (used to control skipping of the first two arguments, // e.g. "sqoop <tool>") boolean skipToken = false; // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; if (variableSpace != null) { s = variableSpace.environmentSubstitute(s); } if (ignoreSqoopCommand && args.isEmpty()) { // If we encounter "sqoop <name>" we should skip the first two arguments so we can support copy/paste of // arguments directly // from a working command line if ("sqoop".equals(s)) { skipToken = true; continue; // skip this one and the next } else if (skipToken) { ignoreSqoopCommand = false; // Don't attempt to ignore any more commands // Skip this token too, reset the flag so we no longer skip any tokens, and continue parsing skipToken = false; continue; } } if (s.startsWith(ARG_D)) { handleCustomOption(args, s, tokenizer, variableSpace); continue; } args.add(escapeEscapeSequences(s)); } } } finally { reader.close(); } return args; }
From source file:org.pentaho.di.job.entries.spark.JobEntrySparkSubmit.java
/** * Parse a string into arguments as if it were provided on the command line. * * @param commandLineString A command line string. * @return List of parsed arguments//w ww. j a v a 2 s .c om * @throws IOException when the command line could not be parsed */ public List<String> parseCommandLine(String commandLineString) throws IOException { List<String> args = new ArrayList<String>(); StringReader reader = new StringReader(commandLineString); try { StreamTokenizer tokenizer = new StreamTokenizer(reader); // Treat a dash as an ordinary character so it gets included in the token tokenizer.ordinaryChar('-'); tokenizer.ordinaryChar('.'); tokenizer.ordinaryChars('0', '9'); // Treat all characters as word characters so nothing is parsed out tokenizer.wordChars('\u0000', '\uFFFF'); // Re-add whitespace characters tokenizer.whitespaceChars(0, ' '); // Use " and ' as quote characters tokenizer.quoteChar('"'); tokenizer.quoteChar('\''); // Add all non-null string values tokenized from the string to the argument list while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) { if (tokenizer.sval != null) { String s = tokenizer.sval; s = environmentSubstitute(s); args.add(s); } } } finally { reader.close(); } return args; }
From source file:org.xwiki.extension.internal.ExtensionUtils.java
/** * @param str the String to parse// ww w .j av a 2s . c o m * @param trim true if the passed String should be trimmed * @return the collection of Strings extracted from the passed String * @since 8.3M1 */ public static List<String> importPropertyStringList(String str, boolean trim) { try { String cleanedString = str; // Trim if (trim) { cleanedString = cleanedString.trim(); } // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters st.whitespaceChars(',', ','); st.whitespaceChars(' ', ' '); st.whitespaceChars('\t', '\t'); st.whitespaceChars('\n', '\n'); st.whitespaceChars('\r', '\r'); // Split comma-delimited tokens into a List List<String> collection = new ArrayList<>(); while (true) { int ttype = st.nextToken(); if (ttype == StreamTokenizer.TT_WORD || ttype > 0) { if (st.sval != null) { collection.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype + " parsing elements."); } } // Return the completed list return collection; } catch (Exception e) { // Log ? } return Collections.emptyList(); }
From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java
/** * Create and initializer a {@link StreamTokenizer} to parse the value. * //from www .j av a 2 s .c o m * @param value the string to parse * @return the {@link StreamTokenizer} used to parse the string */ protected StreamTokenizer createStreamTokenizer(String value) { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters for (char c : getDelimiters().toCharArray()) { st.whitespaceChars(c, c); } return st; }