List of usage examples for java.io StreamTokenizer wordChars
public void wordChars(int low, int hi)
low <= c <= high are word constituents. From source file:org.structr.core.function.Functions.java
public static Object evaluate(final ActionContext actionContext, final GraphObject entity, final String expression) throws FrameworkException { final String expressionWithoutNewlines = expression.replace('\n', ' '); final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(expressionWithoutNewlines)); tokenizer.eolIsSignificant(true);/*from w ww. j a v a 2 s . co m*/ tokenizer.ordinaryChar('.'); tokenizer.wordChars('_', '_'); tokenizer.wordChars('.', '.'); tokenizer.wordChars('!', '!'); Expression root = new RootExpression(); Expression current = root; Expression next = null; String lastToken = null; int token = 0; int level = 0; while (token != StreamTokenizer.TT_EOF) { token = nextToken(tokenizer); switch (token) { case StreamTokenizer.TT_EOF: break; case StreamTokenizer.TT_EOL: break; case StreamTokenizer.TT_NUMBER: if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before NUMBER"); } next = new ConstantExpression(tokenizer.nval); current.add(next); lastToken += "NUMBER"; break; case StreamTokenizer.TT_WORD: if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before " + tokenizer.sval); } next = checkReservedWords(tokenizer.sval); Expression previousExpression = current.getPrevious(); if (tokenizer.sval.startsWith(".") && previousExpression != null && previousExpression instanceof FunctionExpression && next instanceof ValueExpression) { final FunctionExpression previousFunctionExpression = (FunctionExpression) previousExpression; final ValueExpression valueExpression = (ValueExpression) next; current.replacePrevious( new FunctionValueExpression(previousFunctionExpression, valueExpression)); } else { current.add(next); } lastToken = tokenizer.sval; break; case '(': if (((current == null || current instanceof RootExpression) && next == null) || current == next) { // an additional bracket without a new function, // this can only be an execution group. next = new GroupExpression(); current.add(next); } current = next; lastToken += "("; level++; break; case ')': if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before " + lastToken); } current = current.getParent(); if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken); } lastToken += ")"; level--; break; case '[': // bind directly to the previous expression next = new ArrayExpression(); current.add(next); current = next; lastToken += "["; level++; break; case ']': if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket before " + lastToken); } current = current.getParent(); if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken); } lastToken += "]"; level--; break; case ';': next = null; lastToken += ";"; break; case ',': next = current; lastToken += ","; break; default: if (current == null) { throw new FrameworkException(422, "Invalid expression: mismatched opening bracket before " + tokenizer.sval); } current.add(new ConstantExpression(tokenizer.sval)); lastToken = tokenizer.sval; } } if (level > 0) { throw new FrameworkException(422, "Invalid expression: mismatched closing bracket after " + lastToken); } return root.evaluate(actionContext, entity); }
From source file:org.xwiki.extension.internal.ExtensionUtils.java
/** * @param str the String to parse/* www. j a va 2 s. c o m*/ * @param trim true if the passed String should be trimmed * @return the collection of Strings extracted from the passed String * @since 8.3M1 */ public static List<String> importPropertyStringList(String str, boolean trim) { try { String cleanedString = str; // Trim if (trim) { cleanedString = cleanedString.trim(); } // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(cleanedString)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters st.whitespaceChars(',', ','); st.whitespaceChars(' ', ' '); st.whitespaceChars('\t', '\t'); st.whitespaceChars('\n', '\n'); st.whitespaceChars('\r', '\r'); // Split comma-delimited tokens into a List List<String> collection = new ArrayList<>(); while (true) { int ttype = st.nextToken(); if (ttype == StreamTokenizer.TT_WORD || ttype > 0) { if (st.sval != null) { collection.add(st.sval); } } else if (ttype == StreamTokenizer.TT_EOF) { break; } else { throw new ConversionException("Encountered token of type " + ttype + " parsing elements."); } } // Return the completed list return collection; } catch (Exception e) { // Log ? } return Collections.emptyList(); }
From source file:org.xwiki.properties.converter.AbstractCollectionConverter.java
/** * Create and initializer a {@link StreamTokenizer} to parse the value. * /*from w w w . j a va2 s . c o m*/ * @param value the string to parse * @return the {@link StreamTokenizer} used to parse the string */ protected StreamTokenizer createStreamTokenizer(String value) { // Set up a StreamTokenizer on the characters in this String StreamTokenizer st = new StreamTokenizer(new StringReader(value)); // Everything is word st.ordinaryChars(0, 255); st.wordChars(0, 255); // Except quote chars st.quoteChar('"'); st.quoteChar('\''); // And delimiters for (char c : getDelimiters().toCharArray()) { st.whitespaceChars(c, c); } return st; }
From source file:org.yestech.jmlnitrate.transformation.inbound.BaseInboundTransformation.java
/** * Parses a Parameter Request String and return a array of the tokens in the * string. The default delimeters are ^ and ; * * @param rawRequest the Parameter Request * @return Array of the tokens//from w ww .j av a 2 s.com * @throws Exception if an error happens */ private Object[] parseParameterRequest(String rawRequest) throws Exception { StreamTokenizer tokenStream = new StreamTokenizer( new BufferedReader(new StringReader(rawRequest), BUFFER_SIZE)); //reset the stream tokenStream.resetSyntax(); //configure tokens tokenStream.lowerCaseMode(false); tokenStream.eolIsSignificant(false); //add word chars tokenStream.wordChars(32, 58); tokenStream.wordChars(60, 93); tokenStream.wordChars(95, 126); //add <CR>\r <LF>\n tokenStream.wordChars(10, 10); tokenStream.wordChars(13, 13); //set ^ AND ; as delimeters to string tokens tokenStream.quoteChar(94); //removed ; //tokenStream.quoteChar(59); //set up the temp arraylist ArrayList tokens = new ArrayList(); int result = tokenStream.ttype; while (result != StreamTokenizer.TT_EOF) { result = tokenStream.nextToken(); switch (result) { case StreamTokenizer.TT_EOF: break; default: tokens.add(tokenStream.sval); break; } } return tokens.toArray(); }
From source file:uk.ac.leeds.ccg.andyt.projects.fluvialglacial.SlopeAreaAnalysis.java
protected TreeMap<Integer, Object[]> readSwissData(File fileIn) { TreeMap<Integer, Object[]> result; result = new TreeMap<Integer, Object[]>(); BufferedReader br;/*w w w . j a v a2s .co m*/ br = Generic_StaticIO.getBufferedReader(fileIn); StreamTokenizer st; st = new StreamTokenizer(br); Generic_StaticIO.setStreamTokenizerSyntax5(st); st.wordChars('(', '('); st.wordChars(')', ')'); st.wordChars('%', '%'); Generic_StaticIO.skipline(st); int token; String line = ""; String[] fields; try { token = st.nextToken(); int ID; //int pointID; while (token != StreamTokenizer.TT_EOF) { switch (token) { case StreamTokenizer.TT_EOL: //flowacc,area (km2),slope_25_(%),proglac_ID,COUNT //12.11111069,0.00756944,32.33880000000,0,250631 fields = line.split(sComma); ID = Integer.valueOf(fields[3]); if (ID > 0) { //BigDecimal flowacc; BigDecimal area; BigDecimal slope; Object[] data; BigDecimal maxx; BigDecimal maxy; BigDecimal minx; BigDecimal miny; data = result.get(ID); ArrayList<Generic_XYNumericalData> theGeneric_XYNumericalData; if (data == null) { data = new Object[5]; theGeneric_XYNumericalData = new ArrayList<Generic_XYNumericalData>(); maxx = BigDecimal.ZERO; maxy = BigDecimal.ZERO; minx = BigDecimal.valueOf(Double.MAX_VALUE); miny = BigDecimal.valueOf(Double.MAX_VALUE); data[0] = theGeneric_XYNumericalData; data[1] = maxx; data[2] = minx; data[3] = maxy; data[4] = miny; result.put(ID, data); } else { theGeneric_XYNumericalData = (ArrayList<Generic_XYNumericalData>) data[0]; maxx = (BigDecimal) data[1]; minx = (BigDecimal) data[2]; maxy = (BigDecimal) data[3]; miny = (BigDecimal) data[4]; } //pointID = Integer.valueOf(fields[4]); //flowacc = new BigDecimal(fields[0]); area = new BigDecimal(fields[1]); if (area.compareTo(BigDecimal.ZERO) == 1) { area = Generic_BigDecimal.log(10, area, 10, RoundingMode.HALF_UP); } else { area = BigDecimal.ZERO; } slope = new BigDecimal(fields[2]); if (slope.compareTo(BigDecimal.ZERO) == 1) { slope = Generic_BigDecimal.log(10, slope, 10, RoundingMode.HALF_UP); } else { slope = BigDecimal.ZERO; } Generic_XYNumericalData point; point = new Generic_XYNumericalData(slope, area); theGeneric_XYNumericalData.add(point); data[0] = theGeneric_XYNumericalData; data[1] = maxx.max(slope); data[2] = minx.min(slope); data[3] = maxy.max(area); data[4] = miny.min(area); } break; case StreamTokenizer.TT_WORD: line = st.sval; break; } token = st.nextToken(); } } catch (IOException ex) { Logger.getLogger(SlopeAreaAnalysis.class.getName()).log(Level.SEVERE, null, ex); } return result; }
From source file:uk.ac.leeds.ccg.andyt.projects.fluvialglacial.SlopeAreaAnalysis.java
protected TreeMap<Integer, Object[]> readAustriaData(File fileIn) { TreeMap<Integer, Object[]> result; result = new TreeMap<Integer, Object[]>(); BufferedReader br;/*from w w w . java2 s. com*/ br = Generic_StaticIO.getBufferedReader(fileIn); StreamTokenizer st; st = new StreamTokenizer(br); Generic_StaticIO.setStreamTokenizerSyntax5(st); st.wordChars('(', '('); st.wordChars(')', ')'); st.wordChars('%', '%'); Generic_StaticIO.skipline(st); int token; String line = ""; String[] fields; try { token = st.nextToken(); int ID; //int pointID; while (token != StreamTokenizer.TT_EOF) { switch (token) { case StreamTokenizer.TT_EOL: //flowacc,area (km2),slope_25_(%),proglac_ID,COUNT //12.11111069,0.00756944,32.33880000000,0,250631 fields = line.split(sComma); ID = Double.valueOf(fields[1]).intValue(); if (ID > 0) { //BigDecimal flowacc; BigDecimal area; BigDecimal slope; Object[] data; BigDecimal maxx; BigDecimal maxy; BigDecimal minx; BigDecimal miny; data = result.get(ID); ArrayList<Generic_XYNumericalData> theGeneric_XYNumericalData; if (data == null) { data = new Object[5]; theGeneric_XYNumericalData = new ArrayList<Generic_XYNumericalData>(); maxx = BigDecimal.ZERO; maxy = BigDecimal.ZERO; minx = BigDecimal.valueOf(Double.MAX_VALUE); miny = BigDecimal.valueOf(Double.MAX_VALUE); data[0] = theGeneric_XYNumericalData; data[1] = maxx; data[2] = minx; data[3] = maxy; data[4] = miny; result.put(ID, data); } else { theGeneric_XYNumericalData = (ArrayList<Generic_XYNumericalData>) data[0]; maxx = (BigDecimal) data[1]; minx = (BigDecimal) data[2]; maxy = (BigDecimal) data[3]; miny = (BigDecimal) data[4]; } //pointID = Integer.valueOf(fields[4]); //flowacc = new BigDecimal(fields[0]); area = new BigDecimal(fields[3]); if (area.compareTo(BigDecimal.ZERO) == 1) { area = Generic_BigDecimal.log(10, area, 10, RoundingMode.HALF_UP); } else { area = BigDecimal.ZERO; } slope = new BigDecimal(fields[2]); if (slope.compareTo(BigDecimal.ZERO) == 1) { slope = Generic_BigDecimal.log(10, slope, 10, RoundingMode.HALF_UP); } else { slope = BigDecimal.ZERO; } Generic_XYNumericalData point; point = new Generic_XYNumericalData(slope, area); theGeneric_XYNumericalData.add(point); data[0] = theGeneric_XYNumericalData; data[1] = maxx.max(slope); data[2] = minx.min(slope); data[3] = maxy.max(area); data[4] = miny.min(area); } break; case StreamTokenizer.TT_WORD: line = st.sval; break; } token = st.nextToken(); } } catch (IOException ex) { Logger.getLogger(SlopeAreaAnalysis.class.getName()).log(Level.SEVERE, null, ex); } return result; }