List of usage examples for java.io StreamTokenizer TT_EOL
int TT_EOL
To view the source code for java.io StreamTokenizer TT_EOL.
Click Source Link
From source file:ScanStreamTok.java
protected void process() throws IOException { String s = null;//from w ww . j a va 2 s .c o m int i; while ((i = tf.nextToken()) != StreamTokenizer.TT_EOF) { switch (i) { case StreamTokenizer.TT_EOF: System.out.println("End of file"); break; case StreamTokenizer.TT_EOL: System.out.println("End of line"); break; case StreamTokenizer.TT_NUMBER: System.out.println("Number " + tf.nval); break; case StreamTokenizer.TT_WORD: System.out.println("Word, length " + tf.sval.length() + "->" + tf.sval); break; default: System.out.println("What is it? i = " + i); } } }
From source file:net.duckling.ddl.service.render.dml.ParseHtmlImg.java
public Map parseArgs(String argstring) throws IOException { HashMap<String, String> arglist = new HashMap<String, String>(); ///*from w ww .j ava 2 s. c o m*/ // Protection against funny users. // if (argstring == null) { return arglist; } StringReader in = new StringReader(argstring); StreamTokenizer tok = new StreamTokenizer(in); int type; String param = null; String value = null; tok.eolIsSignificant(true); boolean potentialEmptyLine = false; boolean quit = false; while (!quit) { String s; type = tok.nextToken(); switch (type) { case StreamTokenizer.TT_EOF: quit = true; s = null; break; case StreamTokenizer.TT_WORD: s = tok.sval; potentialEmptyLine = false; break; case StreamTokenizer.TT_EOL: quit = potentialEmptyLine; potentialEmptyLine = true; s = null; break; case StreamTokenizer.TT_NUMBER: s = Integer.toString(new Double(tok.nval).intValue()); potentialEmptyLine = false; break; case '\'': s = tok.sval; break; default: s = null; } // // Assume that alternate words on the line are // parameter and value, respectively. // if (s != null) { if (param == null) { param = s; } else { value = s; arglist.put(param, value); param = null; } } } // // Now, we'll check the body. // if (potentialEmptyLine) { StringWriter out = new StringWriter(); FileUtil.copyContents(in, out); String bodyContent = out.toString(); if (bodyContent != null) { arglist.put(PARAM_BODY, bodyContent); } } return arglist; }
From source file:keel.Algorithms.Decision_Trees.C45.C45.java
/** Function to read the options from the execution file and assign the values to the parameters. * * @param options The StreamTokenizer that reads the parameters file. * * @throws Exception If the format of the file is not correct. *///from www. j a va 2 s . co m protected void setOptions(StreamTokenizer options) throws Exception { options.nextToken(); /* Checks that the file starts with the token algorithm */ if (options.sval.equalsIgnoreCase("algorithm")) { options.nextToken(); options.nextToken(); //if (!options.sval.equalsIgnoreCase( "C4.5" ) ) // throw new Exception( "The name of the algorithm is not correct." ); options.nextToken(); options.nextToken(); options.nextToken(); options.nextToken(); /* Reads the names of the input files*/ if (options.sval.equalsIgnoreCase("inputData")) { options.nextToken(); options.nextToken(); modelFileName = options.sval; if (options.nextToken() != StreamTokenizer.TT_EOL) { trainFileName = options.sval; options.nextToken(); testFileName = options.sval; if (options.nextToken() != StreamTokenizer.TT_EOL) { trainFileName = modelFileName; options.nextToken(); } } } else { throw new Exception("No file test provided."); } /* Reads the names of the output files*/ while (true) { if (options.nextToken() == StreamTokenizer.TT_EOF) { throw new Exception("No output file provided."); } if (options.sval == null) { continue; } else if (options.sval.equalsIgnoreCase("outputData")) { break; } } options.nextToken(); options.nextToken(); trainOutputFileName = options.sval; options.nextToken(); testOutputFileName = options.sval; options.nextToken(); resultFileName = options.sval; if (!getNextToken(options)) { return; } while (options.ttype != StreamTokenizer.TT_EOF) { /* Reads the prune parameter */ if (options.sval.equalsIgnoreCase("pruned")) { options.nextToken(); options.nextToken(); if (options.sval.equalsIgnoreCase("TRUE")) { prune = true; } else { //prune = false; prune = true; } } /* Reads the confidence parameter */ if (options.sval.equalsIgnoreCase("confidence")) { if (!prune) { throw new Exception("Doesn't make sense to change confidence for prune " + "tree!"); } options.nextToken(); options.nextToken(); /* Checks that the confidence threshold is between 0 and 1. */ float cf = Float.parseFloat(options.sval); if (cf <= 1 || cf >= 0) { confidence = Float.parseFloat(options.sval); } } /* Reads the itemsets per leaf parameter */ if (options.sval.equalsIgnoreCase("itemsetsPerLeaf")) { options.nextToken(); options.nextToken(); if (Integer.parseInt(options.sval) > 0) { minItemsets = Integer.parseInt(options.sval); } } getNextToken(options); } } }
From source file:keel.Algorithms.ImbalancedClassification.CSMethods.C45CS.C45CS.java
/** Function to read the options from the execution file and assign the values to the parameters. * * @param options The StreamTokenizer that reads the parameters file. * * @throws Exception If the format of the file is not correct. *///from www.j a v a 2 s . com protected void setOptions(StreamTokenizer options) throws Exception { options.nextToken(); /* Checks that the file starts with the token algorithm */ if (options.sval.equalsIgnoreCase("algorithm")) { options.nextToken(); options.nextToken(); //if (!options.sval.equalsIgnoreCase( "C4.5" ) ) // throw new Exception( "The name of the algorithm is not correct." ); options.nextToken(); options.nextToken(); options.nextToken(); options.nextToken(); /* Reads the names of the input files*/ if (options.sval.equalsIgnoreCase("inputData")) { options.nextToken(); options.nextToken(); modelFileName = options.sval; if (options.nextToken() != StreamTokenizer.TT_EOL) { trainFileName = options.sval; options.nextToken(); testFileName = options.sval; if (options.nextToken() != StreamTokenizer.TT_EOL) { trainFileName = modelFileName; options.nextToken(); } } } else { throw new Exception("No file test provided."); } /* Reads the names of the output files*/ while (true) { if (options.nextToken() == StreamTokenizer.TT_EOF) { throw new Exception("No output file provided."); } if (options.sval == null) { continue; } else if (options.sval.equalsIgnoreCase("outputData")) { break; } } options.nextToken(); options.nextToken(); trainOutputFileName = options.sval; options.nextToken(); testOutputFileName = options.sval; options.nextToken(); resultFileName = options.sval; if (!getNextToken(options)) { return; } while (options.ttype != StreamTokenizer.TT_EOF) { /* Reads the prune parameter */ if (options.sval.equalsIgnoreCase("pruned")) { options.nextToken(); options.nextToken(); if (options.sval.equalsIgnoreCase("TRUE")) { prune = true; } else { //prune = false; prune = true; } } /* Reads the confidence parameter */ if (options.sval.equalsIgnoreCase("confidence")) { if (!prune) { throw new Exception("Doesn't make sense to change confidence for prune " + "tree!"); } options.nextToken(); options.nextToken(); /* Checks that the confidence threshold is between 0 and 1. */ float cf = Float.parseFloat(options.sval); if (cf <= 1 || cf >= 0) { confidence = Float.parseFloat(options.sval); } } /* Reads the itemsets per leaf parameter */ if (options.sval.equalsIgnoreCase("itemsetsPerLeaf")) { options.nextToken(); options.nextToken(); if (Integer.parseInt(options.sval) > 0) { minItemsets = Integer.parseInt(options.sval); } } /* Reads the minimum expected cost parameter */ if (options.sval.equalsIgnoreCase("minimumExpectedCost")) { options.nextToken(); options.nextToken(); if (options.sval.equalsIgnoreCase("TRUE")) { minimumExpectedCost = true; } else { minimumExpectedCost = false; } } getNextToken(options); } } }
From source file:cross.io.xml.FragmentXMLSerializer.java
/** * @param name//from w w w . jav a2 s .com * @param dims * @param data * @return */ private Array handleData(final String name, final Dimension[] dims, final Element data, final Range[] ranges) { EvalTools.notNull(dims, this); final String dec = new String(Base64.decode(data.getText(), Base64.GZIP)); final StreamTokenizer st = new StreamTokenizer(new StringReader(dec)); final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US); final int[] shape = new int[dims.length]; int d = 0; for (final Dimension dim : dims) { shape[d++] = dim.getLength(); } Array a = null; IndexIterator idx = null; int tok = -1; // log.info("DataType of array: {}", // dt.getPrimitiveClassType().getName()); Mode m = Mode.UNDEF; Object o = null; try { while ((tok = st.nextToken()) != StreamTokenizer.TT_EOL) { if (tok == StreamTokenizer.TT_WORD) { if (m == Mode.UNDEF) { try { o = nf.parse(st.sval); if (o instanceof Double) { m = Mode.DOUBLE; a = Array.factory(DataType.DOUBLE, shape); } else if (o instanceof Float) { m = Mode.FLOAT; a = Array.factory(DataType.FLOAT, shape); } else if (o instanceof Long) { m = Mode.LONG; a = Array.factory(DataType.LONG, shape); } else if (o instanceof Integer) { m = Mode.INTEGER; a = Array.factory(DataType.INT, shape); } else if (o instanceof Byte) { m = Mode.BYTE; a = Array.factory(DataType.BYTE, shape); } else if (o instanceof Short) { m = Mode.SHORT; a = Array.factory(DataType.SHORT, shape); } } catch (final ParseException pe) { if (st.sval.equalsIgnoreCase("true") || st.sval.equalsIgnoreCase("false")) { m = Mode.BOOLEAN; a = Array.factory(DataType.BOOLEAN, shape); } else { m = Mode.STRING; a = Array.factory(DataType.STRING, shape); } } } else { if (idx == null) { idx = a.getIndexIterator(); } switch (m) { case DOUBLE: { idx.setDoubleNext((Double) o); break; } case FLOAT: { idx.setFloatNext((Float) o); break; } case INTEGER: { idx.setIntNext((Integer) o); break; } case LONG: { idx.setLongNext((Long) o); break; } case BYTE: { idx.setByteNext((Byte) o); break; } case SHORT: { idx.setShortNext((Short) o); break; } case BOOLEAN: { idx.setBooleanNext(Boolean.parseBoolean(st.sval)); break; } case STRING: { idx.setObjectNext(st.sval); break; } case OBJECT: { throw new IllegalArgumentException("Could not handle type"); } } } } } } catch (final IOException e) { log.warn("Could not read data for {}", name); } if (a != null && ranges != null && ranges.length != 0) { try { return a.section(Arrays.asList(ranges)); } catch (InvalidRangeException ex) { log.warn("Invalid range while trying to subset array: ", ex); } } return a; }
From source file:keel.Algorithms.Genetic_Rule_Learning.PART.C45.java
/** Function to read the options from the execution file and assign the values to the parameters. * * @param options The StreamTokenizer that reads the parameters file. * * @throws Exception If the format of the file is not correct. *///from w w w . j a v a2 s .c o m protected void setOptions(StreamTokenizer options) throws Exception { options.nextToken(); /* Checks that the file starts with the token algorithm */ if (options.sval.equalsIgnoreCase("algorithm")) { options.nextToken(); options.nextToken(); //if (!options.sval.equalsIgnoreCase( "C4.5" ) ) // throw new Exception( "The name of the algorithm is not correct." ); options.nextToken(); System.out.println(options.sval + "\n"); options.nextToken(); System.out.println(options.sval + "\n"); //options.nextToken(); //System.out.println(options.sval+"\n"); //options.nextToken(); //System.out.println(options.sval+"\n"); /* Reads the names of the input files*/ if (options.sval.equalsIgnoreCase("inputData")) { options.nextToken(); options.nextToken(); modelFileName = options.sval; System.out.println("Hay inputs\n"); if (options.nextToken() != StreamTokenizer.TT_EOL) { trainFileName = options.sval; options.nextToken(); testFileName = options.sval; if (options.nextToken() != StreamTokenizer.TT_EOL) { trainFileName = modelFileName; options.nextToken(); } System.out.println(trainFileName + "\n"); System.out.println(testFileName + "\n"); } } else throw new Exception("No file test provided."); /* Reads the names of the output files*/ while (true) { if (options.nextToken() == StreamTokenizer.TT_EOF) throw new Exception("No output file provided."); if (options.sval == null) continue; else if (options.sval.equalsIgnoreCase("outputData")) break; } options.nextToken(); options.nextToken(); trainOutputFileName = options.sval; options.nextToken(); testOutputFileName = options.sval; options.nextToken(); resultFileName = options.sval; System.out.println(trainOutputFileName + "\n"); System.out.println(testOutputFileName + "\n"); System.out.println(resultFileName + "\n"); if (!getNextToken(options)) return; while (options.ttype != StreamTokenizer.TT_EOF) { /* Reads the prune parameter */ if (options.sval.equalsIgnoreCase("pruned")) { options.nextToken(); options.nextToken(); if (options.sval.equalsIgnoreCase("TRUE")) prune = true; else prune = false; //prune = true; } /* Reads the confidence parameter */ if (options.sval.equalsIgnoreCase("confidence")) { if (!prune) throw new Exception("Doesn't make sense to change confidence for prune tree!"); options.nextToken(); options.nextToken(); /* Checks that the confidence threshold is between 0 and 1. */ float cf = Float.parseFloat(options.sval); if (cf <= 1 || cf >= 0) confidence = Float.parseFloat(options.sval); } /* Reads the itemsets per leaf parameter */ if (options.sval.equalsIgnoreCase("itemsetsPerLeaf")) { options.nextToken(); options.nextToken(); if (Integer.parseInt(options.sval) > 0) minItemsets = Integer.parseInt(options.sval); } getNextToken(options); } } }
From source file:com.fluffypeople.managesieve.ManageSieveClient.java
/** * "This command lists the scripts the user has on the server". The results * are stored into the @code{List<SieveScript>} passed in. Any existing * contents of this list will be lost. Up to one of the scripts listed will * be marked active.//from www .j av a2s.c o m * * @param scripts @code{List<SieveScript>} non-null List of scripts. Will be * cleared if not zero length, even if there is a problem * @return ManageSieveResponse OK - list was fetched, NO - there was a * problem. * @throws IOException * @throws ParseException */ public synchronized ManageSieveResponse listscripts(List<SieveScript> scripts) throws IOException, ParseException { if (!scripts.isEmpty()) { scripts.clear(); } sendCommand("LISTSCRIPTS"); while (true) { int token = in.nextToken(); switch (token) { case DQUOTE: case LEFT_CURRLY_BRACE: in.pushBack(); String scriptName = parseString(); boolean isActive = false; token = in.nextToken(); if (token == StreamTokenizer.TT_WORD) { if (in.sval.equals("ACTIVE")) { // active script; isActive = true; } else { throw new ParseException("Unexpected word " + in.sval + " at line " + in.lineno()); } token = in.nextToken(); } if (token == StreamTokenizer.TT_EOL) { scripts.add(new SieveScript(scriptName, null, isActive)); } else { throw new ParseException( "Expected EOL, got " + tokenToString(token) + " at line " + in.lineno()); } break; case StreamTokenizer.TT_WORD: in.pushBack(); return parseResponse(); default: throw new ParseException("Unexpected token " + tokenToString(token) + " at line " + in.lineno()); } } }
From source file:com.ecyrd.jspwiki.plugin.PluginManager.java
/** * Parses plugin arguments. Handles quotes and all other kewl stuff. * * <h3>Special parameters</h3> * The plugin body is put into a special parameter defined by {@link #PARAM_BODY}; * the plugin's command line into a parameter defined by {@link #PARAM_CMDLINE}; * and the bounds of the plugin within the wiki page text by a parameter defined * by {@link #PARAM_BOUNDS}, whose value is stored as a two-element int[] array, * i.e., <tt>[start,end]</tt>. * * @param argstring The argument string to the plugin. This is * typically a list of key-value pairs, using "'" to escape * spaces in strings, followed by an empty line and then the * plugin body. In case the parameter is null, will return an * empty parameter list.//from w ww. ja v a 2 s. c om * * @return A parsed list of parameters. * * @throws IOException If the parsing fails. */ public Map parseArgs(String argstring) throws IOException { HashMap<String, Object> arglist = new HashMap<String, Object>(); // // Protection against funny users. // if (argstring == null) return arglist; arglist.put(PARAM_CMDLINE, argstring); StringReader in = new StringReader(argstring); StreamTokenizer tok = new StreamTokenizer(in); int type; String param = null; String value = null; tok.eolIsSignificant(true); boolean potentialEmptyLine = false; boolean quit = false; while (!quit) { String s; type = tok.nextToken(); switch (type) { case StreamTokenizer.TT_EOF: quit = true; s = null; break; case StreamTokenizer.TT_WORD: s = tok.sval; potentialEmptyLine = false; break; case StreamTokenizer.TT_EOL: quit = potentialEmptyLine; potentialEmptyLine = true; s = null; break; case StreamTokenizer.TT_NUMBER: s = Integer.toString((int) tok.nval); potentialEmptyLine = false; break; case '\'': s = tok.sval; break; default: s = null; } // // Assume that alternate words on the line are // parameter and value, respectively. // if (s != null) { if (param == null) { param = s; } else { value = s; arglist.put(param, value); // log.debug("ARG: "+param+"="+value); param = null; } } } // // Now, we'll check the body. // if (potentialEmptyLine) { StringWriter out = new StringWriter(); FileUtil.copyContents(in, out); String bodyContent = out.toString(); if (bodyContent != null) { arglist.put(PARAM_BODY, bodyContent); } } return arglist; }
From source file:com.fluffypeople.managesieve.ManageSieveClient.java
/** * "This command gets the contents of the specified script". The name of the * script is taken from the script parameter, and the body is stored in the * object/*from w w w . j a v a2 s. c o m*/ * * @param script SieveScript to fetch/update * @return OK or NO response. */ public synchronized ManageSieveResponse getScript(SieveScript script) throws IOException, ParseException { String encodedName = encodeString(script.getName()); sendCommand("GETSCRIPT", encodedName); script.setBody(parseString()); int token = in.nextToken(); if (token != StreamTokenizer.TT_EOL) { throw new ParseException("Expecting EOL but got " + tokenToString(token) + " at line " + in.lineno()); } return parseResponse(); }
From source file:com.redskyit.scriptDriver.RunTests.java
private void parseBlock(ExecutionContext script, StreamTokenizer tokenizer, BlockHandler handler) throws Exception { tokenizer.nextToken();// ww w . ja v a2 s . c o m if (tokenizer.ttype == '{') { int braceLevel = 1; System.out.print(" {"); tokenizer.eolIsSignificant(true); tokenizer.nextToken(); while (tokenizer.ttype == StreamTokenizer.TT_WORD || tokenizer.ttype == '"' || tokenizer.ttype == StreamTokenizer.TT_NUMBER || tokenizer.ttype == '\n' || tokenizer.ttype == '{' || tokenizer.ttype == '}' || tokenizer.ttype == ',' || tokenizer.ttype == '*' || tokenizer.ttype == ':') { System.out.print(' '); String arg; switch (tokenizer.ttype) { case '{': braceLevel++; arg = "{"; break; case '}': if (--braceLevel == 0) { System.out.print("}"); tokenizer.eolIsSignificant(false); return; } arg = "}"; break; case StreamTokenizer.TT_NUMBER: arg = String.valueOf(tokenizer.nval); break; case StreamTokenizer.TT_EOL: arg = "\n"; break; case '"': // 5 backslashed required in replace string because its processed once // as a string (yielding \\") and then again by the replace method // of the regular expression (so \\" becomes \") arg = '"' + script.getExpandedString(tokenizer).replaceAll("\"", "\\\\\"") + '"'; break; case ',': arg = ","; break; case ':': arg = ":"; break; case '*': arg = "*"; break; default: arg = script.getExpandedString(tokenizer); } System.out.print(arg); handler.parseToken(tokenizer, arg); tokenizer.nextToken(); } System.out.println(); throw new Exception("args unexpectd token " + tokenizer.ttype); } tokenizer.pushBack(); // no arguments }