Example usage for java.io StreamTokenizer nextToken

Introduction

In this page you can find the example usage for java.io StreamTokenizer nextToken.

Prototype

public int nextToken() throws IOException 

Source Link

Document

Parses the next token from the input stream of this tokenizer.

Usage

From source file:com.rapidminer.tools.Tools.java

/** Delivers the next token and checks if its the end of line. */
public static void getLastToken(StreamTokenizer tokenizer, boolean endOfFileOk) throws IOException {
    if (tokenizer.nextToken() != StreamTokenizer.TT_EOL
            && (tokenizer.ttype != StreamTokenizer.TT_EOF || !endOfFileOk)) {
        throw new IOException("expected the end of the line " + tokenizer.lineno());
    }/*  w  ww. ja va2s. c  om*/
}

---

From source file:com.rapidminer.tools.Tools.java

/** Delivers the next token and skip empty lines. */
public static void getFirstToken(StreamTokenizer tokenizer) throws IOException {
    // skip empty lines
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
    }//from   w  w w . j a  v a2  s .  com
    ;

    if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
        tokenizer.ttype = '?';
    }
}

---

From source file:com.rapidminer.tools.Tools.java

/** Skips all tokens before next end of line (EOL). */
public static void waitForEOL(StreamTokenizer tokenizer) throws IOException {
    // skip everything until EOL
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
    }/*from  w w  w.  j  a  v a2 s .  c o m*/
    ;
    tokenizer.pushBack();
}

---

From source file:cross.io.xml.FragmentXMLSerializer.java

/**
 * @param name/*from   ww  w.jav  a 2  s . c o m*/
 * @param dims
 * @param data
 * @return
 */
private Array handleData(final String name, final Dimension[] dims, final Element data, final Range[] ranges) {
    EvalTools.notNull(dims, this);
    final String dec = new String(Base64.decode(data.getText(), Base64.GZIP));
    final StreamTokenizer st = new StreamTokenizer(new StringReader(dec));
    final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
    final int[] shape = new int[dims.length];
    int d = 0;
    for (final Dimension dim : dims) {
        shape[d++] = dim.getLength();
    }
    Array a = null;
    IndexIterator idx = null;
    int tok = -1;
    // log.info("DataType of array: {}",
    // dt.getPrimitiveClassType().getName());
    Mode m = Mode.UNDEF;
    Object o = null;
    try {
        while ((tok = st.nextToken()) != StreamTokenizer.TT_EOL) {
            if (tok == StreamTokenizer.TT_WORD) {

                if (m == Mode.UNDEF) {
                    try {
                        o = nf.parse(st.sval);

                        if (o instanceof Double) {
                            m = Mode.DOUBLE;
                            a = Array.factory(DataType.DOUBLE, shape);
                        } else if (o instanceof Float) {
                            m = Mode.FLOAT;
                            a = Array.factory(DataType.FLOAT, shape);
                        } else if (o instanceof Long) {
                            m = Mode.LONG;
                            a = Array.factory(DataType.LONG, shape);
                        } else if (o instanceof Integer) {
                            m = Mode.INTEGER;
                            a = Array.factory(DataType.INT, shape);
                        } else if (o instanceof Byte) {
                            m = Mode.BYTE;
                            a = Array.factory(DataType.BYTE, shape);
                        } else if (o instanceof Short) {
                            m = Mode.SHORT;
                            a = Array.factory(DataType.SHORT, shape);
                        }
                    } catch (final ParseException pe) {
                        if (st.sval.equalsIgnoreCase("true") || st.sval.equalsIgnoreCase("false")) {
                            m = Mode.BOOLEAN;
                            a = Array.factory(DataType.BOOLEAN, shape);
                        } else {
                            m = Mode.STRING;
                            a = Array.factory(DataType.STRING, shape);
                        }
                    }
                } else {
                    if (idx == null) {
                        idx = a.getIndexIterator();
                    }
                    switch (m) {
                    case DOUBLE: {
                        idx.setDoubleNext((Double) o);
                        break;
                    }
                    case FLOAT: {
                        idx.setFloatNext((Float) o);
                        break;
                    }
                    case INTEGER: {
                        idx.setIntNext((Integer) o);
                        break;
                    }
                    case LONG: {
                        idx.setLongNext((Long) o);
                        break;
                    }
                    case BYTE: {
                        idx.setByteNext((Byte) o);
                        break;
                    }
                    case SHORT: {
                        idx.setShortNext((Short) o);
                        break;
                    }
                    case BOOLEAN: {
                        idx.setBooleanNext(Boolean.parseBoolean(st.sval));
                        break;
                    }
                    case STRING: {
                        idx.setObjectNext(st.sval);
                        break;
                    }
                    case OBJECT: {
                        throw new IllegalArgumentException("Could not handle type");
                    }
                    }
                }
            }
        }
    } catch (final IOException e) {
        log.warn("Could not read data for {}", name);
    }
    if (a != null && ranges != null && ranges.length != 0) {
        try {
            return a.section(Arrays.asList(ranges));
        } catch (InvalidRangeException ex) {
            log.warn("Invalid range while trying to subset array: ", ex);
        }
    }
    return a;
}

---

From source file:com.jcraft.weirdx.XColormap.java

static void init(Map<String, Color> table) {
    if (_rgbtxt == null)
        return;/*from w  w  w.jav  a 2  s .  co  m*/
    StringBuffer foo = new StringBuffer();
    for (int i = 0; i < _rgbtxt.length; i++) {
        foo.append(_rgbtxt[i]);
    }
    rgbtxt = foo.toString();
    _rgbtxt = null;
    foo = null;

    try {
        InputStream is = new ByteArrayInputStream(RGBTXT.rgbtxt.getBytes());
        //      StreamTokenizer st=new StreamTokenizer(is);
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        StreamTokenizer st = new StreamTokenizer(br);

        st.ordinaryChar('!');
        //    st.ordinaryChar('\n');
        //    st.ordinaryChar('\t');
        //String token=null;
        char c;
        int r, g, b;
        byte[] buf = new byte[1024];
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            //System.out.println("type="+st.ttype+", "+st.sval);
            if (st.ttype == '!') {
                //     while((c=(char)is.read())!='\n');
                while ((c = (char) br.read()) != '\n')
                    ;
                continue;
            }
            if (st.ttype == StreamTokenizer.TT_NUMBER) {
                r = (int) st.nval;
                st.nextToken();
                g = (int) st.nval;
                st.nextToken();
                b = (int) st.nval;
                //System.out.print("r, g, b="+r+", "+g+", "+b);
                int i = 0;
                //     while((c=(char)is.read())!='\n'){
                while ((c = (char) br.read()) != '\n') {
                    if (c == '\t')
                        continue;
                    if (c == ' ')
                        continue;
                    if ('A' <= c && c <= 'Z') {
                        c = (char) ('a' + c - 'A');
                    }
                    buf[i] = (byte) c;
                    i++;
                }
                table.put(new String(buf, 0, i), new Color(r, g, b));
                //System.out.println(" -> "+new String(buf, 0, i));
                continue;
            }
        }
        st = null;
        buf = null;
        //    table.put("slategrey", rgbTable.get("slate grey"));
        //    table.put("Black", rgbTable.get("black"));
        //    table.put("White", rgbTable.get("white"));
    } catch (Exception e) {
        //System.out.println(e);
    }
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private File runScript(String filename) throws Exception {
    File file = new File(filename);
    StreamTokenizer tokenizer = openScript(file);
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
        if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
            runCommand(tokenizer, file, file.getName(), new ExecutionContext());
        }/*  w  ww  . j a  v  a2s.c o  m*/
    }
    return file;
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private boolean runString(String source, File file, String cmd) throws Exception {
    StreamTokenizer tokenizer = openString(source);
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
        if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
            runCommand(tokenizer, file, cmd, null);
        }//  w w w  .ja va2 s.c o m
    }
    return true;
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private void parseParams(StreamTokenizer tokenizer, ParamHandler handler) throws Exception {
    tokenizer.nextToken();
    if (tokenizer.ttype == '(') {
        System.out.print(" (");
        tokenizer.nextToken();/*from   www.j  a  va2s .com*/
        while (tokenizer.ttype == StreamTokenizer.TT_WORD || tokenizer.ttype == ',' || tokenizer.ttype == ')') {
            System.out.print(' ');
            String arg;
            switch (tokenizer.ttype) {
            case ',':
                arg = ",";
                break;
            case ')':
                System.out.print(")");
                return;
            default:
                arg = tokenizer.sval;
            }
            System.out.print(arg);
            handler.processParam(tokenizer, arg);
            tokenizer.nextToken();
        }
        System.out.println();
        throw new Exception("args unexpectd token " + tokenizer.ttype);
    }
    tokenizer.pushBack(); // no arguments
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private void parseBlock(ExecutionContext script, StreamTokenizer tokenizer, BlockHandler handler)
        throws Exception {
    tokenizer.nextToken();
    if (tokenizer.ttype == '{') {
        int braceLevel = 1;
        System.out.print(" {");
        tokenizer.eolIsSignificant(true);
        tokenizer.nextToken();//www.  j  a  v  a2 s . c om
        while (tokenizer.ttype == StreamTokenizer.TT_WORD || tokenizer.ttype == '"'
                || tokenizer.ttype == StreamTokenizer.TT_NUMBER || tokenizer.ttype == '\n'
                || tokenizer.ttype == '{' || tokenizer.ttype == '}' || tokenizer.ttype == ','
                || tokenizer.ttype == '*' || tokenizer.ttype == ':') {
            System.out.print(' ');
            String arg;
            switch (tokenizer.ttype) {
            case '{':
                braceLevel++;
                arg = "{";
                break;
            case '}':
                if (--braceLevel == 0) {
                    System.out.print("}");
                    tokenizer.eolIsSignificant(false);
                    return;
                }
                arg = "}";
                break;
            case StreamTokenizer.TT_NUMBER:
                arg = String.valueOf(tokenizer.nval);
                break;
            case StreamTokenizer.TT_EOL:
                arg = "\n";
                break;
            case '"':
                // 5 backslashed required in replace string because its processed once
                // as a string (yielding \\") and then again by the replace method
                // of the regular expression (so \\" becomes \")
                arg = '"' + script.getExpandedString(tokenizer).replaceAll("\"", "\\\\\"") + '"';
                break;
            case ',':
                arg = ",";
                break;
            case ':':
                arg = ":";
                break;
            case '*':
                arg = "*";
                break;
            default:
                arg = script.getExpandedString(tokenizer);
            }
            System.out.print(arg);
            handler.parseToken(tokenizer, arg);
            tokenizer.nextToken();
        }
        System.out.println();
        throw new Exception("args unexpectd token " + tokenizer.ttype);
    }
    tokenizer.pushBack(); // no arguments
}

---

From source file:com.redskyit.scriptDriver.RunTests.java

private boolean executeFunction(String name, File file, StreamTokenizer parent, ExecutionContext script)
        throws Exception {
    ExecutionContext context = functions.get(name);
    if (null != context) {
        // If this context has parameters then gather argument values
        if (null != parent && null != script) {
            List<String> params = context.getParams();
            if (null != params && params.size() > 0) {
                context.clearArgs();/*from w  ww . j  a va2s  . c om*/
                int count = params.size();
                while (count-- > 0) {
                    // get argument
                    parent.nextToken();
                    System.out.print(' ');
                    switch (parent.ttype) {
                    case StreamTokenizer.TT_NUMBER:
                        System.out.print(parent.nval);
                        context.addArg(parent.nval);
                        break;
                    default:
                        System.out.print(parent.sval);
                        context.addArg(script.getExpandedString(parent));
                        break;
                    }
                }
            }
            System.out.println();
        }

        // Get function body and execute it
        String code = context.getBody();
        if (null != code) {
            StreamTokenizer tokenizer = openString(code);
            while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
                if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
                    runCommand(tokenizer, file, name, context);
                }
            }
            return true;
        }
    }
    return false;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »