Java String Tokenize getTokenTypes(CommonTree tree)

Description

get Token Types

License

Apache License

Declaration

public static List<Integer> getTokenTypes(CommonTree tree) 

Method Source Code

//package com.java2s;
/*/*from   w ww.  j  a  v  a 2 s.  co m*/
 * This file is part of the TSPHP project published under the Apache License 2.0
 * For the full copyright and license information, please have a look at LICENSE in the
 * root folder or visit the project's website http://tsphp.ch/wiki/display/TSPHP/License
 */

import java.util.ArrayList;

import java.util.List;
import org.antlr.runtime.tree.CommonTree;

public class Main {
    public static final int DOWN = -2;
    public static final int UP = -3;
    private static List<Integer> tokenTypes;

    public static List<Integer> getTokenTypes(CommonTree tree) {
        tokenTypes = new ArrayList<>();
        generateTokenTypes(tree);
        return tokenTypes;
    }

    private static void generateTokenTypes(CommonTree tree) {
        int numChildren = tree.getChildCount();
        if (numChildren == 0) {
            tokenTypes.add(tree.token.getType());
        } else {

            if (!tree.isNil()) {
                tokenTypes.add(DOWN);
                tokenTypes.add(tree.token.getType());
            }
            for (int i = 0; i < numChildren; i++) {
                CommonTree t = (CommonTree) tree.getChild(i);
                generateTokenTypes(t);
            }
            if (!tree.isNil()) {
                tokenTypes.add(UP);
            }
        }

    }
}

Related

1. getTokens(String string)
2. getTokens(String string)
3. getTokens(String string, String tokenSeparator)
4. getTokens(String value)
5. getTokens(String vbt)
6. getUnitType(String typeToken)
7. hasSuffix(String _token)
8. isPOSTag(String token)
9. isStringFunction(String token)