Example usage for com.google.common.base Splitter split

List of usage examples for com.google.common.base Splitter split

  1. HOME
  2. Java
  3. com.google.common
  4. com.google.common.base.*
  5. Splitter
  6. split

Introduction

In this page you can find the example usage for com.google.common.base Splitter split.

Prototype

@CheckReturnValue
public Iterable<String> split(final CharSequence sequence)

Source Link

Document

Splits sequence into string components and makes them available through an Iterator , which may be lazily evaluated.

Usage

From source file:com.attribyte.essem.BasicAuth.java

/**
 * Create from properties of the form username:password=index0,index1,...
 * @param props The properties./*from w  ww  .  ja v  a2s . co  m*/
 */
public BasicAuth(final Properties props) {

    Map<HashCode, Set<String>> authMap = Maps.newHashMapWithExpectedSize(16);
    Set<String> indexSet = Sets.newLinkedHashSetWithExpectedSize(16);
    Splitter indexSplitter = Splitter.on(CharMatcher.anyOf(", \t")).trimResults().omitEmptyStrings();

    Enumeration names = props.propertyNames();
    while (names.hasMoreElements()) {

        String key = ((String) names.nextElement()).trim();
        String up = key.replace('|', ':');
        String expectedValue = "Basic " + EncodingUtil.encodeBase64(up.getBytes(Charsets.UTF_8));

        HashCode expectedValueCode = hashFunction.hashString(expectedValue, Charsets.UTF_8);
        Set<String> allowSet = authMap.get(expectedValueCode);
        if (allowSet == null) {
            allowSet = Sets.newLinkedHashSetWithExpectedSize(16);
            authMap.put(expectedValueCode, allowSet);
        }

        String indexStr = props.getProperty(key).trim();
        for (String index : indexSplitter.split(indexStr)) {
            if (!index.equals("*")) {
                indexSet.add(index);
            }
            allowSet.add(index);
        }
    }

    this.authorizedIndexes = ImmutableSet.copyOf(indexSet);

    ImmutableMap.Builder<HashCode, ImmutableSet<String>> builder = ImmutableMap.builder();
    for (Map.Entry<HashCode, Set<String>> entry : authMap.entrySet()) {
        builder.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue()));
    }
    this.authMap = builder.build();
}

From source file:ml.shifu.shifu.core.processor.stats.MapReducerStatsWorker.java

/**
 * Calculate the PSI//  w  w  w. ja va2 s.c  o m
 * 
 * @throws IOException
 *             in scanners read exception
 */
public void runPSI() throws IOException {
    log.info("Run PSI to use {} to compute the PSI ", modelConfig.getPsiColumnName());
    ColumnConfig columnConfig = CommonUtils.findColumnConfigByName(columnConfigList,
            modelConfig.getPsiColumnName());

    if (columnConfig == null || (!columnConfig.isMeta() && !columnConfig.isCategorical())) {
        log.warn(
                "Unable to use the PSI column {} specify in ModelConfig to compute PSI\n"
                        + "neither meta nor categorical type",
                columnConfig != null ? columnConfig.getColumnName() : "unknown");

        return;
    }

    log.info("Start to use {} to compute the PSI ", columnConfig.getColumnName());

    Map<String, String> paramsMap = new HashMap<>();
    paramsMap.put("delimiter", CommonUtils.escapePigString(modelConfig.getDataSetDelimiter()));
    paramsMap.put("PSIColumn", modelConfig.getPsiColumnName().trim());
    paramsMap.put("column_parallel", Integer.toString(columnConfigList.size() / 10));
    paramsMap.put("value_index", "2");

    PigExecutor.getExecutor().submitJob(modelConfig, pathFinder.getScriptPath("scripts/PSI.pig"), paramsMap);

    List<Scanner> scanners = ShifuFileUtils.getDataScanners(pathFinder.getPSIInfoPath(),
            modelConfig.getDataSet().getSource());
    if (CollectionUtils.isEmpty(scanners)) {
        log.info("The PSI got failure during the computation");
        return;
    }

    String delimiter = Environment.getProperty(Constants.SHIFU_OUTPUT_DATA_DELIMITER,
            Constants.DEFAULT_DELIMITER);
    Splitter splitter = Splitter.on(delimiter).trimResults();

    List<String> unitStats = new ArrayList<String>(this.columnConfigList.size());
    for (Scanner scanner : scanners) {
        while (scanner.hasNext()) {
            //String[] output = scanner.nextLine().trim().split("\\|");
            String[] output = Lists.newArrayList(splitter.split(scanner.nextLine())).toArray(new String[0]);
            try {
                int columnNum = Integer.parseInt(output[0]);
                ColumnConfig config = this.columnConfigList.get(columnNum);
                config.setPSI(Double.parseDouble(output[1]));
                unitStats.add(output[0] + "|" + output[2]);
                // config.setUnitStats(
                //        Arrays.asList(StringUtils.split(output[2], CalculateStatsUDF.CATEGORY_VAL_SEPARATOR)));
            } catch (Exception e) {
                log.error("error in parsing", e);
            }
        }
        // close scanner
        IOUtils.closeQuietly(scanner);
    }

    // write unit stat into a temporary file
    ShifuFileUtils.createDirIfNotExists(new SourceFile(Constants.TMP, RawSourceData.SourceType.LOCAL));

    String ccUnitStatsFile = this.pathFinder.getColumnConfigUnitStatsPath();
    ShifuFileUtils.writeLines(unitStats, ccUnitStatsFile, RawSourceData.SourceType.LOCAL);

    log.info("The Unit Stats is stored in - {}.", ccUnitStatsFile);
    log.info("Run PSI - done.");
}

From source file:annis.AnnisBaseRunner.java

protected void runInteractive() throws IOException {
    System.out.println(helloMessage + " " + VersionInfo.getReleaseName());
    System.out.println();//w w w  . ja  va  2 s . c om
    System.out.println("Use \"help\" for a list of all commands.");
    System.out.println();

    ConsoleReader console = new ConsoleReader();
    File annisDir = new File(System.getProperty("user.home") + "/.annis/");
    String annisDirPath = annisDir.getAbsolutePath();
    if (!annisDir.exists()) {
        log.info("Creating directory: " + annisDirPath);
        if (!annisDir.mkdirs()) {
            log.warn("Could not create directory: " + annisDirPath);
        }
    } else if (!annisDir.isDirectory()) {
        log.warn(
                "Could not create directory because a file with the same name already exists: " + annisDirPath);
    }

    history = new FileHistory(new File(System.getProperty("user.home") + "/.annis/shellhistory.txt"));
    console.setHistory(history);
    console.setHistoryEnabled(true);
    console.setBellEnabled(true);
    console.setExpandEvents(false);

    List<String> commands = detectAvailableCommands();
    Collections.sort(commands);
    console.addCompleter(new StringsCompleter(commands));

    Splitter argSplitter = Splitter.on(' ').limit(2);
    String line;
    StringBuilder input = new StringBuilder();
    prompt = "no corpus>";
    console.setPrompt(prompt + " ");
    while ((line = console.readLine()) != null) {
        if (line.endsWith("\\")) {
            // multi-line input
            input.append(line.substring(0, line.length() - 1)).append("\n");
            // notifiy user by changing the prompt
            console.setPrompt("> ");
        } else {
            // input finished, run command
            input.append(line);

            ArrayList<String> splitted = Lists.newArrayList(argSplitter.split(input.toString()));
            String command = splitted.get(0);
            String args = "";

            if ("help".equalsIgnoreCase(command)) {
                System.out.println("Available commands:");
                System.out.println(StringUtils.join(commands, "\n"));
            } else {
                if (splitted.size() > 1) {
                    args = splitted.get(1);
                }
            }
            try {
                if (!command.isEmpty()) {
                    runCommand(command, args);
                }
            } catch (UsageException e) {
                error(e);
            }
            // reset the current prompt
            console.setPrompt(prompt + " ");
            // empty input
            input = new StringBuilder();
        }
    } // end while
}

From source file:com.android.tools.lint.client.api.DefaultConfiguration.java

private void readConfig() {
    mSuppressed = new HashMap<String, List<String>>();
    mSeverity = new HashMap<String, Severity>();

    if (!mConfigFile.exists()) {
        return;//  w  ww .ja v a  2 s .  c o  m
    }

    try {
        Document document = XmlUtils.parseUtfXmlFile(mConfigFile, false);
        NodeList issues = document.getElementsByTagName(TAG_ISSUE);
        Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings();
        for (int i = 0, count = issues.getLength(); i < count; i++) {
            Node node = issues.item(i);
            Element element = (Element) node;
            String idList = element.getAttribute(ATTR_ID);
            if (idList.isEmpty()) {
                formatError("Invalid lint config file: Missing required issue id attribute");
                continue;
            }
            Iterable<String> ids = splitter.split(idList);

            NamedNodeMap attributes = node.getAttributes();
            for (int j = 0, n = attributes.getLength(); j < n; j++) {
                Node attribute = attributes.item(j);
                String name = attribute.getNodeName();
                String value = attribute.getNodeValue();
                if (ATTR_ID.equals(name)) {
                    // already handled
                } else if (ATTR_SEVERITY.equals(name)) {
                    for (Severity severity : Severity.values()) {
                        if (value.equalsIgnoreCase(severity.name())) {
                            for (String id : ids) {
                                mSeverity.put(id, severity);
                            }
                            break;
                        }
                    }
                } else {
                    formatError("Unexpected attribute \"%1$s\"", name);
                }
            }

            // Look up ignored errors
            NodeList childNodes = element.getChildNodes();
            if (childNodes.getLength() > 0) {
                for (int j = 0, n = childNodes.getLength(); j < n; j++) {
                    Node child = childNodes.item(j);
                    if (child.getNodeType() == Node.ELEMENT_NODE) {
                        Element ignore = (Element) child;
                        String path = ignore.getAttribute(ATTR_PATH);
                        if (path.isEmpty()) {
                            String regexp = ignore.getAttribute(ATTR_REGEXP);
                            if (regexp.isEmpty()) {
                                formatError("Missing required attribute %1$s or %2$s under %3$s", ATTR_PATH,
                                        ATTR_REGEXP, idList);
                            } else {
                                addRegexp(idList, ids, n, regexp, false);
                            }
                        } else {
                            // Normalize path format to File.separator. Also
                            // handle the file format containing / or \.
                            if (File.separatorChar == '/') {
                                path = path.replace('\\', '/');
                            } else {
                                path = path.replace('/', File.separatorChar);
                            }

                            if (path.indexOf('*') != -1) {
                                String regexp = globToRegexp(path);
                                addRegexp(idList, ids, n, regexp, false);
                            } else {
                                for (String id : ids) {
                                    List<String> paths = mSuppressed.get(id);
                                    if (paths == null) {
                                        paths = new ArrayList<String>(n / 2 + 1);
                                        mSuppressed.put(id, paths);
                                    }
                                    paths.add(path);
                                }
                            }
                        }
                    }
                }
            }
        }
    } catch (SAXParseException e) {
        formatError(e.getMessage());
    } catch (Exception e) {
        mClient.log(e, null);
    }
}

From source file:org.opendaylight.controller.sal.restconf.impl.RestconfImpl.java

private QName getModuleNameAndRevision(final String identifier) {
    final int mountIndex = identifier.indexOf(ControllerContext.MOUNT);
    String moduleNameAndRevision = "";
    if (mountIndex >= 0) {
        moduleNameAndRevision = identifier.substring(mountIndex + ControllerContext.MOUNT.length());
    } else {/*from w w  w  .j  av a 2s .  com*/
        moduleNameAndRevision = identifier;
    }

    final Splitter splitter = Splitter.on("/").omitEmptyStrings();
    final Iterable<String> split = splitter.split(moduleNameAndRevision);
    final List<String> pathArgs = Lists.<String>newArrayList(split);
    if (pathArgs.size() < 2) {
        LOG.debug("URI has bad format. It should be \'moduleName/yyyy-MM-dd\' " + identifier);
        throw new RestconfDocumentedException(
                "URI has bad format. End of URI should be in format \'moduleName/yyyy-MM-dd\'",
                ErrorType.PROTOCOL, ErrorTag.INVALID_VALUE);
    }

    try {
        final String moduleName = pathArgs.get(0);
        final String revision = pathArgs.get(1);
        final Date moduleRevision = SimpleDateFormatUtil.getRevisionFormat().parse(revision);
        return QName.create(null, moduleRevision, moduleName);
    } catch (final ParseException e) {
        LOG.debug("URI has bad format. It should be \'moduleName/yyyy-MM-dd\' " + identifier);
        throw new RestconfDocumentedException("URI has bad format. It should be \'moduleName/yyyy-MM-dd\'",
                ErrorType.PROTOCOL, ErrorTag.INVALID_VALUE);
    }
}

From source file:com.android.tools.lint.detector.api.Project.java

/**
 * Returns the proguard files configured for this project, if any
 *
 * @return the proguard files, if any//from w ww  .j  a v  a  2s.c  o m
 */
@NonNull
public List<File> getProguardFiles() {
    if (mProguardFiles == null) {
        List<File> files = new ArrayList<File>();
        if (mProguardPath != null) {
            Splitter splitter = Splitter.on(CharMatcher.anyOf(":;")); //$NON-NLS-1$
            for (String path : splitter.split(mProguardPath)) {
                if (path.contains("${")) { //$NON-NLS-1$
                    // Don't analyze the global/user proguard files
                    continue;
                }
                File file = new File(path);
                if (!file.isAbsolute()) {
                    file = new File(getDir(), path);
                }
                if (file.exists()) {
                    files.add(file);
                }
            }
        }
        if (files.isEmpty()) {
            File file = new File(getDir(), OLD_PROGUARD_FILE);
            if (file.exists()) {
                files.add(file);
            }
            file = new File(getDir(), FN_PROJECT_PROGUARD_FILE);
            if (file.exists()) {
                files.add(file);
            }
        }
        mProguardFiles = files;
    }
    return mProguardFiles;
}

From source file:ml.shifu.shifu.core.ConfusionMatrix.java

public PerformanceResult bufferedComputeConfusionMatrixAndPerformance(long pigPosTags, long pigNegTags,
        double pigPosWeightTags, double pigNegWeightTags, long records, double maxPScore, double minPScore,
        String scoreDataPath, String evalPerformancePath, boolean isPrint, boolean isGenerateChart,
        int targetColumnIndex, int scoreColumnIndex, int weightColumnIndex, boolean isUseMaxMinScore)
        throws IOException {
    // 1. compute maxScore and minScore in case some cases score are not in [0, 1]
    double maxScore = 1d * scoreScale, minScore = 0d;

    if (isGBTNeedConvertScore()) {
        // if need convert to [0, 1], just keep max score to 1 and min score to 0 without doing anything
    } else {/*from   ww w . ja v  a  2s. c  o  m*/
        if (isUseMaxMinScore) {
            // TODO some cases maxPScore is already scaled, how to fix that issue
            maxScore = maxPScore;
            minScore = minPScore;
        } else {
            // otherwise, keep [0, 1]
        }
    }

    LOG.info("{} Transformed (scale included) max score is {}, transformed min score is {}",
            evalConfig.getGbtScoreConvertStrategy(), maxScore, minScore);

    SourceType sourceType = evalConfig.getDataSet().getSource();
    List<Scanner> scanners = ShifuFileUtils.getDataScanners(scoreDataPath, sourceType);
    LOG.info("Number of score files is {} in eval {}.", scanners.size(), evalConfig.getName());

    int numBucket = evalConfig.getPerformanceBucketNum();
    boolean hasWeight = StringUtils.isNotBlank(evalConfig.getDataSet().getWeightColumnName());
    boolean isDir = ShifuFileUtils.isDir(pathFinder.getEvalScorePath(evalConfig, sourceType), sourceType);

    List<PerformanceObject> FPRList = new ArrayList<PerformanceObject>(numBucket + 1);
    List<PerformanceObject> catchRateList = new ArrayList<PerformanceObject>(numBucket + 1);
    List<PerformanceObject> gainList = new ArrayList<PerformanceObject>(numBucket + 1);
    List<PerformanceObject> modelScoreList = new ArrayList<PerformanceObject>(numBucket + 1);
    List<PerformanceObject> FPRWeightList = new ArrayList<PerformanceObject>(numBucket + 1);
    List<PerformanceObject> catchRateWeightList = new ArrayList<PerformanceObject>(numBucket + 1);
    List<PerformanceObject> gainWeightList = new ArrayList<PerformanceObject>(numBucket + 1);

    double binScore = (maxScore - minScore) * 1d / numBucket, binCapacity = 1.0 / numBucket, scoreBinCount = 0,
            scoreBinWeigthedCount = 0;
    int fpBin = 1, tpBin = 1, gainBin = 1, fpWeightBin = 1, tpWeightBin = 1, gainWeightBin = 1,
            modelScoreBin = 1;
    long index = 0, cnt = 0, invalidTargetCnt = 0, invalidWgtCnt = 0;

    ConfusionMatrixObject prevCmo = buildInitalCmo(pigPosTags, pigNegTags, pigPosWeightTags, pigNegWeightTags,
            maxScore);
    PerformanceObject po = buildFirstPO(prevCmo);

    FPRList.add(po);
    catchRateList.add(po);
    gainList.add(po);
    FPRWeightList.add(po);
    catchRateWeightList.add(po);
    gainWeightList.add(po);
    modelScoreList.add(po);

    boolean isGBTScoreHalfCutoffStreategy = isGBTScoreHalfCutoffStreategy();
    boolean isGBTScoreMaxMinScaleStreategy = isGBTScoreMaxMinScaleStreategy();

    Splitter splitter = Splitter.on(delimiter).trimResults();

    for (Scanner scanner : scanners) {
        while (scanner.hasNext()) {
            if ((++cnt) % 100000L == 0L) {
                LOG.info("Loaded {} records.", cnt);
            }
            if ((!isDir) && cnt == 1) {
                // if the evaluation score file is the local file, skip the first line since we add
                continue;
            }

            // score is separated by default delimiter in our pig output format
            String[] raw = Lists.newArrayList(splitter.split(scanner.nextLine())).toArray(new String[0]);

            // tag check
            String tag = raw[targetColumnIndex];
            if (StringUtils.isBlank(tag) || (!posTags.contains(tag) && !negTags.contains(tag))) {
                invalidTargetCnt += 1;
                continue;
            }

            double weight = 1d;
            // if has weight
            if (weightColumnIndex > 0) {
                try {
                    weight = Double.parseDouble(raw[weightColumnIndex]);
                } catch (NumberFormatException e) {
                    invalidWgtCnt += 1;
                }
                if (weight < 0d) {
                    invalidWgtCnt += 1;
                    weight = 1d;
                }
            }

            double score = 0.0;
            try {
                score = Double.parseDouble(raw[scoreColumnIndex]);
            } catch (NumberFormatException e) {
                // user set the score column wrong ?
                if (Math.random() < 0.05) {
                    LOG.warn("The score column - {} is not number. Is score column set correctly?",
                            raw[scoreColumnIndex]);
                }
                continue;
            }

            scoreBinCount += 1;
            scoreBinWeigthedCount += weight;

            ConfusionMatrixObject cmo = new ConfusionMatrixObject(prevCmo);
            if (posTags.contains(tag)) {
                // Positive Instance
                cmo.setTp(cmo.getTp() + 1);
                cmo.setFn(cmo.getFn() - 1);
                cmo.setWeightedTp(cmo.getWeightedTp() + weight * 1.0);
                cmo.setWeightedFn(cmo.getWeightedFn() - weight * 1.0);
            } else {
                // Negative Instance
                cmo.setFp(cmo.getFp() + 1);
                cmo.setTn(cmo.getTn() - 1);
                cmo.setWeightedFp(cmo.getWeightedFp() + weight * 1.0);
                cmo.setWeightedTn(cmo.getWeightedTn() - weight * 1.0);
            }

            if (isGBTScoreHalfCutoffStreategy) {
                // half cut off means score <0 then set to 0 and then min score is 0, max score is raw max score,
                // use max min scale to rescale to [0, 1]
                if (score < 0d) {
                    score = 0d;
                }
                score = ((score - 0) * scoreScale) / (maxPScore - 0);
            } else if (isGBTScoreMaxMinScaleStreategy) {
                // use max min scaler to make score in [0, 1], don't foget to time scoreScale
                score = ((score - minPScore) * scoreScale) / (maxPScore - minPScore);
            } else {
                // do nothing, use current score
            }

            cmo.setScore(Double.parseDouble(SCORE_FORMAT.format(score)));

            ConfusionMatrixObject object = cmo;
            po = PerformanceEvaluator.setPerformanceObject(object);
            if (po.fpr >= fpBin * binCapacity) {
                po.binNum = fpBin++;
                FPRList.add(po);
            }

            if (po.recall >= tpBin * binCapacity) {
                po.binNum = tpBin++;
                catchRateList.add(po);
            }

            // prevent 99%
            double validRecordCnt = (double) (index + 1);
            if (validRecordCnt / (pigPosTags + pigNegTags) >= gainBin * binCapacity) {
                po.binNum = gainBin++;
                gainList.add(po);
            }

            if (po.weightedFpr >= fpWeightBin * binCapacity) {
                po.binNum = fpWeightBin++;
                FPRWeightList.add(po);
            }

            if (po.weightedRecall >= tpWeightBin * binCapacity) {
                po.binNum = tpWeightBin++;
                catchRateWeightList.add(po);
            }

            if ((object.getWeightedTp() + object.getWeightedFp()) / object.getWeightedTotal() >= gainWeightBin
                    * binCapacity) {
                po.binNum = gainWeightBin++;
                gainWeightList.add(po);
            }

            if ((maxScore - (modelScoreBin * binScore)) >= score) {
                po.binNum = modelScoreBin++;
                po.scoreCount = scoreBinCount;
                po.scoreWgtCount = scoreBinWeigthedCount;
                // System.out.println("score count is " + scoreBinCount);
                // reset to 0 for next bin score cnt stats
                scoreBinCount = scoreBinWeigthedCount = 0;
                modelScoreList.add(po);
            }
            index += 1;
            prevCmo = cmo;
        }
        scanner.close();
    }
    LOG.info(
            "Totally loading {} records with invalid target records {} and invalid weight records {} in eval {}.",
            cnt, invalidTargetCnt, invalidWgtCnt, evalConfig.getName());

    PerformanceResult result = buildPerfResult(FPRList, catchRateList, gainList, modelScoreList, FPRWeightList,
            catchRateWeightList, gainWeightList);

    synchronized (this.lock) {
        if (isPrint) {
            PerformanceEvaluator.logResult(FPRList, "Bucketing False Positive Rate");

            if (hasWeight) {
                PerformanceEvaluator.logResult(FPRWeightList, "Bucketing Weighted False Positive Rate");
            }

            PerformanceEvaluator.logResult(catchRateList, "Bucketing Catch Rate");

            if (hasWeight) {
                PerformanceEvaluator.logResult(catchRateWeightList, "Bucketing Weighted Catch Rate");
            }

            PerformanceEvaluator.logResult(gainList, "Bucketing Action Rate");

            if (hasWeight) {
                PerformanceEvaluator.logResult(gainWeightList, "Bucketing Weighted Action Rate");
            }

            PerformanceEvaluator.logAucResult(result, hasWeight);
        }

        writePerResult2File(evalPerformancePath, result);

        if (isGenerateChart) {
            generateChartAndJsonPerfFiles(hasWeight, result);
        }
    }

    if (cnt == 0) {
        LOG.error("No score read, the EvalScore did not genernate or is null file");
        throw new ShifuException(ShifuErrorCode.ERROR_EVALSCORE);
    }
    return result;
}

From source file:org.pshdl.model.simulation.codegenerator.CCodeGenerator.java

public Unit getUnit(final ExecutableModel model) {
    try {/*from  www .j  a va 2s .c o  m*/
        Unit unit = null;
        final Splitter annoSplitter = Splitter.on(SimulationTransformationExtension.ANNO_VALUE_SEP);
        boolean _tripleNotEquals = (this.em.annotations != null);
        if (_tripleNotEquals) {
            for (final String a : this.em.annotations) {
                boolean _startsWith = a.startsWith("busDescription");
                if (_startsWith) {
                    Splitter _limit = annoSplitter.limit(2);
                    Iterable<String> _split = _limit.split(a);
                    final String value = IterableExtensions.<String>last(_split);
                    LinkedHashSet<Problem> _linkedHashSet = new LinkedHashSet<Problem>();
                    Unit _parseUnit = MemoryModelAST.parseUnit(value, _linkedHashSet, 0);
                    unit = _parseUnit;
                }
            }
        }
        return unit;
    } catch (Throwable _e) {
        throw Exceptions.sneakyThrow(_e);
    }
}