Example usage for com.google.common.base Splitter on

List of usage examples for com.google.common.base Splitter on

  1. HOME
  2. Java
  3. com.google.common
  4. com.google.common.base.*
  5. Splitter
  6. on

Introduction

In this page you can find the example usage for com.google.common.base Splitter on.

Prototype

@CheckReturnValue
@GwtIncompatible("java.util.regex")
public static Splitter on(final Pattern separatorPattern)

Source Link

Document

Returns a splitter that considers any subsequence matching pattern to be a separator.

Usage

From source file:yrun.commands.Ykill.java

public static void main(String[] args) throws IOException {
    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);// w w w  .  j a  v a  2  s  .  co  m
    yarnClient.start();
    boolean debug = false;
    try {
        Splitter splitter = Splitter.on('_');
        for (String arg : args) {
            List<String> list = toList(splitter.split(arg));
            if (list.size() != 3) {
                System.err.println("Application Id " + arg + " is not a valid application id.");
            } else {
                String prefix = list.get(0);
                if (!prefix.equals("application")) {
                    System.err.println("Application Id " + arg + " is not a valid application id.");
                } else {
                    try {
                        long clusterTimestamp = Long.parseLong(list.get(1));
                        int id = Integer.parseInt(list.get(2));
                        ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp, id);
                        yarnClient.killApplication(applicationId);
                        System.out.println("Killed\t" + arg + "");
                    } catch (Exception e) {
                        if (debug) {
                            e.printStackTrace();
                        }
                        System.err.println("Error while trying to kill " + arg + ".");
                    }
                }
            }
        }
    } finally {
        yarnClient.stop();
        yarnClient.close();
    }
}

From source file:org.lendingclub.mercator.demo.Main.java

public static void main(String[] args) {
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();/*from ww w.j a v a  2s .c  om*/
    Projector projector = new Projector.Builder().build();

    Runnable awsTask = new Runnable() {

        public void run() {
            try {
                List<String> regionList = Splitter.on(",").omitEmptyStrings().trimResults()
                        .splitToList(Strings.nullToEmpty(System.getenv("AWS_REGIONS")));

                if (regionList == null || regionList.isEmpty()) {
                    regionList = Lists.newArrayList();
                    regionList.add("us-west-2");
                    regionList.add("us-east-1");
                }
                LoggerFactory.getLogger(Main.class).info("scanning regions: {}", regionList);
                regionList.forEach(it -> {
                    try {
                        org.slf4j.LoggerFactory.getLogger(Main.class).info("scanning region: {}", it);
                        Regions region = Regions.fromName(it);
                        projector.createBuilder(AWSScannerBuilder.class).withRegion(region)
                                .build(AllEntityScanner.class).scan();
                    } catch (Exception e) {
                        LoggerFactory.getLogger(Main.class).warn("problem scanning " + it, e);
                    }
                });
            } catch (Exception e) {
                LoggerFactory.getLogger(Main.class).warn("problem", e);
            }
        }

    };

    Runnable dockerTask = new Runnable() {
        public void run() {

            projector.createBuilder(DockerScannerBuilder.class).withLocalDockerDaemon().build().scan();
        }
    };

    ScheduledExecutorService exec = Executors.newScheduledThreadPool(5);
    exec.scheduleWithFixedDelay(awsTask, 0, 1, TimeUnit.MINUTES);
    exec.scheduleWithFixedDelay(dockerTask, 0, 10, TimeUnit.SECONDS);

    while (true == true) {
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }
    }
}

From source file:tv.icntv.log.tools.Tools.java

public static void main(String[] args) throws ParseException, IOException {

    CommandLineParser parser = new PosixParser();
    CommandLine line = parser.parse(init(), args);
    Api api = new FileApi();
    long time = System.nanoTime();
    String input = line.getOptionValue("I");
    List<String> inputStr = Lists.newArrayList(Splitter.on(",").omitEmptyStrings().split(input));
    //        List<Path> paths = Lists.transform(inputStr,new Function<String, Path>() {
    //            @Override
    //            public Path apply(java.lang.String input) {
    //                System.out.println("input ="+input);
    //                try{
    //                Path path = new Path(input);  //To change body of implemented methods use File | Settings | File Templates.
    //                }catch (Exception e){
    //                    return null;
    //                }
    //                if()
    //            }
    //        });
    FileSystem fileSystem = FileSystem.get(new Configuration());
    List<Path> paths = Lists.newArrayList();
    for (String str : inputStr) {
        Path path = null;/*from   w  w w  .  j av a2s . c  o  m*/
        try {
            path = new Path(str);
            if (fileSystem.exists(path)) {
                System.out.println(" add path .." + str);
                paths.add(path);
            }
        } catch (Exception e) {
            continue;
        }

    }
    if (null == paths || paths.isEmpty()) {
        System.out.println("input 2 path null");
    }
    String regular = line.getOptionValue("R");
    if (Strings.isNullOrEmpty(regular)) {
        regular = "part-m-\\d*.lzo";
    }
    System.out.println("regular = " + regular);
    boolean test = api.writeDat(paths.toArray(new Path[paths.size()]), regular,
            new Path(line.getOptionValue("Out")));
    System.out.println(test + "\t" + (System.nanoTime() - time) / Math.pow(10, 9));
}

From source file:com.cedarsoft.serialization.SplittingPerformanceRunner.java

public static void main(String[] args) throws Exception {
    final String uri = "http://www.cedarsoft.com/some/slashes/1.0.0";

    run("String.plit", new Callable<String>() {
        @Override/*ww  w  . j ava2 s .  c om*/
        public String call() throws Exception {
            String[] parts = uri.split("/");
            return parts[parts.length - 1];
        }
    });

    run("Splitter", new Callable<String>() {
        @Override
        public String call() throws Exception {
            Splitter splitter = Splitter.on("/");
            Iterable<String> parts = splitter.split(uri);

            Iterator<String> iterator = parts.iterator();
            while (true) {
                String current = iterator.next();
                if (!iterator.hasNext()) {
                    return current;
                }
            }
        }
    });

    run("static Splitter", new Callable<String>() {
        @Override
        public String call() throws Exception {
            Iterable<String> parts = SPLITTER.split(uri);

            Iterator<String> iterator = parts.iterator();
            while (true) {
                String current = iterator.next();
                if (!iterator.hasNext()) {
                    return current;
                }
            }
        }
    });

    run("indexOf", new Callable<String>() {
        @Override
        public String call() throws Exception {
            int index = uri.lastIndexOf("/");
            return uri.substring(index + 1);
        }
    });
}

From source file:tv.icntv.log.crawl2.SendClient.java

/**
 * 0--source  ;example /data/chinacache/2014-03-03/abc.gz
 * 1--lzoPath ;example /data/lzo/chinacache/2014-03-03/
 * 1--hdfs url ;example hdfs://uri/// w ww .  j  a va 2 s.  com
 *
 * @param args
 */
public static void main(String[] args) throws ParseException {

    if (args == null || args.length == 0) {
        help();
        return;
    }
    CommandLineParser parser = new PosixParser();
    CommandLine line = parser.parse(init(), args);
    if (line.hasOption("h")) {
        help();
        return;
    }
    String source = "";
    String fileName = "";
    if (line.hasOption("s")) {
        source = line.getOptionValue("s");
        List<String> sources = Splitter.on(File.separator).splitToList(source);
        fileName = sources.get(sources.size() - 1);
    } else {
        System.out.println("source null");
        return;
    }
    String lzoFile = "";
    String lzoFileName = "";
    if (line.hasOption("r")) {
        String lzoPath = line.getOptionValue("r", lzoDirectory);
        lzoFileName = fileName.substring(0, fileName.lastIndexOf(".")) + suffix;
        lzoFile = lzoPath + File.separator + lzoFileName;
    }
    String hdfsUrl = "";
    if (line.hasOption("t")) {
        hdfsUrl = line.getOptionValue("t");
    } else {
        System.out.println("target url null");
        return;
    }

    //        logger.info("source = "+source+"\r\n lzo="+lzoFile+"\r\n target = "+hdfsUrl);

    File file = new File(source);
    if (!file.exists()) {
        System.out.println("file =" + source + " not exist!");
        return;
    }

    //reCompress from gz to lzo
    long start = System.nanoTime();
    ReCompress reCompress = new Gz2LzoReCompress(source, lzoFile, new LzoCompressImpl(),
            new GzUnCompressImpl());
    try {
        reCompress.reCompress();
    } catch (IOException e) {
        System.out.println(e);
        return;

    }
    logger.info("use time = " + (System.nanoTime() - start) / Math.pow(10, 9));
    //        logger.info("source size = "+file.length()/1024/1024/1024+" G; lzo recompress size " + new File(lzoFile).length()/2014/1024/1024+" G");
    //get target
    HdfsDispatcher hdfsDispatcher = new HdfsDispatcher(lzoFile, hdfsUrl + File.separator + lzoFileName);
    hdfsDispatcher.send();
}

From source file:com.sina.dip.twill.HelloWorld.java

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/* w  ww  .  j  a v a2s  .  c o  m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

From source file:org.apache.twill.example.yarn.HelloWorld.java

public static void main(String[] args) {
    if (args.length < 1) {
        System.err.println("Arguments format: <host:port of zookeeper server>");
        System.exit(1);/*w  w w.  j a v  a2 s  .c  om*/
    }

    String zkStr = args[0];
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
    twillRunner.start();

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
    List<String> applicationClassPaths = Lists.newArrayList();
    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

From source file:com.mapr.PurchaseLog.java

public static void main(String[] args) throws IOException {
    Options opts = new Options();
    CmdLineParser parser = new CmdLineParser(opts);
    try {//from   w  w w .  j  av  a  2 s.c om
        parser.parseArgument(args);
    } catch (CmdLineException e) {
        System.err.println("Usage: -count <number>G|M|K [ -users number ]  log-file user-profiles");
        return;
    }

    Joiner withTab = Joiner.on("\t");

    // first generate lots of user definitions
    SchemaSampler users = new SchemaSampler(
            Resources.asCharSource(Resources.getResource("user-schema.txt"), Charsets.UTF_8).read());
    File userFile = File.createTempFile("user", "tsv");
    BufferedWriter out = Files.newBufferedWriter(userFile.toPath(), Charsets.UTF_8);
    for (int i = 0; i < opts.users; i++) {
        out.write(withTab.join(users.sample()));
        out.newLine();
    }
    out.close();

    // now generate a session for each user
    Splitter onTabs = Splitter.on("\t");
    Splitter onComma = Splitter.on(",");

    Random gen = new Random();
    SchemaSampler intermediate = new SchemaSampler(
            Resources.asCharSource(Resources.getResource("hit_step.txt"), Charsets.UTF_8).read());

    final int COUNTRY = users.getFieldNames().indexOf("country");
    final int CAMPAIGN = intermediate.getFieldNames().indexOf("campaign_list");
    final int SEARCH_TERMS = intermediate.getFieldNames().indexOf("search_keywords");
    Preconditions.checkState(COUNTRY >= 0, "Need country field in user schema");
    Preconditions.checkState(CAMPAIGN >= 0, "Need campaign_list field in step schema");
    Preconditions.checkState(SEARCH_TERMS >= 0, "Need search_keywords field in step schema");

    out = Files.newBufferedWriter(new File(opts.out).toPath(), Charsets.UTF_8);

    for (String line : Files.readAllLines(userFile.toPath(), Charsets.UTF_8)) {
        long t = (long) (TimeUnit.MILLISECONDS.convert(30, TimeUnit.DAYS) * gen.nextDouble());
        List<String> user = Lists.newArrayList(onTabs.split(line));

        // pick session length
        int n = (int) Math.floor(-30 * Math.log(gen.nextDouble()));

        for (int i = 0; i < n; i++) {
            // time on page
            int dt = (int) Math.floor(-20000 * Math.log(gen.nextDouble()));
            t += dt;

            // hit specific values
            JsonNode step = intermediate.sample();

            // check for purchase
            double p = 0.01;
            List<String> campaigns = Lists.newArrayList(onComma.split(step.get("campaign_list").asText()));
            List<String> keywords = Lists.newArrayList(onComma.split(step.get("search_keywords").asText()));
            if ((user.get(COUNTRY).equals("us") && campaigns.contains("5"))
                    || (user.get(COUNTRY).equals("jp") && campaigns.contains("7")) || keywords.contains("homer")
                    || keywords.contains("simpson")) {
                p = 0.5;
            }

            String events = gen.nextDouble() < p ? "1" : "-";

            out.write(Long.toString(t));
            out.write("\t");
            out.write(line);
            out.write("\t");
            out.write(withTab.join(step));
            out.write("\t");
            out.write(events);
            out.write("\n");
        }
    }
    out.close();
}

From source file:com.sina.dip.twill.HelloWorldResourcesSpecification.java

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();// w  w w.j  a v a 2s .  c om

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner
            .prepare(new HelloWorldRunnable(),
                    ResourceSpecification.Builder.with().setVirtualCores(1).setMemory(2, SizeUnit.GIGA)
                            .setInstances(3).build())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

From source file:com.x.app.test.HelloWorld.java

public static void main(String[] args) {
    if (args.length < 1) {
        System.err.println("Arguments format: <host:port of zookeeper server>");
        System.exit(1);/*from w ww.j ava 2 s  .  c o  m*/
    }

    String zkStr = args[0];
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.setSocketAddr("yarn.resourcemanager.address",
            new InetSocketAddress("192.168.80.103", 8032));
    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
    twillRunner.start();

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
    List<String> applicationClassPaths = Lists.newArrayList();
    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}