Example usage for org.apache.hadoop.conf Configuration clear

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration clear.

Prototype

public void clear() 

Source Link

Document

Clears all keys from the configuration.

Usage

From source file:com.quixey.hadoop.fs.oss.OSSCredentialsTest.java

License:Apache License

@Test
public void testThrowIAEOnMissingSecretKey() {
    Configuration config = new Configuration();
    config.clear();
    config.set("fs.oss.secretAccessKey", "abc");

    OSSCredentials oss = new OSSCredentials();

    thrown.expect(IllegalArgumentException.class);
    oss.initialize(URI.create("oss://bucket"), config);
}

---

From source file:com.streamsets.pipeline.stage.destination.hdfs.TestHdfsTarget.java

License:Apache License

@Test
public void testOnlyConfDirectory() throws Exception {
    // Create custom core-site.xml
    Configuration configuration = new Configuration();
    configuration.clear();
    configuration.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "file:///");
    FileOutputStream configOut = FileUtils.openOutputStream(new File(getTestDir() + "/conf-dir/core-site.xml"));
    configuration.writeXml(configOut);/* w w w .  ja  v a  2s  . c o  m*/
    configOut.close();

    HdfsTarget hdfsTarget = HdfsTargetUtil.newBuilder().hdfsUri("").hdfsConfDir(getTestDir() + "/conf-dir/")
            .build();

    TargetRunner runner = new TargetRunner.Builder(HdfsDTarget.class, hdfsTarget)
            .setOnRecordError(OnRecordError.STOP_PIPELINE).build();

    runner.runInit();

    // The configuration object should have the FS config from core-site.xml
    Assert.assertEquals("file:///",
            hdfsTarget.getHdfsConfiguration().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));

    runner.runDestroy();
}

---

From source file:edu.mit.ll.graphulo.pig.backend.AbstractGraphuloStorage.java

License:Apache License

/**
 * Replaces the given entries in the configuration by clearing the
 * Configuration and re-adding the elements that aren't in the Map of
 * entries to unset//from  ww  w . j  a  v  a  2 s  .  com
 * 
 * @param conf
 * @param entriesToUnset
 */
protected void clearUnset(Configuration conf, Map<String, String> entriesToUnset) {
    // Gets a copy of the entries
    Iterator<Entry<String, String>> originalEntries = conf.iterator();
    conf.clear();

    while (originalEntries.hasNext()) {
        Entry<String, String> originalEntry = originalEntries.next();

        // Only re-set() the pairs that aren't in our collection of keys to
        // unset
        if (!entriesToUnset.containsKey(originalEntry.getKey())) {
            conf.set(originalEntry.getKey(), originalEntry.getValue());
        }
    }
}

---

From source file:edu.upc.bip.batch.Batch.java

License:Apache License

public static void main(String[] args) throws Exception {
    //HBaseUtils.deleteTable(tablename);
    HBaseUtils.creatTable(tablename, familys);

    /*            JavaSparkContext spark = new JavaSparkContext(
            new SparkConf().setAppName("JavaWordCount").setMaster("local[20]").set("spark.executor.memory","8g").set("spark.driver.maxResultSize","2g")
    );/*  w w  w . j  a  v a  2 s  .  c  o  m*/
            
    JavaRDD<String> lines = spark.textFile("hdfs://localhost:54310/kafka/plt-input/test/trajectories.*");
            
    //lines to Transactions
    JavaRDD<Transaction> transactions = lines.map(
            new Function<String, Transaction>() {
                @Override
                public Transaction call(String row) throws Exception {
                    String[] words = row.split(";");
                    LocalDateTime time = LocalDateTime.parse(words[TimeStamp], formatter);
                    Double latitude = Double.parseDouble(words[Latitude]);
                    Double longitude = Double.parseDouble(words[Longtitude]);
                    String userID = words[UserID];
                    Coordinate coordinate = new Coordinate(latitude, longitude);
                    return new Transaction(coordinate,time,userID);
                }
            }
    );*/
    localDateTimeList.add(LocalDateTime.now());
    JavaSparkContext spark = new JavaSparkContext(
            new SparkConf().setAppName("JavaWordCount").setMaster("local[200]")
                    .set("spark.executor.memory", "10g").set("spark.driver.maxResultSize", "4g"));

    Configuration conf = null;
    Connection connection = null;

    conf = HBaseConfiguration.create();
    conf.clear();
    conf.set("hbase.zookeeper.quorum", "127.0.0.1");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    connection = ConnectionFactory.createConnection(conf);
    Scan s = new Scan();
    //            FilterList filterList = new FilterList(new PageFilter(400));
    //            s.setFilter(filterList);
    //    s.setStartRow(Bytes.toBytes("t2008-10-23 02:53:151"));
    //    s.setStopRow(Bytes.toBytes("t2008-10-23 02:53:161"));
    conf.set(TableInputFormat.SCAN, convertScanToString(s));

    localDateTimeList.add(LocalDateTime.now());
    conf.set(TableInputFormat.INPUT_TABLE, "csvtohbase");
    JavaPairRDD<ImmutableBytesWritable, Result> hbaseRDD = spark.newAPIHadoopRDD(conf, TableInputFormat.class,
            ImmutableBytesWritable.class, Result.class);
    localDateTimeList.add(LocalDateTime.now());
    JavaRDD<Transaction> transactions = hbaseRDD
            .map(new Function<Tuple2<ImmutableBytesWritable, Result>, Transaction>() {
                public Transaction call(Tuple2<ImmutableBytesWritable, Result> tuple) throws Exception {

                    try {
                        String[] values = Bytes
                                .toString(tuple._2.getValue(Bytes.toBytes("data"), Bytes.toBytes("")))
                                .split(";");
                        return new Transaction(
                                new Coordinate(Double.parseDouble(values[0]), Double.parseDouble(values[1])),
                                LocalDateTime.parse(values[2], formatter), values[3]);
                    } catch (Exception e) {
                        e.printStackTrace();
                        return null;
                    }
                }
            });
    //System.out.print(transactions.count());
    localDateTimeList.add(LocalDateTime.now());
    JavaRDD<Transaction> transactionstsWithRoundedCoordinates = BatchUtils.roundCoordinates(transactions, 1000)
            .persist(StorageLevel.MEMORY_ONLY());
    ;
    localDateTimeList.add(LocalDateTime.now());

    LocalDateTime minTimestamp = transactionstsWithRoundedCoordinates.min(new TimestampComparator())
            .getTimestamp();
    LocalDateTime maxTimestamp = transactionstsWithRoundedCoordinates.max(new TimestampComparator())
            .getTimestamp();
    long duration = minTimestamp.until(maxTimestamp, ChronoUnit.SECONDS);
    int maxDetail = (int) duration;
    long[] steps = { 5, 10, 15 };
    for (long step : steps) {
        long timeStep = step;
        for (int i = 0, j = 0; j < 3600 * 4; i++, j += timeStep) {

            if (i % 60 == 0) {
                int cccccc = 4;
            }
            LocalDateTime start = minTimestamp.plus(timeStep * i, ChronoUnit.SECONDS);
            LocalDateTime end = minTimestamp.plus(timeStep * (i + 1), ChronoUnit.SECONDS);
            JavaRDD<Transaction> measurementsFilteredByTime = BatchUtils
                    .filterByTime(transactionstsWithRoundedCoordinates, start, end);
            JavaPairRDD<Coordinate, Integer> counts = BatchUtils.countPerGridBox(measurementsFilteredByTime);

            BatchUtils.writeJsonToHbase(counts, tablename, familys[0], step + "s" + start.toString(),
                    objectMapper);
        }

    }
    localDateTimeList.add(LocalDateTime.now());
    spark.stop();
    localDateTimeList.add(LocalDateTime.now());
    for (LocalDateTime localDateTime : localDateTimeList) {
        System.out.println(localDateTime);
    }
}

---

From source file:edu.upc.bip.batch.BatchMLLIB.java

License:Apache License

public static void main(String[] args) throws Exception {

    HBaseUtils.deleteTable("dataminingupc");
    HBaseUtils.creatTable("dataminingupc", new String[] { "data" });
    localDateTimeList.add(LocalDateTime.now());
    JavaSparkContext spark = new JavaSparkContext(new SparkConf().setAppName("MLLIB").setMaster("local[500]")
            .set("spark.executor.memory", "12g").set("spark.driver.maxResultSize", "4g"));

    Configuration conf = null;
    Connection connection = null;

    conf = HBaseConfiguration.create();//from ww  w .  j  a  v  a2 s . co  m
    conf.clear();
    conf.set("hbase.zookeeper.quorum", "127.0.0.1");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    connection = ConnectionFactory.createConnection(conf);
    Scan s = new Scan();
    //        FilterList filterList = new FilterList(new PageFilter(4000000));
    //        s.setFilter(filterList);
    //    s.setStartRow(Bytes.toBytes("t2008-10-23 02:53:151"));
    //    s.setStopRow(Bytes.toBytes("t2008-10-23 02:53:161"));
    conf.set(TableInputFormat.SCAN, convertScanToString(s));

    conf.set(TableInputFormat.INPUT_TABLE, "csvtohbase");
    JavaPairRDD<ImmutableBytesWritable, Result> hbaseRDD = spark.newAPIHadoopRDD(conf, TableInputFormat.class,
            ImmutableBytesWritable.class, Result.class);
    JavaRDD<Transaction> transactions = hbaseRDD
            .map(new Function<Tuple2<ImmutableBytesWritable, Result>, Transaction>() {
                public Transaction call(Tuple2<ImmutableBytesWritable, Result> tuple) throws Exception {

                    try {
                        String[] values = Bytes
                                .toString(tuple._2.getValue(Bytes.toBytes("data"), Bytes.toBytes("")))
                                .split(";");
                        return new Transaction(
                                new Coordinate(Double.parseDouble(values[0]), Double.parseDouble(values[1])),
                                LocalDateTime.parse(values[2], formatter), values[3]);
                    } catch (Exception e) {
                        e.printStackTrace();
                        return null;
                    }
                }
            });
    localDateTimeList.add(LocalDateTime.now());
    JavaRDD<Transaction> transactionstsWithRoundedCoordinates = BatchUtils.roundCoordinates(transactions, 10000)
            .cache();
    LocalDateTime minTimestamp = transactionstsWithRoundedCoordinates.min(new TimestampComparator())
            .getTimestamp();
    LocalDateTime maxTimestamp = transactionstsWithRoundedCoordinates.max(new TimestampComparator())
            .getTimestamp();
    localDateTimeList.add(LocalDateTime.now());

    JavaPairRDD<String, List<Transaction>> trajectories = transactionstsWithRoundedCoordinates
            .mapToPair(new PairFunction<Transaction, String, List<Transaction>>() {
                @Override
                public Tuple2<String, List<Transaction>> call(Transaction s) {
                    List<Transaction> ss = new ArrayList<>();
                    ss.add(s);
                    return new Tuple2<>(s.getUserID(), ss);
                }
            }).reduceByKey(new Function2<List<Transaction>, List<Transaction>, List<Transaction>>() {
                @Override
                public List<Transaction> call(List<Transaction> i1, List<Transaction> i2) {
                    List<Transaction> ss = new ArrayList<>();
                    ss.addAll(i1);
                    ss.addAll(i2);
                    ss.sort(new TimestampComparator());
                    return ss;
                }
            });

    JavaRDD<List<Coordinate>> anTrajectories = trajectories
            .map(new Function<Tuple2<String, List<Transaction>>, List<Coordinate>>() {
                @Override
                public List<Coordinate> call(Tuple2<String, List<Transaction>> st) throws Exception {
                    List<Coordinate> c = new ArrayList<Coordinate>();
                    for (Transaction t : st._2) {
                        if (!c.contains(t.getRoundedCoordinate()))
                            c.add(t.getRoundedCoordinate());
                    }
                    return c;
                }
            }).persist(StorageLevel.MEMORY_ONLY());
    localDateTimeList.add(LocalDateTime.now());
    FPGrowth fpg = new FPGrowth().setMinSupport(0.01).setNumPartitions(2000);
    FPGrowthModel<Coordinate> model = fpg.run(anTrajectories);

    List<FPGrowthLocalModel> fpGrowths = new ArrayList<>();
    //.filter(coordinateFreqItemset -> coordinateFreqItemset.javaItems().size()>38)
    for (FPGrowth.FreqItemset<Coordinate> itemset : model.freqItemsets().toJavaRDD().collect()) {
        fpGrowths.add(new FPGrowthLocalModel(itemset.javaItems(), itemset.freq()));
    }

    try {
        HBaseUtils.addRecord("dataminingupc", "s" + minTimestamp + "e" + maxTimestamp, "data", "",
                objectMapper.writeValueAsString(fpGrowths));
    } catch (Exception e) {
        e.printStackTrace();
    }
    localDateTimeList.add(LocalDateTime.now());
    List<AssociationRulesLocalModel> associationRulesLocalModels = new ArrayList<>();
    double minConfidence = 0.8;
    for (AssociationRules.Rule<Coordinate> rule : model.generateAssociationRules(minConfidence).toJavaRDD()
            .collect()) {
        associationRulesLocalModels.add(new AssociationRulesLocalModel(rule.javaAntecedent(),
                rule.javaConsequent(), rule.confidence()));
    }
    try {
        HBaseUtils.addRecord("dataminingupc", "r" + minTimestamp + "e" + maxTimestamp, "data", "",
                objectMapper.writeValueAsString(associationRulesLocalModels));
    } catch (Exception e) {
        e.printStackTrace();
    }
    localDateTimeList.add(LocalDateTime.now());
    spark.stop();
    localDateTimeList.add(LocalDateTime.now());
    for (LocalDateTime localDateTime : localDateTimeList) {
        System.out.println(localDateTime);
    }
}

---

From source file:edu.upc.bip.batch.BatchMllibNEW.java

License:Apache License

public void run(JavaSparkContext spark) throws Exception {

    Configuration conf = null;
    conf = HBaseConfiguration.create();//w w  w  .  j  a va  2 s  .c om
    conf.clear();
    conf.set("hbase.zookeeper.quorum", "127.0.0.1");
    conf.set("hbase.zookeeper.property.clientPort", "2181");
    Scan s = new Scan();
    //    s.setStartRow(Bytes.toBytes("t2008-10-23 02:53:151"));
    //    s.setStopRow(Bytes.toBytes("t2008-10-23 02:53:161"));
    conf.set(TableInputFormat.SCAN, convertScanToString(s));

    conf.set(TableInputFormat.INPUT_TABLE, "transactions");
    JavaPairRDD<ImmutableBytesWritable, Result> hbaseRDD = spark.newAPIHadoopRDD(conf, TableInputFormat.class,
            ImmutableBytesWritable.class, Result.class);
    JavaRDD<Transaction> transactions = hbaseRDD
            .map(new Function<Tuple2<ImmutableBytesWritable, Result>, Transaction>() {
                public Transaction call(Tuple2<ImmutableBytesWritable, Result> tuple) throws Exception {

                    try {
                        String[] values = Bytes
                                .toString(tuple._2.getValue(Bytes.toBytes("data"), Bytes.toBytes("")))
                                .split(";");
                        return new Transaction(
                                new Coordinate(Double.parseDouble(values[0]), Double.parseDouble(values[1])),
                                LocalDateTime.parse(values[2], formatter), values[3]);
                    } catch (Exception e) {
                        e.printStackTrace();
                        return null;
                    }
                }
            });

    JavaRDD<Transaction> transactionstsWithRoundedCoordinates = BatchUtils.roundCoordinates(transactions, 1000)
            .cache();

    if (transactionstsWithRoundedCoordinates.count() > 0) {
        LocalDateTime minTimestamp = transactions.min(new TimestampComparator()).getTimestamp();
        LocalDateTime maxTimestamp = transactions.max(new TimestampComparator()).getTimestamp();

        JavaPairRDD<String, List<Transaction>> trajectories = transactionstsWithRoundedCoordinates
                .mapToPair(new PairFunction<Transaction, String, List<Transaction>>() {
                    @Override
                    public Tuple2<String, List<Transaction>> call(Transaction s) {
                        List<Transaction> ss = new ArrayList<>();
                        ss.add(s);
                        return new Tuple2<>(s.getUserID(), ss);
                    }
                }).reduceByKey(new Function2<List<Transaction>, List<Transaction>, List<Transaction>>() {
                    @Override
                    public List<Transaction> call(List<Transaction> i1, List<Transaction> i2) {
                        List<Transaction> ss = new ArrayList<>();
                        ss.addAll(i1);
                        ss.addAll(i2);
                        ss.sort(new TimestampComparator());
                        return ss;
                    }
                });

        JavaRDD<List<Coordinate>> anTrajectories = trajectories
                .map(new Function<Tuple2<String, List<Transaction>>, List<Coordinate>>() {
                    @Override
                    public List<Coordinate> call(Tuple2<String, List<Transaction>> st) throws Exception {
                        List<Coordinate> c = new ArrayList<Coordinate>();
                        for (Transaction t : st._2) {
                            if (!c.contains(t.getRoundedCoordinate()))
                                c.add(t.getRoundedCoordinate());
                        }
                        return c;
                    }
                }).persist(StorageLevel.MEMORY_ONLY());

        FPGrowth fpg = new FPGrowth().setMinSupport(0.05).setNumPartitions(100);
        FPGrowthModel<Coordinate> model = fpg.run(anTrajectories);

        List<FPGrowthLocalModel> fpGrowths = new ArrayList<>();
        for (FPGrowth.FreqItemset<Coordinate> itemset : model.freqItemsets().toJavaRDD().collect()) {
            fpGrowths.add(new FPGrowthLocalModel(itemset.javaItems(), itemset.freq()));
        }

        try {
            HBaseUtils.addRecord("datamining", "s" + minTimestamp + "e" + maxTimestamp, "data", "",
                    objectMapper.writeValueAsString(fpGrowths));
        } catch (Exception e) {
            e.printStackTrace();
        }

        List<AssociationRulesLocalModel> associationRulesLocalModels = new ArrayList<>();
        double minConfidence = 0.8;
        for (AssociationRules.Rule<Coordinate> rule : model.generateAssociationRules(minConfidence).toJavaRDD()
                .collect()) {
            associationRulesLocalModels.add(new AssociationRulesLocalModel(rule.javaAntecedent(),
                    rule.javaConsequent(), rule.confidence()));
        }
        try {
            HBaseUtils.addRecord("datamining", "r" + minTimestamp + "e" + maxTimestamp, "data", "",
                    objectMapper.writeValueAsString(associationRulesLocalModels));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

---

From source file:edu.upc.bip.batch.BatchMllibNEW.java

License:Apache License

public void run(JavaSparkContext spark, String startRow, String endRow) throws Exception {

    Configuration conf = null;
    conf = HBaseConfiguration.create();//from   w ww  .  j  a  va  2s .c  om
    conf.clear();
    conf.set("hbase.zookeeper.quorum", "127.0.0.1");
    conf.set("hbase.zookeeper.property.clientPort", "2181");
    Scan s = new Scan();
    s.setStartRow(Bytes.toBytes(startRow));
    s.setStopRow(Bytes.toBytes(endRow));
    conf.set(TableInputFormat.SCAN, convertScanToString(s));

    conf.set(TableInputFormat.INPUT_TABLE, "transactions");
    JavaPairRDD<ImmutableBytesWritable, Result> hbaseRDD = spark.newAPIHadoopRDD(conf, TableInputFormat.class,
            ImmutableBytesWritable.class, Result.class);
    JavaRDD<Transaction> transactions = hbaseRDD
            .map(new Function<Tuple2<ImmutableBytesWritable, Result>, Transaction>() {
                public Transaction call(Tuple2<ImmutableBytesWritable, Result> tuple) throws Exception {

                    try {
                        String[] values = Bytes
                                .toString(tuple._2.getValue(Bytes.toBytes("data"), Bytes.toBytes("")))
                                .split(";");
                        return new Transaction(
                                new Coordinate(Double.parseDouble(values[0]), Double.parseDouble(values[1])),
                                LocalDateTime.parse(values[2], formatter), values[3]);
                    } catch (Exception e) {
                        e.printStackTrace();
                        return null;
                    }
                }
            });
    JavaRDD<Transaction> transactionstsWithRoundedCoordinates = BatchUtils.roundCoordinates(transactions, 1000)
            .cache();

    if (transactionstsWithRoundedCoordinates.count() > 0) {

        LocalDateTime minTimestamp = transactions.min(new TimestampComparator()).getTimestamp();
        LocalDateTime maxTimestamp = transactions.max(new TimestampComparator()).getTimestamp();

        JavaPairRDD<String, List<Transaction>> trajectories = transactionstsWithRoundedCoordinates
                .mapToPair(new PairFunction<Transaction, String, List<Transaction>>() {
                    @Override
                    public Tuple2<String, List<Transaction>> call(Transaction s) {
                        List<Transaction> ss = new ArrayList<>();
                        ss.add(s);
                        return new Tuple2<>(s.getUserID(), ss);
                    }
                }).reduceByKey(new Function2<List<Transaction>, List<Transaction>, List<Transaction>>() {
                    @Override
                    public List<Transaction> call(List<Transaction> i1, List<Transaction> i2) {
                        List<Transaction> ss = new ArrayList<>();
                        ss.addAll(i1);
                        ss.addAll(i2);
                        ss.sort(new TimestampComparator());
                        return ss;
                    }
                });

        JavaRDD<List<Coordinate>> anTrajectories = trajectories
                .map(new Function<Tuple2<String, List<Transaction>>, List<Coordinate>>() {
                    @Override
                    public List<Coordinate> call(Tuple2<String, List<Transaction>> st) throws Exception {
                        List<Coordinate> c = new ArrayList<Coordinate>();
                        for (Transaction t : st._2) {
                            if (!c.contains(t.getRoundedCoordinate()))
                                c.add(t.getRoundedCoordinate());
                        }
                        return c;
                    }
                }).persist(StorageLevel.MEMORY_ONLY());

        FPGrowth fpg = new FPGrowth().setMinSupport(0.05).setNumPartitions(100);
        FPGrowthModel<Coordinate> model = fpg.run(anTrajectories);

        List<FPGrowthLocalModel> fpGrowths = new ArrayList<>();
        for (FPGrowth.FreqItemset<Coordinate> itemset : model.freqItemsets().toJavaRDD().collect()) {
            fpGrowths.add(new FPGrowthLocalModel(itemset.javaItems(), itemset.freq()));
        }

        try {
            HBaseUtils.addRecord("datamining", "s" + minTimestamp + "e" + maxTimestamp, "data", "",
                    objectMapper.writeValueAsString(fpGrowths));
        } catch (Exception e) {
            e.printStackTrace();
        }

        List<AssociationRulesLocalModel> associationRulesLocalModels = new ArrayList<>();
        double minConfidence = 0.8;
        for (AssociationRules.Rule<Coordinate> rule : model.generateAssociationRules(minConfidence).toJavaRDD()
                .collect()) {
            associationRulesLocalModels.add(new AssociationRulesLocalModel(rule.javaAntecedent(),
                    rule.javaConsequent(), rule.confidence()));
        }
        try {
            HBaseUtils.addRecord("datamining", "r" + minTimestamp + "e" + maxTimestamp, "data", "",
                    objectMapper.writeValueAsString(associationRulesLocalModels));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

---

From source file:edu.upc.bip.batch.BatchNEW.java

License:Apache License

public void run(JavaSparkContext spark) throws Exception {

    Configuration conf = null;
    conf = HBaseConfiguration.create();//  w  w w .ja  v a  2s. c om
    conf.clear();
    conf.set("hbase.zookeeper.quorum", "127.0.0.1");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    Scan s = new Scan();

    conf.set(TableInputFormat.SCAN, convertScanToString(s));

    conf.set(TableInputFormat.INPUT_TABLE, "transactions");
    JavaPairRDD<ImmutableBytesWritable, Result> hbaseRDD = spark.newAPIHadoopRDD(conf, TableInputFormat.class,
            ImmutableBytesWritable.class, Result.class);
    JavaRDD<Transaction> transactions = hbaseRDD
            .map(new Function<Tuple2<ImmutableBytesWritable, Result>, Transaction>() {
                public Transaction call(Tuple2<ImmutableBytesWritable, Result> tuple) throws Exception {

                    try {
                        String[] values = Bytes
                                .toString(tuple._2.getValue(Bytes.toBytes("data"), Bytes.toBytes("")))
                                .split(";");
                        return new Transaction(
                                new Coordinate(Double.parseDouble(values[0]), Double.parseDouble(values[1])),
                                LocalDateTime.parse(values[2], formatter), values[3]);
                    } catch (Exception e) {
                        e.printStackTrace();
                        return null;
                    }
                }
            });

    JavaRDD<Transaction> transactionstsWithRoundedCoordinates = BatchUtils.roundCoordinates(transactions, 1000)
            .cache();

    LocalDateTime minTimestamp = transactions.min(new TimestampComparator()).getTimestamp();
    LocalDateTime maxTimestamp = transactions.max(new TimestampComparator()).getTimestamp();
    long duration = minTimestamp.until(maxTimestamp, ChronoUnit.SECONDS);
    int maxDetail = (int) duration;
    long[] steps = { 5, 10, 15 };
    for (long step : steps) {
        long timeStep = step;
        for (int i = 0, j = 0; j < maxDetail; i++, j += timeStep) {
            LocalDateTime start = minTimestamp.plus(timeStep * i, ChronoUnit.SECONDS);
            LocalDateTime end = minTimestamp.plus(timeStep * (i + 1), ChronoUnit.SECONDS);
            JavaRDD<Transaction> measurementsFilteredByTime = BatchUtils
                    .filterByTime(transactionstsWithRoundedCoordinates, start, end);
            JavaPairRDD<Coordinate, Integer> counts = BatchUtils.countPerGridBox(measurementsFilteredByTime);

            BatchUtils.writeJsonToHbase(counts, tablename, familys[0], step + "s" + start.toString(),
                    objectMapper);
        }

    }
}

---

From source file:edu.upc.bip.batch.BatchNEW.java

License:Apache License

public void run(JavaSparkContext spark, String startRow, String endRow) throws Exception {

    Configuration conf = null;

    conf = HBaseConfiguration.create();//  w  ww . ja  v  a2s  . c  om
    conf.clear();
    conf.set("hbase.zookeeper.quorum", "127.0.0.1");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    Scan s = new Scan();
    s.setStartRow(Bytes.toBytes(startRow));
    s.setStopRow(Bytes.toBytes(endRow));
    conf.set(TableInputFormat.SCAN, convertScanToString(s));

    conf.set(TableInputFormat.INPUT_TABLE, "transactions");
    JavaPairRDD<ImmutableBytesWritable, Result> hbaseRDD = spark.newAPIHadoopRDD(conf, TableInputFormat.class,
            ImmutableBytesWritable.class, Result.class);
    JavaRDD<Transaction> transactions = hbaseRDD
            .map(new Function<Tuple2<ImmutableBytesWritable, Result>, Transaction>() {
                public Transaction call(Tuple2<ImmutableBytesWritable, Result> tuple) throws Exception {

                    try {
                        String[] values = Bytes
                                .toString(tuple._2.getValue(Bytes.toBytes("data"), Bytes.toBytes("")))
                                .split(";");
                        return new Transaction(
                                new Coordinate(Double.parseDouble(values[0]), Double.parseDouble(values[1])),
                                LocalDateTime.parse(values[2], formatter), values[3]);
                    } catch (Exception e) {
                        e.printStackTrace();
                        return null;
                    }
                }
            });

    JavaRDD<Transaction> transactionstsWithRoundedCoordinates = BatchUtils.roundCoordinates(transactions, 1000)
            .cache();

    if (transactionstsWithRoundedCoordinates.count() > 0) {
        LocalDateTime minTimestamp = transactions.min(new TimestampComparator()).getTimestamp();
        LocalDateTime maxTimestamp = transactions.max(new TimestampComparator()).getTimestamp();
        long duration = minTimestamp.until(maxTimestamp, ChronoUnit.SECONDS);
        int maxDetail = (int) duration;
        long[] steps = { 5, 10, 15 };
        for (long step : steps) {
            long timeStep = step;
            for (int i = 0, j = 0; j < maxDetail; i++, j += timeStep) {
                LocalDateTime start = minTimestamp.plus(timeStep * i, ChronoUnit.SECONDS);
                LocalDateTime end = minTimestamp.plus(timeStep * (i + 1), ChronoUnit.SECONDS);
                JavaRDD<Transaction> measurementsFilteredByTime = BatchUtils
                        .filterByTime(transactionstsWithRoundedCoordinates, start, end);
                JavaPairRDD<Coordinate, Integer> counts = BatchUtils
                        .countPerGridBox(measurementsFilteredByTime);

                BatchUtils.writeJsonToHbase(counts, tablename, familys[0], step + "s" + start.toString(),
                        objectMapper);
            }

        }
    }
}

---

From source file:hbaseweb.controllers.CreateTableController.java

@RequestMapping(value = "/deletetable/{tablename}", method = RequestMethod.GET)
public String delete(@PathVariable(value = "tablename") String tablename, ModelMap map) {

    try {/*w w w.  j a v  a 2s  .  c om*/

        Configuration config = HBaseConfiguration.create();
        config.clear();
        config.set("hbase.zookeeper.quorum", "192.168.10.50");
        config.set("hbase.zookeeper.property.clientPort", "2181");
        HBaseAdmin.checkHBaseAvailable(config);
        Connection connection = ConnectionFactory.createConnection(config);
        Admin admin = connection.getAdmin();
        TableName tableName = TableName.valueOf(tablename);
        HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
        admin.deleteTable(tableName);
    } catch (Exception ce) {
        ce.printStackTrace();
        map.put("error", ce);
        return "delete";
    }

    return "delete";
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »