Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:gov.tva.sparky.util.indexer.IndexingAgent.java

public static void DebugSparkyConfig() {

    Configuration conf = new Configuration(false);
    conf.addResource("hadoop-default.xml");
    conf.addResource("sparky-site.xml");

    System.out.println("Debugging Sparky Configuration ------ ");
    System.out.println("> sparky.http.port: " + conf.get("sparky.http.port"));
    System.out.println("> fs.default.name: " + conf.get("fs.default.name"));

}

---

From source file:gr.ntua.h2rdf.byteImport.HexastoreBulkImport.java

License:Open Source License

public Job createSubmittableJob(String[] args) {
    TABLE_NAME = args[1];/*  w w w.  j  av  a 2  s  .co  m*/
    Job job = null;
    try {
        Configuration conf = new Configuration();
        conf.addResource("hbase-default.xml");
        conf.addResource("hbase-site.xml");
        job = new Job(conf, NAME);
        job.setJarByClass(HexastoreBulkImport.class);
        job.setMapperClass(TotalOrderPrep.Map.class);
        job.setReducerClass(Reduce.class);//sampler.HamaReducer.class);
        job.setCombinerClass(Combiner.class);
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(ImmutableBytesWritable.class);
        job.setPartitionerClass(TotalOrderPartitioner.class);
        //TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), new Path("/user/npapa/"+regions+"partitions/part-r-00000"));
        TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), new Path("partitions/part-r-00000"));
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(HFileOutputFormat.class);
        Path out = new Path("out");
        FileOutputFormat.setOutputPath(job, out);
        FileSystem fs;
        try {
            fs = FileSystem.get(conf);
            if (fs.exists(out)) {
                fs.delete(out, true);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

        // c.addResource(new Path("/0/arcomemDB/hadoop-0.20.2-cdh3u3/conf/hbase-site.xml"));
        HBaseAdmin hadmin = new HBaseAdmin(conf);
        HTableDescriptor desc = new HTableDescriptor(TABLE_NAME + "_stats");
        HColumnDescriptor family = new HColumnDescriptor("size");
        desc.addFamily(family);
        conf.setInt("zookeeper.session.timeout", 600000);
        if (hadmin.tableExists(TABLE_NAME + "_stats")) {
            //hadmin.disableTable(TABLE_NAME+"_stats");
            //hadmin.deleteTable(TABLE_NAME+"_stats");
        } else {
            hadmin.createTable(desc);
        }

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        //job.getConfiguration().setInt("mapred.map.tasks", 18);
        job.getConfiguration().set("h2rdf.tableName", TABLE_NAME);
        job.getConfiguration().setInt("mapred.reduce.tasks", (int) TotalOrderPrep.regions);
        job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
        job.getConfiguration().setBoolean("mapred.reduce.tasks.speculative.execution", false);
        job.getConfiguration().setInt("io.sort.mb", 100);
        job.getConfiguration().setInt("io.file.buffer.size", 131072);
        job.getConfiguration().setInt("mapred.job.reuse.jvm.num.tasks", -1);
        //job.getConfiguration().setInt("hbase.hregion.max.filesize", 67108864);
        job.getConfiguration().setInt("hbase.hregion.max.filesize", 33554432);
        //job.getConfiguration().setInt("io.sort.mb", 100);

    } catch (IOException e2) {
        e2.printStackTrace();
    }

    return job;
}

---

From source file:gr.ntua.h2rdf.loadTriples.TranslateAndImport.java

License:Apache License

private void loadHFiles() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.addResource("hbase-default.xml");
    conf.addResource("hbase-site.xml");
    HBaseAdmin hadmin = new HBaseAdmin(conf);
    Path hfofDir = new Path("out/I");
    FileSystem fs = hfofDir.getFileSystem(conf);
    //if (!fs.exists(hfofDir)) {
    //  throw new FileNotFoundException("HFileOutputFormat dir " +
    //      hfofDir + " not found");
    //}/*from   www  . j a va2 s .c om*/
    // FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
    //if (familyDirStatuses == null) {
    //  throw new FileNotFoundException("No families found in " + hfofDir);
    //}
    int length = 0;
    byte[][] splits = new byte[18000][];
    Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(hfofDir));
    for (Path hfile : hfiles) {
        if (hfile.getName().startsWith("_"))
            continue;

        HFile.Reader hfr = HFile.createReader(fs, hfile, new CacheConfig(conf));
        //HFile.Reader hfr =    new HFile.Reader(fs, hfile, null, false);
        final byte[] first;
        try {
            hfr.loadFileInfo();
            first = hfr.getFirstRowKey();
        } finally {
            hfr.close();
        }
        //System.out.println("out/I/"+hfile.getName()+" \t "+Bytes.toStringBinary(first));
        splits[length] = first.clone();
        length++;
    }
    byte[][] splits1 = new byte[length][];

    for (int i = 0; i < splits1.length; i++) {
        splits1[i] = splits[i];
    }
    Arrays.sort(splits1, Bytes.BYTES_COMPARATOR);

    HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);

    HColumnDescriptor family = new HColumnDescriptor("I");
    family.setCompressionType(Algorithm.SNAPPY);
    desc.addFamily(family);
    family = new HColumnDescriptor("S");
    family.setCompressionType(Algorithm.SNAPPY);
    desc.addFamily(family);
    family = new HColumnDescriptor("T");
    family.setCompressionType(Algorithm.SNAPPY);
    desc.addFamily(family);
    //family= new HColumnDescriptor("C");
    //desc.addFamily(family); 
    //for (int i = 0; i < splits.length; i++) {
    //   System.out.println(Bytes.toStringBinary(splits[i]));
    //}
    conf.setInt("zookeeper.session.timeout", 600000);
    if (hadmin.tableExists(TABLE_NAME)) {
        //hadmin.disableTable(TABLE_NAME);
        //hadmin.deleteTable(TABLE_NAME);
    } else {
        hadmin.createTable(desc, splits1);
    }
    //hadmin.createTable(desc);
    String[] args1 = new String[2];
    args1[0] = "out";
    args1[1] = TABLE_NAME;
    //args1[1]="new2";

    ToolRunner.run(new LoadIncrementalHFiles(HBaseConfiguration.create()), args1);

}

---

From source file:gws.WordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    conf.addResource(new Path("/home/ucas/bigdata/hadoop-2.6.2/etc/hadoop/core-site.xml"));
    conf.addResource(new Path("/home/ucas/bigdata/hadoop-2.6.2/etc/hadoop/hdfs-site.xml"));
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: wordcount <in> [<in>...] <out>");
        System.exit(2);/*from  w  w  w  . j ava  2 s .  c  o m*/
    }

    Job job = Job.getInstance(conf, "word count");

    job.setJarByClass(WordCount.class);

    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumCombiner.class);
    job.setReducerClass(IntSumReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    // add the input paths as given by command line
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }

    // add the output path as given by the command line
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:hadoopdemo.Hadoop.java

private void addFolderButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_addFolderButtonActionPerformed
    try {/*from   ww w.ja  v  a2s  . c om*/

        String newFolder = newFolderTextField.getText();
        System.out.println(newFolder);

        Configuration conf = new Configuration();
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/core-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/hdfs-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/mapred-site.xml"));

        FileSystem fileSystem = FileSystem.get(conf);
        fileSystem.mkdirs(new Path(newFolder));
    } catch (IOException ex) {
        Logger.getLogger(Hadoop.class.getName()).log(Level.SEVERE, null, ex);
    }
}

---

From source file:hadoopdemo.Hadoop.java

private void deleteFolderButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteFolderButtonActionPerformed
    try {//from   w ww. ja  v a  2 s .c o  m
        String deleteFolder = deleteFolderTextField.getText();
        System.out.println(deleteFolder);

        Configuration conf = new Configuration();
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/core-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/hdfs-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/mapred-site.xml"));

        FileSystem fileSystem = FileSystem.get(conf);
        fileSystem.delete(new Path(deleteFolder));
    } catch (IOException ex) {
        Logger.getLogger(Hadoop.class.getName()).log(Level.SEVERE, null, ex);
    }
}

---

From source file:hadoopdemo.Hadoop.java

private void addFileButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_addFileButtonActionPerformed
    try {//  w  w  w.j ava2  s .c o m
        String addFile = addFileTextField.getText();
        String hadoopPath = hadoopPathTextField.getText();
        System.out.println(addFile);

        Configuration conf = new Configuration();
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/core-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/hdfs-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/mapred-site.xml"));

        FileSystem fileSystem = FileSystem.get(conf);
        fileSystem.copyFromLocalFile(new Path(addFile), new Path(hadoopPath));
    } catch (IOException ex) {
        Logger.getLogger(Hadoop.class.getName()).log(Level.SEVERE, null, ex);
    }
}

---

From source file:hadoopdemo.Hadoop.java

private void renameButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_renameButtonActionPerformed
    try {/*from w  w w  .j av a 2 s  . c om*/
        String oldName = oldNameTextField.getText();
        String newName = newNameTextFIeld.getText();

        Configuration conf = new Configuration();
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/core-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/hdfs-site.xml"));
        conf.addResource(new Path("/home/ado/hadoop-2.7.3/etc/hadoop/mapred-site.xml"));

        FileSystem fileSystem = FileSystem.get(conf);
        fileSystem.rename(new Path(oldName), new Path(newName));
    } catch (IOException ex) {
        Logger.getLogger(Hadoop.class.getName()).log(Level.SEVERE, null, ex);
    }
}

---

From source file:hitune.analysis.mapreduce.AnalysisConfiguration.java

License:Apache License

/**
 * Load the current configuration folder recursively
 * /*w ww  .ja v a2 s . c  o m*/
 * if existing list.xml, go step forward
 * 
 * @param folder
 */
public void LoadConfiguration(String folder, Configuration conf) {
    log.debug("scan folder: " + folder);
    File listfile = new File(folder + "/list.xml");

    File conffile = new File(folder + "/conf.xml");
    Configuration newconf = new Configuration(conf);
    newconf.addResource(new Path(conffile.getAbsolutePath()));

    try {

        if (listfile.exists()) {
            Configuration tempconf = new Configuration(newconf);
            tempconf.addResource(new Path(listfile.getAbsolutePath()));

            Configuration _conf = new Configuration(false);
            _conf.addResource(new Path(listfile.getAbsolutePath()));
            Iterator<Map.Entry<String, String>> iter = _conf.iterator();
            while (iter.hasNext()) {
                Map.Entry<String, String> pairs = (Map.Entry<String, String>) iter.next();
                String key = pairs.getKey();
                LoadConfiguration(tempconf.get(key), newconf);
            }
        } else {
            String datasource = newconf.get(AnalysisProcessorConfiguration.datasource);
            log.debug("datasource: " + datasource);
            List<Configuration> cflist = null;
            if (conflist.containsKey(datasource)) {
                cflist = conflist.get(datasource);
            } else {
                cflist = new ArrayList<Configuration>();
            }
            cflist.add(newconf);
            log.debug("add conf: " + newconf);
            conflist.put(datasource, cflist);
        }
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

---

From source file:hydrograph.engine.cascading.assembly.InputFileHiveParquetAssembly.java

License:Apache License

@Override
protected void prepareScheme() {
    LOG.debug("Applying HiveParquetScheme to read data from Hive");

    // HiveParquetTableDescriptor is developed specifically for handling
    // Parquet File format with Hive. Hence, the object of table descriptor
    // is created in its respective assembly and not in its base class.

    Configuration conf = new Configuration();
    conf.addResource(new Path(HiveConfigurationMapping.getHiveConf("path_to_hive_site_xml")));

    Factory factory = new Factory(conf);
    HiveTableDescriptor tb = factory.newInstance(inputFileHiveParquetEntity.getDatabaseName(),
            inputFileHiveParquetEntity.getTableName());

    tableDesc = new HiveParquetTableDescriptor(tb.getDatabaseName(), tb.getTableName(), tb.getColumnNames(),
            tb.getColumnTypes(), tb.getPartitionKeys(), getHiveExternalTableLocationPath());
    scheme = new HiveParquetScheme(tableDesc);
    scheme.setSourceFields(tableDesc.toFields());
    scheme.setSinkFields(tableDesc.toFields());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »