Example usage for org.apache.hadoop.conf Configuration Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration() 

Source Link

Document

A new configuration.

Usage

From source file:ImageDuplicatesRemover.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    //This is the line that makes the hadoop run locally
    //conf.set("mapred.job.tracker", "local");

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: wordcount <in> <out>");
        System.exit(2);//from  www  .  j  a  v a 2 s .c o m
    }
    Job job = new Job(conf, "image dups remover");
    job.setJarByClass(ImageDuplicatesRemover.class);
    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setMapperClass(ImageMd5Mapper.class);
    job.setReducerClass(ImageDupsReducer.class);
    //job.setNumReduceTasks(2);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);

}

---

From source file:FlinkBootstrap.java

License:Apache License

public static void main(String[] args) throws Exception {

    if (args.length != 2) {
        throw new IllegalArgumentException(
                "Provide `TaskManager` or `JobManager` parameter with config folder");
    }//from w  ww.  j  a  va 2s.  c om

    //Load Hadoop S3 wrapper classes, due to ClassNotFound Exception without
    Class.forName("org.apache.flink.runtime.fs.hdfs.HadoopFileSystem");
    Class.forName("org.apache.hadoop.fs.s3a.S3AFileSystem");

    //Verify s3 is accessible
    Configuration conf = new Configuration();
    conf.addResource(new Path("config/hadoop/core-site.xml"));
    conf.addResource(new Path("config/hadoop/hdfs-site.xml"));
    FileSystem fs = FileSystem.get(conf);
    fs.listStatus(new Path("s3://dir"));

    if (args[0].equals("TaskManager")) {
        TaskManager.main(new String[] { "--configDir", args[1], });
    } else if (args[0].equals("JobManager")) {
        JobManager.main(new String[] { "--configDir", args[1], "--executionMode", "cluster", });
    } else {
        throw new IllegalArgumentException("Unknown parameter `" + args[0] + "`");
    }
}

---

From source file:SBP.java

License:Apache License

public static void main(final String[] args) throws Exception {
    final int result = ToolRunner.run(new Configuration(), new SBP(), args);

    System.exit(result);/*  ww  w.j  av  a2 s.c om*/
}

---

From source file:ReverseIndexer.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: ReverseIndexer <output> <input file(s)>");
        System.exit(2);/* ww w .java 2 s.c om*/
    }
    Job job = new Job(conf, "reverse indexer");
    job.setJarByClass(ReverseIndexer.class);
    job.setMapperClass(IndexerMapper.class);
    job.setReducerClass(IndexerReducer.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LineRecWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 1; i < otherArgs.length; i++) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[0]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Authset.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: wordcount <in> [<in>...] <out>");
        System.exit(2);/*from w  w  w .j a va2s  . co m*/
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(Authset.class);
    job.setMapperClass(TokenizerMapper.class);
    //job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(NullWritable.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(10);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    //DistributedCache.addCacheFile(new Path(otherArgs[0]).toUri(),
    //job.getConfiguration());

    //DistributedCache.setLocalFiles(job.getConfiguration(), otherArgs[0]);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:ConfTest.java

License:Open Source License

public static void main(String[] args) {
    Configuration conf = new Configuration();
    conf.addResource(new Path("d:\\test\\a.xml"));
    System.out.println(conf.get("aaa"));
}

---

From source file:RHBlockToKeyRangeIndex.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: rhblockindex <in> <out>");
        System.exit(2);//from ww  w .  j a va 2 s .co  m
    }
    Job job = new Job(conf, "rhblockindex");
    job.setJarByClass(RHBlockToKeyRangeIndex.class);
    job.setMapperClass(RMapper.class);
    job.setCombinerClass(RReducer.class);
    job.setReducerClass(RReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:DumpRecordsExtended.java

License:Apache License

/**
 * Runs this tool./*  w w  w .  j  a va  2s .c o m*/
 */
@SuppressWarnings({ "static-access" })
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();

    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        return -1;
    }

    if (!cmdline.hasOption(INPUT) || !cmdline.hasOption(OUTPUT)) {
        System.out.println("args: " + Arrays.toString(args));
        HelpFormatter formatter = new HelpFormatter();
        formatter.setWidth(120);
        formatter.printHelp(this.getClass().getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        return -1;
    }

    String inputPath = cmdline.getOptionValue(INPUT);
    String outputPath = cmdline.getOptionValue(OUTPUT);

    LOG.info("Tool name: " + DumpRecordsExtended.class.getSimpleName());
    LOG.info(" - input: " + inputPath);
    LOG.info(" - output: " + outputPath);

    Configuration conf = new Configuration();
    conf.setInt("mapred.min.split.size", 1024 * 1024 * 1024);

    Job job = Job.getInstance(conf);
    job.setJobName(DumpRecordsExtended.class.getSimpleName());
    job.setJarByClass(DumpRecordsExtended.class);

    job.setNumReduceTasks(0);

    FileInputFormat.addInputPath(job, new Path(inputPath));
    FileOutputFormat.setOutputPath(job, new Path(outputPath));

    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(PageRankNode.class);

    // Delete the output directory if it exists already.
    FileSystem.get(conf).delete(new Path(outputPath), true);

    job.waitForCompletion(true);

    return 0;
}

---

From source file:LookupQuery.java

License:Apache License

public static void initQuery(String[] args) throws IOException {
    indexPath = args[0];// ww w. ja v  a 2  s . c  o m
    collectionPath = args[1];

    config = new Configuration();
    fs = FileSystem.get(config);
    reader = new MapFile.Reader(fs, indexPath, config);

    key = new Text();
    value = new ArrayListWritable<PairOfInts>();
    areThereMoreLookups = true;
    query = "";
    Qvalue = 0;

}

---

From source file:TCoffee.java

/**
 * @param args the command line arguments
 *//*from w  ww . j  a  v  a 2  s .  c  o  m*/
public static void main(String[] args) throws IOException {

    HBaseConfiguration hconfig = new HBaseConfiguration(new Configuration());
    System.out.println("Connecting...");
    HBaseAdmin hbase_admin = new HBaseAdmin(hconfig);
    if (!hbase_admin.tableExists("TCoffee")) {
        HTableDescriptor htable = new HTableDescriptor("TCoffee");
        htable.addFamily(new HColumnDescriptor("key"));
        htable.addFamily(new HColumnDescriptor("value"));
        System.out.println("Creating Table...");
        hbase_admin.createTable(htable);

    } else {
        hbase_admin.disableTable("TCoffee");
        hbase_admin.deleteTable("TCoffee");
        HTableDescriptor htable = new HTableDescriptor("TCoffee");
        htable.addFamily(new HColumnDescriptor("key"));
        htable.addFamily(new HColumnDescriptor("value"));
        System.out.println("Delete and creating table...");
        hbase_admin.createTable(htable);
    }
    HTable hTable = new HTable(hconfig, "TCoffee");
    BufferedReader reader = null;
    int count = 0;
    File file = new File("full_lib.txt");
    reader = new BufferedReader(new FileReader(file));
    String line = null;
    String[] seq = null;
    String[] split = null;
    boolean cond = false;
    while ((line = reader.readLine()) != null) {
        if (line.indexOf('#') >= 0) {
            line = line.replace("#", "");
            seq = line.split("\\s+");
            cond = true;
        } else if (cond && line.indexOf('!') >= 0) {
            break;
        } else if (cond) {
            split = line.split("\\s+");
            Put put = new Put(Bytes.toBytes("row" + count));
            put.addColumn(Bytes.toBytes("key"), Bytes.toBytes("seq1"), Bytes.toBytes(seq[0]));
            put.addColumn(Bytes.toBytes("key"), Bytes.toBytes("res1"), Bytes.toBytes(split[1]));
            put.addColumn(Bytes.toBytes("value"), Bytes.toBytes("seq2"), Bytes.toBytes(seq[1]));
            put.addColumn(Bytes.toBytes("value"), Bytes.toBytes("res2"), Bytes.toBytes(split[2]));
            put.addColumn(Bytes.toBytes("value"), Bytes.toBytes("pes"), Bytes.toBytes(split[3]));
            hTable.put(put);
            count++;
            put = new Put(Bytes.toBytes("row" + count));
            put.addColumn(Bytes.toBytes("key"), Bytes.toBytes("seq1"), Bytes.toBytes(seq[1]));
            put.addColumn(Bytes.toBytes("key"), Bytes.toBytes("res1"), Bytes.toBytes(split[2]));
            put.addColumn(Bytes.toBytes("value"), Bytes.toBytes("seq2"), Bytes.toBytes(seq[0]));
            put.addColumn(Bytes.toBytes("value"), Bytes.toBytes("res2"), Bytes.toBytes(split[1]));
            put.addColumn(Bytes.toBytes("value"), Bytes.toBytes("pes"), Bytes.toBytes(split[3]));
            hTable.put(put);
            count++;
        }
    }
    hTable.close();
    System.out.println("Done!");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »