Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:Distinct.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: Distinct <in> <out>");
        System.exit(2);/*from  ww  w .j a va 2 s .  c  om*/
    }
    Job job = Job.getInstance(conf, "distinct1");
    job.setJarByClass(Distinct.class);
    job.setMapperClass(TokenizerMapper1.class);
    job.setReducerClass(Reducer1.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(tempDir));
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.waitForCompletion(true);

    Configuration conf2 = new Configuration();
    Job job2 = Job.getInstance(conf2, "distinct2");
    job2.setJarByClass(Distinct.class);
    job2.setMapperClass(TokenizerMapper2.class);
    job2.setReducerClass(Reducer2.class);
    FileInputFormat.addInputPath(job2, new Path(tempDir));
    FileOutputFormat.setOutputPath(job2, new Path(otherArgs[1]));
    job2.setOutputKeyClass(Text.class);
    job2.setOutputValueClass(IntWritable.class);
    System.exit(job2.waitForCompletion(true) ? 0 : 1);
}

From source file:WriteFDFFixedLengthRecord.java

License:Open Source License

public static void main(String[] args) throws Exception {
    FormatDataFile fdf = new FormatDataFile(new Configuration());
    String fileName = "/indextest/testfile1";
    FileSystem.get(new Configuration()).delete(new Path(fileName), true);
    Head head = new Head();
    FieldMap fieldMap = new FieldMap();
    fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
    fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Byte, (short) 1));
    head.setFieldMap(fieldMap);// www. ja v a 2  s  . co m
    head.setPrimaryIndex((short) 0);

    fdf.create(fileName, head);
    for (int i = 0; i < 200; i++) {
        Record record = new Record(2);
        record.addValue(new FieldValue((byte) i, (short) 0));
        record.addValue(new FieldValue((byte) i, (short) 1));
        fdf.addRecord(record);
    }
    fdf.close();
}

From source file:DocToSeq.java

License:Apache License

public static void main(String args[]) throws Exception {
    if (args.length != 2) {
        System.err.println("Arguments: [input tsv file] [output sequence file]");
        return;//w w w  .  j  ava 2  s  .  com
    }
    String inputFileName = args[0];
    String outputDirName = args[1];
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(configuration);
    Writer writer = new SequenceFile.Writer(fs, configuration, new Path(outputDirName + "/chunk-0"), Text.class,
            Text.class);

    int count = 0;
    BufferedReader reader = new BufferedReader(new FileReader(inputFileName));
    Text key = new Text();
    Text value = new Text();
    while (true) {
        String line = reader.readLine();
        if (line == null) {
            break;
        }
        String[] tokens = line.split("\t", 3);
        if (tokens.length != 3) {
            System.out.println("Skip line: " + line);
            continue;
        }
        String category = tokens[0];
        String id = tokens[1];
        String message = tokens[2];
        key.set("/" + category + "/" + id);
        value.set(message);
        writer.append(key, value);
        count++;
    }
    reader.close();
    writer.close();
    System.out.println("Wrote " + count + " entries.");
}

From source file:BigBWA.java

License:Open Source License

public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub
    int res = ToolRunner.run(new Configuration(), new BigBWA(), args);
    System.exit(res);//from w ww  .  ja  v a2s  .  c  o  m

}

From source file:TestOpenFile.java

License:Open Source License

static void getRecordByLine(String filename, int line) throws Exception {
    Configuration conf = new Configuration();
    FormatDataFile fd2 = new FormatDataFile(conf);
    fd2.open(filename);/*from w  ww  .j  a  v  a 2  s .c o m*/
    Record record = fd2.getRecordByLine(-1);
    if (record != null) {
        System.out.println("should get null, line -1");
        fd2.close();
        return;
    }

    Record re = fd2.getRecordByLine(line);
    ArrayList<FieldValue> vals = re.fieldValues();
    for (int i = 0; i < vals.size(); i++) {
        System.out.print(vals.get(i).toObject() + "\t");
    }
    System.out.println();

}

From source file:TestOpenFile.java

License:Open Source License

static void writeFile(String filename, int recnum) throws Exception {

    FieldMap fieldMap = new FieldMap();
    fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1));
    fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 3));
    fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 5));
    fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 7));
    fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 9));
    fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 11));
    fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 13));

    head.setFieldMap(fieldMap);//from   w w w. ja v  a  2 s .  com

    Configuration conf = new Configuration();
    FormatDataFile fd = new FormatDataFile(conf);
    fd.create(filename, head);

    for (int i = 0; i < recnum; i++) {
        Record record = new Record((short) 7);
        record.addValue(new FieldValue((byte) (1 + i), (short) 0));
        record.addValue(new FieldValue((short) (2 + i), (short) 1));
        record.addValue(new FieldValue((int) (3 + i), (short) 2));
        record.addValue(new FieldValue((long) (4 + i), (short) 3));
        record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
        record.addValue(new FieldValue((double) (6.6 + i), (short) 5));
        record.addValue(new FieldValue("hello konten" + i, (short) 6));
        fd.addRecord(record);
    }

    fd.close();
}

From source file:WordCountA.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: wordcount <in> [<in>...] <out>");
        System.exit(2);/*from   ww w .  j  ava 2  s . c o m*/
    }
    Job job = Job.getInstance(conf, "word count");
    job.setJarByClass(WordCountA.class);
    job.setMapperClass(TokenizerMapper.class);
    // Disable the combiner
    // job.setCombinerClass(IntSumReducer.class);
    // Setup the Partitioner
    job.setPartitionerClass(Letterpartitioner.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:$.WordCount.java

License:Open Source License

public static void main(String[] args) throws Exception {
        ToolRunner.run(new Configuration(), new WordCount(), args);
    }

From source file:accesslog2.Accesslog2.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path inputPath = new Path(args[0]);
    Path outputPath = new Path(args[1]);
    Job job = Job.getInstance(conf, "Accesslog2");
    job.setJarByClass(Accesslog2.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, inputPath);
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:accesslog3.Accesslog3.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path inputPath = new Path(args[0]);
    Path outputPath = new Path(args[1]);
    Job job = Job.getInstance(conf, "Accesslog3");
    job.setJarByClass(Accesslog3.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, inputPath);
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}