Example usage for org.apache.hadoop.conf Configuration Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration() 

Source Link

Document

A new configuration.

Usage

From source file:BMTColumnLoader.java

License:Apache License

public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new BMTColumnLoader(), args);
    System.exit(res);//www .ja va 2s.  com
}

---

From source file:ReadFDFPerformance.java

License:Open Source License

static void readseqfdf(String filename, int num) throws Exception {
    Configuration conf = new Configuration();
    FormatDataFile fd2 = new FormatDataFile(conf);
    fd2.open(filename);//from ww  w  . ja va2s. c  o m

    for (int i = 0; i < num; i++) {
        fd2.getRecordByLine(i);
    }

}

---

From source file:ReadFDFPerformance.java

License:Open Source License

static void readranfdf(String filename, int num, int size) throws Exception {
    Random r = new Random();
    Configuration conf = new Configuration();
    FormatDataFile fd2 = new FormatDataFile(conf);
    fd2.open(filename);// ww w .  j  ava 2 s . co  m

    for (int i = 0; i < num; i++) {
        fd2.getRecordByLine(r.nextInt((int) size));
    }
}

---

From source file:FormatStroageStabilityTest.java

License:Open Source License

public static void doWrite(int count, boolean var, byte compress) {
    try {/*from w w  w. j  a v a2s.  c  o m*/
        long begin = System.currentTimeMillis();

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        if (var) {
            fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        }

        Head head = new Head();
        head.setFieldMap(fieldMap);

        if (compress == 1) {
            head.setCompress((byte) 1);
            head.setCompressStyle(ConstVar.LZOCompress);
        }

        String fileName = "MR_input/testMassRecord";
        if (var) {
            fileName += "_var";
        }

        Configuration conf = new Configuration();
        conf.setInt("dfs.replication", 1);
        FormatDataFile fd = new FormatDataFile(conf);

        fd.create(fileName, head);

        File file1 = new File("testWriteMassRecord.result");
        FileOutputStream out1 = new FileOutputStream(file1);

        short fieldNum = 6;
        if (var) {
            fieldNum = 7;
        }

        for (int i = 0; i < count; i++) {
            Record record = new Record(fieldNum);

            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));

            /*
            record.addValue(new FieldValue((byte)(1), (short)0));
                record.addValue(new FieldValue((short)(2), (short)1));
                record.addValue(new FieldValue((int)(3), (short)2));
                record.addValue(new FieldValue((long)(4), (short)3));
                record.addValue(new FieldValue((float)(5.5), (short)4));
                record.addValue(new FieldValue((double)(6.6), (short)5));
                */
            if (var) {
                record.addValue(new FieldValue("hello konten" + i, (short) 6));
            }

            fd.addRecord(record);

            if (i % (1000 * 10000) == 0) {
                String string = "write " + i + " record, delay:" + ((System.currentTimeMillis() - begin) / 1000)
                        + " s . file size:" + fd.getFileLen() + "\n";
                out1.write(string.getBytes());
            }
        }

        fd.close();

        long end = System.currentTimeMillis();

        String string = "write " + count + " record over, delay: " + ((end - begin) / 1000) + " s . file size:"
                + fd.getFileLen() + "\n";
        out1.write(string.getBytes());
        out1.close();
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

---

From source file:FormatStroageStabilityTest.java

License:Open Source License

static void doReadRand(int count, boolean var) {
    try {/*from w w w  .  j av a  2  s .co m*/
        String fileName = "MR_input/testMassRecord";
        if (var) {
            fileName += "_var";
        }

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setOptimize(true);
        fd.open(fileName);

        File file = new File("testReadMassRecord.result");
        FileOutputStream out = new FileOutputStream(file);

        long begin = System.currentTimeMillis();

        int totalCount = 100000000;
        Record valueRecord = new Record();
        for (int i = 0; i < count; i++) {
            int rand = (int) (Math.random() * totalCount);

            Record record = fd.getRecordByLine(rand, valueRecord);
            if (record == null) {
                String string = "record no:" + rand + " return null";
                out.write(string.getBytes());

                continue;
            }

            Error error = new Error();
            /*if(!judgeRecord(record, i, error))
            {
               String string = "record no:"+rand + " value error:"+error.msg+"\n";
               out.write(string.getBytes());
            }*/

            if (i % (10 * 10000) == 0) {
                String string = "read rand " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                out.write(string.getBytes());
            }
        }
        long end = System.currentTimeMillis();
        String string = "Read Rand over, count:" + count + ",totalCount:" + totalCount + ", delay:"
                + (long) ((end - begin) / 1000) + " s";
        out.write(string.getBytes());

        out.close();
        System.out.println(string);
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

---

From source file:FormatStroageStabilityTest.java

License:Open Source License

static void doReadSeq(int count, boolean var, byte compress) {
    int i = 0;// w w w. j  a  va  2s. c om
    try {
        String fileName = "MR_input/testMassRecord";
        if (var) {
            fileName += "_var";
        }

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setOptimize(true);
        fd.open(fileName);

        File file = new File("testReadMassRecord.result");
        FileOutputStream out = new FileOutputStream(file);

        long begin = System.currentTimeMillis();

        Record record = new Record();
        for (i = 0; i < count; i++) {
            if (compress == 1) {
                record = fd.getNextRecord(record);
            } else {
                record = fd.getRecordByLine(i, record);
            }

            if (record == null) {
                String string = "record no:" + i + " return null";
                out.write(string.getBytes());
            }

            /*
            Error error = new Error();
            if (!judgeRecord(record, i, error))
            {
            String string = "record no:" + i + " value error:"+error.msg +"\n";
            out.write(string.getBytes());
            }
            */

            if (i % (10 * 10000) == 0) {
                String string = "read seq " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                out.write(string.getBytes());
            }
        }

        long end = System.currentTimeMillis();
        String string = "Read Seq over, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s";
        out.write(string.getBytes());

        System.out.println(string);
        out.close();
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage() + ",i:" + i);
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage() + ",i:" + i);
    }

}

---

From source file:TestDistinct.java

License:Apache License

public void testDistinct() throws IOException {
    FileSystem fs = FileSystem.get(new Configuration());
    fs.delete(new Path("/tmp/test_distinct_file"), true);
    fs.delete(new Path("/tmp/test_distinct_file_results"), true);

    FSDataOutputStream out = fs.create(new Path("/tmp/test_distinct_file"));
    PrintWriter pw = new PrintWriter(out);
    pw.println("distinct1");
    pw.println("distinct2");
    pw.println("distinct2");
    pw.println("distinct3");
    pw.println("distinct2");
    pw.flush();//from   www  . j a  v a 2 s  .com
    out.close();

    Map<String, Tap> sources = new HashMap<String, Tap>();
    Map<String, Tap> sinks = new HashMap<String, Tap>();

    Tap inTap = new Hfs(new TextLine(new Fields("line")), "/tmp/test_distinct_file");
    Pipe inPipe = new Pipe("inPipe");
    sources.put("inPipe", inTap);

    Distinct distinct = new Distinct(inPipe);

    Tap outTap = new Hfs(new TextLine(new Fields("line")), "/tmp/test_distinct_file_results");
    Pipe outPipe = new Pipe("outPipe", distinct);
    sinks.put("outPipe", outTap);

    Flow flow = new FlowConnector().connect(sources, sinks, inPipe, outPipe);
    flow.complete();

    FSDataInputStream in = fs.open(new Path("/tmp/test_distinct_file_results/part-00000"));
    BufferedReader reader = new BufferedReader(new InputStreamReader(in));

    ArrayList<String> results = new ArrayList<String>();
    results.add("distinct1");
    results.add("distinct2");
    results.add("distinct3");

    try {
        while (true) {
            String s = reader.readLine();
            if (s == null) {
                break;
            }

            assertEquals(results.remove(0), s);
        }
    } catch (Exception e) {
        fail("Got an exception while trying to verify the results: " + e.toString());
    }

    assertEquals("All results must be consumed!", 0, results.size());
}

---

From source file:ConnectHBase124ClusterExample.java

@Test
public void test() throws Exception {
    final String tableName = "test_table";
    final String family = "family";
    final String columnQualifier = "qual";
    final String rowValue = "foo";
    final String cellValue = "bar";

    Configuration baseDefaults = new Configuration();
    baseDefaults.set("hbase.defaults.for.version", "1.2.4");
    Configuration configuration = HBaseConfiguration.create(baseDefaults);
    configuration.set("hbase.zookeeper.quorum", "new-hbase-server:2181");
    UserGroupInformation.setConfiguration(configuration);
    try (HBaseAdmin admin = new HBaseAdmin(configuration);) {
        HColumnDescriptor col = new HColumnDescriptor(family);
        if (!admin.isTableAvailable(tableName)) {
            System.out.println("Table " + tableName + " does not exist. Creating");
            HTableDescriptor desc = new HTableDescriptor(tableName);
            desc.addFamily(col);/*w  ww. j av  a 2s  .c om*/
            admin.createTable(desc);
            System.out.println("Table " + tableName + " created");
        }
        if (!admin.isTableEnabled(tableName)) {
            System.out.println("table " + tableName + " is not enabled. enabling");
            admin.enableTable(tableName);
        }
        try (HTablePool pool = new HTablePool(configuration, 1);
                HTableInterface table = pool.getTable(tableName)) {
            //                Put put = new Put(rowValue.getBytes());
            //                put.add(family.getBytes(), columnQualifier.getBytes(), cellValue.getBytes());
            //                table.put(put);

            Scan scan = new Scan();
            scan.setCacheBlocks(false);
            scan.setCaching(1000);
            scan.setBatch(1000);
            scan.setMaxVersions(1);
            try (ResultScanner scanner = table.getScanner(scan);) {
                Result result = scanner.next();
                while (result != null) {
                    KeyValue cell = result.getColumnLatest(family.getBytes(), columnQualifier.getBytes());
                    System.out.println("row:" + new String(cell.getRow()));
                    System.out.println("value:" + new String(cell.getValue()));
                    result = scanner.next();
                }
            }
        }

    }

}

---

From source file:DateExample_Month.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: wordcount <in> <out>");
        System.exit(2);/*from  w w w  .  j a  v  a2s  .  c  o  m*/
    }
    Job job = new Job(conf, "word count fs");
    job.setJarByClass(DateExample_Month.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setInputFormatClass(IsValidKeyFormat.class);

    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:WordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    // Let ToolRunner handle generic command-line options
    ToolRunner.run(new Configuration(), new WordCount(), args);
    System.exit(0);/*from w  w w  .j  a v a 2 s .c  om*/
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »