Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStoragePerformanceTest.java

License:Open Source License

static void doFormatReadSeq(int count, boolean var) {
    try {//from   www.j a va 2s .c  o  m
        String fileName = "MR_input/testMassRecord";
        if (var) {
            fileName += "_var";
        }
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);

        boolean compressed = (fd.head().compress() == ConstVar.Compressed);
        long begin = System.currentTimeMillis();

        for (int i = 0; i < count; i++) {
            Record valueRecord = new Record();
            valueRecord.clear();
            if (compressed) {
                Record record = fd.getNextRecord(valueRecord);
                if (record == null) {
                }
            } else {
                Record record = fd.getRecordByLine(i);
                if (record == null) {
                }
            }

            if (i % (1 * 1000000) == 0) {
                String string = "read format seq " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                output.write(string.getBytes());
            }
        }

        long end = System.currentTimeMillis();
        String string = "Read Foramt Seq over, count:" + count + ", delay:" + (long) ((end - begin) / 1000)
                + " s";
        output.write(string.getBytes());
        System.out.println(string);
        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

From source file:FormatStoragePerformanceTest.java

License:Open Source License

static void doFormatReadSeq2(int count) {
    try {/*  w  w w.ja  v  a2 s  .  c  o m*/
        String fileName = "MR_input/testMassRecord";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);

        long begin = System.currentTimeMillis();

        int ct = 0;
        {
            Record record = null;
            if (record == null) {
                String string = "seq 2 record no:" + ct + " return null";
                output.write(string.getBytes());
            }

            if (ct % (1 * 10000) == 0) {
                String string = "read format seq2 " + ct + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                output.write(string.getBytes());
            }
        }

        long end = System.currentTimeMillis();
        String string = "Read Foramt Seq2 over, count:" + ct + ", delay:" + (long) ((end - begin) / 1000)
                + " s";
        output.write(string.getBytes());
        System.out.println(string);
        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

From source file:FormatStoragePerformanceTest.java

License:Open Source License

static void doTextReadRand(int count) {
    try {//from  www .j  av  a2 s  . c o  m
        String textFile = "MR_input_text/testPerformanceReadText";
        Path path = new Path(textFile);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        InputStream stream = new BufferedInputStream(in);
        BufferedReader reader = new BufferedReader(new InputStreamReader(stream));

        long begin = System.currentTimeMillis();
        count = 1 * 1000;
        for (int i = 0; i < count; i++) {
            int line = (int) (Math.random() * count);
            for (int j = 0; j < line; j++) {
                String value = reader.readLine();
                value = null;
            }
        }
        reader.close();

        long end = System.currentTimeMillis();
        String string = "text read seq, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s";
        output.write(string.getBytes());
        System.out.println(string);

    } catch (Exception e) {
        e.printStackTrace();
        System.out.println(e.getMessage());
    }
}

From source file:FormatStoragePerformanceTest.java

License:Open Source License

static void doTextReadSeq(int count, boolean var) {
    try {/*  www.  j  av  a2 s  .c  o  m*/
        ArrayList<Integer> meta = new ArrayList<Integer>(10);
        for (int i = 0; i < 7; i++) {
            meta.add(i);
        }

        String textFile = "MR_input_text/testPerformanceReadText";
        if (var) {
            textFile += "_var";
        }
        Path path = new Path(textFile);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        InputStream stream = new BufferedInputStream(in);
        BufferedReader reader = new BufferedReader(new InputStreamReader(stream));

        long begin = System.currentTimeMillis();
        for (int i = 0; i < count; i++) {
            String value = reader.readLine();

            String[] fields = value.split(",");

            /*
             ByteArrayInputStream bin = new ByteArrayInputStream(value.getBytes());
                     
             meta.get(0); byte[] bb= new byte[4]; bin.read(bb); 
             meta.get(1); byte[] sb= new byte[6]; bin.read(sb);  
             meta.get(2); byte[] ib= new byte[9]; bin.read(ib);  
             meta.get(3); byte[] lb= new byte[13]; bin.read(lb); 
             meta.get(4); byte[] fb= new byte[13]; bin.read(fb);  
             meta.get(5); byte[] db= new byte[18]; bin.read(db);  
             meta.get(6);
             value = null;
             */
            Byte.valueOf(fields[0]);
            Short.valueOf(fields[1]);
            Integer.valueOf(fields[2]);
            Long.valueOf(fields[3]);
            Float.valueOf(fields[4]);
            Double.valueOf(fields[5]);
            if (var) {
                String.valueOf(fields[6]);
            }
        }
        reader.close();

        long end = System.currentTimeMillis();

        String string = "text read seq " + count + " record over, delay: " + ((end - begin) / 1000) + " s \n";
        System.out.println(string);
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println(e.getMessage());
    }
}

From source file:ColumnStoragePerformanceTest.java

License:Open Source License

public static void main(String[] argv) {
    try {//from  w w w .j  a v  a 2s  .c o  m
        if (argv.length != 3) {
            System.out.println(
                    "usage: ColumnStroageStabilityTest cmd[write | readSeq | readRand] Count Compress\n");
            return;
        }

        String cmd = argv[0];
        count = Integer.valueOf(argv[1]);
        compress = Byte.valueOf(argv[2]);

        System.out.println("cmd:" + cmd + ",count:" + count);

        output = new FileOutputStream(file);

        String writeCmd = "write";
        String readSeq = "readSeq";
        String readRand = "readRand";

        conf = new Configuration();
        conf.setInt("dfs.replication", 1);

        if (cmd.equals(writeCmd)) {
            doInitFile();
        } else if (cmd.equals(readRand)) {
            doReadRand();
        } else {
            doReadSeq();
        }

        output.close();
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

From source file:ColumnStoragePerformanceTest.java

License:Open Source License

public static void doInitFormatStorageFile() throws Exception {
    FieldMap fieldMap = new FieldMap();
    for (short i = 0; i < 5; i++) {
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) (i * 7)));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) (i * 7 + 1)));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) (i * 7 + 2)));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) (i * 7 + 3)));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) (i * 7 + 4)));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) (i * 7 + 5)));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) (i * 7 + 6)));
    }//from  w  ww  .jav  a 2s .  c  om
    Head head = new Head();
    head.setFieldMap(fieldMap);

    System.out.println("write file:" + formatStorageFilename);
    FormatDataFile fd = new FormatDataFile(new Configuration());
    fd.create(formatStorageFilename, head);

    long begin = System.currentTimeMillis();
    for (int i = 0; i < count; i++) {
        Record record = new Record((short) 35);

        for (short j = 0; j < 5; j++) {
            record.addValue(new FieldValue((byte) 1, (short) (j * 7 + 0)));
            record.addValue(new FieldValue((short) 2, (short) (j * 7 + 1)));
            record.addValue(new FieldValue((int) 3, (short) (j * 7 + 2)));
            record.addValue(new FieldValue((long) 4, (short) (j * 7 + 3)));
            record.addValue(new FieldValue((float) 5.5, (short) (j * 7 + 4)));
            record.addValue(new FieldValue((double) 6.6, (short) (j * 7 + 5)));
            record.addValue(new FieldValue("hello konten", (short) (j * 7 + 6)));
        }
        fd.addRecord(record);

        if (i % 1000000 == 0) {
            long end = System.currentTimeMillis();
            String string = i + " record ok(format), delay:" + (end - begin) / 1000 + " s \n";
            output.write(string.getBytes());
        }
    }

    fd.close();
    long end = System.currentTimeMillis();
    String string = count + " record over(formatStorage), delay:" + (end - begin) / 1000 + " s \n";
    output.write(string.getBytes());
    System.out.println(string);

}

From source file:ColumnStoragePerformanceTest.java

License:Open Source License

static void doFormatReadRand(int count) {
    try {//from w  w w .  j  a v  a 2  s  . c o m
        String fileName = "MR_input/testMassRecord";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);

        long begin = System.currentTimeMillis();
        for (int i = 0; i < count; i++) {
            int rand = (int) (Math.random() * count);

            Record record = fd.getRecordByLine(rand);
            if (record == null) {
                String string = "record no:" + rand + " return null";
                output.write(string.getBytes());
            }

            if (i % (1 * 10000) == 0) {
                String string = "read format rand " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                output.write(string.getBytes());
            }
        }
        long end = System.currentTimeMillis();
        String string = "Read Foramt Rand over, count:" + count + ", delay:" + (long) ((end - begin) / 1000)
                + " s";
        output.write(string.getBytes());
        System.out.println(string);
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

From source file:ColumnStoragePerformanceTest.java

License:Open Source License

static void doTextReadSeq(int count) {
    try {//  w  ww. jav  a  2 s . co  m
        ArrayList<Integer> meta = new ArrayList<Integer>(10);
        for (int i = 0; i < 7; i++) {
            meta.add(i);
        }

        Path path = new Path(textFilename);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        InputStream stream = new BufferedInputStream(in);
        BufferedReader reader = new BufferedReader(new InputStreamReader(stream));

        long begin = System.currentTimeMillis();
        for (int i = 0; i < count; i++) {
            String value = reader.readLine();

            String[] fields = value.split(",");

            ByteArrayInputStream bin = new ByteArrayInputStream(value.getBytes());

            meta.get(0);
            byte[] bb = new byte[4];
            bin.read(bb);
            meta.get(1);
            byte[] sb = new byte[6];
            bin.read(sb);
            meta.get(2);
            byte[] ib = new byte[9];
            bin.read(ib);
            meta.get(3);
            byte[] lb = new byte[13];
            bin.read(lb);
            meta.get(4);
            byte[] fb = new byte[13];
            bin.read(fb);
            meta.get(5);
            byte[] db = new byte[18];
            bin.read(db);
            meta.get(6);
            value = null;
        }
        reader.close();

        long end = System.currentTimeMillis();

        String string = "text read seq " + count + " record over, delay: " + ((end - begin) / 1000) + " s \n";
        System.out.println(string);
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println(e.getMessage());
    }
}

From source file:TestUtil.java

License:Open Source License

public static void writeifdfile(String fileName, int num) throws IOException {
    IFormatDataFile ifdf = new IFormatDataFile(new Configuration());
    IHead head = new IHead();
    IFieldMap fieldMap = new IFieldMap();
    fieldMap.addFieldType(new IFieldType.IFieldByteType());
    fieldMap.addFieldType(new IFieldType.IFieldShortType());
    fieldMap.addFieldType(new IFieldType.IFieldIntType());
    fieldMap.addFieldType(new IFieldType.IFieldLongType());
    fieldMap.addFieldType(new IFieldType.IFieldFloatType());
    fieldMap.addFieldType(new IFieldType.IFieldDoubleType());
    head.setFieldMap(fieldMap);/*from w ww.  jav  a  2 s  .c  o  m*/
    head.setPrimaryIndex((short) 2);
    IUserDefinedHeadInfo udi = new IUserDefinedHeadInfo();
    udi.addInfo(0, fileName);

    ifdf.create(fileName, head);

    for (int i = 0; i < num; i++) {
        IRecord record = new IRecord();
        record.addFieldValue(new IFieldValue((byte) i));
        record.addFieldValue(new IFieldValue((short) (2 * i)));
        record.addFieldValue(new IFieldValue(3 * i));
        record.addFieldValue(new IFieldValue((long) 4 * i));
        record.addFieldValue(new IFieldValue((float) 5 * i));
        record.addFieldValue(new IFieldValue((double) 6 * i, (short) 5));
        ifdf.addRecord(record);
    }
    ifdf.close();

}

From source file:TestFormatStorageRecordReader.java

License:Open Source License

private static void initFormatData() throws Exception {
    FieldMap fieldMap = new FieldMap();
    fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) (0)));
    fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) (1)));
    fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) (2)));
    fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) (3)));
    fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) (4)));
    fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) (5)));
    fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) (6)));

    Head head = new Head();
    head.setFieldMap(fieldMap);//  w w  w .  ja  va2  s .c om

    String fileName = "se_test/fs/basic/f1/kt/testGetRecordByLineFDReadManyTimes1";

    Configuration conf = new Configuration();
    FormatDataFile fd = new FormatDataFile(conf);
    fd.create(fileName, head);

    int recordNum = 10;
    for (int i = 0; i < recordNum; i++) {
        Record record = new Record((short) 7);

        int j = 0;
        record.addValue(new FieldValue((byte) (1 + i), (short) (j * 7 + 0)));
        record.addValue(new FieldValue((short) (2 + i), (short) (j * 7 + 1)));
        record.addValue(new FieldValue(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, null, (short) 2));
        record.addValue(new FieldValue((long) (4 + i), (short) (j * 7 + 3)));
        record.addValue(new FieldValue((float) (5.5 + i), (short) (j * 7 + 4)));
        record.addValue(new FieldValue((double) (6.6 + i), (short) (j * 7 + 5)));
        record.addValue(new FieldValue("hello konten" + i, (short) (j * 7 + 6)));
        fd.addRecord(record);
    }
    fd.close();
}