Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testWriteHead() {
    try {/*from w  w  w.  j av a 2 s  . c  om*/
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);

        String fileName = prefix + "testWriteHead";

        Head head = new Head();

        fd.create(fileName, head);
        fd.close();

        long fileLen = fd.getFileLen();
        long realLen = head.len() + ConstVar.IndexMetaOffset;
        if (fileLen != realLen) {
            fail("fileLen != realLen, fileLen:" + fileLen + "realLen:" + realLen);
        }
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (Exception e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddRecord() {
    try {//from   w w  w  . j  ava  2 s .  co m
        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddRecord", head);

        int size = 100;
        for (int i = 0; i < size; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));
            fd.addRecord(record);
        }

        if (fd.recordNum() != size) {
            fail("error record num:" + fd.recordNum());
        }
        if (fd.currentSegment().currentUnit() == null) {
            fail("null current unit");
        }
        if (fd.currentSegment() == null) {
            fail("null current seg");
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddRecordMoreSegment() {
    try {/*from   www .  j a  va 2  s. c o  m*/
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddRecordMoreSegment", head);

        int size = 200 * 10000;
        for (int i = 0; i < size; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));
            fd.addRecord(record);
        }

        if (fd.recordNum() != size) {
            fail("error record num:" + fd.recordNum());
        }
        if (fd.currentSegment().currentUnit() == null) {
            fail("null current unit");
        }
        if (fd.currentSegment() == null) {
            fail("null current seg");
        }
        if (fd.segmentNum() != 1) {
            fail("error segment num:" + fd.segmentNum());
        }

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testClose() {
    try {/* w  w  w  .j ava2  s  .c  o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testClose", head);

        int size = 100 * 10000;

        for (int i = 0; i < size; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));
            fd.addRecord(record);
        }

        if (fd.recordNum() != size) {
            fail("error record num:" + fd.recordNum());
        }
        if (fd.currentSegment().currentUnit() == null) {
            fail("null current unit");
        }
        if (fd.currentSegment() == null) {
            fail("null current seg");
        }
        if (fd.segmentNum() != 0) {
            fail("error segment num:" + fd.segmentNum());
        }

        int headLen = head.len();
        long currentUnitLen = fd.currentSegment().currentUnit().len();
        long segmentLen = fd.currentSegment().len() + currentUnitLen + ConstVar.LineIndexRecordLen;
        long remain = fd.currentSegment().remain();
        int unitNum = fd.currentSegment().unitNum();
        fd.close();

        int indexLen = ConstVar.LineIndexRecordLen * fd.segmentNum();
        int metaLen = ConstVar.IndexMetaOffset;

        long fileLen = fd.getFileLen();

        if (fileLen != headLen + segmentLen + indexLen + metaLen) {
            fail("error file len:" + fileLen);
        }

        if (fd.in() != null) {
            fail("in should set null");
        }
        if (fd.out() != null) {
            fail("out should set null");
        }
        if (fd.recordNum() != 0) {
            fail("record num should set 0");
        }
        if (fd.keyIndexOffset != -1) {
            fail("key index offset not -1");
        }
        if (fd.lineIndexOffset != -1) {
            fail("line index offset not -1");
        }
        if (fd.currentOffset != -1) {
            fail("current offset not -1");
        }
        if (fd.hasLoadAllSegmentDone) {
            fail("has load all segment Done not false");
        }

        String fileName = prefix + "testClose";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        long metaOffset = fileLen - ConstVar.IndexMetaOffset;
        in.seek(metaOffset);

        int recordNum = in.readInt();
        int segNum = in.readInt();
        long keyIndexOffset = in.readLong();
        long lineIndexOffset = in.readLong();

        if (recordNum != size) {
            fail("error record num:" + recordNum);
        }
        if (segNum != 1) {
            fail("error segNum:" + segNum);
        }
        if (keyIndexOffset != -1) {
            fail("error key index offset:" + keyIndexOffset);
        }
        if (lineIndexOffset != (headLen + segmentLen)) {
            fail("error line index offset:" + lineIndexOffset);
        }

        in.seek(lineIndexOffset);
        for (int i = 0; i < segNum; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != 0) {
                fail("error beginLine:" + beginLine);
            }
            if (endLine != size) {
                fail("error end line:" + endLine);
            }
            if (offset != head.len()) {
                fail("error offset:" + offset);
            }
            long tlen = size * full7chunkLen + size * 8 + ConstVar.DataChunkMetaOffset * (unitNum + 1)
                    + 28 * (unitNum + 1) + 24;
            if (len != tlen) {
                fail("error len:" + len);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testNoFillLastSegment() {
    try {/*ww w.  j ava 2  s  .  c  o m*/
        String fileName = prefix + "testNoFillLastSegment";
        Head head = new Head();

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        fd.addRecord(record);

        fd.close();

        FileSystem fs = FileSystem.get(conf);
        long fileLen = fs.getFileStatus(new Path(fileName)).getLen();

        int tlen = head.len() + full7chunkLen + 8 + ConstVar.DataChunkMetaOffset + ConstVar.LineIndexRecordLen
                + ConstVar.IndexMetaOffset + ConstVar.LineIndexRecordLen + ConstVar.IndexMetaOffset;
        if (fileLen != tlen) {
            fail("error file len:" + fileLen);
        }

        FormatDataFile fd2 = new FormatDataFile(new Configuration());
        fd2.open(fileName);
        if (fd2.recordNum() != 1) {
            fail("error record num:" + fd2.recordNum());
        }
        if (fd2.segmentNum() != 1) {
            fail("error segment num:" + fd2.segmentNum());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testOpenNoRecord() {
    try {/*from  w ww. ja v  a 2 s  . co m*/
        String fileName = prefix + "testOpenNoRecord";
        Head head = new Head();
        FormatDataFile fd = new FormatDataFile(new Configuration());
        fd.create(fileName, head);
        fd.close();

        FileSystem fs = FileSystem.get(new Configuration());
        long fileLen = fs.getFileStatus(new Path(fileName)).getLen();
        if (fileLen != head.len() + ConstVar.IndexMetaOffset) {
            fail("error file len:" + fileLen);
        }

        FormatDataFile fd2 = new FormatDataFile(new Configuration());
        fd2.open(fileName);
        if (fd2.recordNum() != 0) {
            fail("error record num:" + fd2.recordNum());
        }
        if (fd2.segmentNum() != 0) {
            fail("error segment num:" + fd2.segmentNum());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testOpen() {
    try {// w ww  .j a va 2  s  .c  o m
        String fileName = prefix + "testAddRecordMoreSegment";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);
        int recordNum = 200 * 10000;
        if (fd.segmentNum() != 2) {
            fail("error segment num:" + fd.segmentNum());
        }
        if (fd.recordNum() != recordNum) {
            fail("error record num:" + recordNum);
        }
        if (fd.segmentIndex().len() != ConstVar.LineIndexRecordLen * 2) {
            fail("error index len:" + fd.segmentIndex().len());
        }
        if (fd.segmentIndex().lineIndexInfos().size() != 2) {
            fail("error unit index size:" + fd.segmentIndex().lineIndexInfos().size());
        }
        if (fd.segmentIndex().keyIndexInfos().size() != 0) {
            fail("error key index size:" + fd.segmentIndex().keyIndexInfos().size());
        }
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (Exception e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFD() {
    Segment segment = null;//w  ww  .java 2s .  c om
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testAddRecordMoreSegment";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);

        int recordNum = 200 * 10000;

        Record record = fd.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }
        record = fd.getRecordByLine(0);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

        record = fd.getRecordByLine(recordNum - 1);
        if (record == null) {
            fail("should not get null");
        }
        judgeFixedRecord(record);

        record = fd.getRecordByLine(recordNum);
        if (record != null) {
            fail("should get null");
        }

        record = fd.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        record = fd.getRecordByLine(45687);
        if (record == null) {
            fail("should not get null");
        }
        judgeFixedRecord(record);

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFDNotVar() {
    Segment segment = null;//from  w  w  w  . j  a v  a2  s . c om
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFDNotVar";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 200 * 10000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(6);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));

            fd.addRecord(record);
        }

        fd.close();

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.open(fileName);
        Record record = fd2.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }
        record = fd2.getRecordByLine(0);
        if (record == null) {
            fail("should not get null");
        }

        judgeNotFixedRecord(record, 0);

        record = fd2.getRecordByLine(recordNum - 1);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(record, recordNum - 1);

        record = fd2.getRecordByLine(recordNum);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(45687);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(record, 45687);

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderFD() {
    Segment segment = null;//from  ww w . j a  v a2s  . c o  m
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testAddRecordMoreSegment";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);

        int recordNum = 200 * 10000;

        FieldValue[] values = new FieldValue[2];
        values[0] = new FieldValue((byte) 1, (short) 2);
        values[1] = new FieldValue((short) 2, (short) 3);

        Record[] records = fd.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null");
        }

        values[0] = new FieldValue((byte) 2, (short) 1);
        values[1] = new FieldValue((short) 2, (short) 3);

        records = fd.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null");
        }

        values[0] = new FieldValue((byte) 1, (short) 0);
        values[1] = new FieldValue((short) 2, (short) 1);

        records = fd.getRecordByOrder(values, values.length);
        if (records == null) {
            fail("should not get null");
        }
        if (records.length != recordNum) {
            fail("error record len:" + records.length);
        }
        for (int i = 0; i < recordNum; i++) {
            judgeFixedRecord(records[i]);
        }

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}