Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddUnitSegment() {
    try {//w  w w.j a  v a  2s.c o m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddUnitSegment_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 10;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            segment.addUnit(unit);
            if (unit.len() != (100 * full7chunkLen + 8 * 100 + ConstVar.DataChunkMetaOffset)) {
                fail("error unit.len:" + unit.len());
            }
        }

        int unitlen = (100 * full7chunkLen + 8 * 100 + ConstVar.DataChunkMetaOffset);

        if (segment.beginLine() != 100) {
            fail("error begin line:" + segment.beginLine());
        }
        if (segment.endLine() != 1100) {
            fail("error end line:" + segment.endLine());
        }
        if (segment.beginKey() != 0) {
            fail("error begin key:" + segment.beginKey());
        }
        if (segment.endKey() != 0) {
            fail("error end key:" + segment.endKey());
        }
        if (segment.offset() != 12) {
            fail("error offset:" + segment.offset());
        }
        if (segment.currentOffset() != 12 + unitlen * 10) {
            fail("error current offset:" + segment.currentOffset());
        }
        if (segment.unitNum() != unitSize) {
            fail("error unitNum:" + segment.unitNum());
        }

        if (segment.unitIndex().len() != unitSize * ConstVar.LineIndexRecordLen) {
            fail("error unit index len:" + segment.unitIndex().len());
        }

        long total = fd.confSegmentSize() - 12;
        if (segment.remain() != total - unitlen * unitSize - unitSize * ConstVar.LineIndexRecordLen
                - ConstVar.IndexMetaOffset) {
            fail("error remain:" + segment.remain());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddUnitSegmentFull() {
    int count = 0;
    Segment segment = null;//  w  ww.  ja va2  s.  c o  m
    try {
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddUnitSegmentFull_tmp", head);

        segment = new Segment(info, fd);

        for (int i = 0;; i++, count++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (SEException.SegmentFullException e) {
        if (segment.unitNum() != count) {
            fail("error unit num:" + segment.unitNum());
        }

        if (segment.remain() > 77 + ConstVar.LineIndexRecordLen) {
            fail("should add, remain:" + segment.remain() + ",need len:" + (77 + ConstVar.LineIndexRecordLen));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddUnitLineIndex() {
    try {/* ww w. ja va 2 s.  c  o m*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddUnitLineIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 10;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        if (segment.unitIndex().len() != ConstVar.LineIndexRecordLen * unitSize) {
            fail("error unitIndex len:" + segment.unitIndex().len());
        }

        if (segment.unitIndex().lineIndexInfos().size() != unitSize) {
            fail("error line index size:" + segment.unitIndex().lineIndexInfos().size());
        }
        if (segment.unitIndex().keyIndexInfos().size() != 0) {
            fail("error key index size:" + segment.unitIndex().keyIndexInfos().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetSegmentLen() {
    try {//from   w ww .  j  a v  a2s.  c o m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testGetSegmentLen_tmp1", head);

        Segment segment = new Segment(info, fd);

        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
        }

        if (segment.recordNum() != 0) {
            fail("error record num:" + segment.recordNum());
        }
        if (segment.len() != ConstVar.IndexMetaOffset) {
            fail("error len:" + segment.len());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:e" + e.getMessage());
    }

    try {
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testGetSegmentLen_tmp2", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 10;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            addRecord2Unit(unit, 100);
            segment.addUnit(unit);
            if (unit.len() != 100 * full7chunkLen + 8 * 100 + ConstVar.DataChunkMetaOffset) {
                fail("error unitlen:" + unit.len());
            }
        }

        int unitlen = 100 * full7chunkLen + 8 * 100 + ConstVar.DataChunkMetaOffset;
        if (segment.len() != unitSize * unitlen + ConstVar.LineIndexRecordLen * unitSize
                + ConstVar.IndexMetaOffset) {
            fail("error segment.len:" + segment.len());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetDummyLen() {
    try {//from w  w w. ja  v a2  s.c o  m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testGetDummyLen_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitlen = 100 * full7chunkLen + 8 * 100 + ConstVar.DataChunkMetaOffset;
        int unitSize = 10;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            addRecord2Unit(unit, 100);

            segment.addUnit(unit);
            if (unit.len() != unitlen) {
                fail("error unit.len:" + unit.len());
            }

        }

        long total = fd.confSegmentSize() - 12;
        if (segment.remain() != total - unitSize * unitlen - ConstVar.LineIndexRecordLen * unitSize
                - ConstVar.IndexMetaOffset) {
            fail("error segment.remain:" + segment.remain());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineUnitIndex() {
    try {//w w  w  .j a v a 2  s .com
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);

            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;

            segment.addUnit(unit);
        }

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);
        if (out.getPos() != unitSize * ConstVar.LineIndexRecordLen) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        if (segment.lineIndexOffset() != 0) {
            fail("error line index offset:" + segment.lineIndexOffset());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment.keyIndexOffset());
        }

        FSDataInputStream in = fs.open(path);

        for (int i = 0; i < unitSize; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != (i + 1) * 100) {
                fail("error begin line:" + beginLine + " i:" + i);
            }
            if (endLine != (i + 2) * 100) {
                fail("error end line:" + endLine + " i:" + i);
            }
            if (offset != i * 100) {
                fail("error offset:" + offset + " i:" + i);
            }
            if (len != unitlen) {
                fail("error len:" + len + " i:" + i);
            }
            if (idx != i) {
                fail("error idx:" + idx + " i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitIndexMeta() {
    try {//from  w  w w.  j a  va2  s  .c  o m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistentUnitIndexMeta(out);
        if (out.getPos() != ConstVar.IndexMetaOffset) {
            fail("error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int recordNum = in.readInt();
        int unitNum = in.readInt();
        long keyIndexOffset = in.readLong();
        long lineIndexOffset = in.readLong();

        if (recordNum != 234) {
            fail("error recordnum:" + recordNum);
        }
        if (unitNum != unitSize) {
            fail("error unitNum:" + unitNum);
        }
        if (keyIndexOffset != -1) {
            fail("error key index offset:" + keyIndexOffset);
        }
        if (lineIndexOffset != -1) {
            fail("error line inded offset:" + lineIndexOffset);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentDummy() {
    try {/* w  w w. ja  v a2 s.  c  om*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentDummy_tmp", head);

        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            unit.setFull();
            unit.setMetaOffset(indexInfo.offset);
            segment.addUnit(unit);
        }

        String fileName = prefix + "testPersistentDummy";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        segment.persistentUnitIndex(out);

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);
        segment.persistentUnitIndexMeta(out);

        segment.persistentDummy(out);
        if (out.getPos() != segment.unitIndex().len() + ConstVar.IndexMetaOffset + segment.remain()) {
            fail("error pos:" + out.getPos());
        }
        out.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentUnitIndexMeta() {
    try {//from   ww  w  . j  a  va 2  s .co m
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnpersistentUnitIndexMeta_tmp", head);

        Segment segment = new Segment(info, fd);

        String fileName = prefix + "testPersistentUnitIndexMeta";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        int unitSize = 100;

        segment.unpersistentIndexMeta(in);

        if (segment.recordNum() != 234) {
            fail("error record num:" + segment.recordNum());
        }
        if (segment.unitNum() != unitSize) {
            fail("error unit num:" + segment.unitNum());
        }
        if (segment.keyIndexOffset() != -1) {
            fail("error keyIndex offset:" + segment.keyIndexOffset());
        }
        if (segment.lineIndexOffset() != -1) {
            fail("error lineIndexOffset:" + segment.lineIndexOffset());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentLineUnitIndex() {
    try {/*from   w w  w .  j  a va 2s  .c  o  m*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnpersistentLineUnitIndex_tmp", head);

        Segment segment = new Segment(info, fd);

        String fileName = prefix + "testPersistentLineUnitIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        int unitSize = 100;

        segment.setUnitIndex(new UnitIndex());
        segment.setUnitNum(unitSize);
        segment.unpersistentLineUnitIndex(in);

        if (segment.unitIndex().len() != ConstVar.LineIndexRecordLen * unitSize) {
            fail("error unitIndex len:" + segment.unitIndex().len());
        }

        if (segment.unitIndex().lineIndexInfos().size() != unitSize) {
            fail("error line index size:" + segment.unitIndex().lineIndexInfos().size());
        }
        if (segment.unitIndex().keyIndexInfos().size() != 0) {
            fail("error key index size:" + segment.unitIndex().keyIndexInfos().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}