Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testCanPersistentSegment() {
    try {//from www.  j  av a 2 s.  c  o m
        IndexInfo info = new IndexInfo();
        info.offset = 0;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testCanPersistentSegment_tmp", head);

        Segment segment = new Segment(info, fd);

        info.len = 64 * 1024 * 1024 - ConstVar.LineIndexRecordLen - ConstVar.IndexMetaOffset;
        Unit unit = new Unit(info, segment);
        unit.setLen(info.len);
        unit.setFull();
        segment.setCurrentUnit(unit);

        if (!segment.canPersistented()) {
            fail("shoud can persistent, remain:" + segment.remain() + "need:"
                    + (unit.len() + ConstVar.LineIndexRecordLen));
        }

        info.len = 64 * 1024 * 1024 - ConstVar.LineIndexRecordLen - ConstVar.IndexMetaOffset + 1;
        Unit unit2 = new Unit(info, segment);
        unit2.setLen(info.len);
        unit2.setFull();
        segment.setCurrentUnit(unit2);

        if (segment.canPersistented()) {
            fail("should not persistent, remain:" + segment.remain() + "need:"
                    + (unit.len() + ConstVar.LineIndexRecordLen));
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentSegment() {
    try {//from www. j a  va2s .com
        IndexInfo info = new IndexInfo();
        info.offset = 0;

        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentSegment_tmp", head);

        String fileName = prefix + "testPersistentSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        fd.setOut(out);
        Segment segment = new Segment(info, fd);

        int unitSize = 100;
        for (int i = 0; i < unitSize; i++) {
            IndexInfo indexInfo = new IndexInfo();
            indexInfo.offset = i * 100;
            indexInfo.len = 77;
            indexInfo.beginLine = (i + 1) * 100;
            indexInfo.endLine = (i + 2) * 100;
            indexInfo.idx = i;

            Unit unit = new Unit(indexInfo, segment);
            addRecord2Unit(unit, 100);
            unit.beginLine = (i + 1) * 100;
            unit.endLine = (i + 2) * 100;
            segment.addUnit(unit);
            if (unit.len() != 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset) {
                fail("error unit.len:" + unit.len());
            }
        }

        segment.recordNum = 234;
        segment.setBeginLine(1);
        segment.setEndLine(235);

        segment.persistent(out);

        if (out.getPos() != fd.confSegmentSize()) {
            System.out.println("seg.len:" + segment.len() + "seg.remain:" + segment.remain() + "index.len"
                    + segment.unitIndex().len());
            fail("error pos:" + out.getPos());
        }
        out.close();

        int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset;
        FSDataInputStream in = fs.open(path);

        in.seek(segment.lineIndexOffset());

        info.offset = 0;
        info.len = segment.len();
        fd.setWorkStatus(ConstVar.WS_Read);
        Segment segment2 = new Segment(info, fd);
        segment2.unpersistentUnitIndex(in);
        if (segment2.recordNum() != 234) {
            fail("error recordnum:" + segment2.recordNum());
        }
        if (segment2.unitNum() != unitSize) {
            fail("error unitNum:" + segment2.unitNum());
        }
        if (segment2.keyIndexOffset() != -1) {
            fail("error key index offset:" + segment2.keyIndexOffset());
        }
        if (segment2.lineIndexOffset() != unitlen * unitSize) {
            fail("error line index offset:" + segment2.lineIndexOffset());
        }
        if (segment2.units().size() != unitSize) {
            fail("error units.size:" + segment2.units().size());
        }

        UnitIndex index = segment2.unitIndex();
        if (index.lineIndexInfos().size() != unitSize) {
            fail("error line unit index size:" + index.lineIndexInfos().size());
        }
        if (index.keyIndexInfos().size() != 0) {
            fail("error key unit index size:" + index.keyIndexInfos().size());
        }

        for (int i = 0; i < unitSize; i++) {
            IndexInfo ii = index.lineIndexInfos().get(i);
            if (ii.beginLine() != (1 + i) * 100) {
                fail("error beginline:" + ii.beginLine() + "i:" + i);
            }
            if (ii.endLine() != (2 + i) * 100) {
                fail("error end line:" + ii.endLine() + "i:" + i);
            }
            if (ii.offset() != i * 100) {
                fail("error offset:" + ii.offset() + "i:" + i);
            }
            if (ii.len != unitlen) {
                fail("error len:" + ii.len() + "i:" + i);
            }
            if (ii.idx() != i) {
                fail("error idx:" + ii.idx() + "i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineSegment() {
    Segment segment = null;/* w  w w.jav  a 2  s  .c  om*/
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Write);
        fd.head = head;
        fd.setOut(out);

        IndexInfo info = new IndexInfo();
        info.offset = 0;

        segment = new Segment(info, fd);

        int recordNum = 150000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
            record = null;
        }

        segment.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        fd.setIn(in);
        fd.setWorkStatus(ConstVar.WS_Read);

        info.offset = 0;
        info.len = segment.len();
        info.beginLine = 0;
        info.endLine = 1500000;
        Segment segment2 = new Segment(info, fd);
        Record record = segment2.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }

        record = segment2.getRecordByLine(150000);
        if (record != null) {
            fail("should get null");
        }
        record = segment2.getRecordByLine(150001);
        if (record != null) {
            fail("should get null");
        }

        record = segment2.getRecordByLine(0);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

        int line = 150000 - 1;
        record = segment2.getRecordByLine(line);
        if (record == null) {
            fail("should not get null");
        }

        judgeFixedRecord(record);

    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderSegment() {
    Segment segment = null;//w  ww . j  a  va 2  s  .co m
    try {
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByValueSegment";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Write);
        fd.head = head;
        fd.setOut(out);

        IndexInfo info = new IndexInfo();
        info.offset = 0;

        segment = new Segment(info, fd);

        int recordNum = 150000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
            record = null;
        }

        segment.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        fd.setIn(in);
        fd.setWorkStatus(ConstVar.WS_Read);

        info.offset = 0;
        info.len = segment.len();

        Segment segment2 = new Segment(info, fd);

        FieldValue[] values = new FieldValue[2];
        values[0] = new FieldValue((byte) 1, (short) 2);
        values[1] = new FieldValue((short) 2, (short) 3);
        Record[] records = segment2.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null, index error, records.len:" + records.length);
        }

        values[0] = new FieldValue((byte) 1, (short) 0);
        values[1] = new FieldValue((short) 3, (short) 1);
        records = segment2.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null, value error");
        }

        values[0] = new FieldValue((byte) 1, (short) 0);
        values[1] = new FieldValue((short) 2, (short) 1);
        records = segment2.getRecordByOrder(values, values.length);
        if (records == null) {
            fail("should not get null");
        }
        if (records.length != 150000) {
            fail("error result size:" + records.length);
        }

        for (int i = 0; i < 150000; i++) {
            judgeFixedRecord(records[i]);
        }

        records = segment2.getRecordByOrder(null, 0);
        if (records == null) {
            fail("should not get null");
        }
        if (records.length != 150000) {
            fail("error result size:" + records.length);
        }

        for (int i = 0; i < 150000; i++) {
            judgeFixedRecord(records[i]);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testSetNoPreFileName() {
    try {//from   w  w  w.  ja  va 2 s  . c  o  m
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        Head head = new Head();

        String fileName = prefix + "testSetNoPreFileName";
        fd.create(fileName, head);

        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        if (!fs.isFile(path)) {
            fail("create file fail");
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testSetPreFileName() {
    try {//from  ww  w .  j av a  2  s . com
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        Head head = new Head();

        //String fileName = "hdfs://tdw-172-25-38-246:54310/user/tdwadmin/testSetPreFileName";
        String fileName = "/user/tdwadmin/se_test/fs/basic/testSetPreFileName";
        fd.create(fileName, head);
        fd.close();

        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        if (!fs.isFile(path)) {
            fail("create file fail");
        }
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testSetInvalidFileName() {
    try {//from   www.j  av  a2 s .co m
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        Head head = new Head();

        String fileName = "hdfs://tdw-172-25-38-246:54310/user/tdwadmin/se_test/testSetInvalidFileName";
        fd.create(fileName, head);

        fail("invalid file name");
    } catch (IOException e) {

    } catch (Exception e) {
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetUnitSize() {
    try {//from  w ww  .j  a  v  a2  s .c  o m
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);

        long unitSize = fd.confUnitSize();
        if (unitSize == 0) {
            fail("get unit size fail:" + unitSize);
        }
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (Exception e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetSegmentSize() {
    try {/*www.ja  va  2 s .  co m*/
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);

        long segSize = fd.confSegmentSize();
        if (segSize == 0) {
            fail("get seg size fail:" + segSize);
        }
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (Exception e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnitSizeGreatThanSegmentSize() {
    try {/*  ww w .  jav  a2  s .c  om*/
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (Exception e) {
        fail(e.getMessage());
    }
}