Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentFieldMap() {
    try {//from  w w  w  .  jav a2 s .  co  m
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataOutputStream out1 = fs.create(path);

        FieldMap fm1 = new FieldMap();
        fm1.persistent(out1);
        if (out1.getPos() != 2) {
            fail("persisitent null fieldmap fail, pos:" + out1.getPos());
        }
        out1.close();

        FSDataInputStream in1 = fs.open(path);
        short fieldNum = in1.readShort();
        if (fieldNum != 0) {
            fail("persistent null fieldmap fail, fieldNum:" + fieldNum);
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataOutputStream out2 = fs.create(path);

        FieldMap fm2 = new FieldMap();
        fm2.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1));
        fm2.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 3));
        fm2.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 5));
        fm2.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 7));
        fm2.persistent(out2);
        if (out2.getPos() != (2 + 4 * 7)) {
            fail("persistent 4 field fail, pos:" + out2.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);
        fieldNum = in2.readShort();
        if (fieldNum != 4) {
            fail("persistent 4 field fail, fieldNum:" + fieldNum);
        }

        byte type = in2.readByte();
        int len = in2.readInt();
        short idx = in2.readShort();
        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Short) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Short) {
            fail("fail len:" + len);
        }
        if (idx != 3) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Int) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Int) {
            fail("fail len:" + len);
        }
        if (idx != 5) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Long) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Long) {
            fail("fail len:" + len);
        }
        if (idx != 7) {
            fail("fail idx:" + idx);
        }
    } catch (IOException e) {
        fail("testPersistentField fail1:" + e.getMessage());
    } catch (Exception e) {
        fail("testPersistentField fail2:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentFieldMap() {
    try {/*from www  .ja v a 2 s  . co  m*/
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataInputStream in1 = fs.open(path);

        FieldMap fm1 = new FieldMap();
        fm1.unpersistent(in1);
        in1.close();

        if (fm1.len() != 2) {
            fail("unpersistent fieldMap fail, len:" + fm1.len());
        }
        if (fm1.fieldNum() != 0) {
            fail("unpersistent fieldMap fail, fieldNum:" + fm1.fieldNum());
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataInputStream in2 = fs.open(path);

        FieldMap fm2 = new FieldMap();
        fm2.unpersistent(in2);
        if (fm2.len() != 2 + 4 * 7) {
            fail("unpersistent fieldMap fail, len:" + fm2.len());
        }
        if (fm2.fieldNum() != 4) {
            fail("unpersistent fieldMap fail, fieldNum:" + fm2.fieldNum());
        }

        Field field = fm2.fields.get((short) 1);
        if (field.type != ConstVar.FieldType_Byte) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Byte) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 1) {
            fail("unpersistent fail idx:" + field.index);
        }

        field = fm2.fields.get((short) 3);
        if (field.type != ConstVar.FieldType_Short) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Short) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 3) {
            fail("unpersistent fail idx:" + field.index);
        }

        field = fm2.fields.get((short) 5);
        if (field.type != ConstVar.FieldType_Int) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Int) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 5) {
            fail("unpersistent fail idx:" + field.index);
        }

        field = fm2.fields.get((short) 7);
        if (field.type != ConstVar.FieldType_Long) {
            fail("unpersistent fail type:" + field.type);
        }
        if (field.len != ConstVar.Sizeof_Long) {
            fail("unpersistent fail len:" + field.len);
        }
        if (field.index != 7) {
            fail("unpersistent fail idx:" + field.index);
        }
    } catch (IOException e) {
        fail("unpersistent fieldMap fail:" + e.getMessage());
    } catch (Exception e) {
        fail("unpersistent failMap fail:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentHead() {

    try {/*from   w  w w  .ja  v a  2  s. c om*/
        String fileName = prefix + "testPersistentHead";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Head head = new Head();
        head.persistent(out);
        if (out.getPos() != 17) {
            fail("persistent error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int magic = in.readInt();
        short ver = in.readShort();
        byte var = in.readByte();
        byte compress = in.readByte();
        byte compressStyle = in.readByte();
        short primaryIndex = in.readShort();
        byte encode = in.readByte();
        byte encodeStyle = in.readByte();
        short keyLen = in.readShort();
        short fieldNum = in.readShort();

        if (magic != head.magic()) {
            fail("error magic:" + magic);
        }
        if (ver != head.ver()) {
            fail("error ver:" + ver);
        }
        if (var != 0) {
            fail("error var:" + var);
        }
        if (compress != head.compress()) {
            fail("error compress:" + compress);
        }
        if (compressStyle != head.compressStyle()) {
            fail("error compressStyle:" + compressStyle);
        }
        if (primaryIndex != head.primaryIndex()) {
            fail("error primaryIndex:" + primaryIndex);
        }
        if (encode != head.encode()) {
            fail("error encode:" + encode);
        }
        if (encodeStyle != head.encodeStyle()) {
            fail("error encodeStyle:" + encodeStyle);
        }
        if (keyLen != 0) {
            fail("error keyLen:" + keyLen);
        }
        if (fieldNum != 0) {
            fail("error fieldNum:" + fieldNum);
        }

        fileName = prefix + "testPersistentHead2";
        path = new Path(fileName);
        FSDataOutputStream out2 = fs.create(path);

        Head head2 = new Head();
        String key = "hello konten";

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        head2.setFieldMap(fieldMap);
        head2.setKey(key);

        head2.persistent(out2);
        if (out2.getPos() != 13 + 2 + key.length() + fieldMap.len()) {
            fail("persistent error pos:" + out.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);

        magic = in2.readInt();
        ver = in2.readShort();
        var = in2.readByte();
        compress = in2.readByte();
        compressStyle = in2.readByte();
        primaryIndex = in2.readShort();
        encode = in2.readByte();
        encodeStyle = in2.readByte();

        keyLen = in2.readShort();
        if (keyLen == 0) {
            fail("error keyLen:" + keyLen);
        }
        byte[] buf = new byte[keyLen];
        in2.readFully(buf);
        String keykey = new String(buf);
        if (!key.equals(keykey)) {
            fail("error key:" + keykey);
        }

        FieldMap fieldMap22 = new FieldMap();
        fieldMap22.unpersistent(in2);
        if (fieldMap22.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap22.fieldNum());
        }
        Field field = fieldMap22.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap22.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentHead() {
    try {//  w  w w.  ja v a2  s  .c  om
        String fileName = prefix + "testPersistentHead2";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        Head head = new Head();

        head.setMagic(in.readInt());
        head.unpersistent(in);

        if (head.compress() != 0) {
            fail("compress not 0");
        }
        if (head.encode() != 0) {
            fail("encode != 0");
        }
        if (head.var() != 0) {
            fail("var != 0");
        }
        if (head.primaryIndex() != -1) {
            fail("primary index != -1");
        }
        if (head.magic() != ConstVar.DataMagic) {
            fail("invalid init magic");
        }
        if (head.ver() != ConstVar.Ver) {
            fail("invalid init ver");
        }

        String key = "hello konten";
        if (!head.key.equals(key)) {
            fail("error key:" + head.key);
        }

        FieldMap fieldMap = head.fieldMap();
        if (fieldMap.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap.fieldNum());
        }
        Field field = fieldMap.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecord() {
    try {/*  w ww.  j  a  v a  2 s. c o m*/
        String fileName = prefix + "testChunkToRecord";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        byte[] fb = new byte[ConstVar.Sizeof_Float];
        float f = (float) 5.5;
        Util.float2bytes(fb, f);
        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, fb,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());
        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        value = new byte[4];
        in.readFully(value);
        float fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;
        if (bitSet.length() != (fieldNum / 8 + 1) * 8) {
            fail("bitSet.len:" + bitSet.length());
        }

        for (int i = 0; i < fieldNum; i++) {
            if (!bitSet.get(i)) {
                fail("bitSet should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }

        {
        }
        lv = Util.bytes2long(value, 0, len);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        fv = Util.bytes2float(value, 0);
        if (fv != 5.5) {
            fail("error float value:" + fv);
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        fail("should not exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testChunkToRecordNull() {
    try {// www .ja v a2 s .  com
        String fileName = prefix + "testChunkToRecord2";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        short fieldNum = 3;
        Record record = new Record(fieldNum);

        byte[] lb = new byte[ConstVar.Sizeof_Long];
        long l = 4;
        Util.long2bytes(lb, l);
        FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13);
        record.addValue(fieldValue4);

        FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, null,
                (short) 14);
        record.addValue(fieldValue5);

        String str = "hello konten";
        FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(),
                (short) 16);
        record.addValue(fieldValue7);

        DataChunk chunk = new DataChunk(record);

        out.write(chunk.values, 0, (int) chunk.len);

        if (out.getPos() != chunk.len) {
            fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len);
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        FixedBitSet bitSet = new FixedBitSet(fieldNum);
        in.read(bitSet.bytes(), 0, bitSet.size());

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }

        byte[] value = new byte[8];
        in.readFully(value);
        long lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        in.readFloat();

        short strLen = in.readShort();
        if (strLen != str.length()) {
            fail("error strLen:" + strLen);
        }
        value = new byte[strLen];
        in.readFully(value);
        String strv = new String(value, 0, strLen);
        if (!strv.equals(str)) {
            fail("error strv:" + strv);
        }

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16));

        in.seek(0);
        int valuelen = 1 + 8 + 4 + 2 + 12;
        DataChunk chunk2 = new DataChunk(fieldNum);

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        DataInputBuffer inputBuffer = new DataInputBuffer();
        byte[] buf = new byte[valuelen];
        in.read(buf, 0, valuelen);
        inputBuffer.reset(buf, 0, valuelen);
        chunk2.unpersistent(0, valuelen, inputBuffer);
        Record record2 = chunk2.toRecord(fieldMap, true, arrayList);

        bitSet = chunk2.fixedBitSet;

        for (int i = 0; i < fieldNum; i++) {
            if (bitSet.get(1)) {
                fail("shoud not set");
            }

            if (!bitSet.get(i) && i != 1) {
                fail("should set:" + i);
            }
        }
        record = record2;

        int index = 0;
        byte type = record2.fieldValues().get(index).type;
        int len = record2.fieldValues().get(index).len;
        short idx = record2.fieldValues().get(index).idx;
        value = record2.fieldValues().get(index).value;
        if (len != ConstVar.Sizeof_Long) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Long) {
            fail("error fieldType:" + type);
        }
        if (idx != 13) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        lv = Util.bytes2long(value, 0, 8);
        if (lv != 4) {
            fail("error long value:" + lv);
        }

        index = 1;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        if (len != ConstVar.Sizeof_Float) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_Float) {
            fail("error fieldType:" + type);
        }
        if (idx != 14) {
            fail("error idx:" + idx);
        }
        if (value != null) {
            fail("error value not null");
        }

        index = 2;
        type = record.fieldValues().get(index).type;
        len = record.fieldValues().get(index).len;
        idx = record.fieldValues().get(index).idx;
        value = record.fieldValues().get(index).value;

        str = "hello konten";
        if (len != str.length()) {
            fail("error len:" + len);
        }
        if (type != ConstVar.FieldType_String) {
            fail("error fieldType:" + type);
        }
        if (idx != 16) {
            fail("error idx:" + idx);
        }
        if (value == null) {
            fail("error value null");
        }
        {
        }
        String sv = new String(value, 0, len);
        if (!str.equals(sv)) {
            fail("error string value:" + sv);
        }

    } catch (Exception e) {
        e.printStackTrace();
        fail("should not exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testInitUnit() {
    try {//www.  j ava 2 s  .c  o m
        IndexInfo info = new IndexInfo();
        info.beginKey = 1;
        info.endKey = 11;
        info.beginLine = 2;
        info.endLine = 22;
        info.offset = 300;
        info.len = 100;
        info.idx = 7;

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.head = new Head();
        Segment seg = new Segment(info, fd);

        Unit unit2 = new Unit(info, seg);
        if (unit2.offset() != 300) {
            fail("error offset:" + unit2.offset());
        }
        if (unit2.len() != 100 + (0 * 8) + ConstVar.DataChunkMetaOffset) {
            fail("error len:" + unit2.len());
        }
        if (unit2.beginKey() != 1) {
            fail("error beginkey:" + unit2.beginKey());
        }
        if (unit2.endKey() != 11) {
            fail("error endKey:" + unit2.endKey());
        }
        if (unit2.beginLine() != 2) {
            fail("error beginLine:" + unit2.beginLine());
        }
        if (unit2.endLine() != 22) {
            fail("error endLine:" + unit2.endLine());
        }

        Head head = new Head();
        fd.create(prefix + "testUnitInit", head);
        for (int i = 0; i < 67; i++) {
            fd.incRecordNum();
        }

        Unit unit3 = new Unit(info, seg);
        if (unit3.beginLine() != 67) {
            fail("error beginLine:" + unit3.beginLine());
        }
        if (unit3.endLine() != 67) {
            fail("error endLine:" + unit3.endLine());
        }
    } catch (IOException e) {
        fail("get IOexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddOneRecordUnit() {
    try {/*from   w w w  . j  a v a2 s . c o m*/
        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitAddOneRecord", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        unit.addRecord(record);

        if (unit.offset() != 0) {
            fail("error offset:" + unit.offset());
        }
        if (unit.len() != full7chunkLen + ConstVar.DataChunkMetaOffset) {
            fail("error len:" + unit.len());
        }
        if (unit.recordNum() != 1) {
            fail("error recordNum:" + unit.recordNum());
        }
        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 1) {
            fail("error endLine:" + unit.endLine());
        }
        if (unit.beginKey() != 0) {
            fail("error beginKey:" + unit.beginKey());
        }
        if (unit.endKey() != 0) {
            fail("error endKey:" + unit.endKey());
        }
    } catch (IOException e) {
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddMuchRecordUnit() {
    try {/*from w  w  w .j a  va2  s  .c  o  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitAddMuchRecord", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        int count = 0;
        try {
            for (count = 1;; count++) {
                unit.addRecord(record);
            }
        } catch (SEException.UnitFullException e) {

        }

        if (unit.offset() != 0) {
            fail("error offset:" + unit.offset());
        }
        if (unit.len() != full7chunkLen * count + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error len:" + unit.len() + "count:" + count);
        }
        if (unit.recordNum() != count) {
            fail("error recordNum:" + unit.recordNum());
        }
        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != count) {
            fail("error endLine:" + unit.endLine());
        }
        if (unit.beginKey() != 0) {
            fail("error beginKey:" + unit.beginKey());
        }
        if (unit.endKey() != 0) {
            fail("error endKey:" + unit.endKey());
        }
    } catch (IOException e) {
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitVar() {
    try {//  ww w.  jav a 2  s. co m
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitVar_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        int count = 100;
        for (int i = 0; i < count; i++) {
            unit.addRecord(record);
        }

        String file = prefix + "testPersistentUnitVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        unit.persistent(out);
        long pos = out.getPos();
        if (pos != full7chunkLen * count + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error pos:" + pos);
        }
        out.close();

        long len = unit.len();
        if (len != count * full7chunkLen + count * 8 + ConstVar.DataChunkMetaOffset) {
            fail("error unit.len" + len);
        }

        FSDataInputStream in = fs.open(path);
        in.seek(len - 8 - 4);
        long metaOffset = in.readLong();
        if (metaOffset != full7chunkLen * count) {
            fail("error metaOffset:" + metaOffset);
        }

        in.seek(len - 8 - 4 - 4);
        int recordNum = in.readInt();
        if (recordNum != count) {
            fail("error recordNum:" + recordNum);
        }

        in.seek(metaOffset);
        for (int i = 0; i < recordNum; i++) {
            long offset = in.readLong();
            if (offset != full7chunkLen * i) {
                fail("error offset:" + offset + "i:" + i);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}