Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentUnitNotVar() {
    try {//from   ww  w  .j  a v a  2s .co  m
        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testPersistentUnitNotVar_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(6);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));

        int count = 100;
        for (int i = 0; i < count; i++) {
            unit.addRecord(record);
        }

        String file = prefix + "testPersistentUnitNotVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        unit.persistent(out);
        long pos = out.getPos();
        if (pos != full6chunkLen * count + ConstVar.DataChunkMetaOffset) {
            fail("error pos:" + pos);
        }
        out.close();

        long len = unit.len();
        if (len != count * full6chunkLen + ConstVar.DataChunkMetaOffset) {
            fail("error unit.len" + len);
        }

        FSDataInputStream in = fs.open(path);
        in.seek(len - 8 - 4);
        long metaOffset = in.readLong();
        if (metaOffset != full6chunkLen * count) {
            fail("error metaOffset:" + metaOffset);
        }

        in.seek(len - 8 - 4 - 4);
        int recordNum = in.readInt();
        if (recordNum != count) {
            fail("error recordNum:" + recordNum);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistenUnitVar() {
    try {//from  w  w w  .  j  a va  2s . c o m
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Read);
        fd.head = head;

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        info.len = 100 * full7chunkLen + 100 * 8 + ConstVar.DataChunkMetaOffset;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        String file = prefix + "testPersistentUnitVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        byte[] buffer = unit.loadUnitBuffer(in);

        unit.loadDataMeta(buffer, true);

        if (unit.recordNum() != 100) {
            fail("error recordNum:" + unit.recordNum());
        }

        if (unit.offsetArray() == null) {
            fail("error offsetArray, null");
        }

        if (unit.offsetArray().length != 100) {
            fail("error offsetArray len:" + unit.offsetArray().length);
        }

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        ByteArrayInputStream stream1 = new ByteArrayInputStream(buffer);
        DataInputStream stream = new DataInputStream(stream1);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(buffer, 0, buffer.length);
        for (int i = 0; i < unit.offsetArray().length; i++) {
            if (unit.offsetArray()[i] != full7chunkLen * i) {
                fail("error meta offset:" + unit.offsetArray()[i] + "i:" + i);
            }

            DataChunk chunk = new DataChunk((short) 7);
            chunk.unpersistent(unit.offsetArray()[i], full7chunkLen, inputBuffer);

            Record record = chunk.toRecord(fieldMap, true, arrayList);
            judgeFixedRecord(record);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistenUnitNotVar() {
    try {/*  ww  w  .  jav a 2 s  .  c  o  m*/
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.setWorkStatus(ConstVar.WS_Read);
        fd.head = head;

        IndexInfo info = new IndexInfo();
        info.offset = 0;
        info.len = 100 * full6chunkLen + ConstVar.DataChunkMetaOffset;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        String file = prefix + "testPersistentUnitNotVar";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        byte[] buffer = unit.loadUnitBuffer(in);

        unit.loadDataMeta(buffer, false);

        if (unit.recordNum() != 100) {
            fail("error recordNum:" + unit.recordNum());
        }

        ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64);

        ByteArrayInputStream stream1 = new ByteArrayInputStream(buffer);
        DataInputStream stream = new DataInputStream(stream1);
        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(buffer, 0, buffer.length);
        for (int i = 0; i < 100; i++) {
            DataChunk chunk = new DataChunk((short) 6);
            chunk.unpersistent(i * 29, full6chunkLen, inputBuffer);

            Record record = chunk.toRecord(fieldMap, true, arrayList);
            judgeFixedRecord(record);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testTransferUnit() {
    try {//  w w w  .j  ava 2  s .c  o m
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testTransferUnitOneRecord_tmp", head);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);

        Record record = new Record(7);
        record.addValue(new FieldValue((byte) 1, (short) 0));
        record.addValue(new FieldValue((short) 2, (short) 1));
        record.addValue(new FieldValue((int) 3, (short) 2));
        record.addValue(new FieldValue((long) 4, (short) 3));
        record.addValue(new FieldValue((float) 5.5, (short) 4));
        record.addValue(new FieldValue((double) 6.6, (short) 5));
        record.addValue(new FieldValue("hello konten", (short) 6));

        for (int i = 0; i < 100; i++) {
            unit.addRecord(record);
        }

        if (unit.offset() != 123) {
            fail("error offset1:" + unit.offset());
        }

        DataInputBuffer inputBuffer = new DataInputBuffer();
        inputBuffer.reset(((DataOutputBuffer) unit.metasBuffer).getData(), 0,
                ((DataOutputBuffer) unit.metasBuffer).getLength());
        for (int i = 0; i < 100; i++) {
            long value = inputBuffer.readLong();
            if (value != 123 + i * full7chunkLen) {
                fail("error data offset1:" + value + "i:" + i);
            }
        }

        if (unit.metaOffset() != 123 + full7chunkLen * 100) {
            fail("error metaOffset1:" + unit.metaOffset());
        }

        unit.transfer(2000);

        if (unit.offset() != 2000) {
            fail("error offset2:" + unit.offset());
        }

        inputBuffer.reset(((DataOutputBuffer) unit.metasBuffer).getData(), 0,
                ((DataOutputBuffer) unit.metasBuffer).getLength());
        for (int i = 0; i < 100; i++) {
            long value = inputBuffer.readLong();
            if (value != 2000 + i * full7chunkLen) {
                fail("error data offset2:" + value + "i:" + i);
            }
        }
        if (unit.metaOffset() != 2000 + full7chunkLen * 100) {
            fail("error metaOffset2:" + unit.metaOffset());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineUnit() {
    try {/*from   w ww  .  j a v a 2  s .  co  m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByLine_tmp", head);

        String fileName = prefix + "testUnitGetRecordByLine";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        try {
            Record record = unit2.getRecordByLine(-1);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }
        try {
            Record record = unit2.getRecordByLine(120);
            if (record != null) {
                fail("should get null");
            }
        } catch (Exception e) {
            fail("get exception:" + e.getMessage());
        }

        for (int i = 0; i < 100; i++) {
            try {
                Record record = unit2.getRecordByLine(i);

                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != i + index) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + lv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value, 0, len);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderUnit() {
    try {/*  w ww.j  a  va 2s .  c o m*/
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByOrder_tmp", head);

        String fileName = prefix + "testUnitGetRecordByOrder";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        Record[] records = unit2.getRecordByValue(null, 0, ConstVar.OP_GetAll);
        if (records.length != 100) {
            fail("error record.len:" + records.length);
        }

        for (int i = 0; i < 100; i++) {
            Record record = records[i];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + i;
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByValueUnit() {
    try {//from ww w.  j a  v  a 2 s .  com
        Head head = new Head();
        head.setVar((byte) 1);
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testUnitGetRecordByValue_tmp", head);

        String fileName = prefix + "testUnitGetRecordByValue";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.offset = 123;
        Segment seg = new Segment(info, fd);
        Unit unit = new Unit(info, seg);
        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (0 + i), (short) 0));
            record.addValue(new FieldValue((short) (1 + i), (short) 1));
            record.addValue(new FieldValue((int) (2 + i), (short) 2));
            record.addValue(new FieldValue((long) (3 + i), (short) 3));
            record.addValue(new FieldValue((float) (4.4 + i), (short) 4));
            record.addValue(new FieldValue((double) (5.55 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            unit.addRecord(record);

        }

        if (unit.beginLine() != 0) {
            fail("error beginLine:" + unit.beginLine());
        }
        if (unit.endLine() != 100) {
            fail("error endLine:" + unit.endLine());
        }
        byte[] buf = new byte[(int) unit.offset()];
        out.write(buf);

        unit.persistent(out);
        out.close();

        info.len = unit.len();
        info.beginLine = unit.beginLine();
        info.endLine = unit.endLine();

        FSDataInputStream in = fs.open(path);

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));
        head.setFieldMap(fieldMap);

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.head = head;
        fd2.setIn(in);
        Segment seg2 = new Segment(info, fd2);
        Unit unit2 = new Unit(info, seg2);

        if (unit2.beginLine() != 0) {
            fail("error begin line:" + unit2.beginLine());
        }
        if (unit2.endLine() != 100) {
            fail("error end line :" + unit2.endLine());
        }

        FieldValue[] values1 = new FieldValue[2];
        values1[0] = new FieldValue((short) (3), (short) 3);
        values1[1] = new FieldValue((int) (3), (short) 5);
        Record[] records1 = unit2.getRecordByValue(values1, values1.length, ConstVar.OP_GetSpecial);
        if (records1 != null) {
            fail("should return null");
        }

        seg2.units().add(unit2);

        for (int i = 0; i < 100; i++) {
            int base = 0;
            FieldValue[] values = new FieldValue[2];
            values[0] = new FieldValue((short) (1 + i), (short) 1);
            values[1] = new FieldValue((int) (2 + i), (short) 2);
            Record[] records = unit2.getRecordByValue(values, values.length, ConstVar.OP_GetSpecial);
            if (i < 100) {
                if (records == null) {
                    fail("records null:" + i);
                }

                if (records.length != 1) {
                    fail("error record.len:" + records.length + "i:" + i);
                }
            } else {
                if (records != null) {
                    fail("should return null:" + i);
                }
            }

            if (records == null) {
                continue;
            }

            Record record = records[0];
            try {
                short index = 0;
                byte type = record.getType(index);
                int len = record.getLen(index);
                byte[] value = record.getValue(index);
                short idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Byte) {
                    fail("fail type:" + type);
                }
                if (idx != 0) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Byte) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                byte bv = value[0];
                if (bv != (byte) (i + index + base)) {
                    fail("error value:" + bv);
                }

                index = 1;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Short) {
                    fail("fail type:" + type);
                }
                if (idx != 1) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Short) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                short sv = Util.bytes2short(value, 0, 2);
                if (sv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 2;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Int) {
                    fail("fail type:" + type);
                }
                if (idx != 2) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Int) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                int iv = Util.bytes2int(value, 0, 4);
                if (iv != i + index + base) {
                    fail("error value:" + iv);
                }

                index = 3;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Long) {
                    fail("fail type:" + type);
                }
                if (idx != 3) {
                    fail("fail idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Long) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                long lv = Util.bytes2long(value, 0, 8);
                if (lv != i + index + base) {
                    fail("error value:" + sv);
                }

                index = 4;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Float) {
                    fail("fail type:" + type);
                }
                if (idx != 4) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Float) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                float fv = Util.bytes2float(value, 0);
                if (fv != (float) (4.4 + i + base)) {
                    fail("error value:" + fv);
                }

                index = 5;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_Double) {
                    fail("fail type:" + type);
                }
                if (idx != 5) {
                    fail("error idx:" + idx);
                }
                if (len != ConstVar.Sizeof_Double) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }
                double dv = Util.bytes2double(value, 0);
                if (dv != (double) (5.55 + i + base)) {
                    fail("error value:" + dv);
                }

                index = 6;
                type = record.getType(index);
                len = record.getLen(index);
                value = record.getValue(index);
                idx = record.getIndex(index);
                if (type != ConstVar.FieldType_String) {
                    fail("fail type:" + type);
                }
                if (idx != 6) {
                    fail("error idx:" + idx);
                }
                String str = "hello konten" + (i + base);
                if (len != str.length()) {
                    fail("fail len:" + len);
                }
                if (value == null) {
                    fail("error value null");
                }
                {
                }

                String strv = new String(value);
                if (!str.equals(strv)) {
                    fail("error value:" + strv);
                }
            } catch (Exception e) {
                e.printStackTrace();
                fail("get exception:" + e.getMessage());
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get Exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testInitSegment() {
    try {/*  w w  w.ja v  a2  s . co  m*/
        IndexInfo info = new IndexInfo();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        Segment segment = new Segment(info, fd);

        if (segment.beginKey() != 0) {
            fail("error begin key:" + segment.beginKey());
        }
        if (segment.endKey() != 0) {
            fail("error end key:" + segment.endKey());
        }
        if (segment.beginLine() != 0) {
            fail("error begin line:" + segment.beginLine());
        }
        if (segment.endLine() != 0) {
            fail("error endLine:" + segment.endLine());
        }

        if (segment.offset() != -1) {
            fail("error offset:" + segment.offset());
        }
        if (segment.currentOffset() != -1) {
            fail("error current offset:" + segment.currentOffset());
        }
        if (segment.recordNum() != 0) {
            fail("error record num :" + segment.recordNum());
        }
        if (segment.total() != 64 * 1024 * 1024) {
            fail("error total:" + segment.total());
        }
        if (segment.remain() != segment.total()) {
            fail("error remain:" + segment.remain());
        }
        if (segment.unitNum() != 0) {
            fail("error unitNum:" + segment.unitNum());
        }
        if (segment.units().size() != 0) {
            fail("error units.size:" + segment.units().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());

    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }

    try {
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testInitSegment_tmp1", new Head());

        Segment segment = new Segment(info, fd);

        if (segment.beginKey() != 0) {
            fail("error begin key:" + segment.beginKey());
        }
        if (segment.endKey() != 0) {
            fail("error end key:" + segment.endKey());
        }
        if (segment.beginLine() != 0) {
            fail("error begin line:" + segment.beginLine());
        }
        if (segment.endLine() != 0) {
            fail("error endLine:" + segment.endLine());
        }

        if (segment.offset() != 12) {
            fail("error offset:" + segment.offset());
        }
        if (segment.currentOffset() != 12) {
            fail("error current offset:" + segment.currentOffset());
        }
        if (segment.recordNum() != 0) {
            fail("error record num :" + segment.recordNum());
        }
        if (segment.total() != 64 * 1024 * 1024 - 12) {
            fail("error total:" + segment.total());
        }
        if (segment.remain() != segment.total() - 24) {
            fail("error remain:" + segment.remain());
        }
        if (segment.unitNum() != 0) {
            fail("error unitNum:" + segment.unitNum());
        }
        if (segment.units().size() != 0) {
            fail("error units.size:" + segment.units().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());

    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddRecordSegment() {
    try {/* w  w  w.  jav  a  2s  .  c  o  m*/
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddRecordSegment_tmp", head);

        Segment segment = new Segment(info, fd);

        for (int i = 0; i < 100; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
        }

        if (segment.beginKey() != 0) {
            fail("error begin key:" + segment.beginKey());
        }
        if (segment.endKey() != 0) {
            fail("error end key:" + segment.endKey());
        }
        if (segment.beginLine() != 0) {
            fail("error begin line:" + segment.beginLine());
        }
        if (segment.endLine() != 0) {
            fail("error endLine:" + segment.endLine());
        }

        if (segment.offset() != 12) {
            fail("error offset:" + segment.offset());
        }
        if (segment.currentOffset() != 12) {
            fail("error current offset:" + segment.currentOffset());
        }
        if (segment.recordNum() != 0) {
            fail("error record num :" + segment.recordNum());
        }
        if (segment.total() != 64 * 1024 * 1024 - 12) {
            fail("error total:" + segment.total());
        }
        if (segment.remain() != segment.total() - 24) {
            fail("error remain:" + segment.remain());
        }
        if (segment.unitNum() != 0) {
            fail("error unitNum:" + segment.unitNum());
        }
        if (segment.units().size() != 0) {
            fail("error units.size:" + segment.units().size());
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testAddRecordSegmentFull() {
    int count = 1;
    Segment segment = null;/*from w ww .  j  a v  a  2 s  .co m*/
    try {
        IndexInfo info = new IndexInfo();
        info.offset = 12;

        Head head = new Head();
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(prefix + "testAddRecordSegmentFull_tmp", head);

        segment = new Segment(info, fd);

        for (int i = 0;; i++, count++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) 1, (short) 0));
            record.addValue(new FieldValue((short) 2, (short) 1));
            record.addValue(new FieldValue((int) 3, (short) 2));
            record.addValue(new FieldValue((long) 4, (short) 3));
            record.addValue(new FieldValue((float) 5.5, (short) 4));
            record.addValue(new FieldValue((double) 6.6, (short) 5));
            record.addValue(new FieldValue("hello konten", (short) 6));

            segment.addRecord(record);
            record = null;

            if (count == Integer.MAX_VALUE)
                fail("should seg full exception");
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (SEException.SegmentFullException e) {
        if (segment.recordNum() != count - segment.currentUnit().recordNum()) {
            fail("error record num:" + segment.recordNum());
        }

        try {
            if (segment.remain() > segment.currentUnit().len()) {
                fail("should add, remain:" + segment.remain() + ",unit.len:" + segment.currentUnit().len());
            }
        } catch (Exception e1) {
            e1.printStackTrace();
            fail("get exception len:" + e1.getMessage());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}