List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f) throws IOException
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testChunkToRecord() { try {//from ww w . j a v a2s . c o m String fileName = prefix + "testChunkToRecord"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); short fieldNum = 3; Record record = new Record(fieldNum); byte[] lb = new byte[ConstVar.Sizeof_Long]; long l = 4; Util.long2bytes(lb, l); FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13); record.addValue(fieldValue4); byte[] fb = new byte[ConstVar.Sizeof_Float]; float f = (float) 5.5; Util.float2bytes(fb, f); FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, fb, (short) 14); record.addValue(fieldValue5); String str = "hello konten"; FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(), (short) 16); record.addValue(fieldValue7); DataChunk chunk = new DataChunk(record); out.write(chunk.values, 0, (int) chunk.len); if (out.getPos() != chunk.len) { fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len); } out.close(); FSDataInputStream in = fs.open(path); FixedBitSet bitSet = new FixedBitSet(fieldNum); in.read(bitSet.bytes(), 0, bitSet.size()); for (int i = 0; i < fieldNum; i++) { if (!bitSet.get(i)) { fail("should set:" + i); } } byte[] value = new byte[8]; in.readFully(value); long lv = Util.bytes2long(value, 0, 8); if (lv != 4) { fail("error long value:" + lv); } value = new byte[4]; in.readFully(value); float fv = Util.bytes2float(value, 0); if (fv != 5.5) { fail("error float value:" + fv); } short strLen = in.readShort(); if (strLen != str.length()) { fail("error strLen:" + strLen); } value = new byte[strLen]; in.readFully(value); String strv = new String(value); if (!strv.equals(str)) { fail("error strv:" + strv); } FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13)); fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14)); fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16)); in.seek(0); int valuelen = 1 + 8 + 4 + 2 + 12; DataChunk chunk2 = new DataChunk(fieldNum); ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64); DataInputBuffer inputBuffer = new DataInputBuffer(); byte[] buf = new byte[valuelen]; in.read(buf, 0, valuelen); inputBuffer.reset(buf, 0, valuelen); chunk2.unpersistent(0, valuelen, inputBuffer); Record record2 = chunk2.toRecord(fieldMap, true, arrayList); bitSet = chunk2.fixedBitSet; if (bitSet.length() != (fieldNum / 8 + 1) * 8) { fail("bitSet.len:" + bitSet.length()); } for (int i = 0; i < fieldNum; i++) { if (!bitSet.get(i)) { fail("bitSet should set:" + i); } } record = record2; int index = 0; byte type = record2.fieldValues().get(index).type; int len = record2.fieldValues().get(index).len; short idx = record2.fieldValues().get(index).idx; value = record2.fieldValues().get(index).value; if (len != ConstVar.Sizeof_Long) { fail("error len:" + len); } if (type != ConstVar.FieldType_Long) { fail("error fieldType:" + type); } if (idx != 13) { fail("error idx:" + idx); } if (value == null) { fail("error value null"); } { } lv = Util.bytes2long(value, 0, len); if (lv != 4) { fail("error long value:" + lv); } index = 1; type = record.fieldValues().get(index).type; len = record.fieldValues().get(index).len; idx = record.fieldValues().get(index).idx; value = record.fieldValues().get(index).value; if (len != ConstVar.Sizeof_Float) { fail("error len:" + len); } if (type != ConstVar.FieldType_Float) { fail("error fieldType:" + type); } if (idx != 14) { fail("error idx:" + idx); } if (value == null) { fail("error value null"); } { } fv = Util.bytes2float(value, 0); if (fv != 5.5) { fail("error float value:" + fv); } index = 2; type = record.fieldValues().get(index).type; len = record.fieldValues().get(index).len; idx = record.fieldValues().get(index).idx; value = record.fieldValues().get(index).value; str = "hello konten"; if (len != str.length()) { fail("error len:" + len); } if (type != ConstVar.FieldType_String) { fail("error fieldType:" + type); } if (idx != 16) { fail("error idx:" + idx); } if (value == null) { fail("error value null"); } { } String sv = new String(value, 0, len); if (!str.equals(sv)) { fail("error string value:" + sv); } } catch (Exception e) { fail("should not exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testChunkToRecordNull() { try {//from w w w. ja v a 2 s . c om String fileName = prefix + "testChunkToRecord2"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); short fieldNum = 3; Record record = new Record(fieldNum); byte[] lb = new byte[ConstVar.Sizeof_Long]; long l = 4; Util.long2bytes(lb, l); FieldValue fieldValue4 = new FieldValue(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, lb, (short) 13); record.addValue(fieldValue4); FieldValue fieldValue5 = new FieldValue(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, null, (short) 14); record.addValue(fieldValue5); String str = "hello konten"; FieldValue fieldValue7 = new FieldValue(ConstVar.FieldType_String, (short) str.length(), str.getBytes(), (short) 16); record.addValue(fieldValue7); DataChunk chunk = new DataChunk(record); out.write(chunk.values, 0, (int) chunk.len); if (out.getPos() != chunk.len) { fail("error pos:" + out.getPos() + "chunk.len:" + chunk.len); } out.close(); FSDataInputStream in = fs.open(path); FixedBitSet bitSet = new FixedBitSet(fieldNum); in.read(bitSet.bytes(), 0, bitSet.size()); for (int i = 0; i < fieldNum; i++) { if (bitSet.get(1)) { fail("shoud not set"); } if (!bitSet.get(i) && i != 1) { fail("should set:" + i); } } byte[] value = new byte[8]; in.readFully(value); long lv = Util.bytes2long(value, 0, 8); if (lv != 4) { fail("error long value:" + lv); } in.readFloat(); short strLen = in.readShort(); if (strLen != str.length()) { fail("error strLen:" + strLen); } value = new byte[strLen]; in.readFully(value); String strv = new String(value, 0, strLen); if (!strv.equals(str)) { fail("error strv:" + strv); } FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Long, 8, (short) 13)); fieldMap.addField(new Field(ConstVar.FieldType_Float, 4, (short) 14)); fieldMap.addField(new Field(ConstVar.FieldType_String, 8, (short) 16)); in.seek(0); int valuelen = 1 + 8 + 4 + 2 + 12; DataChunk chunk2 = new DataChunk(fieldNum); ArrayList<byte[]> arrayList = new ArrayList<byte[]>(64); DataInputBuffer inputBuffer = new DataInputBuffer(); byte[] buf = new byte[valuelen]; in.read(buf, 0, valuelen); inputBuffer.reset(buf, 0, valuelen); chunk2.unpersistent(0, valuelen, inputBuffer); Record record2 = chunk2.toRecord(fieldMap, true, arrayList); bitSet = chunk2.fixedBitSet; for (int i = 0; i < fieldNum; i++) { if (bitSet.get(1)) { fail("shoud not set"); } if (!bitSet.get(i) && i != 1) { fail("should set:" + i); } } record = record2; int index = 0; byte type = record2.fieldValues().get(index).type; int len = record2.fieldValues().get(index).len; short idx = record2.fieldValues().get(index).idx; value = record2.fieldValues().get(index).value; if (len != ConstVar.Sizeof_Long) { fail("error len:" + len); } if (type != ConstVar.FieldType_Long) { fail("error fieldType:" + type); } if (idx != 13) { fail("error idx:" + idx); } if (value == null) { fail("error value null"); } { } lv = Util.bytes2long(value, 0, 8); if (lv != 4) { fail("error long value:" + lv); } index = 1; type = record.fieldValues().get(index).type; len = record.fieldValues().get(index).len; idx = record.fieldValues().get(index).idx; value = record.fieldValues().get(index).value; if (len != ConstVar.Sizeof_Float) { fail("error len:" + len); } if (type != ConstVar.FieldType_Float) { fail("error fieldType:" + type); } if (idx != 14) { fail("error idx:" + idx); } if (value != null) { fail("error value not null"); } index = 2; type = record.fieldValues().get(index).type; len = record.fieldValues().get(index).len; idx = record.fieldValues().get(index).idx; value = record.fieldValues().get(index).value; str = "hello konten"; if (len != str.length()) { fail("error len:" + len); } if (type != ConstVar.FieldType_String) { fail("error fieldType:" + type); } if (idx != 16) { fail("error idx:" + idx); } if (value == null) { fail("error value null"); } { } String sv = new String(value, 0, len); if (!str.equals(sv)) { fail("error string value:" + sv); } } catch (Exception e) { e.printStackTrace(); fail("should not exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentUnitVar() { try {// w ww . jav a 2s. co m Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentUnitVar_tmp", head); IndexInfo info = new IndexInfo(); info.offset = 0; Segment seg = new Segment(info, fd); Unit unit = new Unit(info, seg); Record record = new Record(7); record.addValue(new FieldValue((byte) 1, (short) 0)); record.addValue(new FieldValue((short) 2, (short) 1)); record.addValue(new FieldValue((int) 3, (short) 2)); record.addValue(new FieldValue((long) 4, (short) 3)); record.addValue(new FieldValue((float) 5.5, (short) 4)); record.addValue(new FieldValue((double) 6.6, (short) 5)); record.addValue(new FieldValue("hello konten", (short) 6)); int count = 100; for (int i = 0; i < count; i++) { unit.addRecord(record); } String file = prefix + "testPersistentUnitVar"; Path path = new Path(file); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); unit.persistent(out); long pos = out.getPos(); if (pos != full7chunkLen * count + count * 8 + ConstVar.DataChunkMetaOffset) { fail("error pos:" + pos); } out.close(); long len = unit.len(); if (len != count * full7chunkLen + count * 8 + ConstVar.DataChunkMetaOffset) { fail("error unit.len" + len); } FSDataInputStream in = fs.open(path); in.seek(len - 8 - 4); long metaOffset = in.readLong(); if (metaOffset != full7chunkLen * count) { fail("error metaOffset:" + metaOffset); } in.seek(len - 8 - 4 - 4); int recordNum = in.readInt(); if (recordNum != count) { fail("error recordNum:" + recordNum); } in.seek(metaOffset); for (int i = 0; i < recordNum; i++) { long offset = in.readLong(); if (offset != full7chunkLen * i) { fail("error offset:" + offset + "i:" + i); } } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentUnitNotVar() { try {// ww w . j a va 2 s . c o m Head head = new Head(); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentUnitNotVar_tmp", head); IndexInfo info = new IndexInfo(); info.offset = 0; Segment seg = new Segment(info, fd); Unit unit = new Unit(info, seg); Record record = new Record(6); record.addValue(new FieldValue((byte) 1, (short) 0)); record.addValue(new FieldValue((short) 2, (short) 1)); record.addValue(new FieldValue((int) 3, (short) 2)); record.addValue(new FieldValue((long) 4, (short) 3)); record.addValue(new FieldValue((float) 5.5, (short) 4)); record.addValue(new FieldValue((double) 6.6, (short) 5)); int count = 100; for (int i = 0; i < count; i++) { unit.addRecord(record); } String file = prefix + "testPersistentUnitNotVar"; Path path = new Path(file); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); unit.persistent(out); long pos = out.getPos(); if (pos != full6chunkLen * count + ConstVar.DataChunkMetaOffset) { fail("error pos:" + pos); } out.close(); long len = unit.len(); if (len != count * full6chunkLen + ConstVar.DataChunkMetaOffset) { fail("error unit.len" + len); } FSDataInputStream in = fs.open(path); in.seek(len - 8 - 4); long metaOffset = in.readLong(); if (metaOffset != full6chunkLen * count) { fail("error metaOffset:" + metaOffset); } in.seek(len - 8 - 4 - 4); int recordNum = in.readInt(); if (recordNum != count) { fail("error recordNum:" + recordNum); } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testGetRecordByLineUnit() { try {//from w ww . j ava2 s.c om Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testUnitGetRecordByLine_tmp", head); String fileName = prefix + "testUnitGetRecordByLine"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); IndexInfo info = new IndexInfo(); info.offset = 123; Segment seg = new Segment(info, fd); Unit unit = new Unit(info, seg); for (int i = 0; i < 100; i++) { Record record = new Record(7); record.addValue(new FieldValue((byte) (0 + i), (short) 0)); record.addValue(new FieldValue((short) (1 + i), (short) 1)); record.addValue(new FieldValue((int) (2 + i), (short) 2)); record.addValue(new FieldValue((long) (3 + i), (short) 3)); record.addValue(new FieldValue((float) (4.4 + i), (short) 4)); record.addValue(new FieldValue((double) (5.55 + i), (short) 5)); record.addValue(new FieldValue("hello konten" + i, (short) 6)); unit.addRecord(record); } if (unit.beginLine() != 0) { fail("error beginLine:" + unit.beginLine()); } if (unit.endLine() != 100) { fail("error endLine:" + unit.endLine()); } byte[] buf = new byte[(int) unit.offset()]; out.write(buf); unit.persistent(out); out.close(); info.len = unit.len(); info.beginLine = unit.beginLine(); info.endLine = unit.endLine(); FSDataInputStream in = fs.open(path); FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0)); fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1)); fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2)); fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3)); fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4)); fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5)); fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6)); head.setFieldMap(fieldMap); FormatDataFile fd2 = new FormatDataFile(conf); fd2.head = head; fd2.setIn(in); Segment seg2 = new Segment(info, fd2); Unit unit2 = new Unit(info, seg2); if (unit2.beginLine() != 0) { fail("error begin line:" + unit2.beginLine()); } if (unit2.endLine() != 100) { fail("error end line :" + unit2.endLine()); } try { Record record = unit2.getRecordByLine(-1); if (record != null) { fail("should get null"); } } catch (Exception e) { fail("get exception:" + e.getMessage()); } try { Record record = unit2.getRecordByLine(120); if (record != null) { fail("should get null"); } } catch (Exception e) { fail("get exception:" + e.getMessage()); } for (int i = 0; i < 100; i++) { try { Record record = unit2.getRecordByLine(i); short index = 0; byte type = record.getType(index); int len = record.getLen(index); byte[] value = record.getValue(index); short idx = record.getIndex(index); if (type != ConstVar.FieldType_Byte) { fail("fail type:" + type); } if (idx != 0) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Byte) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } byte bv = value[0]; if (bv != i + index) { fail("error value:" + bv); } index = 1; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Short) { fail("fail type:" + type); } if (idx != 1) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Short) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } short sv = Util.bytes2short(value, 0, 2); if (sv != i + index) { fail("error value:" + sv); } index = 2; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Int) { fail("fail type:" + type); } if (idx != 2) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Int) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } int iv = Util.bytes2int(value, 0, 4); if (iv != i + index) { fail("error value:" + iv); } index = 3; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Long) { fail("fail type:" + type); } if (idx != 3) { fail("fail idx:" + idx); } if (len != ConstVar.Sizeof_Long) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } long lv = Util.bytes2long(value, 0, 8); if (lv != i + index) { fail("error value:" + lv); } index = 4; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Float) { fail("fail type:" + type); } if (idx != 4) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Float) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } float fv = Util.bytes2float(value, 0); if (fv != (float) (4.4 + i)) { fail("error value:" + fv); } index = 5; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Double) { fail("fail type:" + type); } if (idx != 5) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Double) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } double dv = Util.bytes2double(value, 0); if (dv != (double) (5.55 + i)) { fail("error value:" + dv); } index = 6; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_String) { fail("fail type:" + type); } if (idx != 6) { fail("error idx:" + idx); } String str = "hello konten" + i; if (len != str.length()) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } String strv = new String(value, 0, len); if (!str.equals(strv)) { fail("error value:" + strv); } } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testGetRecordByOrderUnit() { try {/* ww w. j a va 2 s. c o m*/ Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testUnitGetRecordByOrder_tmp", head); String fileName = prefix + "testUnitGetRecordByOrder"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); IndexInfo info = new IndexInfo(); info.offset = 123; Segment seg = new Segment(info, fd); Unit unit = new Unit(info, seg); for (int i = 0; i < 100; i++) { Record record = new Record(7); record.addValue(new FieldValue((byte) (0 + i), (short) 0)); record.addValue(new FieldValue((short) (1 + i), (short) 1)); record.addValue(new FieldValue((int) (2 + i), (short) 2)); record.addValue(new FieldValue((long) (3 + i), (short) 3)); record.addValue(new FieldValue((float) (4.4 + i), (short) 4)); record.addValue(new FieldValue((double) (5.55 + i), (short) 5)); record.addValue(new FieldValue("hello konten" + i, (short) 6)); unit.addRecord(record); } if (unit.beginLine() != 0) { fail("error beginLine:" + unit.beginLine()); } if (unit.endLine() != 100) { fail("error endLine:" + unit.endLine()); } byte[] buf = new byte[(int) unit.offset()]; out.write(buf); unit.persistent(out); out.close(); info.len = unit.len(); info.beginLine = unit.beginLine(); info.endLine = unit.endLine(); FSDataInputStream in = fs.open(path); FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0)); fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1)); fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2)); fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3)); fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4)); fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5)); fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6)); head.setFieldMap(fieldMap); FormatDataFile fd2 = new FormatDataFile(conf); fd2.head = head; fd2.setIn(in); Segment seg2 = new Segment(info, fd2); Unit unit2 = new Unit(info, seg2); if (unit2.beginLine() != 0) { fail("error begin line:" + unit2.beginLine()); } if (unit2.endLine() != 100) { fail("error end line :" + unit2.endLine()); } Record[] records = unit2.getRecordByValue(null, 0, ConstVar.OP_GetAll); if (records.length != 100) { fail("error record.len:" + records.length); } for (int i = 0; i < 100; i++) { Record record = records[i]; try { short index = 0; byte type = record.getType(index); int len = record.getLen(index); byte[] value = record.getValue(index); short idx = record.getIndex(index); if (type != ConstVar.FieldType_Byte) { fail("fail type:" + type); } if (idx != 0) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Byte) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } byte bv = value[0]; if (bv != (byte) (i + index)) { fail("error value:" + bv); } index = 1; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Short) { fail("fail type:" + type); } if (idx != 1) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Short) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } short sv = Util.bytes2short(value, 0, 2); if (sv != i + index) { fail("error value:" + sv); } index = 2; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Int) { fail("fail type:" + type); } if (idx != 2) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Int) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } int iv = Util.bytes2int(value, 0, 4); if (iv != i + index) { fail("error value:" + iv); } index = 3; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Long) { fail("fail type:" + type); } if (idx != 3) { fail("fail idx:" + idx); } if (len != ConstVar.Sizeof_Long) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } long lv = Util.bytes2long(value, 0, 8); if (lv != i + index) { fail("error value:" + sv); } index = 4; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Float) { fail("fail type:" + type); } if (idx != 4) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Float) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } float fv = Util.bytes2float(value, 0); if (fv != (float) (4.4 + i)) { fail("error value:" + fv); } index = 5; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Double) { fail("fail type:" + type); } if (idx != 5) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Double) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } double dv = Util.bytes2double(value, 0); if (dv != (double) (5.55 + i)) { fail("error value:" + dv); } index = 6; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_String) { fail("fail type:" + type); } if (idx != 6) { fail("error idx:" + idx); } String str = "hello konten" + i; if (len != str.length()) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } String strv = new String(value); if (!str.equals(strv)) { fail("error value:" + strv); } } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testGetRecordByValueUnit() { try {//w w w.jav a 2s .c o m Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testUnitGetRecordByValue_tmp", head); String fileName = prefix + "testUnitGetRecordByValue"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); IndexInfo info = new IndexInfo(); info.offset = 123; Segment seg = new Segment(info, fd); Unit unit = new Unit(info, seg); for (int i = 0; i < 100; i++) { Record record = new Record(7); record.addValue(new FieldValue((byte) (0 + i), (short) 0)); record.addValue(new FieldValue((short) (1 + i), (short) 1)); record.addValue(new FieldValue((int) (2 + i), (short) 2)); record.addValue(new FieldValue((long) (3 + i), (short) 3)); record.addValue(new FieldValue((float) (4.4 + i), (short) 4)); record.addValue(new FieldValue((double) (5.55 + i), (short) 5)); record.addValue(new FieldValue("hello konten" + i, (short) 6)); unit.addRecord(record); } if (unit.beginLine() != 0) { fail("error beginLine:" + unit.beginLine()); } if (unit.endLine() != 100) { fail("error endLine:" + unit.endLine()); } byte[] buf = new byte[(int) unit.offset()]; out.write(buf); unit.persistent(out); out.close(); info.len = unit.len(); info.beginLine = unit.beginLine(); info.endLine = unit.endLine(); FSDataInputStream in = fs.open(path); FieldMap fieldMap = new FieldMap(); fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0)); fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1)); fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2)); fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3)); fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4)); fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5)); fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6)); head.setFieldMap(fieldMap); FormatDataFile fd2 = new FormatDataFile(conf); fd2.head = head; fd2.setIn(in); Segment seg2 = new Segment(info, fd2); Unit unit2 = new Unit(info, seg2); if (unit2.beginLine() != 0) { fail("error begin line:" + unit2.beginLine()); } if (unit2.endLine() != 100) { fail("error end line :" + unit2.endLine()); } FieldValue[] values1 = new FieldValue[2]; values1[0] = new FieldValue((short) (3), (short) 3); values1[1] = new FieldValue((int) (3), (short) 5); Record[] records1 = unit2.getRecordByValue(values1, values1.length, ConstVar.OP_GetSpecial); if (records1 != null) { fail("should return null"); } seg2.units().add(unit2); for (int i = 0; i < 100; i++) { int base = 0; FieldValue[] values = new FieldValue[2]; values[0] = new FieldValue((short) (1 + i), (short) 1); values[1] = new FieldValue((int) (2 + i), (short) 2); Record[] records = unit2.getRecordByValue(values, values.length, ConstVar.OP_GetSpecial); if (i < 100) { if (records == null) { fail("records null:" + i); } if (records.length != 1) { fail("error record.len:" + records.length + "i:" + i); } } else { if (records != null) { fail("should return null:" + i); } } if (records == null) { continue; } Record record = records[0]; try { short index = 0; byte type = record.getType(index); int len = record.getLen(index); byte[] value = record.getValue(index); short idx = record.getIndex(index); if (type != ConstVar.FieldType_Byte) { fail("fail type:" + type); } if (idx != 0) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Byte) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } byte bv = value[0]; if (bv != (byte) (i + index + base)) { fail("error value:" + bv); } index = 1; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Short) { fail("fail type:" + type); } if (idx != 1) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Short) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } short sv = Util.bytes2short(value, 0, 2); if (sv != i + index + base) { fail("error value:" + sv); } index = 2; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Int) { fail("fail type:" + type); } if (idx != 2) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Int) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } int iv = Util.bytes2int(value, 0, 4); if (iv != i + index + base) { fail("error value:" + iv); } index = 3; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Long) { fail("fail type:" + type); } if (idx != 3) { fail("fail idx:" + idx); } if (len != ConstVar.Sizeof_Long) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } long lv = Util.bytes2long(value, 0, 8); if (lv != i + index + base) { fail("error value:" + sv); } index = 4; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Float) { fail("fail type:" + type); } if (idx != 4) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Float) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } float fv = Util.bytes2float(value, 0); if (fv != (float) (4.4 + i + base)) { fail("error value:" + fv); } index = 5; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_Double) { fail("fail type:" + type); } if (idx != 5) { fail("error idx:" + idx); } if (len != ConstVar.Sizeof_Double) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } double dv = Util.bytes2double(value, 0); if (dv != (double) (5.55 + i + base)) { fail("error value:" + dv); } index = 6; type = record.getType(index); len = record.getLen(index); value = record.getValue(index); idx = record.getIndex(index); if (type != ConstVar.FieldType_String) { fail("fail type:" + type); } if (idx != 6) { fail("error idx:" + idx); } String str = "hello konten" + (i + base); if (len != str.length()) { fail("fail len:" + len); } if (value == null) { fail("error value null"); } { } String strv = new String(value); if (!str.equals(strv)) { fail("error value:" + strv); } } catch (Exception e) { e.printStackTrace(); fail("get exception:" + e.getMessage()); } } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentLineUnitIndex() { try {// w ww . ja v a 2 s. c om IndexInfo info = new IndexInfo(); info.offset = 12; Head head = new Head(); head.setVar((byte) 1); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentLineUnitIndex_tmp", head); Segment segment = new Segment(info, fd); int unitSize = 100; for (int i = 0; i < unitSize; i++) { IndexInfo indexInfo = new IndexInfo(); indexInfo.offset = i * 100; indexInfo.len = 77; indexInfo.beginLine = (i + 1) * 100; indexInfo.endLine = (i + 2) * 100; indexInfo.idx = i; Unit unit = new Unit(indexInfo, segment); addRecord2Unit(unit, 100); unit.beginLine = (i + 1) * 100; unit.endLine = (i + 2) * 100; segment.addUnit(unit); } int unitlen = full7chunkLen * 100 + 8 * 100 + ConstVar.DataChunkMetaOffset; String fileName = prefix + "testPersistentLineUnitIndex"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); segment.persistentUnitIndex(out); if (out.getPos() != unitSize * ConstVar.LineIndexRecordLen) { fail("error pos:" + out.getPos()); } out.close(); if (segment.lineIndexOffset() != 0) { fail("error line index offset:" + segment.lineIndexOffset()); } if (segment.keyIndexOffset() != -1) { fail("error key index offset:" + segment.keyIndexOffset()); } FSDataInputStream in = fs.open(path); for (int i = 0; i < unitSize; i++) { int beginLine = in.readInt(); int endLine = in.readInt(); long offset = in.readLong(); long len = in.readLong(); int idx = in.readInt(); if (beginLine != (i + 1) * 100) { fail("error begin line:" + beginLine + " i:" + i); } if (endLine != (i + 2) * 100) { fail("error end line:" + endLine + " i:" + i); } if (offset != i * 100) { fail("error offset:" + offset + " i:" + i); } if (len != unitlen) { fail("error len:" + len + " i:" + i); } if (idx != i) { fail("error idx:" + idx + " i:" + i); } } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentUnitIndexMeta() { try {/*from www . j a v a 2 s .c o m*/ IndexInfo info = new IndexInfo(); info.offset = 12; Head head = new Head(); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentUnitIndexMeta_tmp", head); Segment segment = new Segment(info, fd); int unitSize = 100; for (int i = 0; i < unitSize; i++) { IndexInfo indexInfo = new IndexInfo(); indexInfo.offset = i * 100; indexInfo.len = 77; indexInfo.beginLine = (i + 1) * 100; indexInfo.endLine = (i + 2) * 100; indexInfo.idx = i; Unit unit = new Unit(indexInfo, segment); unit.beginLine = (i + 1) * 100; unit.endLine = (i + 2) * 100; unit.setFull(); unit.setMetaOffset(indexInfo.offset); segment.addUnit(unit); } String fileName = prefix + "testPersistentUnitIndexMeta"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); segment.recordNum = 234; segment.setBeginLine(1); segment.setEndLine(235); segment.persistentUnitIndexMeta(out); if (out.getPos() != ConstVar.IndexMetaOffset) { fail("error pos:" + out.getPos()); } out.close(); FSDataInputStream in = fs.open(path); int recordNum = in.readInt(); int unitNum = in.readInt(); long keyIndexOffset = in.readLong(); long lineIndexOffset = in.readLong(); if (recordNum != 234) { fail("error recordnum:" + recordNum); } if (unitNum != unitSize) { fail("error unitNum:" + unitNum); } if (keyIndexOffset != -1) { fail("error key index offset:" + keyIndexOffset); } if (lineIndexOffset != -1) { fail("error line inded offset:" + lineIndexOffset); } } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }
From source file:FormatStorageBasicTest.java
License:Open Source License
public void testPersistentDummy() { try {//from w w w . jav a2 s. c o m IndexInfo info = new IndexInfo(); info.offset = 12; Head head = new Head(); Configuration conf = new Configuration(); FormatDataFile fd = new FormatDataFile(conf); fd.create(prefix + "testPersistentDummy_tmp", head); Segment segment = new Segment(info, fd); int unitSize = 100; for (int i = 0; i < unitSize; i++) { IndexInfo indexInfo = new IndexInfo(); indexInfo.offset = i * 100; indexInfo.len = 77; indexInfo.beginLine = (i + 1) * 100; indexInfo.endLine = (i + 2) * 100; indexInfo.idx = i; Unit unit = new Unit(indexInfo, segment); unit.beginLine = (i + 1) * 100; unit.endLine = (i + 2) * 100; unit.setFull(); unit.setMetaOffset(indexInfo.offset); segment.addUnit(unit); } String fileName = prefix + "testPersistentDummy"; Path path = new Path(fileName); FileSystem fs = FileSystem.get(new Configuration()); FSDataOutputStream out = fs.create(path); segment.persistentUnitIndex(out); segment.recordNum = 234; segment.setBeginLine(1); segment.setEndLine(235); segment.persistentUnitIndexMeta(out); segment.persistentDummy(out); if (out.getPos() != segment.unitIndex().len() + ConstVar.IndexMetaOffset + segment.remain()) { fail("error pos:" + out.getPos()); } out.close(); } catch (IOException e) { e.printStackTrace(); fail("get IOException:" + e.getMessage()); } catch (Exception e) { e.printStackTrace(); fail("get Exception:" + e.getMessage()); } }