Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFD2() {
    try {//from   w w w .j  a  va2  s.  co  m
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFD2";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 200 * 10000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            fd.addRecord(record);
        }

        fd.close();

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.open(fileName);
        Record record = fd2.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }
        record = fd2.getRecordByLine(0);
        if (record == null) {
            fail("should not get null");
        }

        judgeNotFixedRecord(record, 0);

        record = fd2.getRecordByLine(recordNum - 1);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(record, recordNum - 1);

        record = fd2.getRecordByLine(recordNum);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(45687);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(record, 45687);

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByOrderFD2() {
    try {/*from ww w .j  a  v  a2s.  c o  m*/
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFD2";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.open(fileName);

        FieldValue[] values = new FieldValue[2];
        values[0] = new FieldValue((byte) 2, (short) 1);
        values[1] = new FieldValue((short) 2, (short) 3);

        Record[] records = fd.getRecordByOrder(values, values.length);
        if (records != null) {
            fail("should get null");
        }

        for (int i = 0; i < 1; i++) {
            values[0] = new FieldValue((byte) (1 + i), (short) 0);
            values[1] = new FieldValue((short) (2 + i), (short) 1);

            records = fd.getRecordByOrder(values, values.length);
            if (records == null) {
                fail("should not get null");
            }
            if (records.length != 31) {
                fail("error record len:" + records.length);
            }

            judgeNotFixedRecord(records[0], i);
        }

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFD3() {
    try {/*w  w  w  . j  a v  a2  s . co  m*/
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFD3";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 200 * 10000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            fd.addRecord(record);
        }

        fd.close();

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.open(fileName);

        Record value = new Record();
        Record record = fd2.getRecordByLine(-1, value);
        if (record != null) {
            fail("should get null");
        }
        record = fd2.getRecordByLine(0, value);
        if (record == null) {
            fail("should not get null");
        }

        judgeNotFixedRecord(value, 0);

        record = fd2.getRecordByLine(recordNum - 1, value);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(value, recordNum - 1);

        record = fd2.getRecordByLine(recordNum, value);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(45687, value);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(value, 45687);

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFD3NotVar() {
    try {//from  w ww .  jav a2  s  . com
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFD3NotVar";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 200 * 10000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(6);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));

            fd.addRecord(record);
        }

        fd.close();

        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.open(fileName);

        Record value = new Record();
        Record record = fd2.getRecordByLine(-1, value);
        if (record != null) {
            fail("should get null");
        }
        record = fd2.getRecordByLine(0, value);
        if (record == null) {
            fail("should not get null");
        }

        judgeNotFixedRecord(value, 0);

        record = fd2.getRecordByLine(recordNum - 1, value);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(value, recordNum - 1);

        record = fd2.getRecordByLine(recordNum, value);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        record = fd2.getRecordByLine(45687, value);
        if (record == null) {
            fail("should not get null");
        }
        judgeNotFixedRecord(value, 45687);

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFDReadManyTimes() {
    try {/*from  w  w  w . j  a  va2s.  c  om*/
        FieldMap fieldMap = new FieldMap();
        for (int i = 0; i < 10; i++) {
            fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) (i * 7 + 0)));
            fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) (i * 7 + 1)));
            fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) (i * 7 + 2)));
            fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) (i * 7 + 3)));
            fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) (i * 7 + 4)));
            fieldMap.addField(
                    new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) (i * 7 + 5)));
            fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) (i * 7 + 6)));
        }
        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFDReadManyTimes1";

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 10;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(70);
            for (int j = 0; j < 10; j++) {
                record.addValue(new FieldValue((byte) (1 + i), (short) (j * 7 + 0)));
                record.addValue(new FieldValue((short) (2 + i), (short) (j * 7 + 1)));
                record.addValue(new FieldValue((int) (3 + i), (short) (j * 7 + 2)));
                record.addValue(new FieldValue((long) (4 + i), (short) (j * 7 + 3)));
                record.addValue(new FieldValue((float) (5.5 + i), (short) (j * 7 + 4)));
                record.addValue(new FieldValue((double) (6.6 + i), (short) (j * 7 + 5)));
                record.addValue(new FieldValue("hello konten" + i, (short) (j * 7 + 6)));
            }
            fd.addRecord(record);
        }
        fd.close();

        FieldMap fieldMap2 = new FieldMap();
        fieldMap2.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) (0)));
        fieldMap2.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) (1)));
        fieldMap2.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) (2)));
        fieldMap2.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) (3)));
        fieldMap2.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) (4)));
        fieldMap2.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) (5)));
        fieldMap2.addField(new Field(ConstVar.FieldType_String, 0, (short) (6)));

        Head head2 = new Head();
        head2.setFieldMap(fieldMap2);
        String fileName2 = prefix + "testGetRecordByLineFDReadManyTimes2";
        FormatDataFile fd2 = new FormatDataFile(conf);
        fd2.create(fileName2, head2);
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            fd2.addRecord(record);
        }

        fd2.close();

        FormatDataFile fd3 = new FormatDataFile(conf);
        fd3.open(fileName);

        Record value = new Record();
        for (int i = 0; i < recordNum; i++) {
            Record record = fd3.getRecordByLine(i, value);
            judgeNotFixedRecordMuchFields(record, i);
        }

        FormatDataFile fd4 = new FormatDataFile(conf);
        fd4.open(fileName2);
        Record value2 = new Record();
        for (int i = 0; i < recordNum; i++) {
            Record record = fd4.getRecordByLine(i, value2);
            judgeNotFixedRecord(record, i);
        }

        Record value3 = new Record();
        for (int i = 0; i < recordNum; i++) {
            Record record = fd3.getRecordByLine(i, value3);
            judgeNotFixedRecordMuchFields(record, i);
        }

        fd3.close();
        fd4.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFDNullField() {
    try {/*from w  ww .  j a  v  a  2  s .  c o m*/
        FieldMap fieldMap = new FieldMap();

        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) (1)));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) (3)));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) (5)));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) (7)));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) (9)));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) (11)));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) (13)));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testGetRecordByLineFDNullField";

        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 10;
        for (int i = 0; i < recordNum; i++) {
            int j = 0;
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (1 + i), (short) (j * 7 + 1)));
            record.addValue(new FieldValue((short) (2 + i), (short) (j * 7 + 3)));
            record.addValue(new FieldValue(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, null, (short) 5));
            record.addValue(new FieldValue((long) (4 + i), (short) (j * 7 + 7)));
            record.addValue(new FieldValue((float) (5.5 + i), (short) (j * 7 + 9)));
            record.addValue(new FieldValue((double) (6.6 + i), (short) (j * 7 + 11)));
            record.addValue(new FieldValue("hello konten" + i, (short) (j * 7 + 13)));

            fd.addRecord(record);
        }
        fd.close();

        FormatDataFile fd3 = new FormatDataFile(conf);
        fd3.open(fileName);

        Record value = new Record();
        for (int i = 0; i < recordNum; i++) {
            Record record = fd3.getRecordByLine(i, value);
            judgeNotFixedRecordNullField(record, i);

        }
        fd3.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testNotRecordFormatDataFile() {
    try {//w  ww .  j av  a  2s .c  om
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);

        String fileName = prefix + "testNotRecordFormatDataFile";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        fd.close();

        fd.open(fileName);
        SegmentIndex segmentIndex = fd.segmentIndex();
        if (segmentIndex == null) {
            fail("segmentIndex should not null");
        }

        ArrayList<IndexInfo> infos = segmentIndex.indexInfo();
        if (infos == null) {
            fail("infos should not null");
        }

        if (infos.size() != 0) {
            fail("infos.size() should 0");
        }

        FormatDataFile fd1 = new FormatDataFile(conf);
        fd1.open(fileName);

        segmentIndex = fd1.segmentIndex();
        if (segmentIndex == null) {
            fail("segmentIndex should not null 2");
        }

        infos = segmentIndex.indexInfo();
        if (infos == null) {
            fail("infos should not null 2");
        }

        if (infos.size() != 0) {
            fail("infos.size() should 0 2");
        }

        fd1.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get ioexception:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFDCompress() {
    try {//from   ww w  .  j  a v a  2s . c o m
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);
        head.setCompress((byte) 1);
        head.setCompressStyle(ConstVar.LZOCompress);

        String fileName = prefix + "testGetRecordByLineFDCompress";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 1000 * 10000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(7);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));
            record.addValue(new FieldValue("hello konten" + i, (short) 6));

            fd.addRecord(record);
        }

        fd.close();

        fd.open(fileName);
        Record record = fd.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }

        for (int i = 0; i < 10000; i++) {
            record = fd.getRecordByLine(i);
            if (record == null) {
                fail("should not get null:" + i);
            }

            judgeNotFixedRecord(record, i);
        }
        record = fd.getRecordByLine(recordNum);
        if (record != null) {
            fail("should get null");
        }

        record = fd.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFDCompressNotVar() {
    try {// w w w  . j  a  v  a2s.c  o  m
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

        Head head = new Head();
        head.setFieldMap(fieldMap);
        head.setCompress((byte) 1);
        head.setCompressStyle(ConstVar.LZOCompress);

        String fileName = prefix + "testGetRecordByLineFDCompressNotVar";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);
        fd.create(fileName, head);

        int recordNum = 1000 * 10000;
        for (int i = 0; i < recordNum; i++) {
            Record record = new Record(6);
            record.addValue(new FieldValue((byte) (1 + i), (short) 0));
            record.addValue(new FieldValue((short) (2 + i), (short) 1));
            record.addValue(new FieldValue((int) (3 + i), (short) 2));
            record.addValue(new FieldValue((long) (4 + i), (short) 3));
            record.addValue(new FieldValue((float) (5.5 + i), (short) 4));
            record.addValue(new FieldValue((double) (6.6 + i), (short) 5));

            fd.addRecord(record);
        }

        fd.close();

        fd.open(fileName);
        Record record = fd.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }

        for (int i = 0; i < 10000; i++) {
            record = fd.getRecordByLine(i);
            if (record == null) {
                fail("should not get null:" + i);
            }

            judgeNotFixedRecord(record, i);
        }
        record = fd.getRecordByLine(recordNum);
        if (record != null) {
            fail("should get null");
        }

        record = fd.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testGetRecordByLineFDCompressMR() {
    try {//  w ww . j av a 2  s  .  co  m
        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
        fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
        fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
        fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));
        fieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

        Head head = new Head();
        head.setFieldMap(fieldMap);
        head.setCompress((byte) 1);
        head.setCompressStyle(ConstVar.LZOCompress);

        String fileName = prefix + "testGetRecordByLineFDCompress";
        Configuration conf = new Configuration();
        FormatDataFile fd = new FormatDataFile(conf);

        fd.open(fileName);
        Record record = fd.getRecordByLine(-1);
        if (record != null) {
            fail("should get null");
        }

        Record valueRecord = new Record();
        int recordNum = 1000 * 10000;
        for (int i = 0; i < 10000; i++) {
            record = fd.getRecordByLine(i, valueRecord);
            if (record == null) {
                fail("should not get null:" + i);
            }

            judgeNotFixedRecord(record, i);
        }
        record = fd.getRecordByLine(recordNum, valueRecord);
        if (record != null) {
            fail("should get null");
        }

        record = fd.getRecordByLine(recordNum + 1);
        if (record != null) {
            fail("should get null");
        }

        fd.close();
    } catch (IOException e) {
        e.printStackTrace();
        fail("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        fail("get exception:" + e.getMessage());
    }
}