Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:IndexMRTest.java

License:Open Source License

public static void main(String[] args) {
    String inputfiles = "hdfs://172.25.38.253:54310/user/tdw/warehouse/default_db/tcssformat/attempt_201011021154_0013_m_000000_0.1288685557438";
    String outputdir = "/se/tmp/test/";
    Configuration conf = new Configuration();
    String ids = "2";
    String indexname = "testindex";

    IndexMR.run(new Configuration(), inputfiles, false, ids, outputdir);
}

From source file:FDFGenData.java

License:Open Source License

public static void testwritefile(String tabledir, int num) throws Exception {

    String rawtmp = "/tmp/raw/rawfile";

    FileSystem fs = FileSystem.get(new Configuration());
    FileStatus[] fss = fs.listStatus(new Path(tabledir));
    int x = 0;/*  w w  w .  j  a va 2  s  .  c om*/
    if (fss != null) {
        x = fss.length;
    }

    PT.testgenrawfiler(rawtmp, num);
    PT.testwritefdf(tabledir + "file" + (x + 1), rawtmp, false, (short) -1);
    PT.testgenrawfiler(rawtmp, num);
    PT.testwritefdf(tabledir + "file" + (x + 2), rawtmp, false, (short) -1);
    PT.testgenrawfiler(rawtmp, num);
    PT.testwritefdf(tabledir + "file" + (x + 3), rawtmp, false, (short) -1);
    PT.testgenrawfiler(rawtmp, num);
    PT.testwritefdf(tabledir + "file" + (x + 4), rawtmp, false, (short) -1);
    PT.testgenrawfiler(rawtmp, num);
    PT.testwritefdf(tabledir + "file" + (x + 5), rawtmp, false, (short) -1);
}

From source file:FDFGenData.java

License:Open Source License

private static FormatDataFile createfdf(String filename, boolean compress, short keyindex) throws Exception {
    Head head = new Head();
    FieldMap fieldMap = new FieldMap();

    fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
    fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
    fieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));
    fieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));
    fieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));
    fieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

    head.setFieldMap(fieldMap);/*www.  java  2 s  . c om*/
    head.setPrimaryIndex(keyindex);

    if (compress) {
        head.setCompress((byte) 1);
        head.setCompressStyle(ConstVar.LZOCompress);
    }

    Configuration conf = new Configuration();
    FormatDataFile fd = new FormatDataFile(conf);
    fd.create(filename, head);
    return fd;

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndexInfo() {
    try {/*from w w w  .  j a  v a  2  s. com*/
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginLine = 11;
        info.endLine = 22;
        info.offset = 33;
        info.len = 44;
        info.idx = 55;

        info.persistentLineIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginLine = in.readInt();
        int endLine = in.readInt();
        long offset = in.readLong();
        long len = in.readLong();
        int idx = in.readInt();
        in.close();

        if (beginLine != 11) {
            fail("beginLine fail:" + beginLine);
        }
        if (endLine != 22) {
            fail("endLine fail:" + endLine);
        }
        if (offset != 33) {
            fail("offset fail:" + offset);
        }
        if (len != 44) {
            fail("len fail:" + len);
        }
        if (idx != 55) {
            fail("idx fail:" + idx);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndexInfo() {
    try {/* w  w w  .ja va 2 s. c o  m*/
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginKey = 111;
        info.endKey = 222;

        info.persistentKeyIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginKey = in.readInt();
        int endKey = in.readInt();
        in.close();

        if (beginKey != 111) {
            fail("beginKey fail:" + beginKey);
        }
        if (endKey != 222) {
            fail("beginKey fail:" + beginKey);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentLineIndexInfo() {
    try {//from   w w w  . j  a  va2 s.co  m
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        IndexInfo info = new IndexInfo();
        info.unpersistentLineIndexInfo(in);
        in.close();

        if (info.beginLine != 11) {
            fail("beginLine fail:" + info.beginLine);
        }
        if (info.endLine != 22) {
            fail("endLine fail:" + info.endLine);
        }
        if (info.offset != 33) {
            fail("offset fail:" + info.offset);
        }
        if (info.len != 44) {
            fail("len fail:" + info.len);
        }
        if (info.idx != 55) {
            fail("idx fail:" + info.idx);
        }

        fs.close();
    } catch (IOException e) {
        fail(e.getMessage());
    }

}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testUnpersistentKeyIndexInfo() {
    try {//ww w .  ja  va 2 s. co m
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        IndexInfo info = new IndexInfo();
        info.unpersistentKeyIndexInfo(in);
        in.close();

        if (info.beginKey != 111) {
            fail("beginKey fail:" + info.beginKey);
        }
        if (info.endKey != 222) {
            fail("beginKey fail:" + info.beginKey);
        }

        fs.close();
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndex() {
    try {/*w  w  w  .j av  a2s .co  m*/
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginLine = i;
            info.endLine = i + 10;
            info.offset = i;
            info.len = i + 20;
            info.idx = i;

            index.addIndexInfo(info, ConstVar.LineMode);
        }

        String fileName = prefix + "testPersistentLineIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != i) {
                fail("beginLine fail:" + beginLine);
            }
            if (endLine != i + 10) {
                fail("endLine fail:" + endLine);
            }
            if (offset != i) {
                fail("offset fail:" + offset);
            }
            if (len != i + 20) {
                fail("len fail:" + len);
            }
            if (idx != i) {
                fail("idx fail:" + idx);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndex() {
    try {// w w w.ja va 2  s .  c  o m
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginKey = i;
            info.endKey = i + 10;

            index.addIndexInfo(info, ConstVar.KeyMode);
        }

        String fileName = prefix + "testPersistentKeyIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginKey = in.readInt();
            int endKey = in.readInt();

            if (beginKey != i) {
                fail("beginKey fail:" + beginKey);
            }
            if (endKey != i + 10) {
                fail("endKey fail:" + endKey);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentField() throws IOException {
    {//from ww  w.java  2 s.  c  o m
        String file = prefix + "testPersistentField";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Field field = new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1);
        field.persistent(out);
        if (out.getPos() != 7) {
            fail("error out.pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);
        byte type = in.readByte();
        int len = in.readInt();
        short idx = in.readShort();

        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

    }
    {
    }
    {
    }
}