Example usage for org.apache.hadoop.fs FileSystem create

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem create.

Prototype

public FSDataOutputStream create(Path f) throws IOException 

Source Link

Document

Create an FSDataOutputStream at the indicated Path.

Usage

From source file:GetRetweetersAndCountPerUser.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 3) {
        System.err.println("Usage: GetRetweetersAndCountPerUser <in> <out> <num_reducers>");
        System.exit(2);/*from w w w .  jav a2s  . co m*/
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(RetweetersPerUser.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    System.out.println(otherArgs[0]);
    job.setMapperClass(TweetMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(Integer.parseInt(args[2]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    if (job.waitForCompletion(true)) {
        FileSystem hdfs = FileSystem.get(new URI(args[1]), conf);
        Path dir = new Path(args[1]);
        PathFilter filter = new PathFilter() {
            public boolean accept(Path file) {
                return file.getName().startsWith("part-r-");
            }
        };

        HashMap<Integer, Integer> counts_for_user = new HashMap<Integer, Integer>();
        FileStatus[] files = hdfs.listStatus(dir, filter);
        Arrays.sort(files);
        for (int i = 0; i != files.length; i++) {
            Path pt = files[i].getPath();
            BufferedReader br = new BufferedReader(new InputStreamReader(hdfs.open(pt)));
            String line = null;
            while ((line = br.readLine()) != null) {
                String[] columns = new String[2];
                columns = line.split("\t");
                int key = Integer.parseInt(columns[0]);
                if (counts_for_user.containsKey(key))
                    counts_for_user.put(key, counts_for_user.get(key) + 1);
                else
                    counts_for_user.put(key, 1);
            }
            br.close();
        }

        FSDataOutputStream fsDataOutputStream = hdfs.create(new Path(otherArgs[1] + "_count"));
        PrintWriter writer = new PrintWriter(fsDataOutputStream);
        for (Entry<Integer, Integer> e : counts_for_user.entrySet()) {
            writer.write(e.getKey() + "\t" + e.getValue() + "\n");
        }
        writer.close();
        fsDataOutputStream.close();
        hdfs.close();
        System.exit(0);
    }
    System.exit(1);
}

---

From source file:ExportStressTest.java

License:Apache License

public void createFile(int fileId) throws IOException {
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    Path dirPath = new Path("ExportStressTest");
    fs.mkdirs(dirPath);/*from w w  w.j a  va  2  s  .co m*/
    Path filePath = new Path(dirPath, "input-" + fileId);

    OutputStream os = fs.create(filePath);
    Writer w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < RECORDS_PER_FILE; i++) {
        long v = (long) i + ((long) RECORDS_PER_FILE * (long) fileId);
        w.write("" + v + "," + ALPHABET + ALPHABET + ALPHABET + ALPHABET + "\n");

    }
    w.close();
    os.close();
}

---

From source file:FDFGenData.java

License:Open Source License

static void testgenrawfile(FileSystem fs, String filename, int recordnum) throws IOException {
    Random r = new Random();
    FSDataOutputStream fos = fs.create(new Path(filename));
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < recordnum; i++) {
        fos.writeByte(i / 1000);/*from   ww w. j a  v  a 2  s .  co m*/
        fos.writeShort(i / 1000);
        fos.writeInt(i / 1000);
        fos.writeLong(i / 1000);
        fos.writeFloat(i / 1000);
        fos.writeDouble(i / 1000);
        int strnum = r.nextInt(12) + 7;
        sb.setLength(0);
        for (int j = 0; j < strnum; j++) {
            sb.append((char) ('a' + j));
        }
        fos.writeUTF(sb.toString());
        if (i % 1000000 == 0) {
        }
    }
    fos.close();
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndexInfo() {
    try {/*from   ww  w  .ja v a 2  s  . c  om*/
        String fileName = prefix + "testPersistentLineIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginLine = 11;
        info.endLine = 22;
        info.offset = 33;
        info.len = 44;
        info.idx = 55;

        info.persistentLineIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginLine = in.readInt();
        int endLine = in.readInt();
        long offset = in.readLong();
        long len = in.readLong();
        int idx = in.readInt();
        in.close();

        if (beginLine != 11) {
            fail("beginLine fail:" + beginLine);
        }
        if (endLine != 22) {
            fail("endLine fail:" + endLine);
        }
        if (offset != 33) {
            fail("offset fail:" + offset);
        }
        if (len != 44) {
            fail("len fail:" + len);
        }
        if (idx != 55) {
            fail("idx fail:" + idx);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndexInfo() {
    try {//w  w  w.j a  v  a  2  s.c om
        String fileName = prefix + "testPersistentKeyIndexInfo";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        IndexInfo info = new IndexInfo();
        info.beginKey = 111;
        info.endKey = 222;

        info.persistentKeyIndexInfo(out);
        out.close();

        FSDataInputStream in = fs.open(path);

        int beginKey = in.readInt();
        int endKey = in.readInt();
        in.close();

        if (beginKey != 111) {
            fail("beginKey fail:" + beginKey);
        }
        if (endKey != 222) {
            fail("beginKey fail:" + beginKey);
        }

    } catch (IOException e) {
        fail(e.getMessage());
    }

}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentLineIndex() {
    try {//from  w ww .j a  v  a2  s  .  c  o  m
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginLine = i;
            info.endLine = i + 10;
            info.offset = i;
            info.len = i + 20;
            info.idx = i;

            index.addIndexInfo(info, ConstVar.LineMode);
        }

        String fileName = prefix + "testPersistentLineIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginLine = in.readInt();
            int endLine = in.readInt();
            long offset = in.readLong();
            long len = in.readLong();
            int idx = in.readInt();

            if (beginLine != i) {
                fail("beginLine fail:" + beginLine);
            }
            if (endLine != i + 10) {
                fail("endLine fail:" + endLine);
            }
            if (offset != i) {
                fail("offset fail:" + offset);
            }
            if (len != i + 20) {
                fail("len fail:" + len);
            }
            if (idx != i) {
                fail("idx fail:" + idx);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentKeyIndex() {
    try {//from  w w  w  . j a v a2s .  com
        BlockIndex index = new BlockIndex();

        for (int i = 0; i < 10; i++) {
            IndexInfo info = new IndexInfo();
            info.beginKey = i;
            info.endKey = i + 10;

            index.addIndexInfo(info, ConstVar.KeyMode);
        }

        String fileName = prefix + "testPersistentKeyIndex";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        index.persistent(out);
        out.close();

        FSDataInputStream in = fs.open(path);
        for (int i = 0; i < 10; i++) {
            int beginKey = in.readInt();
            int endKey = in.readInt();

            if (beginKey != i) {
                fail("beginKey fail:" + beginKey);
            }
            if (endKey != i + 10) {
                fail("endKey fail:" + endKey);
            }
        }
    } catch (IOException e) {
        fail(e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentField() throws IOException {
    {//from w ww  .j ava2 s. c o m
        String file = prefix + "testPersistentField";
        Path path = new Path(file);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Field field = new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1);
        field.persistent(out);
        if (out.getPos() != 7) {
            fail("error out.pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);
        byte type = in.readByte();
        int len = in.readInt();
        short idx = in.readShort();

        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

    }
    {
    }
    {
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentFieldMap() {
    try {//  www  .  ja v  a  2 s . c  o m
        FileSystem fs = FileSystem.get(new Configuration());

        String file1 = prefix + "testPersistentFieldMap1";
        Path path = new Path(file1);
        FSDataOutputStream out1 = fs.create(path);

        FieldMap fm1 = new FieldMap();
        fm1.persistent(out1);
        if (out1.getPos() != 2) {
            fail("persisitent null fieldmap fail, pos:" + out1.getPos());
        }
        out1.close();

        FSDataInputStream in1 = fs.open(path);
        short fieldNum = in1.readShort();
        if (fieldNum != 0) {
            fail("persistent null fieldmap fail, fieldNum:" + fieldNum);
        }

        String file2 = prefix + "testPersistentFieldMap2";
        path = new Path(file2);
        FSDataOutputStream out2 = fs.create(path);

        FieldMap fm2 = new FieldMap();
        fm2.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 1));
        fm2.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 3));
        fm2.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 5));
        fm2.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 7));
        fm2.persistent(out2);
        if (out2.getPos() != (2 + 4 * 7)) {
            fail("persistent 4 field fail, pos:" + out2.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);
        fieldNum = in2.readShort();
        if (fieldNum != 4) {
            fail("persistent 4 field fail, fieldNum:" + fieldNum);
        }

        byte type = in2.readByte();
        int len = in2.readInt();
        short idx = in2.readShort();
        if (type != ConstVar.FieldType_Byte) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Byte) {
            fail("fail len:" + len);
        }
        if (idx != 1) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Short) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Short) {
            fail("fail len:" + len);
        }
        if (idx != 3) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Int) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Int) {
            fail("fail len:" + len);
        }
        if (idx != 5) {
            fail("fail idx:" + idx);
        }

        type = in2.readByte();
        len = in2.readInt();
        idx = in2.readShort();
        if (type != ConstVar.FieldType_Long) {
            fail("fail type:" + type);
        }
        if (len != ConstVar.Sizeof_Long) {
            fail("fail len:" + len);
        }
        if (idx != 7) {
            fail("fail idx:" + idx);
        }
    } catch (IOException e) {
        fail("testPersistentField fail1:" + e.getMessage());
    } catch (Exception e) {
        fail("testPersistentField fail2:" + e.getMessage());
    }
}

---

From source file:FormatStorageBasicTest.java

License:Open Source License

public void testPersistentHead() {

    try {//from   w  w  w  .j a va 2  s . c  om
        String fileName = prefix + "testPersistentHead";
        Path path = new Path(fileName);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        Head head = new Head();
        head.persistent(out);
        if (out.getPos() != 17) {
            fail("persistent error pos:" + out.getPos());
        }
        out.close();

        FSDataInputStream in = fs.open(path);

        int magic = in.readInt();
        short ver = in.readShort();
        byte var = in.readByte();
        byte compress = in.readByte();
        byte compressStyle = in.readByte();
        short primaryIndex = in.readShort();
        byte encode = in.readByte();
        byte encodeStyle = in.readByte();
        short keyLen = in.readShort();
        short fieldNum = in.readShort();

        if (magic != head.magic()) {
            fail("error magic:" + magic);
        }
        if (ver != head.ver()) {
            fail("error ver:" + ver);
        }
        if (var != 0) {
            fail("error var:" + var);
        }
        if (compress != head.compress()) {
            fail("error compress:" + compress);
        }
        if (compressStyle != head.compressStyle()) {
            fail("error compressStyle:" + compressStyle);
        }
        if (primaryIndex != head.primaryIndex()) {
            fail("error primaryIndex:" + primaryIndex);
        }
        if (encode != head.encode()) {
            fail("error encode:" + encode);
        }
        if (encodeStyle != head.encodeStyle()) {
            fail("error encodeStyle:" + encodeStyle);
        }
        if (keyLen != 0) {
            fail("error keyLen:" + keyLen);
        }
        if (fieldNum != 0) {
            fail("error fieldNum:" + fieldNum);
        }

        fileName = prefix + "testPersistentHead2";
        path = new Path(fileName);
        FSDataOutputStream out2 = fs.create(path);

        Head head2 = new Head();
        String key = "hello konten";

        FieldMap fieldMap = new FieldMap();
        fieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));
        fieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));
        head2.setFieldMap(fieldMap);
        head2.setKey(key);

        head2.persistent(out2);
        if (out2.getPos() != 13 + 2 + key.length() + fieldMap.len()) {
            fail("persistent error pos:" + out.getPos());
        }
        out2.close();

        FSDataInputStream in2 = fs.open(path);

        magic = in2.readInt();
        ver = in2.readShort();
        var = in2.readByte();
        compress = in2.readByte();
        compressStyle = in2.readByte();
        primaryIndex = in2.readShort();
        encode = in2.readByte();
        encodeStyle = in2.readByte();

        keyLen = in2.readShort();
        if (keyLen == 0) {
            fail("error keyLen:" + keyLen);
        }
        byte[] buf = new byte[keyLen];
        in2.readFully(buf);
        String keykey = new String(buf);
        if (!key.equals(keykey)) {
            fail("error key:" + keykey);
        }

        FieldMap fieldMap22 = new FieldMap();
        fieldMap22.unpersistent(in2);
        if (fieldMap22.fieldNum() != 2) {
            fail("error fieldNum:" + fieldMap22.fieldNum());
        }
        Field field = fieldMap22.getField((short) 0);
        if (field.type() != ConstVar.FieldType_Byte) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Byte) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 0) {
            fail("fail index:" + field.index());
        }

        field = fieldMap22.getField((short) 1);
        if (field.type() != ConstVar.FieldType_Short) {
            fail("fail field type:" + field.type());
        }
        if (field.len() != ConstVar.Sizeof_Short) {
            fail("fail field len:" + field.len());
        }
        if (field.index() != 1) {
            fail("fail index:" + field.index());
        }

    } catch (Exception e) {
        fail("get exception:" + e.getMessage());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »