Example usage for org.apache.hadoop.fs FileSystem create

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem create.

Prototype

public FSDataOutputStream create(Path f) throws IOException 

Source Link

Document

Create an FSDataOutputStream at the indicated Path.

Usage

From source file:com.ckelsel.hadoop.dfs.Test.Test.java

License:Open Source License

public static void main(String[] args) throws Exception {
    String uri = "hdfs://localhost:9000/";
    Configuration config = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(uri), config);

    // hdfs/user/ckelsel/
    FileStatus[] statuses = fs.listStatus(new Path("/user/ckelsel"));
    for (FileStatus status : statuses) {
        System.out.println(status);
    }/*w  w w.j  a v a  2s.  co m*/

    // hdfs/user/ckelsel
    FSDataOutputStream os = fs.create(new Path("/user/ckelsel/test.log"));
    os.write("Hello World!".getBytes());
    os.flush();
    os.close();

    // hdfs/user/ckelsel
    InputStream is = fs.open(new Path("/user/ckelsel/test.log"));
    IOUtils.copyBytes(is, System.out, 1024, true);
}

---

From source file:com.cloudera.cdk.data.TestDatasetDescriptor.java

License:Apache License

@Test
public void testSchemaFromHdfs() throws IOException {
    FileSystem fs = getDFS();

    // copy a schema to HDFS
    Path schemaPath = fs.makeQualified(new Path("schema.avsc"));
    FSDataOutputStream out = fs.create(schemaPath);
    IOUtils.copyBytes(USER_SCHEMA_URL.toURL().openStream(), out, fs.getConf());
    out.close();/*from   www.j a  v  a 2s  .  c  o m*/

    // build a schema using the HDFS path and check it's the same
    Schema schema = new DatasetDescriptor.Builder().schemaUri(schemaPath.toUri()).build().getSchema();

    Assert.assertEquals(USER_SCHEMA, schema);
}

---

From source file:com.cloudera.circus.test.TestXTest.java

License:Open Source License

@Test
@TestHadoop/*ww w  .j a  va2  s  .  c o m*/
public void testHadoopFileSystem() throws Exception {
    JobConf conf = getHadoopConf();
    FileSystem fs = FileSystem.get(conf);
    try {
        OutputStream os = fs.create(new Path(getHadoopTestDir(), "foo"));
        os.write(new byte[] { 1 });
        os.close();
        InputStream is = fs.open(new Path(getHadoopTestDir(), "foo"));
        Assert.assertEquals(is.read(), 1);
        Assert.assertEquals(is.read(), -1);
        is.close();
    } finally {
        fs.close();
    }
}

---

From source file:com.cloudera.circus.test.TestXTest.java

License:Open Source License

@Test
@TestHadoop//from w  w  w.j  a va  2s  .com
public void testHadoopMapReduce() throws Exception {
    JobConf conf = getHadoopConf();
    FileSystem fs = FileSystem.get(conf);
    JobClient jobClient = new JobClient(conf);
    try {
        Path inputDir = new Path(getHadoopTestDir(), "input");
        Path outputDir = new Path(getHadoopTestDir(), "output");

        fs.mkdirs(inputDir);
        Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
        writer.write("a\n");
        writer.write("b\n");
        writer.write("c\n");
        writer.close();

        JobConf jobConf = getHadoopConf();
        jobConf.setInt("mapred.map.tasks", 1);
        jobConf.setInt("mapred.map.max.attempts", 1);
        jobConf.setInt("mapred.reduce.max.attempts", 1);
        jobConf.set("mapred.input.dir", inputDir.toString());
        jobConf.set("mapred.output.dir", outputDir.toString());
        final RunningJob runningJob = jobClient.submitJob(jobConf);
        waitFor(60 * 1000, true, new Predicate() {
            @Override
            public boolean evaluate() throws Exception {
                return runningJob.isComplete();
            }
        });
        Assert.assertTrue(runningJob.isSuccessful());
        Assert.assertTrue(fs.exists(new Path(outputDir, "part-00000")));
        BufferedReader reader = new BufferedReader(
                new InputStreamReader(fs.open(new Path(outputDir, "part-00000"))));
        Assert.assertTrue(reader.readLine().trim().endsWith("a"));
        Assert.assertTrue(reader.readLine().trim().endsWith("b"));
        Assert.assertTrue(reader.readLine().trim().endsWith("c"));
        Assert.assertNull(reader.readLine());
        reader.close();
    } finally {
        fs.close();
        jobClient.close();
    }
}

---

From source file:com.cloudera.crunch.impl.mem.MemPipeline.java

License:Open Source License

@Override
public void write(PCollection<?> collection, Target target) {
    if (target instanceof PathTarget) {
        Path path = ((PathTarget) target).getPath();
        try {/*from   w  w  w  . ja  v  a 2 s  .c o  m*/
            FileSystem fs = FileSystem.get(conf);
            FSDataOutputStream os = fs.create(path);
            if (collection instanceof PTable) {
                for (Object o : collection.materialize()) {
                    Pair p = (Pair) o;
                    os.writeBytes(p.first().toString());
                    os.writeBytes("\t");
                    os.writeBytes(p.second().toString());
                    os.writeBytes("\r\n");
                }
            } else {
                for (Object o : collection.materialize()) {
                    os.writeBytes(o.toString() + "\r\n");
                }
            }
            os.close();
        } catch (IOException e) {
            LOG.error("Exception writing target: " + target, e);
        }
    } else {
        LOG.error("Target " + target + " is not a PathTarget instance");
    }
}

---

From source file:com.cloudera.earthquake.GetData.java

License:Apache License

public static void main(String[] args) throws IOException {
    URL url = new URL("http://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.csv");
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    conn.connect();/*from   w ww. j a v a2 s.  c  om*/
    InputStream connStream = conn.getInputStream();

    FileSystem hdfs = FileSystem.get(new Configuration());
    FSDataOutputStream outStream = hdfs.create(new Path(args[0], "month.txt"));
    IOUtils.copy(connStream, outStream);

    outStream.close();
    connStream.close();
    conn.disconnect();
}

---

From source file:com.cloudera.flume.handlers.hdfs.TestDFSWrite.java

License:Apache License

@Test
public void testDirectWrite() throws IOException {
    FlumeConfiguration conf = FlumeConfiguration.get();

    Path path = new Path("file:///tmp/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);/*w w  w . j a va2 s  .c om*/

    String STRING = "Hello World";

    // writing
    FSDataOutputStream dos = hdfs.create(path);
    dos.writeUTF(STRING);
    dos.close();

    // reading
    FSDataInputStream dis = hdfs.open(path);
    String s = dis.readUTF();
    System.out.println(s);
    assertEquals(STRING, s);

    dis.close();

    hdfs.close();
}

---

From source file:com.cloudera.flume.handlers.hdfs.TestDFSWrite.java

License:Apache License

@Test
public void testWhyFail() throws IOException {

    // There a was a failure case using :
    FlumeConfiguration conf = FlumeConfiguration.get();
    Path path = new Path("file:///tmp/testfile");
    FileSystem hdfs = path.getFileSystem(conf);

    // writing//w w w  .j a v  a 2 s .c om
    FSDataOutputStream dos = hdfs.create(path);
    hdfs.deleteOnExit(path);

    // this version's Writer has ownOutputStream=false.
    Writer writer = SequenceFile.createWriter(conf, dos, WriteableEventKey.class, WriteableEvent.class,
            SequenceFile.CompressionType.NONE, new DefaultCodec());

    Event e = new EventImpl("EVENT".getBytes());

    writer.append(new WriteableEventKey(e), new WriteableEvent(e));
    writer.sync();
    writer.close();

    dos.close(); // It is strange that I have to close the underlying
    // FSDataOutputStream.

    // WTF: nothing written by this writer!
    FileStatus stats = hdfs.getFileStatus(path);
    assertTrue(stats.getLen() > 0);
    // it should have written something but it failed.
}

---

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

private boolean writeElasticSearchToMarkerFolder(StringBuilder httpQuery) {
    FileSystem hdfs;
    try {/*from   w  w w .  j  a v  a2 s  . c  o  m*/
        String markerFolder = conf.getElasticSearchDefaultMarkerFolder();
        dstPath = new Path(markerFolder);
        hdfs = dstPath.getFileSystem(conf);
        if (!hdfs.exists(dstPath)) {
            hdfs.mkdirs(dstPath);
        }

        dstPath = new Path(markerFolder + "/es-" + System.currentTimeMillis() + ".marker");
        System.out.println("creating file at: " + dstPath.toString());
        FSDataOutputStream writer_marker = hdfs.create(dstPath);
        writer_marker.writeBytes(httpQuery + "\n");
        writer_marker.close();
        dstPath = null;
        writer_marker = null;
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        return false;
    }
    return true;
}

---

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

public boolean writeHiveMarker(String hqlQuery, String filePath, String hiveMarkerFolder,
        String hiveMarkerPath) {// www.  j  av  a2 s . c  om
    LOG.debug("writing to hiveMarker: " + hiveMarkerFolder);

    FileSystem hdfs;
    dstPath = new Path(hiveMarkerFolder);
    try {
        hdfs = dstPath.getFileSystem(conf);

        if (!hdfs.exists(dstPath)) {
            hdfs.mkdirs(dstPath);
        }
        dstPath = new Path(hiveMarkerPath);
        FSDataOutputStream writer = hdfs.create(dstPath);
        writer.writeBytes(filePath + "\t" + hqlQuery + "\n");
        writer.close();
        dstPath = null;
        writer = null;

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    /*
      dstPath = new Path(hiveMarkerPath);
      hdfs = dstPath.getFileSystem(conf);
            
      writer = hdfs.create(dstPath);      
      writer.writeUTF(hqlQuery);
      writer.close();
      writer = null;
     */
    return true;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »