Example usage for org.apache.hadoop.fs FileSystem create

List of usage examples for org.apache.hadoop.fs FileSystem create

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. create

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem create.

Prototype

public FSDataOutputStream create(Path f) throws IOException

Source Link

Document

Create an FSDataOutputStream at the indicated Path.

Usage

From source file:com.github.sakserv.minicluster.impl.HdfsLocalClusterIntegrationTest.java

License:Apache License

@Test
public void testDfsClusterStart() throws Exception {

    // Write a file to HDFS containing the test string
    FileSystem hdfsFsHandle = dfsCluster.getHdfsFileSystemHandle();
    FSDataOutputStream writer = hdfsFsHandle
            .create(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    writer.writeUTF(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY));
    writer.close();/*  ww  w . java2s. co m*/

    // Read the file and compare to test string
    FSDataInputStream reader = hdfsFsHandle
            .open(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    assertEquals(reader.readUTF(), propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY));
    reader.close();
    hdfsFsHandle.close();

    URL url = new URL(String.format("http://localhost:%s/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest",
            propertyParser.getProperty(ConfigVars.HDFS_NAMENODE_HTTP_PORT_KEY)));
    URLConnection connection = url.openConnection();
    connection.setRequestProperty("Accept-Charset", "UTF-8");
    BufferedReader response = new BufferedReader(new InputStreamReader(connection.getInputStream()));
    String line = response.readLine();
    response.close();
    assertEquals("{\"Path\":\"/user/guest\"}", line);

}

From source file:com.github.sakserv.minicluster.impl.KdcLocalClusterHdfsIntegrationTest.java

License:Apache License

@Test
public void testHdfs() throws Exception {
    FileSystem hdfsFsHandle = hdfsLocalCluster.getHdfsFileSystemHandle();

    UserGroupInformation.loginUserFromKeytab(kdcLocalCluster.getKrbPrincipalWithRealm("hdfs"),
            kdcLocalCluster.getKeytabForPrincipal("hdfs"));

    assertTrue(UserGroupInformation.isSecurityEnabled());
    assertTrue(UserGroupInformation.isLoginKeytabBased());

    // Write a file to HDFS containing the test string
    FSDataOutputStream writer = hdfsFsHandle
            .create(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    writer.writeUTF(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY));
    writer.close();/*www  .  j a va 2 s  . com*/

    // Read the file and compare to test string
    FSDataInputStream reader = hdfsFsHandle
            .open(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    assertEquals(reader.readUTF(), propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY));
    reader.close();

    // Log out
    UserGroupInformation.getLoginUser().logoutUserFromKeytab();

    UserGroupInformation.reset();

    try {
        Configuration conf = new Configuration();
        UserGroupInformation.setConfiguration(conf);
        FileSystem.get(hdfsFsHandle.getUri(), conf)
                .open(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
        fail();
    } catch (AccessControlException e) {
        LOG.info("Not authenticated!");
    }
}

From source file:com.github.sakserv.minicluster.impl.KnoxLocalClusterIntegrationTest.java

License:Apache License

@Test
public void testKnoxWithWebhdfs() throws Exception {

    // Write a file to HDFS containing the test string
    FileSystem hdfsFsHandle = dfsCluster.getHdfsFileSystemHandle();
    try (FSDataOutputStream writer = hdfsFsHandle
            .create(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)))) {
        writer.write(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY).getBytes("UTF-8"));
        writer.flush();//from   w w  w. j a v  a2 s  .  c  o  m
    }

    // Read the file throught webhdfs
    URL url = new URL(String.format("http://localhost:%s/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest",
            propertyParser.getProperty(ConfigVars.HDFS_NAMENODE_HTTP_PORT_KEY)));
    URLConnection connection = url.openConnection();
    connection.setRequestProperty("Accept-Charset", "UTF-8");
    try (BufferedReader response = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
        String line = response.readLine();
        assertEquals("{\"Path\":\"/user/guest\"}", line);
    }

    url = new URL(String.format("http://localhost:%s/webhdfs/v1%s?op=OPEN&user.name=guest",
            propertyParser.getProperty(ConfigVars.HDFS_NAMENODE_HTTP_PORT_KEY),
            propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    connection = url.openConnection();
    connection.setRequestProperty("Accept-Charset", "UTF-8");
    try (BufferedReader response = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
        String line = response.readLine();
        response.close();
        assertEquals(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY), line);
    }

    // Knox clients need self trusted certificates in tests
    defaultBlindTrust();

    // Read the file throught Knox
    url = new URL(String.format(
            "https://localhost:%s/gateway/mycluster/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest",
            propertyParser.getProperty(ConfigVars.KNOX_PORT_KEY)));
    connection = url.openConnection();
    connection.setRequestProperty("Accept-Charset", "UTF-8");
    try (BufferedReader response = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
        String line = response.readLine();
        assertEquals("{\"Path\":\"/user/guest\"}", line);
    }

    url = new URL(String.format("https://localhost:%s/gateway/mycluster/webhdfs/v1/%s?op=OPEN&user.name=guest",
            propertyParser.getProperty(ConfigVars.KNOX_PORT_KEY),
            propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    connection = url.openConnection();
    connection.setRequestProperty("Accept-Charset", "UTF-8");
    try (BufferedReader response = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
        String line = response.readLine();
        response.close();
        assertEquals(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY), line);
    }
}

From source file:com.github.sakserv.minicluster.impl.MRLocalClusterIntegrationTest.java

License:Apache License

private void writeFileToHdfs(String fileName, String contents) throws Exception {
    // Write a file to HDFS containing the test string
    FileSystem hdfsFsHandle = dfsCluster.getHdfsFileSystemHandle();
    FSDataOutputStream writer = hdfsFsHandle.create(new Path(fileName));
    writer.writeUTF(contents);//from  ww  w.j av  a  2s  .c  o m
    writer.close();
    hdfsFsHandle.close();
}

From source file:com.google.mr4c.content.HDFSContentFactory.java

License:Open Source License

public OutputStream getOutputStreamForContent(URI uri) throws IOException {
    Path path = toPath(uri);/*  ww w  . j  a  va 2s.  c  o m*/
    FileSystem fs = FileSystem.get(uri, m_config);
    return fs.create(path);
}

From source file:com.hadoop.compression.lzo.LzoIndex.java

License:Open Source License

/**
 * Index an lzo file to allow the input format to split them into separate map
 * jobs./*from  ww  w  .j  a  v a  2 s .c  o  m*/
 *
 * @param fs File system that contains the file.
 * @param lzoFile the lzo file to index.  For filename.lzo, the created index file will be
 * filename.lzo.index.
 * @throws IOException
 */
public static void createIndex(FileSystem fs, Path lzoFile) throws IOException {

    Configuration conf = fs.getConf();
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodec(lzoFile);
    if (null == codec) {
        throw new IOException("Could not find codec for file " + lzoFile
                + " - you may need to add the LZO codec to your io.compression.codecs "
                + "configuration in core-site.xml");
    }
    ((Configurable) codec).setConf(conf);

    FSDataInputStream is = null;
    FSDataOutputStream os = null;
    Path outputFile = lzoFile.suffix(LZO_INDEX_SUFFIX);
    Path tmpOutputFile = lzoFile.suffix(LZO_TMP_INDEX_SUFFIX);

    // Track whether an exception was thrown or not, so we know to either
    // delete the tmp index file on failure, or rename it to the new index file on success.
    boolean indexingSucceeded = false;
    try {
        is = fs.open(lzoFile);
        os = fs.create(tmpOutputFile);
        LzopDecompressor decompressor = (LzopDecompressor) codec.createDecompressor();
        // Solely for reading the header
        codec.createInputStream(is, decompressor);
        int numCompressedChecksums = decompressor.getCompressedChecksumsCount();
        int numDecompressedChecksums = decompressor.getDecompressedChecksumsCount();

        while (true) {
            // read and ignore, we just want to get to the next int
            int uncompressedBlockSize = is.readInt();
            if (uncompressedBlockSize == 0) {
                break;
            } else if (uncompressedBlockSize < 0) {
                throw new EOFException();
            }

            int compressedBlockSize = is.readInt();
            if (compressedBlockSize <= 0) {
                throw new IOException("Could not read compressed block size");
            }

            // See LzopInputStream.getCompressedData
            boolean isUncompressedBlock = (uncompressedBlockSize == compressedBlockSize);
            int numChecksumsToSkip = isUncompressedBlock ? numDecompressedChecksums
                    : numDecompressedChecksums + numCompressedChecksums;
            long pos = is.getPos();
            // write the pos of the block start
            os.writeLong(pos - 8);
            // seek to the start of the next block, skip any checksums
            is.seek(pos + compressedBlockSize + (4 * numChecksumsToSkip));
        }
        // If we're here, indexing was successful.
        indexingSucceeded = true;
    } finally {
        // Close any open streams.
        if (is != null) {
            is.close();
        }

        if (os != null) {
            os.close();
        }

        if (!indexingSucceeded) {
            // If indexing didn't succeed (i.e. an exception was thrown), clean up after ourselves.
            fs.delete(tmpOutputFile, false);
        } else {
            // Otherwise, rename filename.lzo.index.tmp to filename.lzo.index.
            fs.rename(tmpOutputFile, outputFile);
        }
    }
}

From source file:com.hadoop.mapreduce.LzoTextInputFormat.java

License:Open Source License

/**
 * Index an lzo file to allow the input format to split them into separate map
 * jobs.//  ww  w . ja  va  2  s .c  o  m
 * 
 * @param fs
 *          File system that contains the file.
 * @param lzoFile
 *          the lzo file to index.
 * @throws IOException
 */
public static void createIndex(FileSystem fs, Path lzoFile) throws IOException {

    Configuration conf = fs.getConf();
    CompressionCodecFactory factory = new CompressionCodecFactory(fs.getConf());
    CompressionCodec codec = factory.getCodec(lzoFile);
    ((Configurable) codec).setConf(conf);

    InputStream lzoIs = null;
    FSDataOutputStream os = null;
    Path outputFile = new Path(lzoFile.toString() + LzoTextInputFormat.LZO_INDEX_SUFFIX);
    Path tmpOutputFile = outputFile.suffix(".tmp");

    try {
        FSDataInputStream is = fs.open(lzoFile);
        os = fs.create(tmpOutputFile);
        LzopDecompressor decompressor = (LzopDecompressor) codec.createDecompressor();
        // for reading the header
        lzoIs = codec.createInputStream(is, decompressor);

        int numChecksums = decompressor.getChecksumsCount();

        while (true) {
            // read and ignore, we just want to get to the next int
            int uncompressedBlockSize = is.readInt();
            if (uncompressedBlockSize == 0) {
                break;
            } else if (uncompressedBlockSize < 0) {
                throw new EOFException();
            }

            int compressedBlockSize = is.readInt();
            if (compressedBlockSize <= 0) {
                throw new IOException("Could not read compressed block size");
            }

            long pos = is.getPos();
            // write the pos of the block start
            os.writeLong(pos - 8);
            // seek to the start of the next block, skip any checksums
            is.seek(pos + compressedBlockSize + (4 * numChecksums));
        }
    } finally {
        if (lzoIs != null) {
            lzoIs.close();
        }

        if (os != null) {
            os.close();
        }
    }

    fs.rename(tmpOutputFile, outputFile);
}

From source file:com.hdfs.concat.clean.TestClean.java

License:Apache License

@Test
public void testAge() throws Exception {
    Configuration conf = createJobConf();
    FileSystem fs = FileSystem.get(conf);
    fs.mkdirs(new Path(ROOT_DIR, "a"));
    fs.mkdirs(new Path(new Path(ROOT_DIR, "a"), "1"));
    fs.mkdirs(new Path(ROOT_DIR, "b"));
    fs.mkdirs(new Path(ROOT_DIR, "c"));
    fs.mkdirs(new Path(new Path(ROOT_DIR, "c"), "2"));

    Path oldFile = new Path(new Path(new Path(ROOT_DIR, "a"), "1"), "oldfile");
    FSDataOutputStream out = fs.create(oldFile);
    out.write("bla".getBytes());
    out.close();//w w  w .  jav  a2 s . co m

    Path cFile = new Path(new Path(new Path(ROOT_DIR, "c"), "1"), "cfile");
    FSDataOutputStream out2 = fs.create(cFile);
    out2.write("wah".getBytes());
    out2.close();

    assertEquals(true, fs.exists(cFile));
    assertEquals(true, fs.exists(oldFile));

    Clean cleanWarn = new Clean();
    Configuration warnConf = createJobConf();
    warnConf.set(Clean.TARGET_DIR, ROOT_DIR.toString());
    warnConf.set(Clean.TARGET_EXPR, "cfile");
    warnConf.set(Clean.WARN_MODE, "true");
    ToolRunner.run(warnConf, cleanWarn, new String[] {});
    assertEquals(true, fs.exists(cFile));
    assertEquals(true, fs.exists(oldFile));

    Clean cleanReg = new Clean();
    Configuration regConf = createJobConf();
    regConf.set(Clean.TARGET_DIR, ROOT_DIR.toString());
    regConf.set(Clean.TARGET_EXPR, "cfile");
    ToolRunner.run(regConf, cleanReg, new String[] {});
    assertEquals(false, fs.exists(cFile));
    assertEquals(true, fs.exists(oldFile));

    Clean clean = new Clean();
    Configuration cleanConf = createJobConf();
    cleanConf.setLong(Clean.CUTTOFF_MILLIS, 20000);
    cleanConf.set(Clean.TARGET_DIR, ROOT_DIR.toString());
    ToolRunner.run(cleanConf, clean, new String[] {});
    assertEquals(true, fs.exists(oldFile));
    Thread.sleep(3);

    Clean clean2 = new Clean();
    Configuration cleanConf2 = createJobConf();
    cleanConf2.setLong(Clean.CUTTOFF_MILLIS, 1);
    cleanConf2.set(Clean.TARGET_DIR, ROOT_DIR.toString());
    ToolRunner.run(cleanConf2, clean2, new String[] {});
    assertEquals(false, fs.exists(oldFile));

}

From source file:com.hortonworks.etl.talend.oozie.AbstractOozieJobSubmission.java

License:Apache License

protected void serializeToHDFS(String toSerialize, String asFile, JobContext jobContext) throws IOException {
    org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
    configuration.set("fs.default.name", jobContext.getNameNodeEndPoint());
    FileSystem fs = FileSystem.get(configuration);

    Path wfFile = new Path(jobContext.getNameNodeEndPoint() + jobContext.getJobPathOnHDFS() + asFile);
    FSDataOutputStream outputStream = null;
    try {// ww w .  j a  v a  2  s  .c  om
        outputStream = fs.create(wfFile);
        outputStream.writeBytes(toSerialize);
    } finally {
        if (outputStream != null) {
            outputStream.close();
        }
    }
}

From source file:com.huayu.metis.flume.sink.hdfs.HDFSSequenceFile.java

License:Apache License

@Override
public void open(String filePath) throws IOException {
    Configuration conf = new Configuration();
    Path dstPath = new Path(filePath);

    FileSystem fileSystem = dstPath.getFileSystem(conf);
    //2.2Hadoop, dfs.append.support ???
    if (fileSystem.exists(dstPath) && fileSystem.isFile(dstPath)) {
        outStream = fileSystem.append(dstPath);
    } else {/*from  w w w  .java2s. c  o m*/
        outStream = fileSystem.create(dstPath);
    }

    writer = SequenceFile.createWriter(conf, SequenceFile.Writer.stream(outStream),
            SequenceFile.Writer.keyClass(serializer.getKeyClass()),
            SequenceFile.Writer.valueClass(serializer.getValueClass()));
    registerCurrentStream(outStream, fileSystem, dstPath);
}