Example usage for org.apache.hadoop.fs FileSystem delete

List of usage examples for org.apache.hadoop.fs FileSystem delete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyCommitter.java

License:Apache License

/**
 * Cleanup meta folder and other temporary files
 *
 * @param conf - Job Configuration/*  ww w .  j  a  v a 2  s .c  o m*/
 */
private void cleanup(Configuration conf) {
    Path metaFolder = new Path(conf.get(DistCpConstants.CONF_LABEL_META_FOLDER));
    try {
        FileSystem fs = metaFolder.getFileSystem(conf);
        LOG.info("Cleaning up temporary work folder: " + metaFolder);
        fs.delete(metaFolder, true);
    } catch (IOException ignore) {
        LOG.error("Exception encountered ", ignore);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyCommitter.java

License:Apache License

private void deleteMissing(Configuration conf) throws IOException {
    LOG.info("-delete option is enabled. About to remove entries from " + "target that are missing in source");

    Path sourceListing = new Path(conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH));
    FileSystem clusterFS = sourceListing.getFileSystem(conf);
    Path sortedSourceListing = DistCpUtils.sortListing(clusterFS, conf, sourceListing);

    Path targetListing = new Path(sourceListing.getParent(), "targetListing.seq");
    CopyListing target = new GlobbedCopyListing(conf, null);

    List<Path> targets = new ArrayList<Path>(1);
    Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
    targets.add(targetFinalPath);// ww  w.  j a v  a 2s  .  c  om
    DistCpOptions options = new DistCpOptions(targets, new Path("/NONE"));

    target.buildListing(targetListing, options);
    Path sortedTargetListing = DistCpUtils.sortListing(clusterFS, conf, targetListing);
    long totalLen = clusterFS.getFileStatus(sortedTargetListing).getLen();

    SequenceFile.Reader sourceReader = new SequenceFile.Reader(clusterFS, sortedSourceListing, conf);
    SequenceFile.Reader targetReader = new SequenceFile.Reader(clusterFS, sortedTargetListing, conf);

    long deletedEntries = 0;
    try {
        FileStatus srcFileStatus = new FileStatus();
        Text srcRelPath = new Text();
        FileStatus trgtFileStatus = new FileStatus();
        Text trgtRelPath = new Text();

        FileSystem targetFS = targetFinalPath.getFileSystem(conf);
        boolean srcAvailable = sourceReader.next(srcRelPath, srcFileStatus);
        while (targetReader.next(trgtRelPath, trgtFileStatus)) {
            while (srcAvailable && trgtRelPath.compareTo(srcRelPath) > 0) {
                srcAvailable = sourceReader.next(srcRelPath, srcFileStatus);
            }

            if (srcAvailable && trgtRelPath.equals(srcRelPath))
                continue;

            boolean result = (!targetFS.exists(trgtFileStatus.getPath())
                    || targetFS.delete(trgtFileStatus.getPath(), true));
            if (result) {
                LOG.info("Deleted " + trgtFileStatus.getPath() + " - Missing at source");
                deletedEntries++;
            } else {
                throw new IOException("Unable to delete " + trgtFileStatus.getPath());
            }
            HadoopCompat.progress(taskAttemptContext);
            HadoopCompat.setStatus(taskAttemptContext, "Deleting missing files from target. ["
                    + targetReader.getPosition() * 100 / totalLen + "%]");
        }
    } finally {
        IOUtils.closeStream(sourceReader);
        IOUtils.closeStream(targetReader);
    }
    LOG.info("Deleted " + deletedEntries + " from target: " + targets.get(0));
}

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyCommitter.java

License:Apache License

private void commitData(Configuration conf) throws IOException {

    Path workDir = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
    Path finalDir = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
    FileSystem targetFS = workDir.getFileSystem(conf);

    LOG.info("Atomic commit enabled. Moving " + workDir + " to " + finalDir);
    if (targetFS.exists(finalDir) && targetFS.exists(workDir))
        if (!targetFS.delete(finalDir, true)) {
            LOG.error("Unable to delete pre-existing final-data at " + finalDir);
            throw new IOException("Atomic commit failed. Pre-existing final data" + " in " + finalDir
                    + " could not be cleared, before commit.");
        }//from   w  ww.j av a  2 s  .  co m

    boolean result = targetFS.rename(workDir, finalDir);
    if (!result) {
        LOG.warn("Rename failed. Perhaps data already moved. Verifying...");
        result = targetFS.exists(finalDir) && !targetFS.exists(workDir);
    }
    if (result) {
        LOG.info("Data committed successfully to " + finalDir);
        HadoopCompat.setStatus(taskAttemptContext, "Data committed successfully to " + finalDir);
    } else {
        LOG.error("Unable to commit data to " + finalDir);
        throw new IOException(
                "Atomic commit failed. Temporary data in " + workDir + ", Unable to move to " + finalDir);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private long doCopy(FileStatus sourceFileStatus, Path target, Mapper.Context context,
        EnumSet<FileAttribute> fileAttributes, Map<Long, Long> received) throws IOException {

    Path tmpTargetPath = getTmpFile(target, context);
    final Configuration configuration = HadoopCompat.getTaskConfiguration(context);
    FileSystem targetFS = target.getFileSystem(configuration);
    compressionCodecs = new CompressionCodecFactory(context.getConfiguration());
    try {//from w ww . j a  v  a  2  s .  c om
        if (LOG.isDebugEnabled()) {
            LOG.debug("Copying " + sourceFileStatus.getPath() + " to " + target);
            LOG.debug("Tmp-file path: " + tmpTargetPath);
        }
        FileSystem sourceFS = sourceFileStatus.getPath().getFileSystem(configuration);
        long bytesRead = copyToTmpFile(tmpTargetPath, targetFS, sourceFileStatus, context, fileAttributes,
                received);

        compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead);
        if (bytesRead > 0) {
            compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath);
        }
        promoteTmpToTarget(tmpTargetPath, target, targetFS);
        return bytesRead;

    } finally {
        if (targetFS.exists(tmpTargetPath))
            targetFS.delete(tmpTargetPath, false);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private void promoteTmpToTarget(Path tmpTarget, Path target, FileSystem fs) throws IOException {
    if ((fs.exists(target) && !fs.delete(target, false))
            || (!fs.exists(target.getParent()) && !fs.mkdirs(target.getParent()))
            || !fs.rename(tmpTarget, target)) {
        throw new IOException("Failed to promote tmp-file:" + tmpTarget + " to: " + target);
    }/*from   w ww. j a  va 2s  .c  o  m*/
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private void doTestIgnoreFailures(boolean ignoreFailures) {
    try {/*  www  .j  a  v a 2  s .  c o  m*/
        deleteState();
        createSourceData();

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        Configuration configuration = context.getConfiguration();
        configuration.setBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), ignoreFailures);
        configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), true);
        configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), true);
        copyMapper.setup(context);

        for (Path path : pathList) {
            final FileStatus fileStatus = fs.getFileStatus(path);
            if (!fileStatus.isDir()) {
                fs.delete(path, true);
                copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), fileStatus,
                        context);
            }
        }
        if (ignoreFailures) {
            for (Text value : writer.values()) {
                Assert.assertTrue(value.toString() + " is not skipped", value.toString().startsWith("FAIL:"));
            }
        }
        Assert.assertTrue("There should have been an exception.", ignoreFailures);
    } catch (Exception e) {
        Assert.assertTrue("Unexpected exception: " + e.getMessage(), !ignoreFailures);
        e.printStackTrace();
    }
}

From source file:com.inmobi.conduit.distcp.tools.util.DistCpUtils.java

License:Apache License

/**
 * Sort sequence file containing FileStatus and Text as key and value respecitvely
 *
 * @param fs - File System/*from   w  w w . j  ava 2 s .c o m*/
 * @param conf - Configuration
 * @param sourceListing - Source listing file
 * @return Path of the sorted file. Is source file with _sorted appended to the name
 * @throws IOException - Any exception during sort.
 */
public static Path sortListing(FileSystem fs, Configuration conf, Path sourceListing) throws IOException {
    SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, Text.class, FileStatus.class, conf);
    Path output = new Path(sourceListing.toString() + "_sorted");

    if (fs.exists(output)) {
        fs.delete(output, false);
    }

    sorter.sort(sourceListing, output);
    return output;
}

From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java

License:Apache License

@Test
public void testPreserve() {
    try {/*from www.j a  v a2 s  .c  om*/
        FileSystem fs = FileSystem.get(config);
        EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);

        Path path = new Path("/tmp/abc");
        Path src = new Path("/tmp/src");
        fs.mkdirs(path);
        fs.mkdirs(src);
        FileStatus srcStatus = fs.getFileStatus(src);

        FsPermission noPerm = new FsPermission((short) 0);
        fs.setPermission(path, noPerm);
        fs.setOwner(path, "nobody", "nobody");

        DistCpUtils.preserve(fs, path, srcStatus, attributes);
        FileStatus target = fs.getFileStatus(path);
        Assert.assertEquals(target.getPermission(), noPerm);
        Assert.assertEquals(target.getOwner(), "nobody");
        Assert.assertEquals(target.getGroup(), "nobody");

        attributes.add(FileAttribute.PERMISSION);
        DistCpUtils.preserve(fs, path, srcStatus, attributes);
        target = fs.getFileStatus(path);
        Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
        Assert.assertEquals(target.getOwner(), "nobody");
        Assert.assertEquals(target.getGroup(), "nobody");

        attributes.add(FileAttribute.GROUP);
        attributes.add(FileAttribute.USER);
        DistCpUtils.preserve(fs, path, srcStatus, attributes);
        target = fs.getFileStatus(path);
        Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
        Assert.assertEquals(target.getOwner(), srcStatus.getOwner());
        Assert.assertEquals(target.getGroup(), srcStatus.getGroup());

        fs.delete(path, true);
        fs.delete(src, true);
    } catch (IOException e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Preserve test failure");
    }
}

From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java

License:Apache License

public static void delete(FileSystem fs, String path) {
    try {/*from  w  w  w .j  a  v a 2s. co  m*/
        if (fs != null) {
            if (path != null) {
                fs.delete(new Path(path), true);
            }
        }
    } catch (IOException e) {
        LOG.warn("Exception encountered ", e);
    }
}

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

private void cleanUpTmp(FileSystem fs) throws Exception {
    if (fs.exists(tmpPath)) {
        LOG.info("Deleting tmpPath recursively [" + tmpPath + "]");
        fs.delete(tmpPath, true);
    }/*from   www  .  j a v a 2s .  c o m*/
}