Example usage for org.apache.hadoop.fs FileSystem delete

List of usage examples for org.apache.hadoop.fs FileSystem delete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.nearinfinity.blur.utils.BlurUtil.java

License:Apache License

public static void removeIndexFiles(String uri) throws IOException {
    Path tablePath = new Path(uri);
    FileSystem fileSystem = FileSystem.get(tablePath.toUri(), new Configuration());
    fileSystem.delete(tablePath, true);
}

From source file:com.netflix.aegisthus.tools.StorageHelper.java

License:Apache License

public void deleteBaseTempLocation() throws IOException {
    String base = getBaseTempLocation();
    Path path = new Path(base);
    FileSystem fs = path.getFileSystem(config);
    LOG.info(String.format("deleting: %s", base));
    fs.delete(path, true);
}

From source file:com.netflix.aegisthus.tools.Utils.java

License:Apache License

public static boolean delete(Configuration config, Path path, boolean recursive) throws IOException {
    FileSystem fs = path.getFileSystem(config);
    return fs.delete(path, recursive);
}

From source file:com.netflix.bdp.s3.S3DirectoryOutputCommitter.java

License:Apache License

@Override
public void commitJob(JobContext context) throws IOException {
    Path outputPath = getOutputPath(context);
    // use the FS implementation because it will check for _$folder$
    FileSystem fs = outputPath.getFileSystem(context.getConfiguration());
    if (fs.exists(outputPath)) {
        switch (getMode(context)) {
        case FAIL:
            // this was checked in setupJob, but this avoids some cases where
            // output was created while the job was processing
            throw new AlreadyExistsException("Output path already exists: " + outputPath);
        case APPEND:
            // do nothing
            break;
        case REPLACE:
            LOG.info("Removing output path to be replaced: " + outputPath);
            if (!fs.delete(outputPath, true /* recursive */ )) {
                throw new IOException("Failed to delete existing output directory for replace:" + outputPath);
            }/*from w ww  .  jav  a 2s  .  com*/
            break;
        default:
            throw new RuntimeException("Unknown conflict resolution mode: " + getMode(context));
        }
    }

    super.commitJob(context);
}

From source file:com.netflix.bdp.s3.S3MultipartOutputCommitter.java

License:Apache License

protected void commitTaskInternal(final TaskAttemptContext context, Iterable<FileStatus> taskOutput)
        throws IOException {
    Configuration conf = context.getConfiguration();
    final AmazonS3 client = getClient(getOutputPath(context), conf);

    final Path attemptPath = getTaskAttemptPath(context);
    FileSystem attemptFS = attemptPath.getFileSystem(conf);

    // add the commits file to the wrapped commiter's task attempt location.
    // this complete file will be committed by the wrapped committer at the end
    // of this method.
    Path commitsAttemptPath = wrappedCommitter.getTaskAttemptPath(context);
    FileSystem commitsFS = commitsAttemptPath.getFileSystem(conf);

    // keep track of unfinished commits in case one fails. if something fails,
    // we will try to abort the ones that had already succeeded.
    final List<S3Util.PendingUpload> commits = Lists.newArrayList();

    boolean threw = true;
    ObjectOutputStream completeUploadRequests = new ObjectOutputStream(
            commitsFS.create(commitsAttemptPath, false));
    try {/*  w ww .j a  va2  s. c o m*/
        Tasks.foreach(taskOutput).stopOnFailure().throwFailureWhenFinished().executeWith(threadPool)
                .run(new Task<FileStatus, IOException>() {
                    @Override
                    public void run(FileStatus stat) throws IOException {
                        File localFile = new File(URI.create(stat.getPath().toString()).getPath());
                        if (localFile.length() <= 0) {
                            return;
                        }
                        String relative = Paths.getRelativePath(attemptPath, stat.getPath());
                        String partition = getPartition(relative);
                        String key = getFinalKey(relative, context);
                        S3Util.PendingUpload commit = S3Util.multipartUpload(client, localFile, partition,
                                getBucket(context), key, uploadPartSize);
                        commits.add(commit);
                    }
                });

        for (S3Util.PendingUpload commit : commits) {
            completeUploadRequests.writeObject(commit);
        }

        threw = false;

    } finally {
        if (threw) {
            Tasks.foreach(commits).run(new Task<S3Util.PendingUpload, RuntimeException>() {
                @Override
                public void run(S3Util.PendingUpload commit) {
                    S3Util.abortCommit(client, commit);
                }
            });
            try {
                attemptFS.delete(attemptPath, true);
            } catch (Exception e) {
                LOG.error("Failed while cleaning up failed task commit: ", e);
            }
        }
        Closeables.close(completeUploadRequests, threw);
    }

    wrappedCommitter.commitTask(context);

    attemptFS.delete(attemptPath, true);
}

From source file:com.netflix.bdp.s3.S3MultipartOutputCommitter.java

License:Apache License

@Override
public void abortTask(TaskAttemptContext context) throws IOException {
    // the API specifies that the task has not yet been committed, so there are
    // no uploads that need to be cancelled. just delete files on the local FS.
    Path attemptPath = getTaskAttemptPath(context);
    FileSystem fs = attemptPath.getFileSystem(context.getConfiguration());
    if (!fs.delete(attemptPath, true)) {
        LOG.error("Failed to delete task attempt data: " + attemptPath);
    }// ww w. ja  v a2s.  c o  m
    wrappedCommitter.abortTask(context);
}

From source file:com.netflix.bdp.s3.S3PartitionedOutputCommitter.java

License:Apache License

@Override
public void commitJob(JobContext context) throws IOException {
    List<S3Util.PendingUpload> pending = getPendingUploads(context);

    FileSystem s3 = getOutputPath(context).getFileSystem(context.getConfiguration());
    Set<Path> partitions = Sets.newLinkedHashSet();
    LOG.info("The partitions are: " + partitions);
    for (S3Util.PendingUpload commit : pending) {
        Path filePath = new Path("s3://" + commit.getBucketName() + "/" + commit.getKey());
        partitions.add(filePath.getParent());
    }/*from w  ww.  jav a 2  s.  c o  m*/

    // enforce conflict resolution
    boolean threw = true;
    try {
        switch (getMode(context)) {
        case FAIL:
            // FAIL checking is done on the task side, so this does nothing
            break;
        case APPEND:
            // no check is needed because the output may exist for appending
            break;
        case REPLACE:
            for (Path partitionPath : partitions) {
                if (s3.exists(partitionPath)) {
                    LOG.info("Removing partition path to be replaced: " + partitionPath);
                    if (!s3.delete(partitionPath, true /* recursive */)) {
                        throw new IOException("Failed to delete existing " + "partition directory for replace:"
                                + partitionPath);
                    }
                }
            }
            break;
        default:
            throw new RuntimeException("Unknown conflict resolution mode: " + getMode(context));
        }

        threw = false;

    } catch (IOException e) {
        throw new IOException("Failed to enforce conflict resolution", e);

    } finally {
        if (threw) {
            abortJobInternal(context, pending, threw);
        }
    }

    commitJobInternal(context, pending);
}

From source file:com.netflix.bdp.s3.TestS3DirectoryOutputCommitter.java

License:Apache License

@Test
public void testReplaceConflictResolution() throws Exception {
    FileSystem mockS3 = getMockS3();

    when(mockS3.exists(OUTPUT_PATH)).thenReturn(true);

    getJob().getConfiguration().set(S3Committer.CONFLICT_MODE, "replace");

    final S3DirectoryOutputCommitter committer = newJobCommitter();

    committer.setupJob(getJob());/*from  ww w. j a v a 2 s  .  c  o m*/
    verify(mockS3).exists(OUTPUT_PATH);
    verifyNoMoreInteractions(mockS3);

    Mockito.reset(mockS3);
    when(mockS3.exists(OUTPUT_PATH)).thenReturn(true);
    when(mockS3.delete(OUTPUT_PATH, true)).thenReturn(true);

    committer.commitJob(getJob());
    verify(mockS3).exists(OUTPUT_PATH);
    verify(mockS3).delete(OUTPUT_PATH, true);
    verifyNoMoreInteractions(mockS3);
}

From source file:com.netflix.bdp.s3.TestS3PartitionedFileListing.java

License:Apache License

@Test
public void testTaskOutputListing() throws Exception {
    S3PartitionedOutputCommitter committer = newTaskCommitter();

    // create files in the attempt path that should be found by getTaskOutput
    Path attemptPath = committer.getTaskAttemptPath(getTAC());
    FileSystem attemptFS = attemptPath.getFileSystem(getTAC().getConfiguration());
    attemptFS.delete(attemptPath, true);

    List<String> expectedFiles = Lists.newArrayList();
    for (String dateint : Arrays.asList("20161115", "20161116")) {
        for (String hour : Arrays.asList("13", "14")) {
            String relative = "dateint=" + dateint + "/hour=" + hour + "/" + UUID.randomUUID().toString()
                    + ".parquet";
            expectedFiles.add(relative);
            attemptFS.create(new Path(attemptPath, relative)).close();
        }//from ww  w  .j  av a 2  s.co m
    }

    List<FileStatus> attemptFiles = committer.getTaskOutput(getTAC());
    List<String> actualFiles = Lists.newArrayList();
    for (FileStatus stat : attemptFiles) {
        String relative = getRelativePath(attemptPath, stat.getPath());
        actualFiles.add(relative);
    }

    Assert.assertEquals("File sets should match", expectedFiles, actualFiles);

    attemptFS.delete(attemptPath, true);
}

From source file:com.netflix.bdp.s3.TestS3PartitionedFileListing.java

License:Apache License

@Test
public void testTaskOutputListingWithHiddenFiles() throws Exception {
    S3PartitionedOutputCommitter committer = newTaskCommitter();

    // create files in the attempt path that should be found by getTaskOutput
    Path attemptPath = committer.getTaskAttemptPath(getTAC());
    FileSystem attemptFS = attemptPath.getFileSystem(getTAC().getConfiguration());
    attemptFS.delete(attemptPath, true);

    List<String> expectedFiles = Lists.newArrayList();
    for (String dateint : Arrays.asList("20161115", "20161116")) {
        String metadata = "dateint=" + dateint + "/" + "_metadata";
        attemptFS.create(new Path(attemptPath, metadata)).close();

        for (String hour : Arrays.asList("13", "14")) {
            String relative = "dateint=" + dateint + "/hour=" + hour + "/" + UUID.randomUUID().toString()
                    + ".parquet";
            expectedFiles.add(relative);
            attemptFS.create(new Path(attemptPath, relative)).close();

            String partial = "dateint=" + dateint + "/hour=" + hour + "/." + UUID.randomUUID().toString()
                    + ".partial";
            attemptFS.create(new Path(attemptPath, partial)).close();
        }/* w ww  .  j a  v  a2  s.c om*/
    }

    List<FileStatus> attemptFiles = committer.getTaskOutput(getTAC());
    List<String> actualFiles = Lists.newArrayList();
    for (FileStatus stat : attemptFiles) {
        String relative = getRelativePath(attemptPath, stat.getPath());
        actualFiles.add(relative);
    }

    Assert.assertEquals("File sets should match", expectedFiles, actualFiles);

    attemptFS.delete(attemptPath, true);
}