Example usage for org.apache.hadoop.fs FileSystem delete

List of usage examples for org.apache.hadoop.fs FileSystem delete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.asakusafw.operation.tools.hadoop.fs.Clean.java

License:Apache License

private boolean remove(FileSystem fs, FileStatus file, Context context) {
    LOG.debug("Attempt to remove {}", file.getPath()); //$NON-NLS-1$
    boolean isSymlink = context.isSymlink(fs, file);
    if (isSymlink) {
        LOG.error(MessageFormat.format("[OT-CLEAN-W01001] Symlink is currenty not supported: {0}",
                file.getPath()));/*from   w ww .  j  ava2 s.c  om*/
        context.setError();
        return false;
    }
    if (file.isDir()) {
        if (context.isRecursive()) {
            List<FileStatus> children;
            try {
                children = asList(fs.listStatus(file.getPath()));
            } catch (IOException e) {
                LOG.error(
                        MessageFormat.format("[OT-CLEAN-E01003] Failed to list directory: {0}", file.getPath()),
                        e);
                context.setError();
                return false;
            }
            boolean deleteChildren = true;
            for (FileStatus child : children) {
                deleteChildren &= remove(fs, child, context);
            }
            if (deleteChildren == false) {
                LOG.info(MessageFormat.format("[OT-CLEAN-I01004] Skipped: {0} (is no-empty directory)",
                        file.getPath(), new Date(file.getModificationTime())));
                return false;
            }
        } else {
            LOG.info(MessageFormat.format("[OT-CLEAN-I01003] Skipped: {0} (is directory)", file.getPath(),
                    new Date(file.getModificationTime())));
            return false;
        }
    }
    if (context.canDelete(file)) {
        LOG.debug("Removing {}", file.getPath()); //$NON-NLS-1$
        if (context.isDryRun() == false) {
            try {
                boolean removed = fs.delete(file.getPath(), false);
                if (removed == false) {
                    LOG.error(MessageFormat.format("[OT-CLEAN-E01004] Failed to remove: {0}", file.getPath()));
                    context.setError();
                    return false;
                }
            } catch (IOException e) {
                LOG.warn(MessageFormat.format("[OT-CLEAN-E01004] Failed to remove: {0}", file.getPath()), e);
                context.setError();
                return false;
            }
        }
        LOG.info(MessageFormat.format("[OT-CLEAN-I01001] Removed: {0} (timestamp={1})", file.getPath(),
                new Date(file.getModificationTime())));
    } else {
        LOG.info(MessageFormat.format("[OT-CLEAN-I01002] Kept: {0} (timestamp={1})", file.getPath(),
                new Date(file.getModificationTime())));
        return false;
    }
    return true;
}

From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

License:Apache License

private boolean doApply(String executionId) throws IOException, InterruptedException {
    assert executionId != null;
    Path transactionInfo = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
    Path commitMark = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
    FileSystem fs = commitMark.getFileSystem(getConf());
    if (fs.exists(transactionInfo) == false) {
        return false;
    }//  w  w  w.  jav a2 s .  com
    boolean succeed = true;
    if (fs.exists(commitMark) == false) {
        // FIXME cleanup
        return false;
    }
    DirectDataSourceRepository repo = getRepository();
    for (String containerPath : repo.getContainerPaths()) {
        String datasourceId = repo.getRelatedId(containerPath);
        try {
            DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
            OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
            datasource.commitTransactionOutput(context);
            datasource.cleanupTransactionOutput(context);
        } catch (IOException e) {
            succeed = false;
            LOG.error(MessageFormat.format("Failed to apply transaction (datastoreId={0}, executionId={1})",
                    datasourceId, executionId));
        }
    }
    if (succeed) {
        LOG.info(MessageFormat.format("Deleting commit mark (executionId={0}, path={1})", executionId,
                commitMark));
        try {
            if (fs.delete(commitMark, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                        executionId, commitMark));
            } else if (fs.delete(transactionInfo, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                        executionId, transactionInfo));
            }
        } catch (FileNotFoundException e) {
            LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                    executionId, commitMark), e);
        }
        return true;
    } else {
        throw new IOException(MessageFormat.format("Failed to apply this transaction (executionId={0});"
                + " if you want to ignore this transaction, please abort this.", executionId));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

License:Apache License

private boolean doAbort(String executionId) throws IOException, InterruptedException {
    assert executionId != null;
    Path transactionInfo = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
    Path commitMark = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
    FileSystem fs = commitMark.getFileSystem(getConf());
    if (fs.exists(transactionInfo) == false) {
        return false;
    }//from w  w w .  ja v  a 2s  .c  o m
    boolean succeed = true;
    if (fs.exists(commitMark)) {
        LOG.info(MessageFormat.format("Deleting commit mark (executionId={0}, path={1})", executionId,
                commitMark));
        if (fs.delete(commitMark, true) == false) {
            succeed = false;
            LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                    executionId, commitMark));
        }
    }
    DirectDataSourceRepository repo = getRepository();
    for (String containerPath : repo.getContainerPaths()) {
        String datasourceId = repo.getRelatedId(containerPath);
        try {
            DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
            OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
            datasource.cleanupTransactionOutput(context);
        } catch (IOException e) {
            succeed = false;
            LOG.error(MessageFormat.format("Failed to abort transaction (datastoreId={0}, executionId={1})",
                    datasourceId, executionId));
        }
    }
    if (succeed) {
        LOG.info(MessageFormat.format("Deleting transaction info (executionId={0}, path={1})", executionId,
                commitMark));
        try {
            if (fs.delete(transactionInfo, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                        executionId, transactionInfo));
            }
        } catch (FileNotFoundException e) {
            LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                    executionId, commitMark), e);
        }
        return true;
    } else {
        throw new IOException(MessageFormat.format(
                "Failed to abort this transaction (executionId={0});"
                        + " if you want to ignore this transaction, please delete {1} manually.",
                executionId, transactionInfo));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public boolean delete(String basePath, ResourcePattern resourcePattern, boolean recursive, Counter counter)
        throws IOException, InterruptedException {
    assert basePath.startsWith("/") == false; //$NON-NLS-1$
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start deleting files (id={0}, path={1}, resource={2}, recursive={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, recursive));
    }//from  www  .  j  a  v a  2  s.c  o  m
    FilePattern pattern = validate(resourcePattern);
    HadoopDataSourceProfile p = profile;
    FileSystem fs = p.getFileSystem();
    Path root = p.getFileSystemPath();
    Path base = append(root, basePath);
    List<FileStatus> stats = HadoopDataSourceUtil.search(fs, base, pattern);
    Path temporary = p.getTemporaryFileSystemPath();
    stats = normalize(stats, root, temporary);
    if (recursive) {
        stats = HadoopDataSourceUtil.onlyMinimalCovered(stats);
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Process deleting files (id={0}, path={1}, resource={2}, files={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, stats.size()));
    }
    boolean succeed = true;
    for (FileStatus stat : stats) {
        if (LOG.isTraceEnabled()) {
            LOG.trace(MessageFormat.format("Deleting file (id={0}, path={1}, recursive={2})", //$NON-NLS-1$
                    profile.getId(), stat.getPath(), recursive));
        }
        if (recursive == false && FileSystemCompatibility.isDirectory(stat)) {
            LOG.info(MessageFormat.format("Skip deleting directory (id={0}, path={1})", profile.getId(),
                    stat.getPath()));
        } else {
            counter.add(1);
            succeed &= fs.delete(stat.getPath(), recursive);
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish deleting files (id={0}, path={1}, resource={2}, files={3})", //$NON-NLS-1$
                profile.getId(), basePath, resourcePattern, stats.size()));
    }
    return succeed;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public void cleanupAttemptOutput(OutputAttemptContext context) throws IOException, InterruptedException {
    if (isLocalAttemptOutput()) {
        Path attempt = getLocalAttemptOutput(context);
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Delete local attempt area (id={0}, path={1})", //$NON-NLS-1$
                    profile.getId(), attempt));
        }/* w  w  w.j a  va 2s.c o m*/
        FileSystem fs = profile.getLocalFileSystem();
        fs.delete(attempt, true);
    } else {
        Path attempt = getAttemptOutput(context);
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Delete attempt area (id={0}, path={1})", //$NON-NLS-1$
                    profile.getId(), attempt));
        }
        FileSystem fs = profile.getFileSystem();
        fs.delete(attempt, true);
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceCore.java

License:Apache License

@Override
public void cleanupTransactionOutput(OutputTransactionContext context)
        throws IOException, InterruptedException {
    FileSystem fs = profile.getFileSystem();
    Path path = getTemporaryOutput(context);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Delete temporary area (id={0}, path={1})", //$NON-NLS-1$
                profile.getId(), path));
    }/*  w  ww.  jav  a  2  s. c  o m*/
    try {
        if (fs.exists(path)) {
            if (fs.delete(path, true) == false) {
                LOG.warn(MessageFormat.format("Failed to delete temporary area (id={0}, path={0})",
                        profile.getId(), path));
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Temporary area is not found (may be not used): {0}", //$NON-NLS-1$
                        path));
            }
        }
    } catch (FileNotFoundException e) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Temporary area is not found (may be not used): {0}", //$NON-NLS-1$
                    path));
        }
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static void move(Counter counter, FileSystem fromFs, Path from, FileSystem toFs, Path to,
        boolean fromLocal) throws IOException {
    if (counter == null) {
        throw new IllegalArgumentException("counter must not be null"); //$NON-NLS-1$
    }//w  w w .  ja va 2 s.  c o m
    if (fromFs == null) {
        throw new IllegalArgumentException("fromFs must not be null"); //$NON-NLS-1$
    }
    if (from == null) {
        throw new IllegalArgumentException("from must not be null"); //$NON-NLS-1$
    }
    if (toFs == null) {
        throw new IllegalArgumentException("toFs must not be null"); //$NON-NLS-1$
    }
    if (to == null) {
        throw new IllegalArgumentException("to must not be null"); //$NON-NLS-1$
    }
    if (fromLocal && isLocalPath(from) == false) {
        throw new IllegalArgumentException("from must be on local file system"); //$NON-NLS-1$
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start moving files (from={0}, to={1})", //$NON-NLS-1$
                from, to));
    }
    Path source = fromFs.makeQualified(from);
    Path target = toFs.makeQualified(to);
    List<Path> list = createFileListRelative(counter, fromFs, source);
    if (list.isEmpty()) {
        return;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Process moving files (from={0}, to={1}, count={2})", //$NON-NLS-1$
                from, to, list.size()));
    }
    Set<Path> directoryCreated = new HashSet<>();
    for (Path path : list) {
        Path sourceFile = new Path(source, path);
        Path targetFile = new Path(target, path);
        if (LOG.isTraceEnabled()) {
            FileStatus stat = fromFs.getFileStatus(sourceFile);
            LOG.trace(MessageFormat.format("Moving file (from={0}, to={1}, size={2})", //$NON-NLS-1$
                    sourceFile, targetFile, stat.getLen()));
        }
        try {
            FileStatus stat = toFs.getFileStatus(targetFile);
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Deleting file: {0}", //$NON-NLS-1$
                        targetFile));
            }
            if (FileSystemCompatibility.isDirectory(stat)) {
                toFs.delete(targetFile, true);
            } else {
                toFs.delete(targetFile, false);
            }
        } catch (FileNotFoundException e) {
            Path targetParent = targetFile.getParent();
            if (directoryCreated.contains(targetParent) == false) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("Creating directory: {0}", //$NON-NLS-1$
                            targetParent));
                }
                toFs.mkdirs(targetParent);
                directoryCreated.add(targetParent);
            }
        }
        counter.add(1);
        if (fromLocal) {
            toFs.moveFromLocalFile(sourceFile, targetFile);
        } else {
            boolean succeed = toFs.rename(sourceFile, targetFile);
            if (succeed == false) {
                throw new IOException(
                        MessageFormat.format("Failed to move file (from={0}, to={1})", sourceFile, targetFile));
            }
        }
        counter.add(1);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish moving files (from={0}, to={1}, count={2})", //$NON-NLS-1$
                from, to, list.size()));
    }
}

From source file:com.asakusafw.runtime.stage.AbstractCleanupStageClient.java

License:Apache License

@Override
protected int execute(String[] args) throws IOException, InterruptedException {
    Configuration conf = getConf();
    Path path = getPath(conf);//from   w ww.j a  v  a 2 s. c  om
    FileSystem fileSystem = FileSystem.get(path.toUri(), conf);
    String info = MessageFormat.format("batchId={0}, flowId={1}, executionId={2}, operationId={3}, path={4}", //$NON-NLS-1$
            getBatchId(), getFlowId(), getExecutionId(), getOperationId(), path);
    try {
        LOG.info(MessageFormat.format("Searching for cleanup target: {0}", info));
        long start = System.currentTimeMillis();
        if (RuntimeContext.get().isSimulation()) {
            LOG.info(MessageFormat.format(
                    "Skip deleting cleanup target because current execution is in simulation mode: {0}", info));
        } else {
            FileStatus stat = fileSystem.getFileStatus(path);
            if (stat == null) {
                throw new FileNotFoundException(path.toString());
            }
            LOG.info(MessageFormat.format("Start deleting cleanup target: {0}", info));
            if (fileSystem.delete(path, true) == false) {
                throw new IOException("FileSystem.delete() returned false");
            }
        }
        long end = System.currentTimeMillis();
        LOG.info(MessageFormat.format("Finish deleting cleanup target: {0}, elapsed={1}ms", info, end - start));
        return 0;
    } catch (FileNotFoundException e) {
        LOG.warn(MessageFormat.format("Cleanup target is missing: {0}", info));
        return 0;
    } catch (IOException e) {
        LOG.warn(MessageFormat.format("Failed to delete cleanup target: {0}", info), e);
        return 1;
    } finally {
        FileSystem.closeAll();
    }
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private void delete(FileSystem fs, Path path) throws IOException {
    if (checkBeforeDelete && fs.exists(path) == false) {
        return;//from  w  w w  . j a v  a 2s.  co m
    }
    fs.delete(path, false);
}

From source file:com.asakusafw.testdriver.executor.DefaultDeleteTaskExecutor.java

License:Apache License

@Override
protected void deleteOnHadoopFileSystem(TaskExecutionContext context, String path) throws IOException {
    Configuration conf = context.findResource(Configuration.class).get();
    Path p = new Path(path);
    FileSystem fs = p.getFileSystem(conf);
    try {//from   w w  w  .  j a v  a 2s .co m
        fs.delete(p, true);
    } catch (FileNotFoundException e) {
        // may not occur
        LOG.debug("unexpected exception was occurred", e);
    }
}