Example usage for org.apache.hadoop.fs FileSystem getFileStatus

List of usage examples for org.apache.hadoop.fs FileSystem getFileStatus

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getFileStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getFileStatus.

Prototype

public abstract FileStatus getFileStatus(Path f) throws IOException;

Source Link

Document

Return a file status object that represents the path.

Usage

From source file:com.alexholmes.hadooputils.combine.seqfile.mapred.CombineSequenceFileInputFormat.java

License:Apache License

@Override
protected FileStatus[] listStatus(JobConf job) throws IOException {
    FileStatus[] files = super.listStatus(job);
    for (int i = 0; i < files.length; i++) {
        FileStatus file = files[i];/*  w  ww.ja v a2 s.  com*/
        if (file.isDir()) { // it's a MapFile
            Path dataFile = new Path(file.getPath(), MapFile.DATA_FILE_NAME);
            FileSystem fs = file.getPath().getFileSystem(job);
            // use the data file
            files[i] = fs.getFileStatus(dataFile);
        }
    }
    return files;
}

From source file:com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileInputFormat.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
protected List<FileStatus> listStatus(JobContext job) throws IOException {

    List<FileStatus> files = super.listStatus(job);

    int len = files.size();
    for (int i = 0; i < len; ++i) {
        FileStatus file = files.get(i);/*from www  .  j a  v a2  s  .  c  o  m*/
        if (file.isDir()) { // it's a MapFile
            Path p = file.getPath();
            FileSystem fs = p.getFileSystem(job.getConfiguration());
            // use the data file
            files.set(i, fs.getFileStatus(new Path(p, MapFile.DATA_FILE_NAME)));
        }
    }
    return files;
}

From source file:com.alexholmes.hdfsslurper.Configurator.java

License:Apache License

public static void testCreateDir(Path p, Configuration conf)
        throws IOException, ConfigSettingException, FileSystemMkdirFailed {
    FileSystem fs = p.getFileSystem(conf);
    if (fs.exists(p) && !fs.getFileStatus(p).isDir()) {
        throw new ConfigSettingException("Directory appears to be a file: '" + p + "'");
    }/*from   w  w w  .  jav  a2s.  c  o  m*/

    if (!fs.exists(p)) {
        log.info("Attempting creation of directory: " + p);
        if (!fs.mkdirs(p)) {
            throw new FileSystemMkdirFailed("Failed to create directory: '" + p + "'");
        }
    }
}

From source file:com.alexholmes.hdfsslurper.WorkerThread.java

License:Apache License

private void process(FileStatus srcFileStatus) throws IOException, InterruptedException {

    Path stagingFile = null;//  w  w w . ja v  a2 s.c om
    FileSystem destFs = null;
    String filenameBatchidDelimiter = config.getFileNameBatchIdDelimiter();

    try {
        FileSystem srcFs = srcFileStatus.getPath().getFileSystem(config.getConfig());

        // run a script which can change the name of the file as well as
        // write out a new version of the file
        //
        if (config.getWorkScript() != null) {
            Path newSrcFile = stageSource(srcFileStatus);
            srcFileStatus = srcFileStatus.getPath().getFileSystem(config.getConfig()).getFileStatus(newSrcFile);
        }

        Path srcFile = srcFileStatus.getPath();

        // get the target HDFS file
        //
        Path destFile = getHdfsTargetPath(srcFileStatus);

        if (config.getCodec() != null) {
            String ext = config.getCodec().getDefaultExtension();
            if (!destFile.getName().endsWith(ext)) {
                destFile = new Path(destFile.toString() + ext);
            }
        }

        destFs = destFile.getFileSystem(config.getConfig());

        // get the staging HDFS file
        //
        stagingFile = fileSystemManager.getStagingFile(srcFileStatus, destFile);
        String batchId = srcFile.toString().substring(
                srcFile.toString().lastIndexOf(filenameBatchidDelimiter) + 1, srcFile.toString().length());

        log.info("event#Copying source file '" + srcFile + "' to staging destination '" + stagingFile + "'"
                + "$batchId#" + batchId);

        // if the directory of the target file doesn't exist, attempt to
        // create it
        //
        Path destParentDir = destFile.getParent();
        if (!destFs.exists(destParentDir)) {
            log.info("event#Attempting creation of target directory: " + destParentDir.toUri());
            if (!destFs.mkdirs(destParentDir)) {
                throw new IOException("event#Failed to create target directory: " + destParentDir.toUri());
            }
        }

        // if the staging directory doesn't exist, attempt to create it
        //
        Path destStagingParentDir = stagingFile.getParent();
        if (!destFs.exists(destStagingParentDir)) {
            log.info("event#Attempting creation of staging directory: " + destStagingParentDir.toUri());
            if (!destFs.mkdirs(destStagingParentDir)) {
                throw new IOException("event#Failed to create staging directory: " + destParentDir.toUri());
            }
        }

        // copy the file
        //
        InputStream is = null;
        OutputStream os = null;
        CRC32 crc = new CRC32();
        try {
            is = new BufferedInputStream(srcFs.open(srcFile));
            if (config.isVerify()) {
                is = new CheckedInputStream(is, crc);
            }
            os = destFs.create(stagingFile);

            if (config.getCodec() != null) {
                os = config.getCodec().createOutputStream(os);
            }

            IOUtils.copyBytes(is, os, 4096, false);
        } finally {
            IOUtils.closeStream(is);
            IOUtils.closeStream(os);
        }

        long srcFileSize = srcFs.getFileStatus(srcFile).getLen();
        long destFileSize = destFs.getFileStatus(stagingFile).getLen();
        if (config.getCodec() == null && srcFileSize != destFileSize) {
            throw new IOException(
                    "event#File sizes don't match, source = " + srcFileSize + ", dest = " + destFileSize);
        }

        log.info("event#Local file size = " + srcFileSize + ", HDFS file size = " + destFileSize + "$batchId#"
                + batchId);

        if (config.isVerify()) {
            verify(stagingFile, crc.getValue());
        }

        if (destFs.exists(destFile)) {
            destFs.delete(destFile, false);
        }

        log.info("event#Moving staging file '" + stagingFile + "' to destination '" + destFile + "'"
                + "$batchId#" + batchId);
        if (!destFs.rename(stagingFile, destFile)) {
            throw new IOException("event#Failed to rename file");
        }

        if (config.isCreateLzopIndex() && destFile.getName().endsWith(lzopExt)) {
            Path lzoIndexPath = new Path(destFile.toString() + LzoIndex.LZO_INDEX_SUFFIX);
            if (destFs.exists(lzoIndexPath)) {
                log.info("event#Deleting index file as it already exists");
                destFs.delete(lzoIndexPath, false);
            }
            indexer.index(destFile);
        }

        fileSystemManager.fileCopyComplete(srcFileStatus);

    } catch (Throwable t) {
        log.error("event#Caught exception working on file " + srcFileStatus.getPath(), t);

        // delete the staging file if it still exists
        //
        try {
            if (destFs != null && destFs.exists(stagingFile)) {
                destFs.delete(stagingFile, false);
            }
        } catch (Throwable t2) {
            log.error("event#Failed to delete staging file " + stagingFile, t2);
        }

        fileSystemManager.fileCopyError(srcFileStatus);
    }

}

From source file:com.alibaba.jstorm.hdfs.blobstore.HdfsBlobStoreFile.java

License:Apache License

@Override
public long getModTime() throws IOException {
    if (_modTime == null) {
        FileSystem fs = _path.getFileSystem(_hadoopConf);
        _modTime = fs.getFileStatus(_path).getModificationTime();
    }/*from w ww. j av a  2s  . c  om*/
    return _modTime;
}

From source file:com.alibaba.jstorm.hdfs.spout.DirLock.java

License:Apache License

/** if the lock on the directory is stale, take ownership */
public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) {
    Path dirLockFile = getDirLockFile(dirToLock);

    long now = System.currentTimeMillis();
    long expiryTime = now - (lockTimeoutSec * 1000);

    try {//  ww w. j  a v  a 2  s. co m
        long modTime = fs.getFileStatus(dirLockFile).getModificationTime();
        if (modTime <= expiryTime) {
            return takeOwnership(fs, dirLockFile);
        }
        return null;
    } catch (IOException e) {
        return null;
    }
}

From source file:com.alibaba.jstorm.hdfs.spout.FileLock.java

License:Apache License

/**
 * checks if lockFile is older than 'olderThan' UTC time by examining the modification time
 * on file and (if necessary) the timestamp in last log entry in the file. If its stale, then
 * returns the last log entry, else returns null.
 * @param fs//from  w  ww  .  ja  v  a2 s .c om
 * @param lockFile
 * @param olderThan  time (millis) in UTC.
 * @return the last entry in the file if its too old. null if last entry is not too old
 * @throws IOException
 */
public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long olderThan) throws IOException {
    long modifiedTime = fs.getFileStatus(lockFile).getModificationTime();
    if (modifiedTime <= olderThan) { // look
        //Impt: HDFS timestamp may not reflect recent appends, so we double check the
        // timestamp in last line of file to see when the last update was made
        LogEntry lastEntry = getLastEntry(fs, lockFile);
        if (lastEntry == null) {
            LOG.warn("Empty lock file found. Deleting it. {}", lockFile);
            try {
                if (!fs.delete(lockFile, false))
                    throw new IOException("Empty lock file deletion failed");
            } catch (Exception e) {
                LOG.error("Unable to delete empty lock file " + lockFile, e);
            }
        }
        if (lastEntry.eventTime <= olderThan)
            return lastEntry;
    }
    return null;
}

From source file:com.asakusafw.cleaner.main.HDFSCleaner.java

License:Apache License

/**
 * ?//w ww  .j a  va2s. c o  m
 * @param fs HDFS?
 * @param cleanPath HDFS??
 * @param isSetExecutionId ID????????
 * @param pattern 
 * @param keepDate ??
 * @param now ?
 * @param recursive ????
 * @return ?
 * @throws CleanerSystemException
 */
private boolean cleanDir(FileSystem fs, Path cleanPath, boolean isSetExecutionId, String pattern, int keepDate,
        Date now, boolean recursive) throws CleanerSystemException {
    try {
        if (!fs.exists(cleanPath)) {
            // ??????
            Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_ERROR,
                    "??????", cleanPath.toString());
            return false;
        }
        if (!fs.getFileStatus(cleanPath).isDir()) {
            // ??????
            Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_ERROR,
                    "??????", cleanPath.toString());
            return false;
        }

        // ?
        Log.log(CLASS, MessageIdConst.HCLN_FILE_DELETE, cleanPath.toString());
        int cleanFileCount = 0;
        int cleanDirCount = 0;
        boolean result = true;
        FileStatus[] dirStatus = getListStatus(fs, cleanPath);
        Path[] listedPaths = FileUtil.stat2Paths(dirStatus);
        for (Path path : listedPaths) {
            FileStatus status = fs.getFileStatus(path);
            long lastModifiedTime = status.getModificationTime();
            if (status.isDir() && recursive) {
                // ????????
                if (isSetExecutionId) {
                    // ID??????MM???????
                    String executionId = path.getName();
                    if (isRunningJobFlow(executionId)) {
                        // ???????
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_DIR_EXEC, path.toString());
                        continue;
                    }
                }
                FileStatus[] childdirStatus = getListStatus(fs, path);
                if (childdirStatus.length == 0) {
                    // ???????
                    if (isExpired(lastModifiedTime, keepDate, now)) {
                        if (!fs.delete(path, false)) {
                            Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "",
                                    path.toString());
                            result = false;
                        } else {
                            cleanDirCount++;
                            Log.log(CLASS, MessageIdConst.HCLN_DIR_DELETE, path.toString());
                        }
                    }
                } else {
                    // ?????????
                    if (cleanDir(fs, path, false, pattern, keepDate, now, recursive)) {
                        // ????????
                        childdirStatus = getListStatus(fs, path);
                        if (childdirStatus.length == 0) {
                            if (isExpired(lastModifiedTime, keepDate, now)) {
                                if (!fs.delete(path, false)) {
                                    Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "",
                                            path.toString());
                                    result = false;
                                } else {
                                    cleanDirCount++;
                                    Log.log(CLASS, MessageIdConst.HCLN_DIR_DELETE, path.toString());
                                }
                            }
                        }
                    } else {
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "", path.toString());
                        result = false;
                    }
                }
            } else if (!status.isDir()) {
                // ???????????
                if (isExpired(lastModifiedTime, keepDate, now) && isMatchPattern(path, pattern)) {
                    if (!fs.delete(path, false)) {
                        Log.log(CLASS, MessageIdConst.HCLN_CLEN_FAIL, "", path.toString());
                        result = false;
                    } else {
                        Log.log(CLASS, MessageIdConst.HCLN_DELETE_FILE, path.toString());
                        cleanFileCount++;
                    }
                }
            }
        }

        Log.log(CLASS, MessageIdConst.HCLN_FILE_DELETE_SUCCESS, cleanPath.toString(), cleanDirCount,
                cleanFileCount);

        return result;
    } catch (IOException e) {
        Log.log(e, CLASS, MessageIdConst.HCLN_CLEN_DIR_EXCEPTION, cleanPath.getName());
        return false;
    }
}

From source file:com.asakusafw.dmdl.directio.sequencefile.driver.SequenceFileFormatEmitterTest.java

License:Apache License

/**
 * A simple case.//  w  ww . java2  s .  c om
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    File tempFile = folder.newFile("tempfile");
    Path path = new Path(tempFile.toURI());

    ModelLoader loaded = generateJava("simple");
    ModelWrapper model = loaded.newModel("Simple");
    DataFormat<?> support = (DataFormat<?>) loaded.newObject("sequencefile", "SimpleSequenceFileFormat");
    assertThat(support, is(instanceOf(Configurable.class)));
    Thread.currentThread().setContextClassLoader(support.getClass().getClassLoader());

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(tempFile.toURI(), conf);
    if (support instanceof Configurable) {
        ((Configurable) support).setConf(conf);
    }

    assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass()));

    HadoopFileFormat<Object> unsafe = unsafe(support);

    model.set("value", new Text("Hello, world!"));

    try (ModelOutput<Object> writer = unsafe.createOutput(model.unwrap().getClass(), fs, path, new Counter())) {
        writer.write(model.unwrap());
    }

    try (ModelInput<Object> reader = unsafe.createInput(model.unwrap().getClass(), fs, path, 0,
            fs.getFileStatus(path).getLen(), new Counter())) {
        Object buffer = loaded.newModel("Simple").unwrap();
        assertThat(reader.readTo(buffer), is(true));
        assertThat(buffer, is(buffer));
        assertThat(reader.readTo(buffer), is(false));
    }
}

From source file:com.asakusafw.operation.tools.directio.file.FileMakeDirectoryCommand.java

License:Apache License

static Optional<FileStatus> stat(FileSystem fs, Path path) {
    try {/*  w ww  .  j a v  a 2 s  . c  o  m*/
        return Optional.of(fs.getFileStatus(path));
    } catch (IOException e) {
        LOG.trace("exception: {}", path, e);
        return Optional.empty();
    }
}