Example usage for org.apache.hadoop.fs FileSystem setTimes

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem setTimes.

Prototype

public void setTimes(Path p, long mtime, long atime) throws IOException 

Source Link

Document

Set access time of a file.

Usage

From source file:org.apache.ivory.service.SharedLibraryHostingService.java

License:Apache License

public static void pushLibsToHDFS(String path, Cluster cluster, PathFilter pathFilter) throws IOException {
    Configuration conf = ClusterHelper.getConfiguration(cluster);
    FileSystem fs = FileSystem.get(conf);
    String localPaths = StartupProperties.get().getProperty("system.lib.location");
    assert localPaths != null && !localPaths.isEmpty() : "Invalid value for system.lib.location";
    if (!new File(localPaths).isDirectory()) {
        LOG.warn(localPaths + " configured for system.lib.location doesn't contain any valid libs");
        return;//from   ww  w .  ja  v a2s.  co m
    }
    for (File localFile : new File(localPaths).listFiles()) {
        Path clusterFile = new Path(path, localFile.getName());
        if (!pathFilter.accept(clusterFile))
            continue;

        if (fs.exists(clusterFile)) {
            FileStatus fstat = fs.getFileStatus(clusterFile);
            if (fstat.getLen() == localFile.length() && fstat.getModificationTime() == localFile.lastModified())
                continue;
        }
        fs.copyFromLocalFile(false, true, new Path(localFile.getAbsolutePath()), clusterFile);
        fs.setTimes(clusterFile, localFile.lastModified(), System.currentTimeMillis());
        LOG.info("Copied " + localFile.getAbsolutePath() + " to " + path + " in " + fs.getUri());
    }
}

---

From source file:org.apache.kylin.job.tools.DeployCoprocessorCLI.java

License:Apache License

public static Path uploadCoprocessorJar(String localCoprocessorJar, FileSystem fileSystem,
        Set<String> oldJarPaths) throws IOException {
    Path uploadPath = null;//from www . jav  a 2  s .  c o m
    File localCoprocessorFile = new File(localCoprocessorJar);

    // check existing jars
    if (oldJarPaths == null) {
        oldJarPaths = new HashSet<String>();
    }
    Path coprocessorDir = getCoprocessorHDFSDir(fileSystem, KylinConfig.getInstanceFromEnv());
    for (FileStatus fileStatus : fileSystem.listStatus(coprocessorDir)) {
        if (fileStatus.getLen() == localCoprocessorJar.length()
                && fileStatus.getModificationTime() == localCoprocessorFile.lastModified()) {
            uploadPath = fileStatus.getPath();
            break;
        }
        String filename = fileStatus.getPath().toString();
        if (filename.endsWith(".jar")) {
            oldJarPaths.add(filename);
        }
    }

    // upload if not existing
    if (uploadPath == null) {
        // figure out a unique new jar file name
        Set<String> oldJarNames = new HashSet<String>();
        for (String path : oldJarPaths) {
            oldJarNames.add(new Path(path).getName());
        }
        String baseName = getBaseFileName(localCoprocessorJar);
        String newName = null;
        int i = 0;
        while (newName == null) {
            newName = baseName + "-" + (i++) + ".jar";
            if (oldJarNames.contains(newName))
                newName = null;
        }

        // upload
        uploadPath = new Path(coprocessorDir, newName);
        FileInputStream in = null;
        FSDataOutputStream out = null;
        try {
            in = new FileInputStream(localCoprocessorFile);
            out = fileSystem.create(uploadPath);
            IOUtils.copy(in, out);
        } finally {
            IOUtils.closeQuietly(in);
            IOUtils.closeQuietly(out);
        }

        fileSystem.setTimes(uploadPath, localCoprocessorFile.lastModified(), -1);

    }

    uploadPath = uploadPath.makeQualified(fileSystem.getUri(), null);
    return uploadPath;
}

---

From source file:org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI.java

License:Apache License

public synchronized static Path uploadCoprocessorJar(String localCoprocessorJar, FileSystem fileSystem,
        Set<String> oldJarPaths) throws IOException {
    Path uploadPath = null;//w  w w .j  av a 2s  . co m
    File localCoprocessorFile = new File(localCoprocessorJar);

    // check existing jars
    if (oldJarPaths == null) {
        oldJarPaths = new HashSet<String>();
    }
    Path coprocessorDir = getCoprocessorHDFSDir(fileSystem, KylinConfig.getInstanceFromEnv());
    for (FileStatus fileStatus : fileSystem.listStatus(coprocessorDir)) {
        if (isSame(localCoprocessorFile, fileStatus)) {
            uploadPath = fileStatus.getPath();
            break;
        }
        String filename = fileStatus.getPath().toString();
        if (filename.endsWith(".jar")) {
            oldJarPaths.add(filename);
        }
    }

    // upload if not existing
    if (uploadPath == null) {
        // figure out a unique new jar file name
        Set<String> oldJarNames = new HashSet<String>();
        for (String path : oldJarPaths) {
            oldJarNames.add(new Path(path).getName());
        }
        String baseName = getBaseFileName(localCoprocessorJar);
        String newName = null;
        int i = 0;
        while (newName == null) {
            newName = baseName + "-" + (i++) + ".jar";
            if (oldJarNames.contains(newName))
                newName = null;
        }

        // upload
        uploadPath = new Path(coprocessorDir, newName);
        FileInputStream in = null;
        FSDataOutputStream out = null;
        try {
            in = new FileInputStream(localCoprocessorFile);
            out = fileSystem.create(uploadPath);
            IOUtils.copy(in, out);
        } finally {
            IOUtils.closeQuietly(in);
            IOUtils.closeQuietly(out);
        }

        fileSystem.setTimes(uploadPath, localCoprocessorFile.lastModified(), -1);

    }

    uploadPath = uploadPath.makeQualified(fileSystem.getUri(), null);
    return uploadPath;
}

---

From source file:org.apache.kylin.storage.hbase.util.IIDeployCoprocessorCLI.java

License:Apache License

private static Path uploadCoprocessorJar(String localCoprocessorJar, FileSystem fileSystem,
        Set<String> oldJarPaths) throws IOException {
    Path uploadPath = null;//w w  w.  j a  v  a  2  s . c  om
    File localCoprocessorFile = new File(localCoprocessorJar);

    // check existing jars
    if (oldJarPaths == null) {
        oldJarPaths = new HashSet<String>();
    }
    Path coprocessorDir = getCoprocessorHDFSDir(fileSystem, KylinConfig.getInstanceFromEnv());
    for (FileStatus fileStatus : fileSystem.listStatus(coprocessorDir)) {
        if (isSame(localCoprocessorFile, fileStatus)) {
            uploadPath = fileStatus.getPath();
            break;
        }
        String filename = fileStatus.getPath().toString();
        if (filename.endsWith(".jar")) {
            oldJarPaths.add(filename);
        }
    }

    // upload if not existing
    if (uploadPath == null) {
        // figure out a unique new jar file name
        Set<String> oldJarNames = new HashSet<String>();
        for (String path : oldJarPaths) {
            oldJarNames.add(new Path(path).getName());
        }
        String baseName = getBaseFileName(localCoprocessorJar);
        String newName = null;
        int i = 0;
        while (newName == null) {
            newName = baseName + "-" + (i++) + ".jar";
            if (oldJarNames.contains(newName))
                newName = null;
        }

        // upload
        uploadPath = new Path(coprocessorDir, newName);
        FileInputStream in = null;
        FSDataOutputStream out = null;
        try {
            in = new FileInputStream(localCoprocessorFile);
            out = fileSystem.create(uploadPath);
            IOUtils.copy(in, out);
        } finally {
            IOUtils.closeQuietly(in);
            IOUtils.closeQuietly(out);
        }

        fileSystem.setTimes(uploadPath, localCoprocessorFile.lastModified(), -1);

    }

    uploadPath = uploadPath.makeQualified(fileSystem.getUri(), null);
    return uploadPath;
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler.java

License:Apache License

private static Path getFromCache(PigContext pigContext, Configuration conf, URL url) throws IOException {
    InputStream is1 = null;//www . ja v  a2 s  .c om
    InputStream is2 = null;
    OutputStream os = null;

    try {
        Path stagingDir = getCacheStagingDir(conf);
        String filename = FilenameUtils.getName(url.getPath());

        is1 = url.openStream();
        String checksum = DigestUtils.shaHex(is1);
        FileSystem fs = FileSystem.get(conf);
        Path cacheDir = new Path(stagingDir, checksum);
        Path cacheFile = new Path(cacheDir, filename);
        if (fs.exists(cacheFile)) {
            log.debug("Found " + url + " in jar cache at " + cacheDir);
            long curTime = System.currentTimeMillis();
            fs.setTimes(cacheFile, -1, curTime);
            return cacheFile;
        }
        log.info("Url " + url + " was not found in jarcache at " + cacheDir);
        // attempt to copy to cache else return null
        fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
        is2 = url.openStream();
        os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
        IOUtils.copyBytes(is2, os, 4096, true);

        return cacheFile;

    } catch (IOException ioe) {
        log.info("Unable to retrieve jar from jar cache ", ioe);
        return null;
    } finally {
        org.apache.commons.io.IOUtils.closeQuietly(is1);
        org.apache.commons.io.IOUtils.closeQuietly(is2);
        // IOUtils should not close stream to HDFS quietly
        if (os != null) {
            os.close();
        }
    }
}

---

From source file:org.janusgraph.hadoop.config.job.AbstractDistCacheConfigurer.java

License:Apache License

protected Path uploadFileIfNecessary(FileSystem localFS, Path localPath, FileSystem destFS) throws IOException {

    // Fast path for local FS -- DistributedCache + local JobRunner seems copy/link files automatically
    if (destFS.equals(localFS)) {
        log.debug("Skipping file upload for {} (destination filesystem {} equals local filesystem)", localPath,
                destFS);//from   w  w  w  .  jav a2  s .c o  m
        return localPath;
    }

    Path destPath = new Path(destFS.getHomeDirectory() + "/" + HDFS_TMP_LIB_DIR + "/" + localPath.getName());

    Stats fileStats = null;

    try {
        fileStats = compareModtimes(localFS, localPath, destFS, destPath);
    } catch (IOException e) {
        log.warn("Unable to read or stat file: localPath={}, destPath={}, destFS={}", localPath, destPath,
                destFS);
    }

    if (fileStats != null && !fileStats.isRemoteCopyCurrent()) {
        log.debug("Copying {} to {}", localPath, destPath);
        destFS.copyFromLocalFile(localPath, destPath);
        if (null != fileStats.local) {
            final long mtime = fileStats.local.getModificationTime();
            log.debug("Setting modtime on {} to {}", destPath, mtime);
            destFS.setTimes(destPath, mtime, -1); // -1 means leave atime alone
        }
    }

    return destPath;
}

---

• «
• 1
• 2
• »