List of usage examples for org.apache.hadoop.fs FileSystem setTimes
public void setTimes(Path p, long mtime, long atime) throws IOException
From source file:cascading.flow.hadoop.util.HadoopUtil.java
License:Open Source License
/** * Copies paths from one local path to a remote path. If syncTimes is true, both modification and access time are * changed to match the local 'from' path. * <p/>/*w w w. ja va 2 s. c o m*/ * Returns a map of file-name to remote modification times if the remote time is different than the local time. * * @param config * @param commonPaths * @param syncTimes */ public static Map<String, Long> syncPaths(Configuration config, Map<Path, Path> commonPaths, boolean syncTimes) { if (commonPaths == null) return Collections.emptyMap(); Map<String, Long> timestampMap = new HashMap<>(); Map<Path, Path> copyPaths = getCopyPaths(config, commonPaths); // tests remote file existence or if stale LocalFileSystem localFS = getLocalFS(config); FileSystem remoteFS = getDefaultFS(config); for (Map.Entry<Path, Path> entry : copyPaths.entrySet()) { Path localPath = entry.getKey(); Path remotePath = entry.getValue(); try { LOG.info("copying from: {}, to: {}", localPath, remotePath); remoteFS.copyFromLocalFile(localPath, remotePath); if (!syncTimes) { timestampMap.put(remotePath.getName(), remoteFS.getFileStatus(remotePath).getModificationTime()); continue; } } catch (IOException exception) { throw new FlowException("unable to copy local: " + localPath + " to remote: " + remotePath, exception); } FileStatus localFileStatus = null; try { // sync the modified times so we can lazily upload jars to hdfs after job is started // otherwise modified time will be local to hdfs localFileStatus = localFS.getFileStatus(localPath); remoteFS.setTimes(remotePath, localFileStatus.getModificationTime(), -1); // don't set the access time } catch (IOException exception) { LOG.info( "unable to set local modification time on remote file: {}, 'dfs.namenode.accesstime.precision' may be set to 0 on HDFS.", remotePath); if (localFileStatus != null) timestampMap.put(remotePath.getName(), localFileStatus.getModificationTime()); } } return timestampMap; }
From source file:com.blackberry.logdriver.LockedFs.java
License:Apache License
public void touch(Configuration conf, String file) throws IOException { FileSystem fs = FileSystem.get(conf); long now = System.currentTimeMillis(); Path path = new Path(file); fs.createNewFile(path);/*from ww w . ja v a 2s . c o m*/ fs.setTimes(path, now, now); }
From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.attrs.SetAccessTimeHandler.java
License:Apache License
@Override public boolean set(NFS4Handler server, Session session, FileSystem fs, FileStatus fileStatus, StateID stateID, SetAccessTime attr) throws NFS4Exception, IOException { if (attr.getHow() == NFS4_SET_TO_CLIENT_TIME4) { fs.setTimes(fileStatus.getPath(), fileStatus.getModificationTime(), attr.getTime().toMilliseconds()); } else {//from www . j ava2 s. c o m fs.setTimes(fileStatus.getPath(), fileStatus.getModificationTime(), System.currentTimeMillis()); } return true; }
From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.attrs.SetModifyTimeHandler.java
License:Apache License
@Override public boolean set(NFS4Handler server, Session session, FileSystem fs, FileStatus fileStatus, StateID stateID, SetModifyTime attr) throws NFS4Exception, IOException { if (attr.getHow() == NFS4_SET_TO_CLIENT_TIME4) { fs.setTimes(fileStatus.getPath(), attr.getTime().toMilliseconds(), fileStatus.getAccessTime()); } else {//www . j a v a2 s . c o m fs.setTimes(fileStatus.getPath(), System.currentTimeMillis(), fileStatus.getAccessTime()); } return true; }
From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java
License:Open Source License
private void testSetTimes() throws Exception { FileSystem fs = FileSystem.get(getHadoopConf()); Path path = new Path(getHadoopTestDir(), "foo.txt"); OutputStream os = fs.create(path); os.write(1);//from w w w. j a v a2s .c om os.close(); FileStatus status1 = fs.getFileStatus(path); fs.close(); long at = status1.getAccessTime(); long mt = status1.getModificationTime(); Configuration conf = new Configuration(); conf.set("fs.http.impl", HoopFileSystem.class.getName()); fs = FileSystem.get(getJettyURL().toURI(), conf); fs.setTimes(path, mt + 10, at + 20); fs.close(); fs = FileSystem.get(getHadoopConf()); status1 = fs.getFileStatus(path); fs.close(); long atNew = status1.getAccessTime(); long mtNew = status1.getModificationTime(); Assert.assertEquals(mtNew, mt + 10); Assert.assertEquals(atNew, at + 20); }
From source file:com.cloudera.hoop.fs.FSSetTimes.java
License:Open Source License
/** * Executes the filesystem operation./*from w w w.j a v a 2s . c o m*/ * * @param fs filesystem instance to use. * @return void. * @throws IOException thrown if an IO error occured. */ @Override public Void execute(FileSystem fs) throws IOException { fs.setTimes(path, mTime, aTime); return null; }
From source file:com.kylinolap.job.tools.DeployCoprocessorCLI.java
License:Apache License
public static Path uploadCoprocessorJar(String localCoprocessorJar, FileSystem fileSystem, Set<String> oldJarPaths) throws IOException { Path uploadPath = null;//w ww . java 2s . c o m File localCoprocessorFile = new File(localCoprocessorJar); // check existing jars if (oldJarPaths == null) { oldJarPaths = new HashSet<String>(); } Path coprocessorDir = getCoprocessorHDFSDir(fileSystem, KylinConfig.getInstanceFromEnv()); for (FileStatus fileStatus : fileSystem.listStatus(coprocessorDir)) { if (fileStatus.getLen() == localCoprocessorJar.length() && fileStatus.getModificationTime() == localCoprocessorFile.lastModified()) { uploadPath = fileStatus.getPath(); break; } String filename = fileStatus.getPath().toString(); if (filename.endsWith(".jar")) { oldJarPaths.add(filename); } } // upload if not existing if (uploadPath == null) { // figure out a unique new jar file name Set<String> oldJarNames = new HashSet<String>(); for (String path : oldJarPaths) { oldJarNames.add(new Path(path).getName()); } String baseName = getBaseFileName(localCoprocessorJar); String newName = null; int i = 0; while (newName == null) { newName = baseName + "-" + (i++) + ".jar"; if (oldJarNames.contains(newName)) newName = null; } // upload uploadPath = new Path(coprocessorDir, newName); FileInputStream in = null; FSDataOutputStream out = null; try { in = new FileInputStream(localCoprocessorFile); out = fileSystem.create(uploadPath); IOUtils.copy(in, out); } finally { IOUtils.closeQuietly(in); IOUtils.closeQuietly(out); } fileSystem.setTimes(uploadPath, localCoprocessorFile.lastModified(), System.currentTimeMillis()); } uploadPath = uploadPath.makeQualified(fileSystem.getUri(), null); return uploadPath; }
From source file:com.mellanox.r4h.DistributedFileSystem.java
License:Apache License
@Override public void setTimes(Path p, final long mtime, final long atime) throws IOException { statistics.incrementWriteOps(1);/*from w ww . j ava 2 s.co m*/ Path absF = fixRelativePart(p); new FileSystemLinkResolver<Void>() { @Override public Void doCall(final Path p) throws IOException, UnresolvedLinkException { dfs.setTimes(getPathName(p), mtime, atime); return null; } @Override public Void next(final FileSystem fs, final Path p) throws IOException { fs.setTimes(p, mtime, atime); return null; } }.resolve(this, absF); }
From source file:fuse4j.hadoopfs.HdfsClientImpl.java
License:Apache License
@Override public boolean utime(int uid, String path, int atime, int mtime) { FileSystem dfs = null; try {/*from ww w. jav a 2 s . co m*/ dfs = getDfs(uid); System.out.println("mtime:" + mtime); long latime = atime * 1000L; long lmtime = mtime * 1000L; dfs.setTimes(new Path(path), lmtime, latime); return true; } catch (Exception ioe) { // fall through to failure } return false; }
From source file:io.hops.erasure_coding.BaseEncodingManager.java
License:Apache License
/** * Generate parity file/*from w w w . j a va 2 s . c o m*/ */ static private void generateParityFile(Configuration conf, FileStatus sourceFile, int targetRepl, Progressable reporter, FileSystem inFs, Path destPath, Codec codec, long blockNum, int srcRepl, int metaRepl, long blockSize) throws IOException { Path inpath = sourceFile.getPath(); FileSystem outFs = inFs; // If the parity file is already upto-date and source replication is set // then nothing to do. try { FileStatus stmp = outFs.getFileStatus(destPath); if (stmp.getModificationTime() == sourceFile.getModificationTime() && srcRepl == targetRepl) { LOG.info("Parity file for " + inpath + "(" + blockNum + ") is " + destPath + " already upto-date and " + "file is at target replication . Nothing more to do."); return; } } catch (IOException e) { // ignore errors because the raid file might not exist yet. } Encoder encoder = new Encoder(conf, codec); FileStatus srcStat = inFs.getFileStatus(inpath); long srcSize = srcStat.getLen(); long numBlocks = (srcSize % blockSize == 0) ? (srcSize / blockSize) : ((srcSize / blockSize) + 1); long numStripes = (numBlocks % codec.stripeLength == 0) ? (numBlocks / codec.stripeLength) : ((numBlocks / codec.stripeLength) + 1); StripeReader sReader = new FileStripeReader(conf, blockSize, codec, inFs, 0, inpath, srcSize); encoder.encodeFile(conf, inFs, inpath, outFs, destPath, (short) metaRepl, numStripes, blockSize, reporter, sReader); // set the modification time of the RAID file. This is done so that the modTime of the // RAID file reflects that contents of the source file that it has RAIDed. This should // also work for files that are being appended to. This is necessary because the time on // on the destination namenode may not be synchronised with the timestamp of the // source namenode. outFs.setTimes(destPath, sourceFile.getModificationTime(), -1); FileStatus outstat = outFs.getFileStatus(destPath); FileStatus inStat = inFs.getFileStatus(inpath); if (sourceFile.getModificationTime() != inStat.getModificationTime()) { String msg = "Source file changed mtime during raiding from " + sourceFile.getModificationTime() + " to " + inStat.getModificationTime(); throw new IOException(msg); } if (outstat.getModificationTime() != inStat.getModificationTime()) { String msg = "Parity file mtime " + outstat.getModificationTime() + " does not match source mtime " + inStat.getModificationTime(); throw new IOException(msg); } LOG.info("Source file " + inpath + " of size " + inStat.getLen() + " Parity file " + destPath + " of size " + outstat.getLen() + " src mtime " + sourceFile.getModificationTime() + " parity mtime " + outstat.getModificationTime()); }