List of usage examples for org.apache.hadoop.fs FileSystem rename
public abstract boolean rename(Path src, Path dst) throws IOException;
From source file:com.ibm.bi.dml.runtime.io.WriterTextCSV.java
License:Open Source License
/** * /*from www .j a v a2 s .c o m*/ * @param srcFileName * @param destFileName * @param csvprop * @param rlen * @param clen * @throws IOException */ @SuppressWarnings("unchecked") public void addHeaderToCSV(String srcFileName, String destFileName, long rlen, long clen) throws IOException { Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf()); Path srcFilePath = new Path(srcFileName); Path destFilePath = new Path(destFileName); FileSystem hdfs = FileSystem.get(conf); if (!_props.hasHeader()) { // simply move srcFile to destFile /* * TODO: Remove this roundabout way! * For example: destFilePath = /user/biadmin/csv/temp/out/file.csv * & the only path that exists already on HDFS is /user/biadmin/csv/. * In this case: the directory structure /user/biadmin/csv/temp/out must be created. * Simple hdfs.rename() does not seem to create this directory structure. */ // delete the destination file, if exists already //boolean ret1 = hdfs.delete(destFilePath, true); // Create /user/biadmin/csv/temp/out/file.csv so that ..../temp/out/ is created. //boolean ret2 = hdfs.createNewFile(destFilePath); // delete the file "file.csv" but preserve the directory structure /user/biadmin/csv/temp/out/ //boolean ret3 = hdfs.delete(destFilePath, true); // finally, move the data to destFilePath = /user/biadmin/csv/temp/out/file.csv //boolean ret4 = hdfs.rename(srcFilePath, destFilePath); //System.out.println("Return values = del:" + ret1 + ", createNew:" + ret2 + ", del:" + ret3 + ", rename:" + ret4); return; } // construct the header line StringBuilder sb = new StringBuilder(); for (int i = 0; i < clen; i++) { sb.append("C" + (i + 1)); if (i < clen - 1) sb.append(_props.getDelim()); } sb.append('\n'); if (hdfs.isDirectory(srcFilePath)) { // compute sorted order among part files ArrayList<Path> files = new ArrayList<Path>(); for (FileStatus stat : hdfs.listStatus(srcFilePath, CSVReblockMR.hiddenFileFilter)) files.add(stat.getPath()); Collections.sort(files); // first part file path Path firstpart = files.get(0); // create a temp file, and add header and contents of first part Path tmp = new Path(firstpart.toString() + ".tmp"); OutputStream out = hdfs.create(tmp, true); out.write(sb.toString().getBytes()); sb.setLength(0); // copy rest of the data from firstpart InputStream in = null; try { in = hdfs.open(firstpart); IOUtils.copyBytes(in, out, conf, true); } finally { IOUtilFunctions.closeSilently(in); IOUtilFunctions.closeSilently(out); } // rename tmp to firstpart hdfs.delete(firstpart, true); hdfs.rename(tmp, firstpart); // rename srcfile to destFile hdfs.delete(destFilePath, true); hdfs.createNewFile(destFilePath); // force the creation of directory structure hdfs.delete(destFilePath, true); // delete the file, but preserve the directory structure hdfs.rename(srcFilePath, destFilePath); // move the data } else if (hdfs.isFile(srcFilePath)) { // create destination file OutputStream out = hdfs.create(destFilePath, true); // write header out.write(sb.toString().getBytes()); sb.setLength(0); // copy the data from srcFile InputStream in = null; try { in = hdfs.open(srcFilePath); IOUtils.copyBytes(in, out, conf, true); } finally { IOUtilFunctions.closeSilently(in); IOUtilFunctions.closeSilently(out); } } else { throw new IOException(srcFilePath.toString() + ": No such file or directory"); } }
From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java
License:Open Source License
/** * /*from w ww . j a va2 s .c om*/ * @param context * @param fs * @param file * @throws IOException */ private void moveFileToDestination(TaskAttemptContext context, FileSystem fs, Path file) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); //get output index and final destination String taskType = (conf.getBoolean(JobContext.TASK_ISMAP, true)) ? "m" : "r"; String name = file.getName(); int charIx = name.indexOf("-" + taskType + "-"); int index = Integer.parseInt(name.substring(0, charIx)); Path finalPath = new Path(outputs[index], file.getName()); //move file from 'file' to 'finalPath' if (!fs.rename(file, finalPath)) { if (!fs.delete(finalPath, true)) throw new IOException("Failed to delete earlier output " + finalPath + " for rename of " + file + " in task " + attemptId); if (!fs.rename(file, finalPath)) throw new IOException( "Failed to save output " + finalPath + " for rename of " + file + " in task: " + attemptId); } }
From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java
License:Open Source License
public static void renameFileOnHDFS(String originalDir, String newDir) throws IOException { Path originalpath = new Path(originalDir); deleteFileIfExistOnHDFS(newDir);// w w w . ja v a2 s.co m Path newpath = new Path(newDir); FileSystem fs = FileSystem.get(_rJob); if (fs.exists(originalpath)) { fs.rename(originalpath, newpath); } else { throw new FileNotFoundException(originalDir); } }
From source file:com.idvp.platform.hdfs.BucketWriter.java
License:Apache License
/** * Rename bucketPath file from .tmp to permanent location. *//*from w ww .jav a 2 s . c o m*/ // When this bucket writer is rolled based on rollCount or // rollSize, the same instance is reused for the new file. But if // the previous file was not closed/renamed, // the bucket writer fields no longer point to it and hence need // to be passed in from the thread attempting to close it. Even // when the bucket writer is closed due to close timeout, // this method can get called from the scheduled thread so the // file gets closed later - so an implicit reference to this // bucket writer would still be alive in the Callable instance. private void renameBucket(String bucketPath, String targetPath, final FileSystem fs) throws IOException, InterruptedException { if (bucketPath.equals(targetPath)) { return; } final Path srcPath = new Path(bucketPath); final Path dstPath = new Path(targetPath); callWithTimeout((CallRunner<Void>) () -> { if (fs.exists(srcPath)) { // could block LOG.info("Renaming " + srcPath + " to " + dstPath); renameTries.incrementAndGet(); fs.rename(srcPath, dstPath); // could block } return null; }); }
From source file:com.iflytek.spider.crawl.CrawlDb.java
License:Apache License
public static void install(Job job, Path crawlDb) throws IOException { Path newCrawlDb = FileOutputFormat.getOutputPath(job); FileSystem fs = FileSystem.get(job.getConfiguration()); Path old = new Path(crawlDb, "old"); Path current = new Path(crawlDb, CURRENT_NAME); if (fs.exists(current)) { if (fs.exists(old)) fs.delete(old, true);/*from www .j a v a 2s . co m*/ fs.rename(current, old); } fs.mkdirs(crawlDb); fs.rename(newCrawlDb, current); if (fs.exists(old)) fs.delete(old, true); Path lock = new Path(crawlDb, LOCK_NAME); LockUtil.removeLockFile(fs, lock); }
From source file:com.iflytek.spider.util.FSUtils.java
License:Apache License
/** * Replaces the current path with the new path and if set removes the old * path. If removeOld is set to false then the old path will be set to the * name current.old.//from w ww.j a va2s. c om * * @param fs The FileSystem. * @param current The end path, the one being replaced. * @param replacement The path to replace with. * @param removeOld True if we are removing the current path. * * @throws IOException If an error occurs during replacement. */ public static void replace(FileSystem fs, Path current, Path replacement, boolean removeOld) throws IOException { // rename any current path to old Path old = new Path(current + ".old"); if (fs.exists(current)) { fs.rename(current, old); } // rename the new path to current and remove the old path if needed fs.rename(replacement, current); if (fs.exists(old) && removeOld) { fs.delete(old, true); } }
From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java
License:Open Source License
private void bringTempOutputToFront(CustomMapReduceJobPojo cmr) throws IOException, SAXException, ParserConfigurationException { // Get the names: Configuration config = HadoopUtils.getConfiguration(prop_custom); FileSystem fs = FileSystem.get(config); Path pathTmp = HadoopUtils.getPathForJob(cmr, config, true); Path pathFinal = HadoopUtils.getPathForJob(cmr, config, false); // OK don't do anything if pathTmp doesn't exist... if (fs.exists(pathTmp)) { // If the final path exists, delete it if (!fs.exists(pathFinal)) { // create it, which guarantees the parent path also exists //(otherwise the rename fails sigh) fs.mkdirs(pathFinal);/* w w w . j a v a 2 s .co m*/ } fs.delete(pathFinal, true); fs.rename(pathTmp, pathFinal); } }
From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java
License:Open Source License
public static void bringTempOutputToFront(CustomMapReduceJobPojo cmr, PropertiesManager prop_custom) throws IOException, SAXException, ParserConfigurationException { // Get the names: Configuration config = HadoopUtils.getConfiguration(prop_custom); FileSystem fs = FileSystem.get(config); Path pathTmp = HadoopUtils.getPathForJob(cmr, config, true); Path pathFinal = HadoopUtils.getPathForJob(cmr, config, false); // OK don't do anything if pathTmp doesn't exist... if (fs.exists(pathTmp)) { // If the final path exists, delete it if (!fs.exists(pathFinal)) { // create it, which guarantees the parent path also exists //(otherwise the rename fails sigh) fs.mkdirs(pathFinal);// w ww .j a v a 2s.c o m } fs.delete(pathFinal, true); fs.rename(pathTmp, pathFinal); } }
From source file:com.indeed.imhotep.builder.tsv.TsvConverter.java
License:Apache License
/** * * @return true if upload succeeded/*from w w w. j a v a2s . co m*/ */ private static boolean uploadShard(String localShardDir, String shardName, String indexName, Path finalIndexPath, FileSystem finalFS, boolean qaMode) { final Path finalIndexDirPath = new Path(finalIndexPath, indexName); final Path finalShardPath = new Path(finalIndexDirPath, shardName + ".sqar"); try { if (!finalFS.exists(finalIndexDirPath)) { finalFS.mkdirs(finalIndexDirPath); if (qaMode) { makeWorldWritable(finalFS, finalIndexDirPath); } } if (finalFS.exists(finalShardPath)) { log.info("File already exists. HDFS upload aborted."); return true; } final String scheme = finalFS.getUri().getScheme(); if (scheme.equals("hdfs")) { /* * upload to temp file then rename, * to avoid having other systems see a partial file */ final String tmpUploadShardName = indexName + "-" + shardName; final Path tempUploadPath = new Path(new Path("/tmp/"), tmpUploadShardName + ".sqar"); final File shardDir = new File(localShardDir, shardName); final SquallArchiveWriter writer = new SquallArchiveWriter(finalFS, tempUploadPath, true, SquallArchiveCompressor.GZIP); writer.batchAppendDirectory(shardDir); writer.commit(); finalFS.rename(tempUploadPath, finalShardPath); } else if (scheme.equals("s3n")) { /* * s3 files are only visible after the upload is complete, * so no need to use a temp file */ final File shardDir = new File(localShardDir, shardName); final SquallArchiveWriter writer = new SquallArchiveWriter(finalFS, finalShardPath, true, SquallArchiveCompressor.GZIP); writer.batchAppendDirectory(shardDir); writer.commit(); } } catch (IOException e) { log.error(e); return false; } if (qaMode) { try { // try to set permissions on the uploaded file makeWorldWritable(finalFS, finalShardPath); } catch (Exception e) { log.warn("Failed to set permissions on the uploaded file " + finalShardPath); } } return true; }
From source file:com.inmobi.conduit.AbstractService.java
License:Apache License
protected boolean retriableRename(FileSystem fs, Path src, Path dst, String streamName) throws Exception { int count = 0; boolean result = false; Exception exception = null;/*from w w w . ja v a2s.c om*/ while (count < numOfRetries) { try { result = fs.rename(src, dst); exception = null; break; } catch (Exception e) { LOG.warn("Moving " + src + " to " + dst + " failed.Retrying ", e); exception = e; if (stopped) break; } count++; if (streamName != null) { ConduitMetrics.updateSWGuage(getServiceType(), RETRY_RENAME, streamName, 1); } else { LOG.warn("Can not increment retriable rename gauge as stream name is null"); } try { Thread.sleep(TIME_RETRY_IN_MILLIS); } catch (InterruptedException e) { LOG.warn(e); } } if (count == numOfRetries) { LOG.error("Max retries done for moving " + src + " to " + dst + " quitting now"); } if (exception == null) { return result; } else { throw exception; } }