List of usage examples for org.apache.hadoop.fs FileSystem deleteOnExit
Set deleteOnExit
To view the source code for org.apache.hadoop.fs FileSystem deleteOnExit.
Click Source Link
From source file:org.apache.carbondata.core.datastore.impl.FileFactory.java
License:Apache License
/** * for creating a new Lock file and if it is successfully created * then in case of abrupt shutdown then the stream to that file will be closed. * * @param filePath/*w w w . j av a 2s . c o m*/ * @param fileType * @return * @throws IOException */ public static boolean createNewLockFile(String filePath, FileType fileType) throws IOException { filePath = filePath.replace("\\", "/"); switch (fileType) { case HDFS: case ALLUXIO: case VIEWFS: Path path = new Path(filePath); FileSystem fs = path.getFileSystem(configuration); if (fs.createNewFile(path)) { fs.deleteOnExit(path); return true; } return false; case LOCAL: default: filePath = getUpdatedFilePath(filePath, fileType); File file = new File(filePath); return file.createNewFile(); } }
From source file:org.apache.drill.exec.store.StorageStrategy.java
License:Apache License
/** * Applies storage strategy to passed path on passed file system. * Sets appropriate permission/*from w ww . ja v a2s . c o m*/ * and adds to file system delete on exit list if needed. * * @param fs file system where path is located * @param path path location * @param permission permission to be applied * @param deleteOnExit if to delete path on exit * @throws IOException is thrown in case of problems while setting permission * or adding path to delete on exit list */ private void applyStrategy(FileSystem fs, Path path, String permission, boolean deleteOnExit) throws IOException { fs.setPermission(path, new FsPermission(permission)); if (deleteOnExit) { fs.deleteOnExit(path); } }
From source file:org.apache.hama.pipes.util.DistributedCacheUtil.java
License:Apache License
/** * Add the Files to HDFS/*w ww. jav a 2s .c o m*/ * * @param conf The job's configuration * @param files Paths that should be transfered to HDFS */ public static String addFilesToHDFS(Configuration conf, String files) { if (files == null) return null; String[] fileArr = files.split(","); String[] finalArr = new String[fileArr.length]; for (int i = 0; i < fileArr.length; i++) { String tmp = fileArr[i]; String finalPath; URI pathURI; try { pathURI = new URI(tmp); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } try { LocalFileSystem local = LocalFileSystem.getLocal(conf); Path pathSrc = new Path(pathURI); // LOG.info("pathSrc: " + pathSrc); if (local.exists(pathSrc)) { FileSystem hdfs = FileSystem.get(conf); Path pathDst = new Path(hdfs.getWorkingDirectory() + "/temp", pathSrc.getName()); // LOG.info("WorkingDirectory: " + hdfs.getWorkingDirectory()); LOG.debug("copyToHDFSFile: " + pathDst); hdfs.copyFromLocalFile(pathSrc, pathDst); hdfs.deleteOnExit(pathDst); finalPath = pathDst.makeQualified(hdfs).toString(); finalArr[i] = finalPath; } } catch (IOException e) { LOG.error(e); } } return StringUtils.arrayToString(finalArr); }
From source file:org.apache.hcatalog.hcatmix.load.hadoop.HadoopUtils.java
License:Apache License
/** * Walks through the classpath and uploads the files in HDFS and also adds them to the JobConf object. * The upload files are marked for deletion upon exit. So they gets deleted when the job finishes * * @param jobConf/*w w w . j ava 2 s . c o m*/ * @param tmpDir where all the files would be copied to * @throws IOException */ public static void uploadClasspathAndAddToJobConf(JobConf jobConf, Path tmpDir) throws IOException { final String[] jars = findFilesInCP( new String[] { System.getenv().get("CLASSPATH"), System.getProperty("java.class.path") }); final FileSystem fs = FileSystem.get(jobConf); for (String jarFile : jars) { Path srcJarFilePath = new Path("file:///" + jarFile); String filename = srcJarFilePath.getName(); Path tmpJarFilePath = getTmpFileName(tmpDir, filename); fs.deleteOnExit(tmpJarFilePath); fs.copyFromLocalFile(srcJarFilePath, tmpJarFilePath); DistributedCache.addFileToClassPath(tmpJarFilePath, jobConf); } DistributedCache.createSymlink(jobConf); }
From source file:org.apache.kylin.source.hive.HiveMRInputTest.java
License:Apache License
@Test public void TestGetJobWorkingDir() throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration()); Path jobWorkDirPath = null;// w w w . ja v a 2 s . c om KylinConfig kylinConfig = mock(KylinConfig.class); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) { when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true); when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/"); DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class); defaultChainedExecutable.setId(RandomUtil.randomUUID().toString()); String jobWorkingDir = HiveInputBase.getJobWorkingDir(defaultChainedExecutable, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); jobWorkDirPath = new Path(jobWorkingDir); Assert.assertTrue(fileSystem.exists(jobWorkDirPath)); } finally { if (jobWorkDirPath != null) fileSystem.deleteOnExit(jobWorkDirPath); } }
From source file:org.apache.kylin.storage.hbase.steps.HFileOutputFormat3.java
License:Apache License
/** * Configure <code>job</code> with a TotalOrderPartitioner, partitioning against * <code>splitPoints</code>. Cleans up the partitions file after job exists. *//* w ww. j a va 2s . co m*/ static void configurePartitioner(Job job, List<ImmutableBytesWritable> splitPoints) throws IOException { Configuration conf = job.getConfiguration(); // create the partitions file FileSystem fs = FileSystem.get(conf); Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + RandomUtil.randomUUID()); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, splitPoints); fs.deleteOnExit(partitionsPath); // configure job to use it job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(conf, partitionsPath); }
From source file:org.apache.mahout.classifier.sequencelearning.baumwelchmapreduce.MapWritableCache.java
License:Apache License
/** * @param key SequenceFile key//from w w w . j a v a 2s . c om * @param map Map to save */ public static void save(Writable key, MapWritable map, Path output, Configuration conf, boolean overwritePath, boolean deleteOnExit) throws IOException { FileSystem fs = FileSystem.get(conf); output = fs.makeQualified(output); if (overwritePath) { HadoopUtil.delete(conf, output); } // set the cache DistributedCache.setCacheFiles(new URI[] { output.toUri() }, conf); // set up the writer SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, output, IntWritable.class, MapWritable.class); try { writer.append(key, new MapWritable(map)); } finally { Closeables.closeQuietly(writer); } if (deleteOnExit) { fs.deleteOnExit(output); } }
From source file:org.apache.mahout.clustering.spectral.common.TestVectorCache.java
License:Apache License
@Test public void testLoad() throws Exception { // save a vector manually Configuration conf = new Configuration(); Writable key = new IntWritable(0); Vector value = new DenseVector(VECTOR); Path path = getTestTempDirPath("output"); FileSystem fs = FileSystem.get(path.toUri(), conf); // write the vector path = fs.makeQualified(path);/*from w w w. j a v a 2 s . c o m*/ fs.deleteOnExit(path); HadoopUtil.delete(conf, path); DistributedCache.setCacheFiles(new URI[] { path.toUri() }, conf); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class); try { writer.append(key, new VectorWritable(value)); } finally { Closeables.closeQuietly(writer); } // load it Vector result = VectorCache.load(conf); // are they the same? assertNotNull("Vector is not null", result); assertEquals("Loaded vector is identical to original", result, value); }
From source file:org.apache.mahout.clustering.spectral.common.VectorCache.java
License:Apache License
/** * /* w ww . j av a 2 s. co m*/ * @param key SequenceFile key * @param vector Vector to save, to be wrapped as VectorWritable */ public static void save(Writable key, Vector vector, Path output, Configuration conf, boolean overwritePath, boolean deleteOnExit) throws IOException { FileSystem fs = FileSystem.get(output.toUri(), conf); output = fs.makeQualified(output); if (overwritePath) { HadoopUtil.delete(conf, output); } // set the cache DistributedCache.setCacheFiles(new URI[] { output.toUri() }, conf); // set up the writer SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, output, IntWritable.class, VectorWritable.class); try { writer.append(key, new VectorWritable(vector)); } finally { Closeables.closeQuietly(writer); } if (deleteOnExit) { fs.deleteOnExit(output); } }
From source file:org.apache.mahout.clustering.spectral.TestVectorCache.java
License:Apache License
@Test public void testLoad() throws Exception { // save a vector manually Configuration conf = getConfiguration(); Writable key = new IntWritable(0); Vector value = new DenseVector(VECTOR); Path path = getTestTempDirPath("output"); FileSystem fs = FileSystem.get(path.toUri(), conf); // write the vector path = fs.makeQualified(path);/*from ww w.j a v a2 s. co m*/ fs.deleteOnExit(path); HadoopUtil.delete(conf, path); SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class); try { writer.append(key, new VectorWritable(value)); } finally { Closeables.close(writer, false); } DistributedCache.setCacheFiles(new URI[] { path.toUri() }, conf); // load it Vector result = VectorCache.load(conf); // are they the same? assertNotNull("Vector is null", result); assertEquals("Loaded vector is not identical to original", result, value); }