List of usage examples for org.apache.hadoop.fs FileSystem close
@Override public void close() throws IOException
From source file:com.davidgildeh.hadoop.utils.FileUtils.java
License:Apache License
/** * Merges a list of input files in a directory to a single file under the * outputpath with a specified filename// ww w. j a v a 2 s .c o m * * @param inputPath The input directory containing all the input files. E.g. /input/dir/on/hdfs/ * @param outputPath The output path to output the file. E.g. /output/dir/on/hdfs/filename * @throws IOException */ public static void mergeFiles(String inputPath, String outputPath) throws IOException { Path inputDir = new Path(inputPath); Path outputFile = new Path(outputPath); FileSystem fileSystem = getFileSystem(outputFile); checkFileExists(fileSystem, inputDir); // Check the input path is a directory if (!fileSystem.getFileStatus(inputDir).isDir()) { LOG.error("Path '" + inputDir.toString() + "' is not a directory."); throw new IOException("Path '" + inputDir.toString() + "' is not a directory."); } // Create Output File OutputStream out = fileSystem.create(outputFile); try { FileStatus contents[] = fileSystem.listStatus(inputDir); // Loop through all files in directory and merge them into one file for (int i = 0; i < contents.length; i++) { if (!contents[i].isDir()) { InputStream in = fileSystem.open(contents[i].getPath()); try { IOUtils.copyBytes(in, out, fileSystem.getConf(), false); } finally { in.close(); } } } } finally { out.close(); fileSystem.close(); LOG.info("Merged input files from '" + inputPath + "' to '" + outputPath + "'"); } }
From source file:com.davidgildeh.hadoop.utils.FileUtils.java
License:Apache License
/** * Check if a file exists, if not will throw a FileNotFoundException * //from ww w .ja v a2 s . c om * @param path The path of the file to check * @throws IOException */ private static void checkFileExists(FileSystem fileSystem, Path path) throws IOException { // Check file exists if (!fileSystem.exists(path)) { LOG.error("Path '" + path.toString() + "' does not exist."); fileSystem.close(); throw new FileNotFoundException("Path '" + path.toString() + "' does not exist."); } }
From source file:com.ebay.erl.mobius.core.criterion.TupleRestrictions.java
License:Apache License
/** * Create a tuple criterion that only accepts tuples when the value * of the <code>column</code> are presented in the given <code>file</code> * <p>/* ww w. j a v a 2 s. co m*/ * * The assumption of the file is that, it's single column and one to many * line text file. Each line is read into a case insensitive set, and * using the set to check the value of the <code>column</code> within * the set or not. * * * @param column the name of a column to be tested that whether its value is in * the given <code>file</code> or not * * @param file a single column and multiple lines of file that contains strings/numbers, * each line is treated as a single unit. * * @return an instance of {@link TupleCriterion} that extracts only the records * when the value of its <code>column</code> are presented in the given * <code>file</code>. * * @throws FileNotFoundException if the given file cannot be found. */ public static TupleCriterion within(final String column, File file) throws FileNotFoundException { final File f = TupleRestrictions.checkFileExist(file); return new TupleCriterion() { private static final long serialVersionUID = -1121221619118915652L; private Set<String> set; @Override public void setConf(Configuration conf) { try { if (conf.get("tmpfiles") == null || conf.get("tmpfiles").trim().length() == 0) { conf.set("tmpfiles", validateFiles(f.getAbsolutePath(), conf)); } else { conf.set("tmpfiles", validateFiles(f.getAbsolutePath(), conf) + "," + conf.get("tmpfiles")); } } catch (IOException e) { throw new IllegalArgumentException(e); } } /** * COPIED FROM org.apache.hadoop.util.GenericOptionsParser */ private String validateFiles(String files, Configuration conf) throws IOException { if (files == null) return null; String[] fileArr = files.split(","); String[] finalArr = new String[fileArr.length]; for (int i = 0; i < fileArr.length; i++) { String tmp = fileArr[i]; String finalPath; Path path = new Path(tmp); URI pathURI = path.toUri(); FileSystem localFs = FileSystem.getLocal(conf); if (pathURI.getScheme() == null) { // default to the local file system // check if the file exists or not first if (!localFs.exists(path)) { throw new FileNotFoundException("File " + tmp + " does not exist."); } finalPath = path.makeQualified(localFs).toString(); } else { // check if the file exists in this file system // we need to recreate this filesystem object to copy // these files to the file system jobtracker is running // on. FileSystem fs = path.getFileSystem(conf); if (!fs.exists(path)) { throw new FileNotFoundException("File " + tmp + " does not exist."); } finalPath = path.makeQualified(fs).toString(); try { fs.close(); } catch (IOException e) { } ; } finalArr[i] = finalPath; } return StringUtils.arrayToString(finalArr); } @Override protected boolean evaluate(Tuple tuple, Configuration configuration) { if (set == null) { set = new CaseInsensitiveTreeSet(); BufferedReader br = null; try { br = new BufferedReader(new FileReader(new File(f.getName()))); String newLine = null; while ((newLine = br.readLine()) != null) { this.set.add(newLine); } } catch (IOException e) { throw new RuntimeException(e); } finally { try { br.close(); } catch (Throwable e) { } } } String value = tuple.getString(column); if (value != null) { return this.set.contains(value); } else { return false; } } @Override public String[] getInvolvedColumns() { return new String[] { column }; } }; }
From source file:com.ema.hadoop.test_hdfs.TestWrite.java
public static void main(String[] args) throws IOException, URISyntaxException { Configuration configuration = new Configuration(); FileSystem hdfs = FileSystem.get(new URI("hdfs://localhost:9000"), configuration); Path file = new Path("hdfs://localhost:9000/user/student/text_file_write.txt"); if (hdfs.exists(file)) { hdfs.delete(file, true);/* w w w . ja v a 2 s . co m*/ } OutputStream os = hdfs.create(file, new Progressable() { @Override public void progress() { out.println("...bytes written"); } }); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); br.write("This is just a test to check if it is possible to write a file on HDFS using the Java API"); br.close(); hdfs.close(); }
From source file:com.ema.hadoop.wordcount.WordCount_cache.java
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: WordCount <input path> <output path>"); System.exit(-1);//ww w . ja v a 2 s.c om } // First we write the stop word list // it could also be a file manually loaded into HDFS String[] stopwords = { "the", "a" }; Configuration configuration = new Configuration(); FileSystem hdfs = FileSystem.get(new URI("hdfs://localhost:9000"), configuration); Path file = new Path("hdfs://localhost:9000/user/student/stop_words.txt"); if (hdfs.exists(file)) { hdfs.delete(file, true); } OutputStream os = hdfs.create(file, new Progressable() { @Override public void progress() { out.println("...bytes written"); } }); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); for (String w : stopwords) { br.write(w + "\n"); } br.close(); hdfs.close(); Job job = Job.getInstance(); job.addCacheFile(new Path("hdfs://localhost:9000/user/student/stop_words.txt").toUri()); job.setJarByClass(WordCount_cache.class); job.setJobName("Word count job"); FileInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setMapperClass(WCMapper_cache.class); job.setReducerClass(WCReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.example.sparkservice.SparkService.java
/** * Metodo che permette l'eliminazione della directory in cui salvato * il modello precedentemente addestrato. * Il server prima di effettuare un nuovo addestramento eliminer tale * directory./*w w w . jav a2 s . c o m*/ * @return True se l'eliminazione viene eseguita con successo; * False se viene sollevata qualche eccezione. */ public boolean deleteModel() { //Configuration di HADOOP Configuration config = new Configuration(); try { //File system HDFS FileSystem fs = FileSystem.get(URI.create(PATH_MODEL), config); fs.delete(new Path(PATH_MODEL), true); fs.close(); } catch (IOException ex) { Logger.getLogger(SparkService.class.getName()).log(Level.SEVERE, null, ex); return false; } return true; }
From source file:com.example.sparkservice.SparkService.java
/** * Metodo che permette l'append della mail classificata erroneamente dal * server nel dataset corretto./* www. j a va 2 s .c o m*/ * I dataset sono due file, uno contenente tutte le mail classificate come * HAM, l'altro tutte le mail classificate come SPAM. * @param uri Path del dataset * @param mail Testo della mail */ public void appendData(String uri, String mail) { Configuration config = new Configuration(); FileSystem fs; try { fs = FileSystem.get(URI.create(uri), config); FSDataOutputStream fsout = fs.append(new Path(uri)); PrintWriter writer = new PrintWriter(fsout); writer.append(mail.replace('\n', ' ').replace('\r', ' ') + "\n"); writer.close(); fs.close(); } catch (IOException ex) { Logger.getLogger(SparkService.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:com.gemstone.gemfire.cache.hdfs.internal.HDFSStoreImpl.java
License:Apache License
private void closeFileSystemIgnoreError(FileSystem fileSystem) { if (fileSystem == null) { logger.debug("{}Trying to close null file system", logPrefix); return;/* w ww . j a va2 s . c o m*/ } try { if (logger.isDebugEnabled()) { logger.debug("{}Closing file system at " + fileSystem.getUri() + " " + fileSystem.hashCode(), logPrefix); } fileSystem.close(); } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug("Failed to close file system at " + fileSystem.getUri() + " " + fileSystem.hashCode(), e); } } }
From source file:com.github.libsml.commons.util.HadoopUtils.java
License:Apache License
public static void mkdir(Path path, boolean overwrite) throws IOException { Configuration config = new Configuration(); FileSystem fs = path.getFileSystem(config); if (fs.exists(path) && !overwrite) { throw new IllegalStateException("Mkdir exception:path=" + path.toString() + " exists"); }/* w w w .j a va 2 s . c o m*/ if (fs.exists(path)) { fs.delete(path, true); } fs.mkdirs(path); fs.close(); }
From source file:com.github.sakserv.minicluster.HdfsLocalClusterIntegrationTest.java
License:Apache License
@Test public void testDfsClusterStart() throws IOException { // Write a file to HDFS containing the test string FileSystem hdfsFsHandle = dfsCluster.getHdfsFileSystemHandle(); FSDataOutputStream writer = hdfsFsHandle .create(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY))); writer.writeUTF(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY)); writer.close();// w w w. j a va 2 s. c o m // Read the file and compare to test string FSDataInputStream reader = hdfsFsHandle .open(new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY))); assertEquals(reader.readUTF(), propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY)); reader.close(); hdfsFsHandle.close(); }