List of usage examples for org.apache.hadoop.fs FileSystem get
public static FileSystem get(URI uri, Configuration conf) throws IOException
From source file:cascading.tap.hadoop.Dfs.java
License:Open Source License
@Override protected FileSystem getDefaultFileSystem(Configuration configuration) { String name = configuration.get("fs.default.name", "hdfs://localhost:5001/"); if (name.equals("local") || name.matches(".*://.*") && !name.startsWith("hdfs://")) name = "hdfs://localhost:5001/"; else if (name.indexOf('/') == -1) name = "hdfs://" + name; try {//from w w w . j a va2 s. co m return FileSystem.get(URI.create(name), configuration); } catch (IOException exception) { throw new TapException("unable to get handle to get filesystem for: " + name, exception); } }
From source file:cascading.tap.hadoop.Hfs.java
License:Open Source License
protected FileSystem getFileSystem(Configuration configuration) { URI scheme = getURIScheme(configuration); try {// ww w . jav a 2 s . c o m return FileSystem.get(scheme, configuration); } catch (IOException exception) { throw new TapException("unable to get handle to get filesystem for: " + scheme.getScheme(), exception); } }
From source file:cascading.tap.Hfs.java
License:Open Source License
protected FileSystem getFileSystem(JobConf jobConf) throws IOException { return FileSystem.get(getURIScheme(jobConf), jobConf); }
From source file:cc.solr.lucene.store.hdfs.ChangeFileExt.java
License:Apache License
public static void main(String[] args) throws IOException { Path p = new Path(args[0]); FileSystem fileSystem = FileSystem.get(p.toUri(), new Configuration()); FileStatus[] listStatus = fileSystem.listStatus(p); for (FileStatus fileStatus : listStatus) { Path path = fileStatus.getPath(); fileSystem.rename(path, new Path(path.toString() + ".lf")); }//from w ww.j ava2s . c o m }
From source file:cc.solr.lucene.store.hdfs.ConvertDirectory.java
License:Apache License
public static void convert(Path path) throws IOException { FileSystem fileSystem = FileSystem.get(path.toUri(), new Configuration()); if (!fileSystem.exists(path)) { System.out.println(path + " does not exists."); return;/*from w w w . jav a 2s . com*/ } FileStatus fileStatus = fileSystem.getFileStatus(path); if (fileStatus.isDir()) { FileStatus[] listStatus = fileSystem.listStatus(path); for (FileStatus status : listStatus) { convert(status.getPath()); } } else { System.out.println("Converting file [" + path + "]"); HdfsMetaBlock block = new HdfsMetaBlock(); block.realPosition = 0; block.logicalPosition = 0; block.length = fileStatus.getLen(); FSDataOutputStream outputStream = fileSystem.append(path); block.write(outputStream); outputStream.writeInt(1); outputStream.writeLong(fileStatus.getLen()); outputStream.writeInt(HdfsFileWriter.VERSION); outputStream.close(); } }
From source file:cc.solr.lucene.store.hdfs.HdfsDirectory.java
License:Apache License
protected void reopenFileSystem() throws IOException { FileSystem fileSystem = FileSystem.get(_hdfsDirPath.toUri(), _configuration); FileSystem oldFs = _fileSystemRef.get(); _fileSystemRef.set(fileSystem);/* w ww . jav a 2s . c o m*/ if (oldFs != null) { oldFs.close(); } }
From source file:ch.epfl.data.squall.utilities.SerializableHDFSFileInputStream.java
License:Open Source License
public SerializableHDFSFileInputStream(String URIstring, int bufferSize, int section, int parts) throws IOException { _uri = URI.create(URIstring); Configuration conf = new Configuration(); _file = FileSystem.get(_uri, conf); _fis = _file.open(new Path(_uri)); _buffer = new byte[bufferSize]; setParameters(section, parts);/* w w w. ja v a2s . c o m*/ fillBuffer(); }
From source file:cmd.freebase2rdf4mr.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName()); ToolRunner.printGenericCommandUsage(System.err); return -1; }//from www . j ava 2 s .c o m Configuration configuration = getConf(); boolean overrideOutput = configuration.getBoolean(OPTION_OVERRIDE_OUTPUT, OPTION_OVERRIDE_OUTPUT_DEFAULT); FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration); if (overrideOutput) { fs.delete(new Path(args[1]), true); } Tool driver = new Freebase2RDFDriver(configuration); driver.run(new String[] { args[0], args[1] }); return 0; }
From source file:cmd.infer.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName()); ToolRunner.printGenericCommandUsage(System.err); return -1; }// www.ja v a 2 s . co m Configuration configuration = getConf(); boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERRIDE_OUTPUT, Constants.OPTION_OVERRIDE_OUTPUT_DEFAULT); FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration); if (overrideOutput) { fs.delete(new Path(args[1]), true); } Tool infer = new InferDriver(configuration); infer.run(new String[] { args[0], args[1] }); return 0; }
From source file:cmd.rdf2adjacencylist.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName()); ToolRunner.printGenericCommandUsage(System.err); return -1; }//from w w w .jav a 2 s.c o m Configuration configuration = getConf(); boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERWRITE_OUTPUT, Constants.OPTION_OVERWRITE_OUTPUT_DEFAULT); FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration); if (overrideOutput) { fs.delete(new Path(args[1]), true); } Tool tool = new Rdf2AdjacencyListDriver(configuration); tool.run(new String[] { args[0], args[1] }); return 0; }