List of usage examples for org.apache.hadoop.fs FileSystem get
public static FileSystem get(URI uri, Configuration conf) throws IOException
From source file:bixo.hadoop.HadoopConfigured.java
License:Apache License
public FileSystem getFileSystem(URI uri) throws IOException { return FileSystem.get(uri, getConf()); }
From source file:bixo.hadoop.HadoopConfigured.java
License:Apache License
/** * If the path is a valid URI we lookup the file system based on the uri, if * it is not we use the configured file system. * Please note that "/something" is a valid uri and will return the local file system. * // w w w .j a v a2 s .c o m * @throws IOException */ public FileSystem getFileSystem(String path) throws IOException { try { URI uri = new URI(path); return FileSystem.get(uri, getConf()); } catch (URISyntaxException e) { LOGGER.warn("The path: " + path + " is not a valid uri. Therefore we use the configured file system.", e); } return getFileSystem(); }
From source file:biz.hangyang.knnspark.spark.KNNClassifySpark.java
public static JavaPairRDD<Entity, Object> calKDistance(final String trainingDataPath, String testingDataPath, final int k, final Map<Object, Double> weightMap, JavaSparkContext sc, int partition, final Accumulator<Integer> accum) { JavaRDD<String> testingDataRDD = sc.textFile(testingDataPath, partition); //?Entity// w w w . java 2 s .c om JavaRDD<Entity> testingEntityRDD = testingDataRDD.map(new Function<String, Entity>() { @Override public Entity call(String line) throws Exception { return new GeneEntity(line); } }); //??????K??KV JavaPairRDD<Entity, KDistance> ekRDD = testingEntityRDD .mapPartitionsToPair(new PairFlatMapFunction<Iterator<Entity>, Entity, KDistance>() { @Override public Iterable<Tuple2<Entity, KDistance>> call(Iterator<Entity> t) throws Exception { //?PARTITION? List<Entity> entityList = new ArrayList<>(); while (t.hasNext()) { entityList.add(t.next()); } //??LIST List<KDistance> kDistanceList = new ArrayList<>(); for (int i = 0; i < entityList.size(); i++) { kDistanceList.add(new KDistance(k)); } //???hdfs Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(trainingDataPath), conf); FSDataInputStream in = fs.open(new Path(trainingDataPath)); BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8")); String line; while ((line = br.readLine()) != null) { Entity lineEntity = new GeneEntity(line); for (int i = 0; i < entityList.size(); i++) { kDistanceList.get(i).add(new DemoDistanceCatagory( lineEntity.distance(entityList.get(i)), lineEntity.category)); } } List<Tuple2<Entity, KDistance>> tList = new ArrayList<>(); for (int i = 0; i < entityList.size(); i++) { tList.add(new Tuple2<>(entityList.get(i), kDistanceList.get(i))); } return tList; } }); JavaPairRDD<Entity, Object> eoRDD = ekRDD .mapToPair(new PairFunction<Tuple2<Entity, KDistance>, Entity, Object>() { @Override public Tuple2<Entity, Object> call(Tuple2<Entity, KDistance> t) throws Exception { KDistance kDistance = t._2(); //??? Object catagory = KDistance.getCatagory(kDistance.get(), weightMap); if (t._1().category.equals(catagory)) { accum.add(1); } return new Tuple2<>(t._1(), catagory); } }); return eoRDD; }
From source file:boa.compiler.Test.java
License:Apache License
public static void main(String[] args) throws IOException, URISyntaxException { Configuration configuration = new Configuration(); FileSystem hdfs = FileSystem.get(new URI("hdfs://localhost:54310"), configuration); FileStatus[] fileStatus = hdfs.listStatus(new Path("hdfs://localhost:54310/ast/")); Path[] paths = FileUtil.stat2Paths(fileStatus); System.out.println("***** Contents of the Directory *****"); for (Path path : paths) { System.out.println(path); }/*from w w w .j av a2 s. com*/ }
From source file:ca.sparkera.adapters.mainframe.CobolSerdeUtils.java
License:Apache License
protected static String getLayoutFromFS(String layoutFSUrl, Configuration conf) throws IOException, URISyntaxException { FSDataInputStream in = null;/*from w ww . j av a2s .c om*/ FileSystem fs = null; try { fs = FileSystem.get(new URI(layoutFSUrl), conf); } catch (IOException ioe) { // return null only if the file system in layout is not recognized String msg = "Failed to open file system for uri " + layoutFSUrl + " assuming it is not a FileSystem url"; LOG.debug(msg, ioe); return null; } try { in = fs.open(new Path(layoutFSUrl)); String s = CobolSerdeUtils.getLayoutFor(in); return s; } finally { if (in != null) in.close(); } }
From source file:ca.uwaterloo.iss4e.spark.pointperrow.CosineMain.java
License:Open Source License
public void fetch(JavaSparkContext sc, String source) { try {//w w w.ja v a2 s . co m FileSystem fs = FileSystem.get(new URI(source), new Configuration()); Path src = new Path(source); if (fs.exists(src)) { FileStatus[] lists = fs.listStatus(src); readFiles(sc, fs, lists); } } catch (IOException e) { e.printStackTrace(); } catch (URISyntaxException e) { e.printStackTrace(); } }
From source file:cascading.flow.hadoop.MapReduceFlowPlatformTest.java
License:Open Source License
private String remove(String path, boolean delete) throws IOException { FileSystem fs = FileSystem.get(URI.create(path), HadoopPlanner.createJobConf(getProperties())); if (delete)/* w w w . ja v a 2 s .com*/ fs.delete(new Path(path), true); return path; }
From source file:cascading.flow.MapReduceFlowTest.java
License:Open Source License
private String remove(String path, boolean delete) throws IOException { FileSystem fs = FileSystem.get(URI.create(path), MultiMapReducePlanner.getJobConf(getProperties())); if (delete)/*from ww w . j a va 2 s . co m*/ fs.delete(new Path(path), true); return path; }
From source file:cascading.platform.hadoop.BaseHadoopPlatform.java
License:Open Source License
public boolean isHDFSAvailable() { try {//from w w w .ja va 2s.c om FileSystem fileSystem = FileSystem.get(new URI("hdfs:", null, null), configuration); return fileSystem != null; } catch (IOException exception) // if no hdfs, a no filesystem for scheme io exception will be caught { LOG.warn("unable to get hdfs filesystem", exception); } catch (URISyntaxException exception) { throw new RuntimeException("internal failure", exception); } return false; }
From source file:cascading.tap.Dfs.java
License:Open Source License
@Override protected FileSystem getDefaultFileSystem(JobConf jobConf) throws IOException { String name = jobConf.get("fs.default.name", "hdfs://localhost:5001/"); if (name.equals("local") || name.matches(".*://.*") && !name.startsWith("hdfs://")) name = "hdfs://localhost:5001/"; else if (name.indexOf('/') == -1) name = "hdfs://" + name; return FileSystem.get(URI.create(name), jobConf); }