List of usage examples for org.apache.hadoop.fs FileSystem getFileStatus
public abstract FileStatus getFileStatus(Path f) throws IOException;
From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java
License:Apache License
public void readRandomHeap() throws Exception { System.out.println("reading random file in heap mode " + path); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); FileStatus status = fs.getFileStatus(path); FSDataInputStream instream = fs.open(path); byte[] buf = new byte[size]; double sumbytes = 0; double ops = 0; long _range = status.getLen() - ((long) buf.length); double range = (double) _range; Random random = new Random(); System.out.println("file capacity " + status.getLen()); System.out.println("read size " + size); System.out.println("operations " + loop); long start = System.currentTimeMillis(); while (ops < loop) { double _offset = range * random.nextDouble(); long offset = (long) _offset; instream.seek(offset);//from www .j a v a2 s.com double ret = (double) this.read(instream, buf); if (ret > 0) { sumbytes = sumbytes + ret; ops = ops + 1.0; } else { break; } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1024.0 / 1024.0; latency = 1000000.0 * executionTime / ops; } System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); System.out.println("closing stream"); instream.close(); fs.close(); }
From source file:com.ibm.jaql.io.hadoop.DefaultHadoopOutputAdapter.java
License:Apache License
@Override public JsonValue expand() throws Exception { BufferedJsonRecord jr = (BufferedJsonRecord) this.options.getCopy(null); BufferedJsonArray ja = new BufferedJsonArray(); String namenode = null;//w ww . j a v a 2s. c o m String dfsport = null; if (this.location != null) { Configuration conf = new Configuration(); namenode = conf.get("fs.default.name"); dfsport = conf.get("dfs.namenode.http-address"); FileSystem fs = FileSystem.get(conf); FileStatus files = fs.getFileStatus(new Path(this.location)); if (files.isDir()) { StringBuilder sb = new StringBuilder(); FileStatus[] dirContent = fs.listStatus(new Path(this.location)); for (FileStatus file : dirContent) { if (!file.isDir()) { ja.add(new JsonString(file.getPath().toString())); } } } else { ja.add(new JsonString(files.getPath().toString())); } } jr.add(Adapter.LOCATION_NAME, ja); jr.add(Adapter.TYPE_NAME, args.get(Adapter.TYPE_NAME)); jr.add(new JsonString("expanded"), JsonBool.make(true)); if (namenode != null) jr.add(new JsonString("fs.default.name"), new JsonString(namenode)); if (dfsport != null) jr.add(new JsonString("dfs.namenode.http-address"), new JsonString(dfsport)); return jr; }
From source file:com.ibm.stocator.fs.swift2d.systemtests.StreamingSwiftTest.java
License:Open Source License
public void accessPublicSwiftContainerWithSpaceTest() throws Exception { FileSystem fs = new ObjectStoreFileSystem(); Configuration conf = new Configuration(); String uriString = conf.get("fs.swift2d.test.uri"); Assume.assumeNotNull(uriString);/*from w w w . j av a 2s .c o m*/ // adding suffix with space to the container name String scheme = "swift2d"; String host = getHost(URI.create(uriString)); // String origContainerName = getContainerName(host); // String newContainerName = origContainerName + " t"; // uriString = uriString.replace(origContainerName, newContainerName); // use URI ctor that encodes authority according to the rules specified // in RFC 2396, section 5.2, step 7 URI publicContainerURI = new URI(scheme, getHost(URI.create(uriString)), "/", null, null); fs.initialize(publicContainerURI, conf); FileStatus objectFS = null; try { objectFS = fs.getFileStatus(new Path(publicContainerURI)); } catch (Exception e) { e.printStackTrace(); Assert.assertNotNull("Unable to access public object ", objectFS); } }
From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java
License:Open Source License
/** * Assert that a path does not exist//from w w w. jav a2s. c om * * @param fileSystem filesystem to examine * @param message message to include in the assertion failure message * @param path path in the filesystem * @throws IOException IO problems */ public static void assertPathDoesNotExist(FileSystem fileSystem, String message, Path path) throws IOException { try { FileStatus status = fileSystem.getFileStatus(path); if (status != null) { fail(message + ": unexpectedly found " + path + " as " + status); } } catch (FileNotFoundException expected) { //this is expected } }
From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java
License:Open Source License
/** * Make an assertion about the length of a file * @param fs filesystem/*from www . j av a 2s . c o m*/ * @param path path of the file * @param expected expected length * @throws IOException on File IO problems */ public static void assertFileHasLength(FileSystem fs, Path path, int expected) throws IOException { FileStatus status = fs.getFileStatus(path); assertEquals("Wrong file length of file " + path + " status: " + status, expected, status.getLen()); }
From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java
License:Open Source License
/** * Assert that a path refers to a directory * @param fs filesystem/* ww w . ja va 2 s . com*/ * @param path path of the directory * @throws IOException on File IO problems */ public static void assertIsDirectory(FileSystem fs, Path path) throws IOException { FileStatus fileStatus = fs.getFileStatus(path); assertIsDirectory(fileStatus); }
From source file:com.iflytek.spider.util.HadoopFSUtil.java
License:Apache License
/** * Returns PathFilter that passes directories through. *//*from ww w. j a va2 s . c om*/ public static PathFilter getPassDirectoriesFilter(final FileSystem fs) { return new PathFilter() { public boolean accept(final Path path) { try { return fs.getFileStatus(path).isDir(); } catch (IOException ioe) { return false; } } }; }
From source file:com.iflytek.spider.util.LockUtil.java
License:Apache License
/** * Create a lock file./*from www . ja v a 2s .co m*/ * @param fs filesystem * @param lockFile name of the lock file * @param accept if true, and the target file exists, consider it valid. If false * and the target file exists, throw an IOException. * @throws IOException if accept is false, and the target file already exists, * or if it's a directory. */ public static void createLockFile(FileSystem fs, Path lockFile, boolean accept) throws IOException { if (fs.exists(lockFile)) { if (!accept) throw new IOException("lock file " + lockFile + " already exists."); if (fs.getFileStatus(lockFile).isDir()) throw new IOException("lock file " + lockFile + " already exists and is a directory."); // do nothing - the file already exists. } else { // make sure parents exist fs.mkdirs(lockFile.getParent()); fs.createNewFile(lockFile); } }
From source file:com.iflytek.spider.util.LockUtil.java
License:Apache License
/** * Remove lock file. NOTE: applications enforce the semantics of this file - * this method simply removes any file with a given name. * @param fs filesystem//www . j a va 2s. co m * @param lockFile lock file name * @return false, if the lock file doesn't exist. True, if it existed and was * successfully removed. * @throws IOException if lock file exists but it is a directory. */ public static boolean removeLockFile(FileSystem fs, Path lockFile) throws IOException { if (!fs.exists(lockFile)) return false; if (fs.getFileStatus(lockFile).isDir()) throw new IOException("lock file " + lockFile + " exists but is a directory!"); return fs.delete(lockFile, false); }
From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java
License:Open Source License
public static Path cacheLocalFile(String localPath, String localName, Configuration config) throws IOException { FileSystem fs = FileSystem.get(config); Path toDir = new Path("cache"); Path destFile = new Path("cache/" + localName); File fromFile = new File(localPath + "/" + localName); if (!fromFile.exists()) { throw new IOException("Source file does not exist: " + fromFile.toString()); }/*from www . j a v a 2 s . c o m*/ boolean needToCopyFile = true; if (!fs.exists(toDir)) { // (ie relative to WD) fs.mkdirs(toDir); } else { // Now check if the file already exists if (fs.exists(destFile)) { FileStatus fsStat = fs.getFileStatus(destFile); if ((fsStat.getLen() == fromFile.length()) && (fromFile.lastModified() <= fsStat.getModificationTime())) { needToCopyFile = false; } } } if (needToCopyFile) { fs.copyFromLocalFile(false, true, new Path(localPath + "/" + localName), destFile); } return new Path(fs.getFileStatus(destFile).getPath().toUri().getPath()); // (apparently the path has to be in absolute format without even the hdfs:// at the front?!) }