List of usage examples for org.apache.hadoop.fs FileSystem newInstance
public static FileSystem newInstance(URI uri, Configuration config) throws IOException
From source file:com.datatorrent.lib.io.jms.FSPsuedoTransactionableStore.java
License:Open Source License
/** * Override this method to change the FileSystem instance that is used by the operator. * This method is mainly helpful for unit testing. * @return A FileSystem object./*from w w w. j a v a 2s . c om*/ * @throws IOException */ protected FileSystem getFSInstance() throws IOException { FileSystem tempFS = FileSystem.newInstance(new Path(recoveryDirectory).toUri(), new Configuration()); if (tempFS instanceof LocalFileSystem) { tempFS = ((LocalFileSystem) tempFS).getRaw(); } return tempFS; }
From source file:com.datatorrent.lib.parser.XmlParser.java
License:Apache License
@Override public void setup(com.datatorrent.api.Context.OperatorContext context) { try {//from w ww.ja v a2s . c o m JAXBContext ctx = JAXBContext.newInstance(getClazz()); unmarshaller = ctx.createUnmarshaller(); if (schemaXSDFile != null) { Path filePath = new Path(schemaXSDFile); Configuration configuration = new Configuration(); FileSystem fs = FileSystem.newInstance(filePath.toUri(), configuration); FSDataInputStream inputStream = fs.open(filePath); SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); Schema schema = factory.newSchema(new StreamSource(inputStream)); unmarshaller.setSchema(schema); validator = schema.newValidator(); fs.close(); } } catch (SAXException e) { DTThrowable.wrapIfChecked(e); } catch (JAXBException e) { DTThrowable.wrapIfChecked(e); } catch (IOException e) { DTThrowable.wrapIfChecked(e); } }
From source file:com.datatorrent.lib.util.FSStorageAgent.java
License:Open Source License
public FSStorageAgent(String path, Configuration conf) { this.path = path; try {//w w w . j ava 2 s .com logger.debug("Initialize storage agent with {}.", path); Path lPath = new Path(path); fs = FileSystem.newInstance(lPath.toUri(), conf == null ? new Configuration() : conf); try { if (fs.mkdirs(lPath)) { fs.setWorkingDirectory(lPath); } } catch (IOException e) { // some file system (MapR) throw exception if folder exists if (!fs.exists(lPath)) { throw e; } } } catch (IOException ex) { throw new RuntimeException(ex); } }
From source file:com.datatorrent.lib.util.FSWindowDataManagerTest.java
License:Apache License
@Test public void testDelete() throws IOException { testMeta.storageManager.setup(testMeta.context); Map<Integer, String> dataOf1 = Maps.newHashMap(); dataOf1.put(1, "one"); dataOf1.put(2, "two"); dataOf1.put(3, "three"); Map<Integer, String> dataOf2 = Maps.newHashMap(); dataOf2.put(4, "four"); dataOf2.put(5, "five"); dataOf2.put(6, "six"); Map<Integer, String> dataOf3 = Maps.newHashMap(); dataOf2.put(7, "seven"); dataOf2.put(8, "eight"); dataOf2.put(9, "nine"); for (int i = 1; i <= 9; ++i) { testMeta.storageManager.save(dataOf1, 1, i); }//from w ww. j ava2s.c o m testMeta.storageManager.save(dataOf2, 2, 1); testMeta.storageManager.save(dataOf3, 3, 1); testMeta.storageManager.partitioned(Lists.<WindowDataManager>newArrayList(testMeta.storageManager), Sets.newHashSet(2, 3)); testMeta.storageManager.setup(testMeta.context); testMeta.storageManager.deleteUpTo(1, 6); Path appPath = new Path(testMeta.applicationPath + '/' + testMeta.storageManager.getRecoveryPath()); FileSystem fs = FileSystem.newInstance(appPath.toUri(), new Configuration()); FileStatus[] fileStatuses = fs.listStatus(new Path(appPath, Integer.toString(1))); Assert.assertEquals("number of windows for 1", 3, fileStatuses.length); TreeSet<String> windows = Sets.newTreeSet(); for (FileStatus fileStatus : fileStatuses) { windows.add(fileStatus.getPath().getName()); } Assert.assertEquals("window list for 1", Sets.newTreeSet(Arrays.asList("7", "8", "9")), windows); Assert.assertEquals("no data for 2", false, fs.exists(new Path(appPath, Integer.toString(2)))); Assert.assertEquals("no data for 3", false, fs.exists(new Path(appPath, Integer.toString(3)))); testMeta.storageManager.teardown(); }
From source file:com.datatorrent.lib.util.WindowDataManagerTest.java
License:Apache License
@Test public void testDelete() throws IOException { Map<Integer, String> dataOf1 = Maps.newHashMap(); dataOf1.put(1, "one"); dataOf1.put(2, "two"); dataOf1.put(3, "three"); Map<Integer, String> dataOf2 = Maps.newHashMap(); dataOf2.put(4, "four"); dataOf2.put(5, "five"); dataOf2.put(6, "six"); Map<Integer, String> dataOf3 = Maps.newHashMap(); dataOf2.put(7, "seven"); dataOf2.put(8, "eight"); dataOf2.put(9, "nine"); for (int i = 1; i <= 9; ++i) { testMeta.storageManager.save(dataOf1, 1, i); }// w w w . j a va2 s .c o m testMeta.storageManager.save(dataOf2, 2, 1); testMeta.storageManager.save(dataOf3, 3, 1); testMeta.storageManager.partitioned(Lists.<WindowDataManager>newArrayList(testMeta.storageManager), Sets.newHashSet(2, 3)); testMeta.storageManager.setup(testMeta.context); testMeta.storageManager.deleteUpTo(1, 6); Path appPath = new Path(testMeta.applicationPath + '/' + testMeta.storageManager.getRecoveryPath()); FileSystem fs = FileSystem.newInstance(appPath.toUri(), new Configuration()); FileStatus[] fileStatuses = fs.listStatus(new Path(appPath, Integer.toString(1))); Assert.assertEquals("number of windows for 1", 3, fileStatuses.length); TreeSet<String> windows = Sets.newTreeSet(); for (FileStatus fileStatus : fileStatuses) { windows.add(fileStatus.getPath().getName()); } Assert.assertEquals("window list for 1", Sets.newLinkedHashSet(Arrays.asList("7", "8", "9")), windows); Assert.assertEquals("no data for 2", false, fs.exists(new Path(appPath, Integer.toString(2)))); Assert.assertEquals("no data for 3", false, fs.exists(new Path(appPath, Integer.toString(3)))); }
From source file:com.datatorrent.stram.cli.ApexCli.java
License:Apache License
@SuppressWarnings("unused") private StramAppLauncher getStramAppLauncher(String jarfileUri, Configuration config, boolean ignorePom) throws Exception { URI uri = new URI(jarfileUri); String scheme = uri.getScheme(); StramAppLauncher appLauncher = null; if (scheme == null || scheme.equals("file")) { File jf = new File(uri.getPath()); appLauncher = new StramAppLauncher(jf, config); } else {/* www .j a v a 2 s . c o m*/ try (FileSystem tmpFs = FileSystem.newInstance(uri, conf)) { Path path = new Path(uri.getPath()); appLauncher = new StramAppLauncher(tmpFs, path, config); } } if (appLauncher != null) { if (verboseLevel > 0) { System.err.print(appLauncher.getMvnBuildClasspathOutput()); } return appLauncher; } else { throw new CliException("Scheme " + scheme + " not supported."); } }
From source file:com.datatorrent.stram.cli.ApexCli.java
License:Apache License
private File copyToLocal(String[] files) throws IOException { File tmpDir = new File(System.getProperty("java.io.tmpdir") + "/datatorrent/" + ManagementFactory.getRuntimeMXBean().getName()); tmpDir.mkdirs();/*from w w w. j a v a 2s. co m*/ for (int i = 0; i < files.length; i++) { try { URI uri = new URI(files[i]); String scheme = uri.getScheme(); if (scheme == null || scheme.equals("file")) { files[i] = uri.getPath(); } else { try (FileSystem tmpFs = FileSystem.newInstance(uri, conf)) { Path srcPath = new Path(uri.getPath()); Path dstPath = new Path(tmpDir.getAbsolutePath(), String.valueOf(i) + srcPath.getName()); tmpFs.copyToLocalFile(srcPath, dstPath); files[i] = dstPath.toUri().getPath(); } } } catch (URISyntaxException ex) { throw new RuntimeException(ex); } } return tmpDir; }
From source file:com.datatorrent.stram.cli.DTCli.java
License:Apache License
@SuppressWarnings("unused") private StramAppLauncher getStramAppLauncher(String jarfileUri, Configuration config, boolean ignorePom) throws Exception { URI uri = new URI(jarfileUri); String scheme = uri.getScheme(); StramAppLauncher appLauncher = null; if (scheme == null || scheme.equals("file")) { File jf = new File(uri.getPath()); appLauncher = new StramAppLauncher(jf, config); } else {/*from w ww . j av a 2 s . c o m*/ FileSystem tmpFs = FileSystem.newInstance(uri, conf); try { Path path = new Path(uri.getPath()); appLauncher = new StramAppLauncher(tmpFs, path, config); } finally { tmpFs.close(); } } if (appLauncher != null) { if (verboseLevel > 0) { System.err.print(appLauncher.getMvnBuildClasspathOutput()); } return appLauncher; } else { throw new CliException("Scheme " + scheme + " not supported."); } }
From source file:com.datatorrent.stram.cli.DTCli.java
License:Apache License
private File copyToLocal(String[] files) throws IOException { File tmpDir = new File("/tmp/datatorrent/" + ManagementFactory.getRuntimeMXBean().getName()); tmpDir.mkdirs();//from w w w .j a va 2 s . co m for (int i = 0; i < files.length; i++) { try { URI uri = new URI(files[i]); String scheme = uri.getScheme(); if (scheme == null || scheme.equals("file")) { files[i] = uri.getPath(); } else { FileSystem tmpFs = FileSystem.newInstance(uri, conf); try { Path srcPath = new Path(uri.getPath()); Path dstPath = new Path(tmpDir.getAbsolutePath(), String.valueOf(i) + srcPath.getName()); tmpFs.copyToLocalFile(srcPath, dstPath); files[i] = dstPath.toUri().getPath(); } finally { tmpFs.close(); } } } catch (URISyntaxException ex) { throw new RuntimeException(ex); } } return tmpDir; }
From source file:com.datatorrent.stram.FSRecoveryHandler.java
License:Apache License
public FSRecoveryHandler(String appDir, Configuration conf) throws IOException { this.basedir = new Path(appDir, DIRECTORY_RECOVERY); fs = FileSystem.newInstance(this.basedir.toUri(), conf); logPath = new Path(basedir, FILE_LOG); logBackupPath = new Path(basedir, FILE_LOG_BACKUP); snapshotPath = new Path(basedir, FILE_SNAPSHOT); snapshotBackupPath = new Path(basedir, FILE_SNAPSHOT_BACKUP); heartbeatPath = new Path(basedir, FILE_HEARTBEATURI); }