List of usage examples for org.apache.hadoop.fs FileSystem newInstance
public static FileSystem newInstance(URI uri, Configuration config) throws IOException
From source file:org.apache.apex.malhar.lib.fs.s3.S3InitiateFileUploadOperatorTest.java
License:Apache License
@After public void afterTest() { Path root = new Path(applicationPath); try {//from www. j a v a 2 s . c om FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration()); fs.delete(root, true); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.apache.apex.malhar.lib.fs.s3.S3OutputModuleMockTest.java
License:Apache License
@Test public void testS3OutputModule() throws Exception { InitiateMultipartUploadResult result = new InitiateMultipartUploadResult(); result.setUploadId(uploadId);//from w w w .jav a 2 s . c o m PutObjectResult objResult = new PutObjectResult(); objResult.setETag("SuccessFullyUploaded"); UploadPartResult partResult = new UploadPartResult(); partResult.setPartNumber(1); partResult.setETag("SuccessFullyPartUploaded"); MockitoAnnotations.initMocks(this); when(client.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(result); when(client.putObject(any(PutObjectRequest.class))).thenReturn(objResult); when(client.uploadPart(any(UploadPartRequest.class))).thenReturn(partResult); when(client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))) .thenReturn(completeMultiPart()); Application app = new S3OutputModuleMockTest.Application(); Configuration conf = new Configuration(); conf.set("dt.operator.HDFSInputModule.prop.files", inputDir); conf.set("dt.operator.HDFSInputModule.prop.blockSize", "10"); conf.set("dt.operator.HDFSInputModule.prop.blocksThreshold", "1"); conf.set("dt.attr.CHECKPOINT_WINDOW_COUNT", "20"); conf.set("dt.operator.S3OutputModule.prop.accessKey", "accessKey"); conf.set("dt.operator.S3OutputModule.prop.secretAccessKey", "secretKey"); conf.set("dt.operator.S3OutputModule.prop.bucketName", "bucketKey"); conf.set("dt.operator.S3OutputModule.prop.outputDirectoryPath", outputDir); Path outDir = new Path("file://" + new File(outputDir).getAbsolutePath()); final Path outputFilePath = new Path(outDir.toString() + File.separator + FILE); final FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration()); LocalMode lma = LocalMode.newInstance(); lma.prepareDAG(app, conf); LocalMode.Controller lc = lma.getController(); lc.setHeartbeatMonitoringEnabled(true); ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() { @Override public Boolean call() throws Exception { return fs.exists(outputFilePath); } }); lc.run(10000); Assert.assertTrue("output file exist", fs.exists(outputFilePath)); }
From source file:org.apache.apex.malhar.lib.fs.s3.S3Reconciler.java
License:Apache License
@Override public void setup(Context.OperatorContext context) { s3client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey)); if (region != null) { s3client.setRegion(Region.getRegion(Regions.fromName(region))); }//w ww .ja va 2s . c o m filePath = context.getValue(DAG.APPLICATION_PATH); try { fs = FileSystem.newInstance(new Path(filePath).toUri(), new Configuration()); } catch (IOException e) { logger.error("Unable to create FileSystem: {}", e.getMessage()); } super.setup(context); }
From source file:org.apache.apex.malhar.lib.io.fs.AbstractFileInputOperator.java
License:Apache License
/** * Override this method to change the FileSystem instance that is used by the operator. * * @return A FileSystem object./* w w w . j a v a 2 s . c o m*/ * @throws IOException */ protected FileSystem getFSInstance() throws IOException { return FileSystem.newInstance(filePath.toUri(), configuration); }
From source file:org.apache.apex.malhar.lib.io.fs.FSInputModuleAppTest.java
License:Apache License
@Test public void testApplication() throws Exception { app = new Application(); Configuration conf = new Configuration(false); conf.set("dt.operator.hdfsInputModule.prop.files", inputDir); conf.set("dt.operator.hdfsInputModule.prop.blockSize", "10"); conf.set("dt.operator.hdfsInputModule.prop.blocksThreshold", "4"); conf.set("dt.operator.hdfsInputModule.prop.scanIntervalMillis", "10000"); conf.set("dt.attr.CHECKPOINT_WINDOW_COUNT", "10"); LocalMode lma = LocalMode.newInstance(); lma.prepareDAG(app, conf);//from w w w . j av a2 s . c o m LocalMode.Controller lc = lma.getController(); lc.setHeartbeatMonitoringEnabled(true); lc.runAsync(); long now = System.currentTimeMillis(); Path outDir = new Path("file://" + new File(outputDir).getAbsolutePath()); FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration()); while (!fs.exists(outDir) && System.currentTimeMillis() - now < 20000) { Thread.sleep(500); LOG.debug("Waiting for {}", outDir); } Thread.sleep(10000); lc.shutdown(); Assert.assertTrue("output dir does not exist", fs.exists(outDir)); File dir = new File(outputDir); FileFilter fileFilter = new WildcardFileFilter(OUT_METADATA_FILE + "*"); verifyFileContents(dir.listFiles(fileFilter), "[fileName=file1.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/file1.txt]"); verifyFileContents(dir.listFiles(fileFilter), "[fileName=file2.txt, numberOfBlocks=6, isDirectory=false, relativePath=input/file2.txt]"); verifyFileContents(dir.listFiles(fileFilter), "[fileName=dir, numberOfBlocks=0, isDirectory=true, relativePath=input/dir]"); verifyFileContents(dir.listFiles(fileFilter), "[fileName=inner.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/dir/inner.txt]"); fileFilter = new WildcardFileFilter(OUT_DATA_FILE + "*"); verifyFileContents(dir.listFiles(fileFilter), FILE_1_DATA); verifyFileContents(dir.listFiles(fileFilter), FILE_2_DATA); }
From source file:org.apache.apex.malhar.lib.io.IdempotentStorageManagerTest.java
License:Apache License
@Test public void testDelete() throws IOException { Map<Integer, String> dataOf1 = Maps.newHashMap(); dataOf1.put(1, "one"); dataOf1.put(2, "two"); dataOf1.put(3, "three"); Map<Integer, String> dataOf2 = Maps.newHashMap(); dataOf2.put(4, "four"); dataOf2.put(5, "five"); dataOf2.put(6, "six"); Map<Integer, String> dataOf3 = Maps.newHashMap(); dataOf2.put(7, "seven"); dataOf2.put(8, "eight"); dataOf2.put(9, "nine"); for (int i = 1; i <= 9; ++i) { testMeta.storageManager.save(dataOf1, 1, i); }/*from ww w . j av a2 s . c o m*/ testMeta.storageManager.save(dataOf2, 2, 1); testMeta.storageManager.save(dataOf3, 3, 1); testMeta.storageManager.partitioned(Lists.<IdempotentStorageManager>newArrayList(testMeta.storageManager), Sets.newHashSet(2, 3)); testMeta.storageManager.setup(testMeta.context); testMeta.storageManager.deleteUpTo(1, 6); Path appPath = new Path(testMeta.applicationPath + '/' + testMeta.storageManager.recoveryPath); FileSystem fs = FileSystem.newInstance(appPath.toUri(), new Configuration()); FileStatus[] fileStatuses = fs.listStatus(new Path(appPath, Integer.toString(1))); Assert.assertEquals("number of windows for 1", 3, fileStatuses.length); TreeSet<String> windows = Sets.newTreeSet(); for (FileStatus fileStatus : fileStatuses) { windows.add(fileStatus.getPath().getName()); } Assert.assertEquals("window list for 1", Sets.newLinkedHashSet(Arrays.asList("7", "8", "9")), windows); Assert.assertEquals("no data for 2", false, fs.exists(new Path(appPath, Integer.toString(2)))); Assert.assertEquals("no data for 3", false, fs.exists(new Path(appPath, Integer.toString(3)))); }
From source file:org.apache.apex.malhar.lib.parser.XmlParser.java
License:Apache License
@Override public void setup(com.datatorrent.api.Context.OperatorContext context) { try {/*from w w w . j a v a 2 s. co m*/ if (schemaXSDFile != null) { Path filePath = new Path(schemaXSDFile); Configuration configuration = new Configuration(); FileSystem fs = FileSystem.newInstance(filePath.toUri(), configuration); FSDataInputStream inputStream = fs.open(filePath); SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); schema = factory.newSchema(new StreamSource(inputStream)); validator = schema.newValidator(); fs.close(); } } catch (SAXException e) { DTThrowable.wrapIfChecked(e); } catch (IOException e) { DTThrowable.wrapIfChecked(e); } }
From source file:org.apache.apex.malhar.sql.FileEndpointTest.java
License:Apache License
private boolean waitTillFileIsPopulated(String outputFolder, int timeout) throws IOException, InterruptedException { boolean result; long now = System.currentTimeMillis(); Path outDir = new Path("file://" + new File(outputFolder).getAbsolutePath()); try (FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration())) { List<String> strings = Lists.newArrayList(); while (System.currentTimeMillis() - now < timeout) { if (fs.exists(outDir)) { File file = new File(outputFolder); if (file.list().length > 0) { File file1 = new File(outputFolder + file.list()[0]); strings = FileUtils.readLines(file1); if (strings.size() != 0) { break; }//from w ww . ja v a 2 s . c o m } } Thread.sleep(500); } result = fs.exists(outDir) && (strings.size() != 0); } return result; }
From source file:org.apache.apex.malhar.sql.sample.PureStyleSQLApplicationTest.java
License:Apache License
public static boolean waitTillFileIsPopulated(String outputFolder, int timeout) throws IOException, InterruptedException { boolean result; long now = System.currentTimeMillis(); Path outDir = new Path("file://" + new File(outputFolder).getAbsolutePath()); try (FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration())) { List<String> strings = Lists.newArrayList(); while (System.currentTimeMillis() - now < timeout) { if (fs.exists(outDir)) { File file = new File(outputFolder); if (file.list().length > 0) { File file1 = new File(outputFolder + file.list()[0]); strings = FileUtils.readLines(file1); if (strings.size() != 0) { break; }// w w w .ja v a2 s . c o m } } Thread.sleep(500); } result = fs.exists(outDir) && (strings.size() != 0); } return result; }
From source file:org.apache.apex.malhar.sql.schema.TupleSchemaRegistry.java
License:Apache License
public String generateCommonJar() throws IOException { File file = File.createTempFile("schemaSQL", ".jar"); FileSystem fs = FileSystem.newInstance(file.toURI(), new Configuration()); FSDataOutputStream out = fs.create(new Path(file.getAbsolutePath())); JarOutputStream jout = new JarOutputStream(out); for (Schema schema : schemas.values()) { jout.putNextEntry(new ZipEntry(schema.fqcn.replace(".", "/") + ".class")); jout.write(schema.beanClassBytes); jout.closeEntry();/*from w w w.j av a 2s .co m*/ } jout.close(); out.close(); return file.getAbsolutePath(); }