Example usage for org.apache.hadoop.fs FileSystem newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem newInstance.

Prototype

public static FileSystem newInstance(URI uri, Configuration config) throws IOException 

Source Link

Document

Returns the FileSystem for this URI's scheme and authority.

Usage

From source file:com.datatorrent.lib.io.fs.AbstractFileOutputOperator.java

License:Open Source License

/**
 * Override this method to change the FileSystem instance that is used by the operator.
 * This method is mainly helpful for unit testing.
 * @return A FileSystem object./*from   w  ww . j a v a2s. c  o m*/
 * @throws IOException
 */
protected FileSystem getFSInstance() throws IOException {
    FileSystem tempFS = FileSystem.newInstance(new Path(filePath).toUri(), new Configuration());

    if (tempFS instanceof LocalFileSystem) {
        tempFS = ((LocalFileSystem) tempFS).getRaw();
    }

    return tempFS;
}

---

From source file:com.datatorrent.lib.io.fs.AbstractFSDirectoryInputOperator.java

License:Open Source License

@Override
public void setup(OperatorContext context) {
    try {// ww w .  j a va  2  s  . c  o  m
        filePath = new Path(directory);
        configuration = new Configuration();
        fs = FileSystem.newInstance(filePath.toUri(), configuration);
        if (!unfinishedFiles.isEmpty()) {
            retryFailedFile(unfinishedFiles.poll());
            skipCount = 0;
        } else if (!failedFiles.isEmpty()) {
            retryFailedFile(failedFiles.poll());
            skipCount = 0;
        }
        long startTime = System.currentTimeMillis();
        LOG.info("Continue reading {} from index {} time={}", currentFile, offset, startTime);
        // fast forward to previous offset
        if (inputStream != null) {
            for (int index = 0; index < offset; index++) {
                readEntity();
            }
        }
        LOG.info("Read offset={} records in setup time={}", offset, System.currentTimeMillis() - startTime);
    } catch (IOException ex) {
        if (maxRetryCount <= 0) {
            throw new RuntimeException(ex);
        }
        LOG.error("FS reader error", ex);
        addToFailedList();
    }
}

---

From source file:com.datatorrent.lib.io.fs.AbstractHdfsTupleFileOutputOperator.java

License:Open Source License

/**
 *
 * @param context/*w w w.jav a 2s.  c o  m*/
 */
@Override
public void setup(Context.OperatorContext context) {
    try {
        fs = FileSystem.newInstance(new Path(filePath).toUri(), new Configuration());
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:com.datatorrent.lib.io.fs.FileSplitterBase.java

License:Apache License

protected FileSystem getFSInstance() throws IOException {
    return FileSystem.newInstance(new Path(file).toUri(), new Configuration());
}

---

From source file:com.datatorrent.lib.io.fs.FileStitcher.java

License:Apache License

/**
 * //from  w ww  . ja v  a2 s .  co  m
 * @return Application FileSystem instance
 * @throws IOException
 */
protected FileSystem getAppFSInstance() throws IOException {
    return FileSystem.newInstance((new Path(blocksDirectoryPath)).toUri(), new Configuration());
}

---

From source file:com.datatorrent.lib.io.fs.FileStitcher.java

License:Apache License

/**
 * //from w  w  w  .  j a va  2 s  .c  om
 * @return Destination FileSystem instance
 * @throws IOException
 */
protected FileSystem getOutputFSInstance() throws IOException {
    return FileSystem.newInstance((new Path(filePath)).toUri(), new Configuration());
}

---

From source file:com.datatorrent.lib.io.fs.FSInputModuleAppTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    app = new Application();
    Configuration conf = new Configuration(false);
    conf.set("dt.operator.hdfsInputModule.prop.files", inputDir);
    conf.set("dt.operator.hdfsInputModule.prop.blockSize", "10");
    conf.set("dt.operator.hdfsInputModule.prop.blocksThreshold", "4");
    conf.set("dt.operator.hdfsInputModule.prop.scanIntervalMillis", "10000");

    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);/*from  www. j  a va  2 s  . c om*/
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(true);
    lc.runAsync();

    long now = System.currentTimeMillis();
    Path outDir = new Path("file://" + new File(outputDir).getAbsolutePath());
    FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration());
    while (!fs.exists(outDir) && System.currentTimeMillis() - now < 20000) {
        Thread.sleep(500);
        LOG.debug("Waiting for {}", outDir);
    }

    Thread.sleep(10000);
    lc.shutdown();

    Assert.assertTrue("output dir does not exist", fs.exists(outDir));

    File dir = new File(outputDir);
    FileFilter fileFilter = new WildcardFileFilter(OUT_METADATA_FILE + "*");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=file1.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/file1.txt]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=file2.txt, numberOfBlocks=6, isDirectory=false, relativePath=input/file2.txt]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=dir, numberOfBlocks=0, isDirectory=true, relativePath=input/dir]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=inner.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/dir/inner.txt]");

    fileFilter = new WildcardFileFilter(OUT_DATA_FILE + "*");
    verifyFileContents(dir.listFiles(fileFilter), FILE_1_DATA);
    verifyFileContents(dir.listFiles(fileFilter), FILE_2_DATA);
}

---

From source file:com.datatorrent.lib.io.fs.HDFSInputModuleAppTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    app = new Application();
    Configuration conf = new Configuration(false);
    conf.set("dt.operator.hdfsInputModule.prop.files", inputDir);
    conf.set("dt.operator.hdfsInputModule.prop.blockSize", "10");
    conf.set("dt.operator.hdfsInputModule.prop.scanIntervalMillis", "10000");

    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);//from ww w .  ja v  a  2s  . c o m
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(true);
    lc.runAsync();

    long now = System.currentTimeMillis();
    Path outDir = new Path("file://" + new File(outputDir).getAbsolutePath());
    FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration());
    while (!fs.exists(outDir) && System.currentTimeMillis() - now < 20000) {
        Thread.sleep(500);
        LOG.debug("Waiting for {}", outDir);
    }

    Thread.sleep(10000);
    lc.shutdown();

    Assert.assertTrue("output dir does not exist", fs.exists(outDir));

    File dir = new File(outputDir);
    FileFilter fileFilter = new WildcardFileFilter(OUT_METADATA_FILE + "*");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=file1.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/file1.txt]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=file2.txt, numberOfBlocks=6, isDirectory=false, relativePath=input/file2.txt]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=dir, numberOfBlocks=0, isDirectory=true, relativePath=input/dir]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=inner.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/dir/inner.txt]");

    fileFilter = new WildcardFileFilter(OUT_DATA_FILE + "*");
    verifyFileContents(dir.listFiles(fileFilter), FILE_1_DATA);
    verifyFileContents(dir.listFiles(fileFilter), FILE_2_DATA);
}

---

From source file:com.datatorrent.lib.io.fs.S3InputModuleAppTest.java

License:Apache License

@Test
public void testS3Application() throws Exception {
    app = new S3InputModuleAppTest.Application();
    Configuration conf = new Configuration(false);
    conf.set("dt.operator.s3InputModule.prop.files", files);
    conf.set("dt.operator.s3InputModule.prop.blockSize", "10");
    conf.set("dt.operator.s3InputModule.prop.scanIntervalMillis", "10000");

    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);/*  w w  w  .ja  va2 s.co m*/
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(true);
    lc.runAsync();

    long now = System.currentTimeMillis();
    Path outDir = new Path("file://" + new File(outputDir).getAbsolutePath());
    FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration());
    while (!fs.exists(outDir) && System.currentTimeMillis() - now < 20000) {
        Thread.sleep(500);
        LOG.debug("Waiting for {}", outDir);
    }

    Thread.sleep(10000);
    lc.shutdown();

    Assert.assertTrue("output dir does not exist", fs.exists(outDir));

    File dir = new File(outputDir);
    FileFilter fileFilter = new WildcardFileFilter(OUT_METADATA_FILE + "*");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=file1.txt, numberOfBlocks=2, isDirectory=false, relativePath=input/file1.txt]");
    verifyFileContents(dir.listFiles(fileFilter),
            "[fileName=file2.txt, numberOfBlocks=6, isDirectory=false, relativePath=input/file2.txt]");

    fileFilter = new WildcardFileFilter(OUT_DATA_FILE + "*");
    verifyFileContents(dir.listFiles(fileFilter), FILE_1_DATA);
    verifyFileContents(dir.listFiles(fileFilter), FILE_2_DATA);
}

---

From source file:com.datatorrent.lib.io.IdempotentStorageManagerTest.java

License:Open Source License

@Test
public void testDelete() throws IOException {
    Map<Integer, String> dataOf1 = Maps.newHashMap();
    dataOf1.put(1, "one");
    dataOf1.put(2, "two");
    dataOf1.put(3, "three");

    Map<Integer, String> dataOf2 = Maps.newHashMap();
    dataOf2.put(4, "four");
    dataOf2.put(5, "five");
    dataOf2.put(6, "six");

    Map<Integer, String> dataOf3 = Maps.newHashMap();
    dataOf2.put(7, "seven");
    dataOf2.put(8, "eight");
    dataOf2.put(9, "nine");

    testMeta.storageManager.save(dataOf1, 1, 1);
    testMeta.storageManager.save(dataOf2, 2, 1);
    testMeta.storageManager.save(dataOf3, 3, 1);

    testMeta.storageManager.partitioned(Lists.<IdempotentStorageManager>newArrayList(testMeta.storageManager),
            Sets.newHashSet(2, 3));//from   w  w  w  .  ja  va 2  s  .com
    testMeta.storageManager.setup(testMeta.context);
    testMeta.storageManager.deleteUpTo(1, 1);

    Path appPath = new Path(testMeta.recoveryPath + '/' + testMeta.context.getValue(DAG.APPLICATION_ID));
    FileSystem fs = FileSystem.newInstance(appPath.toUri(), new Configuration());
    Assert.assertEquals("no data for 1", 0, fs.listStatus(new Path(appPath, Integer.toString(1))).length);
    Assert.assertEquals("no data for 2", false, fs.exists(new Path(appPath, Integer.toString(2))));
    Assert.assertEquals("no data for 3", false, fs.exists(new Path(appPath, Integer.toString(3))));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »