Example usage for org.apache.hadoop.fs FileSystem closeAll

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem closeAll.

Prototype

public static void closeAll() throws IOException

Source Link

Document

Close all cached FileSystem instances.

Usage

From source file:com.uber.hoodie.io.TestHoodieCommitArchiveLog.java

License:Apache License

@BeforeClass
public static void setUpDFS() throws IOException {
    // Need to closeAll to clear FileSystem.Cache, required because DFS and LocalFS used in the
    // same JVM/* ww  w  .j av a2  s .  c o  m*/
    FileSystem.closeAll();
    if (hdfsTestService == null) {
        hdfsTestService = new HdfsTestService();
        dfsCluster = hdfsTestService.start(true);
        // Create a temp folder as the base path
        dfs = dfsCluster.getFileSystem();
    }
}

From source file:com.uber.hoodie.table.TestMergeOnReadTable.java

License:Apache License

@AfterClass
public static void cleanUp() throws Exception {
    if (hdfsTestService != null) {
        hdfsTestService.stop();/*from   w w  w  . j  a  va2 s.  c o  m*/
        dfsCluster.shutdown();
        dfsCluster = null;
        dfs = null;
        hdfsTestService = null;
    }
    // Need to closeAll to clear FileSystem.Cache, required because DFS and LocalFS used in the
    // same JVM
    FileSystem.closeAll();
}

From source file:com.uber.hoodie.TestMultiFS.java

License:Apache License

@AfterClass
public static void cleanupClass() throws Exception {
    if (jsc != null) {
        jsc.stop();//from  w  ww  . ja v a2  s.co m
    }

    if (hdfsTestService != null) {
        hdfsTestService.stop();
        dfsCluster.shutdown();
    }
    // Need to closeAll to clear FileSystem.Cache, required because DFS and LocalFS used in the
    // same JVM
    FileSystem.closeAll();
}

From source file:com.zhangyue.zeus.service.impl.ExportDataServiceImpl.java

License:Open Source License

@Override
public List<String> readResultData(String hdfsResultLocation, boolean isDownload) {
    List<String> resultList = new ArrayList<String>();
    int readedLines = 0;
    String temp = null;/*w  w w.  ja va  2 s .  co m*/
    try {
        Path hdfsPath = new Path(hdfsResultLocation);
        HiveConf hiveConf = new HiveConf(SessionState.class);
        FileSystem fs = hdfsPath.getFileSystem(hiveConf);
        FileStatus[] fss = fs.listStatus(hdfsPath);
        // ??-60000 ?-200
        int numberLimit = (isDownload == true) ? Constants.HIVE_RESULT_DOWNLOAD_LIMIT
                : Constants.HIVE_RESULT_MAX_LIMIT;
        for (FileStatus fileStatus : fss) {
            Path fsPath = fileStatus.getPath();
            if (readedLines >= numberLimit || fs.getFileStatus(fsPath).isDir()) {
                break;
            }
            BufferedReader bf = new BufferedReader(
                    new InputStreamReader(fs.open(fsPath), Constants.UTF_ENCODING));
            while ((temp = bf.readLine()) != null) {
                if (readedLines >= numberLimit) {
                    break;
                }
                resultList.add(temp);
                readedLines++;
            }
            bf.close();
        }
        FileSystem.closeAll();
    } catch (Exception e) {
        LOG.error("don't had your result", e);
    }
    return resultList;
}

From source file:eu.stratosphere.pact.test.cancelling.CancellingTestBase.java

License:Apache License

@After
public void stopCluster() throws Exception {
    LOG.info("######################### stopping cluster config #########################");
    cluster.stopCluster();//w w w . j  av  a2  s.  com
    ClusterProviderPool.removeInstance(Constants.DEFAULT_TEST_CONFIG);
    FileSystem.closeAll();
    System.gc();
}

From source file:eu.stratosphere.pact.test.util.TestBase.java

License:Apache License

@After
public void stopCluster() throws Exception {
    LOG.info("######################### stop - cluster config : " + clusterConfig
            + " #########################");
    cluster.stopCluster();//from ww  w  . ja  v a2s.  c  o m
    ClusterProviderPool.removeInstance(clusterConfig);
    FileSystem.closeAll();
    System.gc();
}

From source file:eu.stratosphere.pact.test.util.TestBase2.java

License:Apache License

@After
public void stopCluster() throws Exception {
    try {//from   www  .  j  ava  2 s  .co  m
        if (this.executer != null) {
            System.err.println(
                    "######################### STOPPING LOCAL EXECUTION CONTEXT #########################");
            this.executer.stop();
            this.executer = null;
            FileSystem.closeAll();
            System.gc();
        }
    } finally {
        deleteAllTempFiles();
    }
}

From source file:eu.stratosphere.sopremo.server.SopremoTestServer.java

License:Apache License

@Override
public void close() throws IOException {
    for (String fileToClean : this.filesToCleanup)
        try {/* w  w  w  .j  av a2s.  c  o m*/
            if (!getFilesystemProvider().delete(fileToClean, false))
                getFilesystemProvider().delete(fileToClean, true);
        } catch (IOException e) {
        }

    try {
        this.cluster.stopCluster();
        ClusterProviderPool.removeInstance(this.configName);
    } catch (Exception e) {
    }
    this.server.close();
    if (this.executor != this.server)
        RPC.stopProxy(this.executor);
    FileSystem.closeAll();
}

From source file:eu.stratosphere.test.cancelling.CancellingTestBase.java

License:Apache License

@After
public void stopCluster() throws Exception {
    if (this.executor != null) {
        this.executor.stop();
        this.executor = null;
        FileSystem.closeAll();
        System.gc();// w w  w  .  jav  a 2s  .c  o  m
    }
}

From source file:eu.stratosphere.test.util.AbstractTestBase.java

License:Apache License

@After
public void stopCluster() throws Exception {
    try {// w  w w  .j  av  a 2  s  . com
        if (this.executor != null) {
            this.executor.stop();
            this.executor = null;
            FileSystem.closeAll();
            System.gc();
        }
    } finally {
        deleteAllTempFiles();
    }
}