Example usage for org.apache.hadoop.fs FileSystem delete

List of usage examples for org.apache.hadoop.fs FileSystem delete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.inmobi.conduit.local.LocalStreamServiceTest.java

License:Apache License

private void testCreateListing() {
    try {//from w w w. j  av a  2s .  c  om
        Cluster cluster = ClusterTest.buildLocalCluster();
        FileSystem fs = mock(FileSystem.class);
        createMockForFileSystem(fs, cluster);

        Map<FileStatus, String> results = new TreeMap<FileStatus, java.lang.String>();
        Set<FileStatus> trashSet = new HashSet<FileStatus>();
        Table<String, String, String> checkpointPaths = HashBasedTable.create();
        fs.delete(cluster.getDataDir(), true);
        FileStatus dataDir = new FileStatus(20, false, 3, 23823, 2438232, cluster.getDataDir());
        fs.delete(new Path(cluster.getRootDir() + "/conduit-checkpoint"), true);

        Set<String> streamsToProcess = new HashSet<String>();
        streamsToProcess.add("stream1");
        streamsToProcess.add("stream2");
        TestLocalStreamService service = new TestLocalStreamService(null, cluster, null,
                new FSCheckpointProvider(cluster.getRootDir() + "/conduit-checkpoint"), streamsToProcess);
        service.createListing(fs, dataDir, results, trashSet, checkpointPaths);

        Set<String> tmpResults = new LinkedHashSet<String>();
        // print the results
        for (FileStatus status : results.keySet()) {
            tmpResults.add(status.getPath().toString());
            LOG.debug("Results [" + status.getPath().toString() + "]");
        }

        // print the trash
        Iterator<FileStatus> it = trashSet.iterator();
        Set<String> tmpTrashPaths = new LinkedHashSet<String>();
        while (it.hasNext()) {
            FileStatus trashfile = it.next();
            tmpTrashPaths.add(trashfile.getPath().toString());
            LOG.debug("trash file [" + trashfile.getPath());
        }

        Map<String, String> tmpCheckPointPaths = new TreeMap<String, String>();
        Set<String> streams = checkpointPaths.rowKeySet();
        for (String streamName : streams) {
            Map<String, String> collectorCheckpointValueMap = checkpointPaths.row(streamName);
            for (String collector : collectorCheckpointValueMap.keySet()) {
                String checkpointKey = AbstractService.getCheckPointKey(service.getName(), streamName,
                        collector);
                LOG.debug("Check Pointing Key [" + checkpointKey + "] with value ["
                        + collectorCheckpointValueMap.get(collector) + "]");
                tmpCheckPointPaths.put(checkpointKey, collectorCheckpointValueMap.get(collector));
            }
        }
        validateExpectedOutput(tmpResults, tmpTrashPaths, tmpCheckPointPaths);
        fs.delete(new Path(cluster.getRootDir() + "/conduit-checkpoint"), true);
        fs.delete(cluster.getDataDir(), true);
        fs.close();
    } catch (Exception e) {
        LOG.debug("Error in running testCreateListing", e);
        assert false;
    }
}

From source file:com.inmobi.conduit.local.LocalStreamServiceTest.java

License:Apache License

@Test
public void testFileUtil() throws Exception {
    String streamName = "test1";
    Path rootDir = new Path("/tmp/localServiceTest/testcluster2/mergeservice");
    Path dataDir = new Path(rootDir, "data/test1/testcluster2");
    FileSystem fs = dataDir.getFileSystem(new Configuration());
    fs.mkdirs(dataDir);/*from  ww w .j a v a2 s .c o m*/
    String filenameStr = new String(
            streamName + "-" + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_00001");
    Path src = new Path(dataDir, filenameStr);

    LOG.debug("Creating Test Data with filename [" + filenameStr + "]");
    FSDataOutputStream streamout = fs.create(src);
    String content = "Creating Test data for teststream";
    Message msg = new Message(content.getBytes());
    long currentTimestamp = new Date().getTime();
    AuditUtil.attachHeaders(msg, currentTimestamp);
    byte[] encodeMsg = Base64.encodeBase64(msg.getData().array());
    streamout.write(encodeMsg);
    streamout.write("\n".getBytes());
    streamout.write(encodeMsg);
    streamout.write("\n".getBytes());
    long nextMinuteTimeStamp = currentTimestamp + 60000;
    // Genearate a msg with different timestamp.  Default window period is 60sec
    AuditUtil.attachHeaders(msg, nextMinuteTimeStamp);
    encodeMsg = Base64.encodeBase64(msg.getData().array());
    streamout.write(encodeMsg);
    streamout.close();
    Map<Long, Long> received = new HashMap<Long, Long>();
    Path target = new Path(new Path(rootDir, "system/tmp/LocalStreamService_testcluster2_test1@/"
            + "job_local_0001/attempt_local_0001_m_000000_0/"), filenameStr + ".gz");
    FileUtil.gzip(src, target, new Configuration(), received);
    Assert.assertEquals(2, received.size());
    // current timestamp window = currentTimestamp - (currentTimestamp % 60000)
    Assert.assertTrue(2 == received.get(currentTimestamp - (currentTimestamp % 60000)));
    // next timestamp window = nextMinuteTimeStamp - (nextMinuteTimeStamp %60000)
    Assert.assertTrue(1 == received.get(nextMinuteTimeStamp - (nextMinuteTimeStamp % 60000)));
    fs.delete(rootDir, true);
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void testPurgerService(String testfilename, int numofhourstoadd, boolean checkifexists,
        boolean checktrashexists) throws Exception {
    ConduitConfigParser configparser = new ConduitConfigParser(testfilename);
    ConduitConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {
        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar todaysdate = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        todaysdate.add(Calendar.HOUR, numofhourstoadd);

        createTestPurgefiles(fs, cluster, todaysdate, false);

        service.runOnce();/* ww  w .  j  a  va2 s  .  c o  m*/

        verifyPurgefiles(fs, cluster, todaysdate, checkifexists, checktrashexists);
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

public void testDataPurgerPartition() throws Exception {
    LOG.info("Check data purger does not stop when unable to delete a path");
    ConduitConfigParser configparser = new ConduitConfigParser("test-dps-conduit_X_hcat_5.xml");
    ConduitConfig config = configparser.getConfig();
    Conduit.setHCatEnabled(true);/*from  w  w  w .j a v a2 s . c o  m*/
    Conduit.setHcatDBName("conduit");

    for (Cluster cluster : config.getClusters().values()) {
        HiveConf hcatConf1 = TestHCatUtil.getHcatConf(20109, "target/metaStore-purger1", "metadb-purger1");
        TestHCatUtil.startMetaStoreServer(hcatConf1, 20109);
        Thread.sleep(10000);

        Conduit.setHiveConf(hcatConf1);

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        TestHCatUtil.createDatabase(Conduit.getHcatDBName());
        TestHCatUtil thutil = new TestHCatUtil();
        Table table = null;
        for (String stream : cluster.getSourceStreams()) {
            table = thutil.createTable(Conduit.getHcatDBName(), "conduit_local_" + stream);
        }
        Calendar date1 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date1.add(Calendar.HOUR, -7);
        createTestPurgePartitionFiles(fs, cluster, date1, table);

        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        service.runOnce();

        verifyPurgePartitionFiles(fs, cluster, date1, false, false, table);
        service.clearStreamHCatEnableMap();
        Conduit.setHCatEnabled(false);
        TestHCatUtil.stop();
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

public void testDataPurger() throws Exception {
    AbstractService.clearHCatInMemoryMaps();
    LOG.info("Check data purger does not stop when unable to delete a path");
    ConduitConfigParser configparser = new ConduitConfigParser("test-dps-conduit_X_5.xml");
    ConduitConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar date1 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date1.add(Calendar.HOUR, -7);
        createTestPurgefiles(fs, cluster, date1, false);
        Calendar date2 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date2.add(Calendar.HOUR, -6);
        createTestPurgefiles(fs, cluster, date2, false);
        ArrayList<Path> pathsToProcess = new ArrayList<Path>();
        Path[] paths = getLocalCommitPath(fs, cluster, date2);
        for (Path path : paths) {
            fs.setPermission(path, new FsPermission("000"));
            pathsToProcess.add(path);/*from   w  ww . jav  a2 s.co  m*/
        }
        paths = getMergeCommitPath(fs, cluster, date2);
        for (Path path : paths) {
            fs.setPermission(path, new FsPermission("000"));
            pathsToProcess.add(path);
        }
        Calendar date3 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date3.add(Calendar.HOUR, -5);
        createTestPurgefiles(fs, cluster, date3, false);

        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        service.runOnce();

        verifyPurgefiles(fs, cluster, date1, false, false);
        verifyPurgefiles(fs, cluster, date2, true, false);
        verifyPurgefiles(fs, cluster, date3, false, false);
        for (Path p : pathsToProcess) {
            fs.setPermission(p, new FsPermission("755"));
        }
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }

    Assert.assertEquals(ConduitMetrics.<SlidingTimeWindowGauge>getMetric("DataPurgerService",
            "purgePaths.count", DataPurgerService.class.getName()).getValue().longValue(), 9);
    Assert.assertEquals(ConduitMetrics.<SlidingTimeWindowGauge>getMetric("DataPurgerService",
            "deleteFailures.count", DataPurgerService.class.getName()).getValue().longValue(), 0);
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

public void testTrashPurging() throws Exception {
    LOG.info("Creating empty data dirs");
    ConduitConfigParser configparser = new ConduitConfigParser("test-dps-conduit_X_6.xml");
    ConduitConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar date1 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date1.add(Calendar.HOUR, -48);
        createTestPurgefiles(fs, cluster, date1, true);
        Calendar date2 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date2.add(Calendar.HOUR, -24);
        createTestPurgefiles(fs, cluster, date2, true);
        Calendar date3 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date3.add(Calendar.HOUR, -1);
        createTestPurgefiles(fs, cluster, date3, false);

        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        service.runOnce();//from  ww w  .  j  av  a 2  s.c  om

        verifyPurgefiles(fs, cluster, date1, false, false);
        verifyPurgefiles(fs, cluster, date2, false, false);
        verifyPurgefiles(fs, cluster, date3, true, true);
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }

    Assert.assertEquals(ConduitMetrics.<SlidingTimeWindowGauge>getMetric("DataPurgerService",
            "purgePaths.count", DataPurgerService.class.getName()).getValue().longValue(), 6);
}

From source file:com.inmobi.databus.local.LocalStreamService.java

License:Apache License

private void processFile(FileStatus file, String currentFile, String checkPointValue, FileSystem fs,
        Map<FileStatus, String> results, Map<String, FileStatus> collectorPaths) throws IOException {

    String fileName = file.getPath().getName();
    if (fileName != null && !fileName.equalsIgnoreCase(currentFile)) {
        if (!isEmptyFile(file, fs)) {
            Path src = file.getPath().makeQualified(fs);
            String destDir = getCategoryJobOutTmpPath(getCategoryFromSrcPath(src)).toString();
            if (aboveCheckpoint(checkPointValue, fileName))
                results.put(file, destDir);
            collectorPaths.put(fileName, file);
        } else {// w  w w . j  a  v  a 2  s.c  o m
            LOG.info("Empty File [" + file.getPath() + "] found. " + "Deleting it");
            fs.delete(file.getPath(), false);
        }
    }
}

From source file:com.inmobi.databus.local.LocalStreamServiceTest.java

License:Apache License

private void testCreateListing() {
    try {/*from  w w  w.  j  a  v a 2s  .  c  o  m*/
        Cluster cluster = ClusterTest.buildLocalCluster();
        FileSystem fs = mock(FileSystem.class);
        createMockForFileSystem(fs, cluster);

        Map<FileStatus, String> results = new TreeMap<FileStatus, java.lang.String>();
        Set<FileStatus> trashSet = new HashSet<FileStatus>();
        Map<String, FileStatus> checkpointPaths = new HashMap<String, FileStatus>();
        fs.delete(cluster.getDataDir(), true);
        FileStatus dataDir = new FileStatus(20, false, 3, 23823, 2438232, cluster.getDataDir());
        fs.delete(new Path(cluster.getRootDir() + "/databus-checkpoint"), true);

        TestLocalStreamService service = new TestLocalStreamService(null, cluster,
                new FSCheckpointProvider(cluster.getRootDir() + "/databus-checkpoint"));
        service.createListing(fs, dataDir, results, trashSet, checkpointPaths);

        Set<String> tmpResults = new LinkedHashSet<String>();
        // print the results
        for (FileStatus status : results.keySet()) {
            tmpResults.add(status.getPath().toString());
            LOG.debug("Results [" + status.getPath().toString() + "]");
        }

        // print the trash
        Iterator<FileStatus> it = trashSet.iterator();
        Set<String> tmpTrashPaths = new LinkedHashSet<String>();
        while (it.hasNext()) {
            FileStatus trashfile = it.next();
            tmpTrashPaths.add(trashfile.getPath().toString());
            LOG.debug("trash file [" + trashfile.getPath());
        }

        Map<String, String> tmpCheckPointPaths = new TreeMap<String, String>();
        // print checkPointPaths
        for (String key : checkpointPaths.keySet()) {
            tmpCheckPointPaths.put(key, checkpointPaths.get(key).getPath().getName());
            LOG.debug("CheckPoint key [" + key + "] value [" + checkpointPaths.get(key).getPath().getName()
                    + "]");
        }
        validateExpectedOutput(tmpResults, tmpTrashPaths, tmpCheckPointPaths);
        fs.delete(new Path(cluster.getRootDir() + "/databus-checkpoint"), true);
        fs.delete(cluster.getDataDir(), true);
        fs.close();
    } catch (Exception e) {
        LOG.debug("Error in running testCreateListing", e);
        assert false;
    }
}

From source file:com.inmobi.databus.purge.DataPurgerServiceTest.java

License:Apache License

private void testPurgerService(String testfilename, int numofhourstoadd, boolean checkifexists,
        boolean checktrashexists) throws Exception {
    DatabusConfigParser configparser = new DatabusConfigParser(testfilename);
    DatabusConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {
        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar todaysdate = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        todaysdate.add(Calendar.HOUR, numofhourstoadd);

        createTestPurgefiles(fs, cluster, todaysdate);

        service.runOnce();/*  w ww  .j av a  2s.co  m*/

        verifyPurgefiles(fs, cluster, todaysdate, checkifexists, checktrashexists);
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }
}

From source file:com.inmobi.messaging.consumer.databus.TestAbstractDatabusConsumer.java

License:Apache License

public void setup(int numFileToMove) throws Exception {

    ClientConfig config = loadConfig();/* ww w .  j  a v a  2s . c om*/
    config.set(DatabusConsumerConfig.hadoopConfigFileKey, "hadoop-conf.xml");
    testConsumer = getConsumerInstance();
    //System.out.println(testConsumer.getClass().getCanonicalName());
    testConsumer.initializeConfig(config);
    conf = testConsumer.getHadoopConf();
    Assert.assertEquals(conf.get("myhadoop.property"), "myvalue");
    // setup stream, collector dirs and data files
    Set<String> sourceNames = new HashSet<String>();
    sourceNames.add(testStream);
    chkpointPathPrefix = config.getString(DatabusConsumerConfig.checkpointDirConfig);
    setUpCheckpointPaths();
    rootDirs = testConsumer.getRootDirs();
    for (int i = 0; i < rootDirs.length; i++) {
        Map<String, String> clusterConf = new HashMap<String, String>();
        FileSystem fs = rootDirs[i].getFileSystem(conf);
        clusterConf.put("hdfsurl", fs.getUri().toString());
        clusterConf.put("jturl", "local");
        clusterConf.put("name", "databusCluster" + i);
        clusterConf.put("jobqueue", "default");

        String rootDir = rootDirs[i].toUri().toString();
        if (rootDirs[i].toString().startsWith("file:")) {
            String[] rootDirSplit = rootDirs[i].toString().split("file:");
            rootDir = rootDirSplit[1];
        }
        ClusterUtil cluster = new ClusterUtil(clusterConf, rootDir, sourceNames);
        fs.delete(new Path(cluster.getRootDir()), true);
        Path streamDir = new Path(cluster.getDataDir(), testStream);
        fs.delete(streamDir, true);
        fs.mkdirs(streamDir);
        for (String collector : collectors) {
            Path collectorDir = new Path(streamDir, collector);
            fs.delete(collectorDir, true);
            fs.mkdirs(collectorDir);
            TestUtil.setUpFiles(cluster, collector, dataFiles, null, null, numFileToMove, numFileToMove);
        }
    }
}