Example usage for org.apache.hadoop.fs FileSystem delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.datasalt.utils.commons.TestHadoopUtils.java

License:Apache License

@Test
public void testStringToFile() throws IOException {
    FileSystem fs = FileSystem.getLocal(getConf());
    Path path = new Path(TestHadoopUtils.class.getCanonicalName());

    try {/*from w w  w .  j a  va 2 s .c om*/

        String text = "String\nDe Prueba";

        for (int i = 0; i < 10; i++) {
            text += text;
        }

        HadoopUtils.stringToFile(fs, path, text);
        String read = HadoopUtils.fileToString(fs, path);

        assertEquals(text, read);
    } finally {
        fs.delete(path, true);
    }
}

From source file:com.datasayer.meerkat.TestMeerkatMain.java

License:Apache License

@After
public void tearDown() throws Exception {
    HamaConfiguration conf = new HamaConfiguration();
    FileSystem fs = FileSystem.get(conf);
    fs.delete(new Path(testLogFileName), true);
}

From source file:com.datatorrent.lib.bucket.BucketManagerTest.java

License:Open Source License

@AfterClass
public static void teardown() throws IOException {
    manager.shutdownService();/*from   w  ww  .j  a  v  a2s  .  c o  m*/
    Path root = new Path(applicationPath);
    FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
    fs.delete(root, true);
}

From source file:com.datatorrent.lib.bucket.CategoricalBucketManagerTest.java

License:Open Source License

@After
public void teardown() throws IOException {
    manager.shutdownService();//from   w  ww.j  av a2  s  . c  om
    Path root = new Path(applicationPath);
    FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
    fs.delete(root, true);
}

From source file:com.datatorrent.lib.bucket.ExpirableHdfsBucketStore.java

License:Open Source License

@Override
public void deleteExpiredBuckets(long time) throws IOException {
    Iterator<Long> iterator = windowToBuckets.keySet().iterator();
    for (; iterator.hasNext();) {
        long window = iterator.next();
        long timestamp = windowToTimestamp.get(window);
        if (timestamp < time) {
            Collection<Integer> indices = windowToBuckets.get(window);
            synchronized (indices) {
                if (indices.size() > 0) {
                    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
                    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
                    try {
                        if (fs.exists(dataFilePath)) {
                            logger.debug("start delete {}", window);
                            fs.delete(dataFilePath, true);
                            logger.debug("end delete {}", window);
                        }/*from   w  w w .  j a  v a 2  s.com*/
                        for (int bucketIdx : indices) {
                            Map<Long, Long> offsetMap = bucketPositions[bucketIdx];
                            if (offsetMap != null) {
                                synchronized (offsetMap) {
                                    offsetMap.remove(window);
                                }
                            }
                        }
                    } finally {
                        fs.close();
                    }
                }
                windowToTimestamp.remove(window);
                iterator.remove();
            }
        }
    }
}

From source file:com.datatorrent.lib.bucket.HdfsBucketStore.java

License:Open Source License

/**
 * {@inheritDoc}/*from   w  w  w  . j  av a2 s  . co  m*/
 */
@Override
public void deleteBucket(int bucketIdx) throws IOException {
    Map<Long, Long> offsetMap = bucketPositions[bucketIdx];
    if (offsetMap != null) {
        for (Long window : offsetMap.keySet()) {
            Collection<Integer> indices = windowToBuckets.get(window);
            synchronized (indices) {
                boolean elementRemoved = indices.remove(bucketIdx);
                if (indices.isEmpty() && elementRemoved) {
                    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
                    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
                    try {
                        if (fs.exists(dataFilePath)) {
                            logger.debug("start delete {}", window);
                            fs.delete(dataFilePath, true);
                            logger.debug("end delete {}", window);
                        }
                        windowToBuckets.removeAll(window);
                        windowToTimestamp.remove(window);
                    } finally {
                        fs.close();
                    }
                }
            }
        }
    }
    bucketPositions[bucketIdx] = null;
}

From source file:com.datatorrent.lib.dedup.DeduperBloomFilterTest.java

License:Open Source License

@After
public void teardown() {
    Path root = new Path(applicationPath);
    try {/*from  w  ww  . j  a  v  a2  s. c  o m*/
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.datatorrent.lib.dedup.DeduperBucketEvictionTest.java

License:Open Source License

@AfterClass
public static void teardown() {
    Path root = new Path(applicationPath);
    try {//from  w ww . j  a  v a  2 s . c om
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.datatorrent.lib.dedup.DedupUsingCategoricalExpiryTest.java

License:Open Source License

@AfterClass
public static void teardown() {
    Path root = new Path(applicationPath);
    try {/*from   w  w w  . jav a2  s . com*/
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
        logger.debug("Deleted path: " + applicationPath);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.datatorrent.stram.StramClient.java

License:Apache License

public void copyInitialState(Path origAppDir) throws IOException {
    // locate previous snapshot
    String newAppDir = this.dag.assertAppPath();

    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
    // read snapshot against new dependencies
    Object snapshot = recoveryHandler.restore();
    if (snapshot == null) {
        throw new IllegalArgumentException("No previous application state found in " + origAppDir);
    }/*www.ja v  a2 s.c om*/
    InputStream logIs = recoveryHandler.getLog();

    // modify snapshot state to switch app id
    ((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf);
    Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);

    FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
    // remove the path that was created by the storage agent during deserialization and replacement
    fs.delete(checkpointPath, true);

    // write snapshot to new location
    recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
    recoveryHandler.save(snapshot);
    OutputStream logOs = recoveryHandler.rotateLog();
    IOUtils.copy(logIs, logOs);
    logOs.flush();
    logOs.close();
    logIs.close();

    // copy sub directories that are not present in target
    FileStatus[] lFiles = fs.listStatus(origAppDir);
    for (FileStatus f : lFiles) {
        if (f.isDirectory()) {
            String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir);
            if (!fs.exists(new Path(targetPath))) {
                LOG.debug("Copying {} to {}", f.getPath(), targetPath);
                FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
                //FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf);
            } else {
                LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
                //FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777));
            }
        }
    }

}