Example usage for org.apache.hadoop.fs FileSystem delete

List of usage examples for org.apache.hadoop.fs FileSystem delete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.sequenceiq.yarntest.mr.QuasiMonteCarlo.java

License:Apache License

/**
 * Run a map/reduce job for estimating Pi.
 *
 * @return the estimated value of Pi//from   w  w w . ja  v a 2 s .c o m
 */
public static JobID submitPiEstimationMRApp(String jobName, int numMaps, long numPoints, Path tmpDir,
        Configuration conf) throws IOException, ClassNotFoundException, InterruptedException {
    Job job = new Job(conf);
    //setup job conf
    job.setJobName(jobName);
    job.setJarByClass(QuasiMonteCarlo.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);

    job.setOutputKeyClass(BooleanWritable.class);
    job.setOutputValueClass(LongWritable.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    job.setMapperClass(QmcMapper.class);

    job.setReducerClass(QmcReducer.class);
    job.setNumReduceTasks(1);

    // turn off speculative execution, because DFS doesn't handle
    // multiple writers to the same file.
    job.setSpeculativeExecution(false);

    //setup input/output directories
    final Path inDir = new Path(tmpDir, "in");
    final Path outDir = new Path(tmpDir, "out");
    FileInputFormat.setInputPaths(job, inDir);
    FileOutputFormat.setOutputPath(job, outDir);

    final FileSystem fs = FileSystem.get(conf);
    if (fs.exists(tmpDir)) {
        fs.delete(tmpDir, true);
        //      throw new IOException("Tmp directory " + fs.makeQualified(tmpDir)
        //          + " already exists.  Please remove it first.");
    }
    if (!fs.mkdirs(inDir)) {
        throw new IOException("Cannot create input directory " + inDir);
    }

    //  try {
    //generate an input file for each map task
    for (int i = 0; i < numMaps; ++i) {
        final Path file = new Path(inDir, "part" + i);
        final LongWritable offset = new LongWritable(i * numPoints);
        final LongWritable size = new LongWritable(numPoints);
        final SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, file, LongWritable.class,
                LongWritable.class, CompressionType.NONE);
        try {
            writer.append(offset, size);
        } finally {
            writer.close();
        }
        System.out.println("Wrote input for Map #" + i);
    }

    //start a map/reduce job
    System.out.println("Starting Job");
    final long startTime = System.currentTimeMillis();
    job.submit();
    //      final double duration = (System.currentTimeMillis() - startTime)/1000.0;
    //      System.out.println("Job Finished in " + duration + " seconds");
    return job.getJobID();

    //    } finally {
    //      fs.delete(tmpDir, true);
    //    }
}

From source file:com.sirius.hadoop.job.InputUpload.java

License:Apache License

public static void main(String[] args) throws Exception {
    hdfs = new URI("hdfs://hadoop1:8020");
    FileSystem fs = FileSystem.get(hdfs, configuration);

    if (fs.exists(input_path)) {
        fs.delete(input_path, true);
    }// www .j  av  a2 s  . c o m

    System.out.println(Arrays.toString(fs.listStatus(new Path("/"))));

    fs.copyFromLocalFile(false, true, new Path("/Users/pippo/Downloads/subscriber_status.statics.input"),
            input_path);
}

From source file:com.skp.experiment.cf.als.hadoop.ParallelALSFactorizationJob.java

License:Apache License

private Pair<Integer, Double> calculateMatrixDistanceSquared(Path oldMatrix, Path newMatrix, int iteration)
        throws IOException, InterruptedException, ClassNotFoundException {
    FileSystem fs = FileSystem.get(getConf());
    Path path = getTempPath("rmse-" + iteration);
    fs.delete(path, true);
    Job rmseJob = MatrixDistanceSquaredJob.createMinusJob(getConf(), oldMatrix, newMatrix, path);
    rmseJob.waitForCompletion(true);/*from   w w  w .  ja  v  a2  s .  c om*/
    Pair<Integer, Double> result = MatrixDistanceSquaredJob.retrieveDistanceSquaredOutput(getConf(), path);
    fs.delete(path, true);
    return result;
}

From source file:com.splicemachine.derby.impl.io.HdfsDirFile.java

License:Apache License

@Override
public boolean delete() {
    try {/*from  w w  w .  jav a  2  s.c om*/
        FileSystem fs = getFileSystem();
        return fs.delete(new Path(path), false);
    } catch (IOException e) {
        LOG.error(String.format("An exception occurred while deleting the path '%s'.", path), e);
        return false;
    }
}

From source file:com.splicemachine.derby.impl.io.HdfsDirFile.java

License:Apache License

@Override
public boolean deleteAll() {
    try {/*from w  ww  .  j  a  va  2s.  c  o  m*/
        FileSystem fs = getFileSystem();
        return fs.delete(new Path(path), true);
    } catch (IOException e) {
        LOG.error(String.format("An exception occurred while deleting the path '%s'.", path), e);
        return false;
    }
}

From source file:com.splicemachine.hbase.MockSnapshot.java

License:Apache License

public static void deleteFile(Path p) throws IOException {
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///tmp");
    FileSystem fs = p.getFileSystem(conf);
    fs.delete(p, true);
}

From source file:com.splicemachine.mrio.api.core.HiveIntegrationIT.java

License:Apache License

@AfterClass
public static void cleanup() throws Exception {
    FileSystem fs = FileSystem.get(URI.create(getHiveWarehouseDirectory()),
            (Configuration) SIDriver.driver().getConfiguration().getConfigSource().unwrapDelegate());
    fs.delete(new Path(getBaseDirectory() + "/user"), true);
    fs.delete(new Path(getBaseDirectory() + "/../target"), true);
}

From source file:com.splout.db.common.SploutHadoopConfiguration.java

License:Apache License

/**
 * Adds the SQLite native libraries to the DistributedCache so that they will be present in the java.library.path
 * of the child's Hadoop task.//from w w  w .j  a va  2  s .  c o  m
 * <p/>
 * Usually you don't need to do this as the task will already try to load them from the job's uncompressed JAR, however
 * it is not assured that all Hadoop versions do the uncompressing of the JAR so in this case it's safer to use this.
 */
public static void addSQLite4JavaNativeLibsToDC(Configuration conf, File nativeLibsLocalPath)
        throws IOException, URISyntaxException {
    Path nativeLibHdfs = new Path("splout-native");
    FileSystem fS = FileSystem.get(conf);
    if (fS.exists(nativeLibHdfs)) {
        fS.delete(nativeLibHdfs, true);
    }
    fS.mkdirs(nativeLibHdfs);
    // Copy native libs to HDFS
    File[] natives = nativeLibsLocalPath.listFiles();
    if (natives == null) {
        throw new RuntimeException(
                "natives lib folder not present in local working directory! Are you in SPLOUT_HOME?");
    }
    for (File nativeLib : natives) {
        FileUtil.copy(nativeLib, fS, nativeLibHdfs, false, conf);
    }
    for (FileStatus nativeLibInHdfs : fS.listStatus(nativeLibHdfs)) {
        // http://hadoop.apache.org/docs/r0.20.2/native_libraries.html#Loading+native+libraries+through+DistributedCache
        DistributedCache.createSymlink(conf);
        URI uriToAdd = new URI(
                nativeLibInHdfs.getPath().makeQualified(fS) + "#" + nativeLibInHdfs.getPath().getName());
        DistributedCache.addCacheFile(uriToAdd, conf);
        log.info("Adding to distributed cache: " + uriToAdd);
    }
}

From source file:com.splout.db.dnode.TestFetcher.java

License:Open Source License

@Test
public void testHdfsFetching() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    FileSystem fS = FileSystem.getLocal(conf);

    SploutConfiguration testConfig = SploutConfiguration.getTestConfig();
    testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName());
    Fetcher fetcher = new Fetcher(testConfig);

    Path path = new Path("tmp-" + TestFetcher.class.getName());
    OutputStream oS = fS.create(path);
    oS.write("This is what happens when you don't know what to write".getBytes());
    oS.close();//from   w w w . jav  a 2  s  .  c  o  m

    File f = fetcher.fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString());

    assertTrue(f.exists());
    assertTrue(f.isDirectory());

    File file = new File(f, "tmp-" + TestFetcher.class.getName());
    assertTrue(file.exists());

    assertEquals("This is what happens when you don't know what to write",
            Files.toString(file, Charset.defaultCharset()));

    fS.delete(path, true);
    FileUtils.deleteDirectory(f);
}

From source file:com.splout.db.dnode.TestFetcher.java

License:Open Source License

@Test
public void testHdfsFetchingAndThrottling() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    FileSystem fS = FileSystem.getLocal(conf);

    SploutConfiguration testConfig = SploutConfiguration.getTestConfig();
    testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName());
    testConfig.setProperty(FetcherProperties.DOWNLOAD_BUFFER, 4);
    testConfig.setProperty(FetcherProperties.BYTES_PER_SEC_THROTTLE, 8);
    Fetcher fetcher = new Fetcher(testConfig);

    final String str = "This is what happens when you don't know what to write";

    Path path = new Path("tmp-" + TestFetcher.class.getName());
    OutputStream oS = fS.create(path);
    oS.write(str.getBytes());/*from  w  ww .  jav  a  2  s .c  om*/
    oS.close();

    long startTime = System.currentTimeMillis();
    File f = fetcher.fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString());
    long endTime = System.currentTimeMillis();

    double bytesPerSec = (str.getBytes().length / (double) (endTime - startTime)) * 1000;
    assertEquals(8, bytesPerSec, 0.5);

    assertTrue(f.exists());
    assertTrue(f.isDirectory());

    File file = new File(f, "tmp-" + TestFetcher.class.getName());
    assertTrue(file.exists());

    assertEquals(str, Files.toString(file, Charset.defaultCharset()));

    fS.delete(path, true);
    FileUtils.deleteDirectory(f);
}