Example usage for org.apache.hadoop.fs FileSystem delete

List of usage examples for org.apache.hadoop.fs FileSystem delete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.ibm.bi.dml.runtime.transform.GenTfMtdMR.java

License:Open Source License

public static long runJob(String inputPath, String txMtdPath, String specFileWithIDs, String smallestFile,
        String partOffsetsFile, CSVFileFormatProperties inputDataProperties, long numCols, int replication,
        String headerLine) throws IOException, ClassNotFoundException, InterruptedException {
    JobConf job = new JobConf(GenTfMtdMR.class);
    job.setJobName("GenTfMTD");

    /* Setup MapReduce Job */
    job.setJarByClass(GenTfMtdMR.class);

    // set relevant classes
    job.setMapperClass(GTFMTDMapper.class);
    job.setReducerClass(GTFMTDReducer.class);

    // set input and output properties
    job.setInputFormat(TextInputFormat.class);
    job.setOutputFormat(NullOutputFormat.class);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(DistinctValue.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setInt("dfs.replication", replication);

    FileInputFormat.addInputPath(job, new Path(inputPath));
    // delete outputPath, if exists already.
    Path outPath = new Path(txMtdPath);
    FileSystem fs = FileSystem.get(job);
    fs.delete(outPath, true);
    FileOutputFormat.setOutputPath(job, outPath);

    job.set(MRJobConfiguration.TF_HAS_HEADER, Boolean.toString(inputDataProperties.hasHeader()));
    job.set(MRJobConfiguration.TF_DELIM, inputDataProperties.getDelim());
    if (inputDataProperties.getNAStrings() != null)
        // Adding "dummy" string to handle the case of na_strings = ""
        job.set(MRJobConfiguration.TF_NA_STRINGS, TfUtils.prepNAStrings(inputDataProperties.getNAStrings()));
    job.set(MRJobConfiguration.TF_SPEC_FILE, specFileWithIDs);
    job.set(MRJobConfiguration.TF_SMALLEST_FILE, smallestFile);
    job.setLong(MRJobConfiguration.TF_NUM_COLS, numCols);
    job.set(MRJobConfiguration.TF_HEADER, headerLine);

    job.set(MRJobConfiguration.OUTPUT_MATRICES_DIRS_CONFIG, txMtdPath);

    // offsets file to store part-file names and offsets for each input split
    job.set(MRJobConfiguration.TF_OFFSETS_FILE, partOffsetsFile);

    //turn off adaptivemr
    job.setBoolean("adaptivemr.map.enable", false);

    // Run the job
    RunningJob runjob = JobClient.runJob(job);

    Counters c = runjob.getCounters();//ww w . j a v a 2s.  c o  m
    long tx_numRows = c.findCounter(MRJobConfiguration.DataTransformCounters.TRANSFORMED_NUM_ROWS).getCounter();

    return tx_numRows;
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static void deleteFileIfExistOnHDFS(String dir) throws IOException {
    Path outpath = new Path(dir);
    FileSystem fs = FileSystem.get(_rJob);
    if (fs.exists(outpath)) {
        //System.err.println("Deleting " + outpath + " ... ");
        fs.delete(outpath, true);
    }//from   w w  w.j a va 2  s.co m
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from  www.j a  va2 s  . c  o  m*/
 * Removes all the directories specified in the array in HDFS
 * </p>
 * 
 * @param directories
 *            directories array
 */
public static void removeHDFSDirectories(String[] directories) {
    try {
        FileSystem fs = FileSystem.get(conf);
        for (String directory : directories) {
            Path dir = new Path(directory);
            if (fs.exists(dir) && fs.getFileStatus(dir).isDirectory()) {
                fs.delete(dir, true);
            }
        }
    } catch (IOException e) {
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>//  w  w  w .ja v a 2  s  .c o  m
 * Removes all the files specified in the array in HDFS
 * </p>
 * 
 * @param files
 *            files array
 */
public static void removeHDFSFiles(String[] files) {
    try {
        FileSystem fs = FileSystem.get(conf);
        for (String directory : files) {
            Path dir = new Path(directory);
            if (fs.exists(dir) && !fs.getFileStatus(dir).isDirectory()) {
                fs.delete(dir, false);
            }
        }
    } catch (IOException e) {
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/*from   w w  w. j  a  v  a  2  s . c o  m*/
 * Clears a complete directory.
 * </p>
 * 
 * @param directory
 *            directory
 */
public static void clearDirectory(String directory) {
    try {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] directoryContent = fs.listStatus(new Path(directory));
        for (FileStatus content : directoryContent) {
            fs.delete(content.getPath(), true);
        }
    } catch (IOException e) {
    }
}

From source file:com.ibm.bi.dml.test.utils.TestUtils.java

License:Open Source License

/**
 * <p>/* ww w  . j av  a 2s .com*/
 * Removes all temporary files and directories in the current working
 * directory.
 * </p>
 */
public static void removeTemporaryFiles() {
    try {
        FileSystem fs = FileSystem.get(conf);
        Path workingDir = new Path(".");
        FileStatus[] files = fs.listStatus(workingDir);
        for (FileStatus file : files) {
            String fileName = file.getPath().toString()
                    .substring(file.getPath().getParent().toString().length() + 1);
            if (fileName.contains("temp"))
                fs.delete(file.getPath(), false);
        }
    } catch (IOException e) {
        e.printStackTrace();
        fail("unable to remove temporary files: " + e.getMessage());
    }
}

From source file:com.ibm.crail.hdfs.tools.HdfsIOBenchmark.java

License:Apache License

void createFile() throws Exception, InterruptedException {
    System.out.println("create file async hdfs, path " + path + ", size " + size + ", loop " + loop);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    int repfactor = 4;
    for (int k = 0; k < repfactor; k++) {
        LinkedBlockingQueue<Path> pathQueue = new LinkedBlockingQueue<Path>();
        fs.mkdirs(path);//from   ww w . j a va2s. c om
        for (int i = 0; i < loop * size; i++) {
            String name = "" + i;
            Path f = new Path(path, name);
            pathQueue.add(f);
        }

        LinkedBlockingQueue<FSDataOutputStream> streamQueue = new LinkedBlockingQueue<FSDataOutputStream>();
        long start = System.currentTimeMillis();
        for (int i = 0; i < size; i++) {
            //single operation == loop
            for (int j = 0; j < loop; j++) {
                Path path = pathQueue.poll();
                fs.create(path).close();
            }
        }
        long end = System.currentTimeMillis();
        double executionTime = ((double) (end - start));
        double latency = executionTime * 1000.0 / ((double) size);
        System.out.println("execution time [ms] " + executionTime);
        System.out.println("latency [us] " + latency);

        while (!streamQueue.isEmpty()) {
            FSDataOutputStream stream = streamQueue.poll();
            stream.close();
        }

        if (k < repfactor - 1) {
            fs.delete(path, true);
            Thread.sleep(2000);
        }
    }
    fs.close();
}

From source file:com.ibm.jaql.io.hadoop.FileOutputConfigurator.java

License:Apache License

public void setSequential(JobConf conf) throws Exception {
    registerSerializers(conf);//from   w  ww.ja va2s.  com

    // For an expression, the location is the final file name
    Path outPath = new Path(location);
    FileSystem fs = outPath.getFileSystem(conf);
    outPath = outPath.makeQualified(fs);
    if (fs.exists(outPath)) {
        // TODO: Jaql currently has overwrite semantics; add flag to control this
        if (fs.isFile(outPath)) {
            fs.delete(outPath, false);
        } else {
            // Look for a map-reduce output directory
            FileStatus[] nonMR = fs.listStatus(outPath, new PathFilter() {
                boolean onlyOne = true;

                public boolean accept(Path path) {
                    String name = path.getName();
                    if (name.matches("([.][.]?)|([.]part-[0-9]+.crc)|(part-[0-9]+)")) {
                        return false;
                    }
                    if (onlyOne) {
                        onlyOne = false;
                        return true;
                    }
                    return false;
                }
            });
            if (nonMR.length > 0) {
                throw new IOException(
                        "directory exists and is not a map-reduce output directory: " + nonMR[0].getPath());
            }
            fs.delete(outPath, true);
        }
    }

    // In sequential mode, we will write directly to the output file
    // and bypass the _temporary directory and rename of the standard 
    // FileOutputCommitter by using our own DirectFileOutputCommitter.
    FileOutputFormat.setOutputPath(conf, outPath.getParent());
    conf.setClass("mapred.output.committer.class", DirectFileOutputCommiter.class, OutputCommitter.class);
}

From source file:com.ibm.jaql.io.hadoop.FileOutputConfigurator.java

License:Apache License

public void setParallel(JobConf conf) throws Exception {
    registerSerializers(conf);//from  w  ww  . ja v a  2  s  .c  o m

    // For map-reduce, multiple files can be produced, so the location is their
    // parent directory.
    Path outPath = new Path(location);
    FileSystem fs = outPath.getFileSystem(conf);
    fs.delete(outPath, true); // TODO: Jaql currently has overwrite semantics; add flag to control this
    FileOutputFormat.setOutputPath(conf, outPath);
}

From source file:com.ibm.jaql.UtilForTest.java

License:Apache License

/**
 * @param dir/*from  w ww.ja va 2 s . c om*/
 * @throws IOException
 */
public static void cleanUpHDFS(String dir) throws IOException {
    if ("true".equals(System.getProperty("test.cleanup"))) {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        Path p = new Path(dir);
        if (fs.exists(p)) {
            fs.delete(p, true);
        }
    }
}