Example usage for org.apache.hadoop.fs FileSystem delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:com.main.MRSearchMain.java

public void searchHBase(int numOfDays) throws IOException, InterruptedException, ClassNotFoundException {
    long startTime;
    long endTime;

    String path = "/home/hadoop/app/hadoop-2.0.0-cdh4.3.0/etc/hadoop/";
    Configuration conf = HBaseConfiguration.create();
    //      conf.set("hbase.zookeeper.quorum", "streamslab.localdomain");
    //      conf.set("fs.default.name", "hdfs://streamslab.localdomain:8020");
    //      conf.set("mapred.job.tracker", "hdfs://streamslab.localdomain:50300");
    //      conf.set("fs.hdfs.impl",
    //            org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
    conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    //?,FileSystem?
    conf.addResource(new Path(path + "core-site.xml"));
    conf.addResource(new Path(path + "hdfs-site.xml"));
    conf.addResource(new Path(path + "mapred-site.xml"));
    /* //  w ww  . java2  s . co  m
     * ?map 
     */
    conf.set("search.license", "C87310");
    conf.set("search.color", "10");
    conf.set("search.direction", "2");

    Job job = new Job(conf, "MRSearchHBase");
    System.out.println("search.license: " + conf.get("search.license"));
    job.setNumReduceTasks(0);
    job.setJarByClass(MRSearchMain.class);
    Scan scan = new Scan();
    scan.addFamily(FAMILY_NAME);
    byte[] startRow = Bytes.toBytes("2011010100000");
    byte[] stopRow;
    switch (numOfDays) {
    case 1:
        stopRow = Bytes.toBytes("2011010200000");
        break;
    case 10:
        stopRow = Bytes.toBytes("2011011100000");
        break;
    case 30:
        stopRow = Bytes.toBytes("2011020100000");
        break;
    case 365:
        stopRow = Bytes.toBytes("2012010100000");
        break;
    default:
        stopRow = Bytes.toBytes("2011010101000");
    }
    // ?key  
    scan.setStartRow(startRow);
    scan.setStopRow(stopRow);

    TableMapReduceUtil.initTableMapperJob(TABLE_NAME, scan, SearchMapper.class, ImmutableBytesWritable.class,
            Text.class, job);
    Path outPath = new Path("searchresult");
    LOG.info("outPath:" + outPath.toString());

    //hdfs
    FileSystem file = null;
    try {
        file = FileSystem.get(conf);
    } catch (IOException e) {
        e.printStackTrace();
    }
    //      HDFS_File file = new HDFS_File();
    //      file.DelFile(conf, outPath.getName(), true); // 
    //"hdfs://streamslab.localdomain:8020/
    if (file.exists(outPath)) {
        file.delete(outPath, true);
        LOG.info("=====delPath " + outPath.toString() + "=====");
    }
    FileOutputFormat.setOutputPath(job, outPath);//   

    startTime = System.currentTimeMillis();
    job.waitForCompletion(true);
    endTime = System.currentTimeMillis();
    LOG.info("Time used: " + (endTime - startTime));
    LOG.info("startRow:" + Text.decode(startRow));
    LOG.info("stopRow: " + Text.decode(stopRow));
}

---

From source file:com.mellanox.r4h.DistributedFileSystem.java

License:Apache License

@Override
public boolean delete(Path f, final boolean recursive) throws IOException {
    statistics.incrementWriteOps(1);/*from  w w  w .j a v  a  2 s .c  om*/
    Path absF = fixRelativePart(f);
    return new FileSystemLinkResolver<Boolean>() {
        @Override
        public Boolean doCall(final Path p) throws IOException, UnresolvedLinkException {
            return dfs.delete(getPathName(p), recursive);
        }

        @Override
        public Boolean next(final FileSystem fs, final Path p) throws IOException {
            return fs.delete(p, recursive);
        }
    }.resolve(this, absF);
}

---

From source file:com.metamx.druid.indexer.Utils.java

License:Open Source License

public static OutputStream makePathAndOutputStream(JobContext job, Path outputPath, boolean deleteExisting)
        throws IOException {
    OutputStream retVal;//  w  w w .  j a va2 s .  c  om
    FileSystem fs = outputPath.getFileSystem(job.getConfiguration());

    if (fs.exists(outputPath)) {
        if (deleteExisting) {
            fs.delete(outputPath, false);
        } else {
            throw new ISE("outputPath[%s] must not exist.", outputPath);
        }
    }

    if (!FileOutputFormat.getCompressOutput(job)) {
        retVal = fs.create(outputPath, false);
    } else {
        Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(job,
                GzipCodec.class);
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration());
        outputPath = new Path(outputPath.toString() + codec.getDefaultExtension());

        retVal = codec.createOutputStream(fs.create(outputPath, false));
    }

    return retVal;
}

---

From source file:com.michaeljones.hellohadoopworldmaven.HelloMapReduceTest.java

/**
 * Test of RunJobAsync method, of class HelloMapReduce.
 * @throws java.lang.Exception/*from w  w w.j  av a2s .  c o m*/
 */
@Test
public void testRunJobAsync() throws Exception {
    System.out.println("RunJobAsync");
    LOGGER.info("RunJobAsync");

    FileSystem hdfs = FileSystem.get(hadoopConfig);
    Path outputPath = new Path(wcOutputPathDir);

    // We need to remove the output directory before running the map reduce job.
    if (hdfs.exists(outputPath)) {
        // remove the directory recursively.
        hdfs.delete(outputPath, true);
    }

    Path inputPath = new Path(wcInputPathDir);
    Job result = HelloMapReduce.RunJobAsync(inputPath, outputPath, hadoopConfig);
    boolean ok = result.waitForCompletion(true);
    assertTrue(ok);
}

---

From source file:com.michaeljones.hellohadoopworldmaven.HelloMapReduceTest.java

/**
 * Test of RunJobAnalysisAsync method, of class HelloMapReduce.
 * @throws java.lang.Exception/* w w  w.  j  a v a 2 s . co  m*/
 */
@Test
public void testRunJobAnalysisAsync() throws Exception {
    System.out.println("RunJobAnalysisAsync");
    LOGGER.info("RunJobAnalysisAsync");
    FileSystem hdfs = FileSystem.get(hadoopConfig);
    Path outputPath = new Path(wcOutputAnalysisPathDir);
    if (hdfs.exists(outputPath)) {
        hdfs.delete(outputPath, true);
    }

    Path inputPath = new Path(wcInputPathDir);
    Job result = HelloMapReduce.RunJobAnalysisAsync(inputPath, outputPath, hadoopConfig);
    boolean ok = result.waitForCompletion(true);
    assertTrue(ok);
}

---

From source file:com.michaeljones.hellohadoopworldmaven.HelloMapReduceTest.java

/**
 * Test of main method, of class HelloMapReduce.
 * @throws java.lang.Exception/*w  w w .  j  a v  a2 s .  co  m*/
 */
@Test
public void testMain() throws Exception {
    System.out.println("main");
    LOGGER.info("testMain");
    FileSystem hdfs = FileSystem.get(hadoopConfig);

    Path outputPath = new Path(wcOutputMainPathDir);
    if (hdfs.exists(outputPath)) {
        hdfs.delete(outputPath, true);
    }

    String[] args = { wcInputPathDir, wcOutputMainPathDir };
    HelloMapReduce.main(args);

    // Assume it is true.
    assertTrue(true);
}

---

From source file:com.ML_Hadoop.K_meansClustering.K_meansClusteringMapReduce.java

public static void main(String[] args) throws Exception {
    int iteration = 0, num_of_iteration = 30;
    int feature_size = 2;
    FileSystem fs;
    int number_of_clusters = 2;

    do {/*from w w w .j  av a2 s . c  o m*/
        Configuration conf = new Configuration();
        fs = FileSystem.get(conf);

        Job job = new Job(conf, "K_meansClusteringMapReduce");
        job.setJarByClass(K_meansClusteringMapReduce.class);

        conf = job.getConfiguration(); // This line is mandatory. 

        job.setOutputKeyClass(LongWritable.class);
        job.setOutputValueClass(FloatArrayWritable.class);

        job.setMapperClass(K_meansClusteringMap.class);
        job.setReducerClass(K_meansClusteringReduce.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        job.setNumReduceTasks(1); // set number of reducers to one.

        FileInputFormat.addInputPath(job, new Path(args[0]));
        Path out = new Path(args[1]);
        if (fs.exists(out))
            fs.delete(out, true);

        FileOutputFormat.setOutputPath(job, out);
        number_of_clusters = Integer.parseInt(args[2]);
        num_of_iteration = Integer.parseInt(args[3]);
        feature_size = Integer.parseInt(args[4]);

        conf.setInt("number_of_clusters", number_of_clusters);
        conf.setInt("feature_size", feature_size);
        conf.setInt("current_iteration_num", iteration);

        try {
            job.waitForCompletion(true);
            iteration++;
        } catch (IOException e) {
            e.printStackTrace();
        }
    } while (iteration < num_of_iteration);

}

---

From source file:com.ML_Hadoop.K_meansClustering.K_meansClusteringReduce.java

@Override
protected void cleanup(Context context) throws IOException {
    String uri = "/user/hduser/k_mean.txt";
    Path path = new Path(uri);

    // Write the latest values of cetroids' of clusters in 'k_mean.txt' file
    try {//from   ww w .  j av  a 2s .c  o m
        FileSystem fs = FileSystem.get(URI.create(uri), context.getConfiguration());
        if (fs.exists(path))
            fs.delete(path, true);
        BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path, true)));
        for (int i = 0; i < number_of_clusters; i++) {
            for (int j = 0; j < feature_size; j++)
                br.write(((Float) ((FloatWritable) cetroids_of_all_clusters.get(i).get()[j]).get()).toString()
                        + ",");
            br.write("\n");
        }
        br.close();
    } catch (Exception e) {
        System.out.println("File k_mean.txt not found");
    }

    // Write the values of cetroids' of clusters for current iteration in directory '/user/hduser/K-means/...'

    uri = "/user/hduser/K-means/means-" + current_iteration_num + ".txt";
    path = new Path(uri);

    try {
        FileSystem fs = FileSystem.get(context.getConfiguration());
        if (current_iteration_num == 0)
            fs.delete(new Path("/user/hduser/K-means"), true);
        OutputStreamWriter osw = new OutputStreamWriter(fs.create(path, true));
        BufferedWriter br = new BufferedWriter(osw);
        for (int i = 0; i < number_of_clusters; i++) {
            for (int j = 0; j < feature_size; j++)
                br.write((Float) ((FloatWritable) cetroids_of_all_clusters.get(i).get()[j]).get() + ",");
            br.write("\n");
        }
        br.close();
    } catch (Exception e) {
        System.out.println("File not found");
    }
}

---

From source file:com.ML_Hadoop.MultipleLinearRegression.MultipleLinearRegressionMapReduce.java

public static void main(String[] args) throws Exception {
    String[] theta;//from ww w. j ava 2s.  com
    int iteration = 0, num_of_iteration = 1;
    int feature_size = 0, input_data_size = 0;
    FileSystem fs;
    Float alpha = 0.1f;

    do {
        Configuration conf = new Configuration();
        fs = FileSystem.get(conf);

        Job job = new Job(conf, "LinearRegressionMapReduce");
        job.setJarByClass(MultipleLinearRegressionMapReduce.class);

        // the following two lines are needed for propagating "theta"
        conf = job.getConfiguration();

        job.setOutputKeyClass(LongWritable.class);
        job.setOutputValueClass(FloatWritable.class);

        job.setMapperClass(MultipleLinearRegressionMap.class);
        job.setReducerClass(MultipleLinearRegressionReduce.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        job.setNumReduceTasks(1); // set mapred.reduce.tasks = 1 (only one reducer)

        FileInputFormat.addInputPath(job, new Path(args[0]));
        Path out = new Path(args[1]);
        if (fs.exists(out))
            fs.delete(out, true);

        FileOutputFormat.setOutputPath(job, out);
        alpha = Float.parseFloat(args[2]);
        num_of_iteration = Integer.parseInt(args[3]);
        feature_size = Integer.parseInt(args[4]);
        input_data_size = Integer.parseInt(args[5]);
        conf.setFloat("alpha", alpha);
        conf.setInt("feature_size", feature_size);
        conf.setInt("input_data_size", input_data_size);
        conf.setInt("iteration", iteration);

        theta = new String[feature_size];

        if (iteration == 0) { // first iteration
            for (int i = 0; i < theta.length; i++)
                theta[i] = "0.0";
            conf.setStrings("theta", theta);
        } else {
            try {
                String uri = "/user/hduser/theta.txt";
                fs = FileSystem.get(conf);
                //FSDataInputStream in = fs.open(new Path(uri));
                BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(uri))));
                theta = br.readLine().split(",");
            } catch (Exception e) {

            }
            conf.setStrings("theta", theta);
        }

        for (int i = 0; i < theta.length; i++)
            System.out.println("In MapRedce main function: theta[ " + i + " ]" + theta[i]);

        try {
            job.waitForCompletion(true);
            iteration++;
        } catch (IOException e) {
            e.printStackTrace();
        }
    } while (iteration < num_of_iteration);

}

---

From source file:com.ML_Hadoop.MultipleLinearRegression.MultipleLinearRegressionReduce.java

@Override
protected void cleanup(Context context) throws IOException {
    String uri = "/user/hduser/theta.txt";
    Path path = new Path(uri);

    try {/*from ww  w .j a v  a  2 s.c o m*/
        FileSystem fs = FileSystem.get(URI.create(uri), context.getConfiguration());
        if (fs.exists(path))
            fs.delete(path, true);
        BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path, true)));
        for (int i = 0; i < theta.length; i++)
            br.write(theta[i].toString() + ",");
        br.write("\n");
        br.close();
    } catch (Exception e) {
        System.out.println("File not found");
    }

    uri = "/user/hduser/LinearReg/theta-" + iteration + ".txt";
    path = new Path(uri);

    try {
        FileSystem fs = FileSystem.get(context.getConfiguration());
        if (iteration == 0)
            fs.delete(new Path("/user/hduser/LinearReg"), true);
        OutputStreamWriter osw = new OutputStreamWriter(fs.create(path, true));
        BufferedWriter br = new BufferedWriter(osw);
        br.write(prediction_error + ", ");
        for (int i = 0; i < theta.length; i++)
            br.write(theta[i].toString() + ", ");
        br.write("\n");
        br.close();
    } catch (Exception e) {
        System.out.println("File not found");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• »