Example usage for org.apache.hadoop.fs FileSystem delete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem delete.

Prototype

public abstract boolean delete(Path f, boolean recursive) throws IOException;

Source Link

Document

Delete a file.

Usage

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFHybridBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {

    verify();//from w  w  w . j a va2s  .co  m

    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFHybridBenchmark.java

License:Apache License

public static List<double[]> generateRandomInputData(Configuration conf, FileSystem fs, Path in, int numBspTask,
        int numGPUBspTask, int userCount, int itemCount, int percentNonZeroValues, int GPUPercentage,
        int maxTestPrefs) throws IOException {

    // Delete input directory if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }//  w w  w  .java2  s . co  m

    Random rand = new Random(32L);
    Set<Map.Entry<Long, Long>> userItemPairs = new HashSet<Map.Entry<Long, Long>>();
    List<double[]> testItems = new ArrayList<double[]>();

    int possibleUserItemRatings = userCount * itemCount;
    int userItemRatings = possibleUserItemRatings * percentNonZeroValues / 100;
    System.out.println("generateRandomInputData possibleRatings: " + possibleUserItemRatings + " ratings: "
            + userItemRatings);

    // Compute work distributions
    int cpuTaskNum = numBspTask - numGPUBspTask;
    long ratingsPerGPUTask = 0;
    long ratingsPerCPU = 0;
    long ratingsPerCPUTask = 0;
    if ((numGPUBspTask > 0) && (GPUPercentage > 0) && (GPUPercentage <= 100)) {
        ratingsPerGPUTask = (userItemRatings * GPUPercentage) / 100;
        ratingsPerCPU = userItemRatings - ratingsPerGPUTask;
    } else {
        ratingsPerCPU = userItemRatings;
    }
    if (cpuTaskNum > 0) {
        ratingsPerCPUTask = ratingsPerCPU / cpuTaskNum;
    }

    System.out.println("generateRandomInputData ratingsPerGPUTask: " + ratingsPerGPUTask + " ratingsPerCPU: "
            + ratingsPerCPU + " ratingsPerCPUTask: " + ratingsPerCPUTask);

    for (int part = 0; part < numBspTask; part++) {
        Path partIn = new Path(in, "part" + part + ".seq");
        final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf, partIn, LongWritable.class,
                PipesVectorWritable.class, CompressionType.NONE);

        long interval = 0;
        if (part > cpuTaskNum) {
            interval = ratingsPerGPUTask;
        } else {
            interval = ratingsPerCPUTask;
        }
        long start = interval * part;
        long end = start + interval - 1;
        if ((numBspTask - 1) == part) {
            end = userItemRatings;
        }
        System.out.println("Partition " + part + ": from " + start + " to " + end);

        for (long i = start; i <= end; i++) {

            // Find new user item rating which was not used before
            Map.Entry<Long, Long> userItemPair;
            do {
                long userId = rand.nextInt(userCount);
                long itemId = rand.nextInt(itemCount);
                userItemPair = new AbstractMap.SimpleImmutableEntry<Long, Long>(userId, itemId);
            } while (userItemPairs.contains(userItemPair));

            // Add user item rating
            userItemPairs.add(userItemPair);

            // Generate rating
            int rating = rand.nextInt(5) + 1; // values between 1 and 5

            // Add user item rating to test data
            if (i < maxTestPrefs) {
                testItems.add(new double[] { userItemPair.getKey(), userItemPair.getValue(), rating });
            }

            // Write out user item rating
            dataWriter.append(new LongWritable(userItemPair.getKey()), new PipesVectorWritable(
                    new DenseDoubleVector(new double[] { userItemPair.getValue(), rating })));
        }
        dataWriter.close();
    }

    return testItems;
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFHybridBenchmark.java

License:Apache License

public static List<double[]> convertInputData(Configuration conf, FileSystem fs, Path in, Path preferencesIn,
        String inputFile, String separator, int maxTestPrefs) throws IOException {

    List<double[]> testItems = new ArrayList<double[]>();

    // Delete input files if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }/*from   ww  w . j  a va  2  s.com*/
    if (fs.exists(preferencesIn)) {
        fs.delete(preferencesIn, true);
    }

    final SequenceFile.Writer prefWriter = SequenceFile.createWriter(fs, conf, preferencesIn,
            LongWritable.class, PipesVectorWritable.class, CompressionType.NONE);

    BufferedReader br = new BufferedReader(new FileReader(inputFile));
    String line;
    while ((line = br.readLine()) != null) {
        String[] values = line.split(separator);
        long userId = Long.parseLong(values[0]);
        long itemId = Long.parseLong(values[1]);
        double rating = Double.parseDouble(values[2]);
        // System.out.println("userId: " + userId + " itemId: " + itemId
        // + " rating: " + rating);

        double vector[] = new double[2];
        vector[0] = itemId;
        vector[1] = rating;
        prefWriter.append(new LongWritable(userId), new PipesVectorWritable(new DenseDoubleVector(vector)));

        // Add test preferences
        maxTestPrefs--;
        if (maxTestPrefs > 0) {
            testItems.add(new double[] { userId, itemId, rating });
        }

    }
    br.close();
    prefWriter.close();

    return testItems;
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

public static List<Preference<Long, Long>> prepareTestInputData(Configuration conf, FileSystem fs, Path in,
        Path preferencesIn) throws IOException {

    Preference[] train_prefs = { new Preference<Integer, Integer>(1, 0, 4),
            new Preference<Integer, Integer>(1, 1, 2.5), new Preference<Integer, Integer>(1, 2, 3.5),

            new Preference<Integer, Integer>(2, 0, 4), new Preference<Integer, Integer>(2, 1, 2.5),
            new Preference<Integer, Integer>(2, 2, 3.5), new Preference<Integer, Integer>(2, 3, 1),
            new Preference<Integer, Integer>(2, 4, 3.5),

            new Preference<Integer, Integer>(3, 0, 4), new Preference<Integer, Integer>(3, 1, 2.5),
            new Preference<Integer, Integer>(3, 2, 3.5), new Preference<Integer, Integer>(3, 3, 1),
            new Preference<Integer, Integer>(3, 4, 3.5) };

    List<Preference<Long, Long>> test_prefs = new ArrayList<Preference<Long, Long>>();
    test_prefs.add(new Preference<Long, Long>(1l, 0l, 4));
    test_prefs.add(new Preference<Long, Long>(1l, 1l, 2.5));
    test_prefs.add(new Preference<Long, Long>(1l, 2l, 3.5));
    test_prefs.add(new Preference<Long, Long>(1l, 3l, 1));
    test_prefs.add(new Preference<Long, Long>(1l, 4l, 3.5));

    // Delete input files if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }/*from  w  w w.jav a  2 s  . c  o m*/
    if (fs.exists(preferencesIn)) {
        fs.delete(preferencesIn, true);
    }

    final SequenceFile.Writer prefWriter = SequenceFile.createWriter(fs, conf, preferencesIn,
            LongWritable.class, PipesVectorWritable.class, CompressionType.NONE);

    for (Preference<Integer, Integer> taste : train_prefs) {
        double values[] = new double[2];
        values[0] = taste.getItemId();
        values[1] = taste.getValue().get();
        prefWriter.append(new LongWritable(taste.getUserId()),
                new PipesVectorWritable(new DenseDoubleVector(values)));
    }
    prefWriter.close();

    return test_prefs;
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

public static List<Preference<Long, Long>> generateRandomInputData(Configuration conf, FileSystem fs, Path in,
        int numBspTask, int numGPUBspTask, int userCount, int itemCount, int percentNonZeroValues,
        int GPUPercentage, int maxTestPrefs) throws IOException {

    // Delete input directory if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }//from ww  w  .ja  va 2s . c o m

    Random rand = new Random(32L);
    Set<Map.Entry<Long, Long>> userItemPairs = new HashSet<Map.Entry<Long, Long>>();
    List<Preference<Long, Long>> testItems = new ArrayList<Preference<Long, Long>>();

    int possibleUserItemRatings = userCount * itemCount;
    int userItemRatings = possibleUserItemRatings * percentNonZeroValues / 100;
    System.out.println("generateRandomInputData possibleRatings: " + possibleUserItemRatings + " ratings: "
            + userItemRatings);

    // Compute work distributions
    int cpuTaskNum = numBspTask - numGPUBspTask;
    long ratingsPerGPUTask = 0;
    long ratingsPerCPU = 0;
    long ratingsPerCPUTask = 0;
    if ((numGPUBspTask > 0) && (GPUPercentage > 0) && (GPUPercentage <= 100)) {
        ratingsPerGPUTask = (userItemRatings * GPUPercentage) / 100;
        ratingsPerCPU = userItemRatings - ratingsPerGPUTask;
    } else {
        ratingsPerCPU = userItemRatings;
    }
    if (cpuTaskNum > 0) {
        ratingsPerCPUTask = ratingsPerCPU / cpuTaskNum;
    }

    System.out.println("generateRandomInputData ratingsPerGPUTask: " + ratingsPerGPUTask + " ratingsPerCPU: "
            + ratingsPerCPU + " ratingsPerCPUTask: " + ratingsPerCPUTask);

    for (int part = 0; part < numBspTask; part++) {
        Path partIn = new Path(in, "part" + part + ".seq");
        final SequenceFile.Writer dataWriter = SequenceFile.createWriter(fs, conf, partIn, LongWritable.class,
                PipesVectorWritable.class, CompressionType.NONE);

        long interval = 0;
        if (part > cpuTaskNum) {
            interval = ratingsPerGPUTask;
        } else {
            interval = ratingsPerCPUTask;
        }
        long start = interval * part;
        long end = start + interval - 1;
        if ((numBspTask - 1) == part) {
            end = userItemRatings;
        }
        LOG.info("Partition " + part + ": from " + start + " to " + end);

        for (long i = start; i <= end; i++) {

            // Find new user item rating which was not used before
            Map.Entry<Long, Long> userItemPair;
            do {
                long userId = rand.nextInt(userCount);
                long itemId = rand.nextInt(itemCount);
                userItemPair = new AbstractMap.SimpleImmutableEntry<Long, Long>(userId, itemId);
            } while (userItemPairs.contains(userItemPair));

            // Add user item rating
            userItemPairs.add(userItemPair);

            // Generate rating
            int rating = rand.nextInt(5) + 1; // values between 1 and 5

            // Add user item rating to test data
            if (i < maxTestPrefs) {
                testItems.add(
                        new Preference<Long, Long>(userItemPair.getKey(), userItemPair.getValue(), rating));
            }

            // Write out user item rating
            dataWriter.append(new LongWritable(userItemPair.getKey()), new PipesVectorWritable(
                    new DenseDoubleVector(new double[] { userItemPair.getValue(), rating })));
        }
        dataWriter.close();
    }

    return testItems;
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

public static List<Preference<Long, Long>> convertInputData(Configuration conf, FileSystem fs, Path in,
        Path preferencesIn, String inputFile, String separator, int maxTestPrefs) throws IOException {

    List<Preference<Long, Long>> test_prefs = new ArrayList<Preference<Long, Long>>();

    // Delete input files if already exist
    if (fs.exists(in)) {
        fs.delete(in, true);
    }/*from   ww  w .  j ava  2s . c  o m*/
    if (fs.exists(preferencesIn)) {
        fs.delete(preferencesIn, true);
    }

    final SequenceFile.Writer prefWriter = SequenceFile.createWriter(fs, conf, preferencesIn,
            LongWritable.class, PipesVectorWritable.class, CompressionType.NONE);

    BufferedReader br = new BufferedReader(new FileReader(inputFile));
    String line;
    while ((line = br.readLine()) != null) {
        String[] values = line.split(separator);
        long userId = Long.parseLong(values[0]);
        long itemId = Long.parseLong(values[1]);
        double rating = Double.parseDouble(values[2]);
        // System.out.println("userId: " + userId + " itemId: " + itemId
        // + " rating: " + rating);

        double vector[] = new double[2];
        vector[0] = itemId;
        vector[1] = rating;
        prefWriter.append(new LongWritable(userId), new PipesVectorWritable(new DenseDoubleVector(vector)));

        // Add test preferences
        maxTestPrefs--;
        if (maxTestPrefs > 0) {
            test_prefs.add(new Preference<Long, Long>(userId, itemId, rating));
        }

    }
    br.close();
    prefWriter.close();

    return test_prefs;
}

---

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {

    // printOutput(m_conf, CONF_OUTPUT_DIR);

    // Cleanup/*w  ww  . j a v a 2s  . c om*/
    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(CONF_TMP_DIR, true);
}

---

From source file:at.illecker.hama.rootbeer.examples.piestimator.PiEstimatorBenchmark.java

License:Apache License

@Override
protected void tearDown() throws Exception {

    printOutput(m_conf, m_OUTPUT_DIR_PATH);

    // Cleanup//ww w. jav a  2  s.co  m
    FileSystem fs = FileSystem.get(m_conf);
    fs.delete(m_OUTPUT_DIR_PATH, true);
}

---

From source file:audr.text.utils.FileUtils.java

License:Open Source License

/**
 * HDFSpath/*from w  w  w . j a v  a2  s. c  o  m*/
 * 
 * @param path
 */
public static void deleteFileFromHDFS(String path) {
    try {
        Configuration conf = new Configuration();
        FileSystem dst = FileSystem.get(conf);
        Path dstpath = new Path(path);
        dst.delete(dstpath, true);
    } catch (Exception e) {
        e.printStackTrace();
    }

}

---

From source file:avro.HadoopAvro.java

License:Open Source License

private void createAvroFile() throws IOException {
    Path inputPath = new Path(INPUT_PATH);
    FileSystem fs = FileSystem.get(new Configuration());
    fs.delete(inputPath, true);

    DataFileWriter<User> fileWriter = new DataFileWriter<>(new GenericDatumWriter<User>(User.SCHEMA));

    fileWriter.create(User.SCHEMA, fs.create(new Path(inputPath, "file.avro")));
    IntStream.range(0, 100).mapToObj(i -> new User("name" + i, "pass" + i, i, i % 2 == 0))
            .forEach(user -> Util.uncheckRun(() -> fileWriter.append(user)));
    fileWriter.close();//ww  w .  j  ava2s .  c  om
    fs.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• »