Example usage for org.apache.hadoop.fs FileSystem rename

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem rename.

Prototype

public abstract boolean rename(Path src, Path dst) throws IOException;

Source Link

Document

Renames Path src to Path dst.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyCommitter.java

License:Apache License

private void commitData(Configuration conf) throws IOException {

    Path workDir = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
    Path finalDir = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
    FileSystem targetFS = workDir.getFileSystem(conf);

    LOG.info("Atomic commit enabled. Moving " + workDir + " to " + finalDir);
    if (targetFS.exists(finalDir) && targetFS.exists(workDir))
        if (!targetFS.delete(finalDir, true)) {
            LOG.error("Unable to delete pre-existing final-data at " + finalDir);
            throw new IOException("Atomic commit failed. Pre-existing final data" + " in " + finalDir
                    + " could not be cleared, before commit.");
        }//  ww w  .j a  v a 2s . c o  m

    boolean result = targetFS.rename(workDir, finalDir);
    if (!result) {
        LOG.warn("Rename failed. Perhaps data already moved. Verifying...");
        result = targetFS.exists(finalDir) && !targetFS.exists(workDir);
    }
    if (result) {
        LOG.info("Data committed successfully to " + finalDir);
        HadoopCompat.setStatus(taskAttemptContext, "Data committed successfully to " + finalDir);
    } else {
        LOG.error("Unable to commit data to " + finalDir);
        throw new IOException(
                "Atomic commit failed. Temporary data in " + workDir + ", Unable to move to " + finalDir);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private void promoteTmpToTarget(Path tmpTarget, Path target, FileSystem fs) throws IOException {
    if ((fs.exists(target) && !fs.delete(target, false))
            || (!fs.exists(target.getParent()) && !fs.mkdirs(target.getParent()))
            || !fs.rename(tmpTarget, target)) {
        throw new IOException("Failed to promote tmp-file:" + tmpTarget + " to: " + target);
    }//www .  j  a  va  2 s  . c  om
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java

License:Apache License

@Test
public void testDeleteMissing() {
    TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
    JobContext jobContext = Mockito.mock(JobContext.class);
    Mockito.when(jobContext.getConfiguration()).thenReturn(config);
    JobID jobID = new JobID();
    Mockito.when(jobContext.getJobID()).thenReturn(jobID);
    Configuration conf = jobContext.getConfiguration();

    String sourceBase;//from  w w  w.j  av  a  2 s. c o  m
    String targetBase;
    FileSystem fs = null;
    try {
        OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
        fs = FileSystem.get(conf);
        sourceBase = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
        targetBase = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
        String targetBaseAdd = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
        fs.rename(new Path(targetBaseAdd), new Path(targetBase));

        DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), new Path("/out"));
        options.setSyncFolder(true);
        options.setDeleteMissing(true);
        options.appendToConf(conf);

        CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
        Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
        listing.buildListing(listingFile, options);

        conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
        conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);

        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
            Assert.fail("Source and target folders are not in sync");
        }

        //Test for idempotent commit
        committer.commitJob(jobContext);
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
        if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, sourceBase, targetBase)) {
            Assert.fail("Source and target folders are not in sync");
        }
    } catch (Throwable e) {
        LOG.error("Exception encountered while testing for delete missing", e);
        Assert.fail("Delete missing failure");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp1");
    }

}

---

From source file:com.inmobi.conduit.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, FileStatus value, Context context) throws IOException, InterruptedException {
    Path src = value.getPath();/*www . j av a 2s. co m*/
    String dest = key.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();
    Map<Long, Long> received = null;
    if (context.getConfiguration().getBoolean(ConduitConstants.AUDIT_ENABLED_KEY, true)) {
        received = new HashMap<Long, Long>();
    }
    Configuration srcConf = new Configuration();
    srcConf.set(FS_DEFAULT_NAME_KEY, context.getConfiguration().get(SRC_FS_DEFAULT_NAME_KEY));

    FileSystem fs = FileSystem.get(srcConf);
    Path target = getTempPath(context, src, category, collector);
    if (FileUtil.gzip(src, target, srcConf, received)) {
        LOG.info("File " + src + " is empty hence returning without compressing");
        return;
    }
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    String destnFilename = collector + "-" + src.getName() + ".gz";
    Path destPath = new Path(dest + File.separator + destnFilename);
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);
    if (received != null) {

        for (Entry<Long, Long> entry : received.entrySet()) {
            String counterNameValue = getCounterNameValue(category, destnFilename, entry.getKey(),
                    entry.getValue());
            context.write(NullWritable.get(), new Text(counterNameValue));
        }
    }

}

---

From source file:com.inmobi.databus.distcp.MergedStreamService.java

License:Apache License

private void commitMirroredConsumerPaths(Map<String, Set<Path>> committedPaths, Path tmp) throws Exception {
    // Map of Stream and clusters where it's mirrored
    Map<String, Set<Cluster>> mirrorStreamConsumers = new HashMap<String, Set<Cluster>>();
    Map<Path, Path> consumerCommitPaths = new LinkedHashMap<Path, Path>();
    // for each stream in committedPaths
    for (String stream : committedPaths.keySet()) {
        // for each cluster
        for (Cluster cluster : getConfig().getClusters().values()) {
            // is this stream to be mirrored on this cluster
            if (cluster.getMirroredStreams().contains(stream)) {
                Set<Cluster> mirrorConsumers = mirrorStreamConsumers.get(stream);
                if (mirrorConsumers == null)
                    mirrorConsumers = new HashSet<Cluster>();
                mirrorConsumers.add(cluster);
                mirrorStreamConsumers.put(stream, mirrorConsumers);
            }//from  ww w  .ja v  a 2 s  .  com
        }
    } // for each stream

    // Commit paths for each consumer
    for (String stream : committedPaths.keySet()) {
        // consumers for this stream
        Set<Cluster> consumers = mirrorStreamConsumers.get(stream);
        Path tmpConsumerPath;
        if (consumers == null || consumers.size() == 0) {
            LOG.warn(" Consumers is empty for stream [" + stream + "]");
            continue;
        }
        for (Cluster consumer : consumers) {
            // commit paths for this consumer, this stream
            // adding srcCluster avoids two Remote Copiers creating same filename
            String tmpPath = "src_" + getSrcCluster().getName() + "_via_" + getDestCluster().getName()
                    + "_mirrorto_" + consumer.getName() + "_" + stream;
            tmpConsumerPath = new Path(tmp, tmpPath);
            FSDataOutputStream out = getDestFs().create(tmpConsumerPath);
            try {
                for (Path path : committedPaths.get(stream)) {
                    LOG.debug("Writing Mirror Commit Path [" + path.toString() + "]");
                    out.writeBytes(path.toString());
                    out.writeBytes("\n");
                }
            } finally {
                out.close();
            }
            // Two MergedStreamConsumers will write file for same consumer within
            // the same time
            // adding srcCLuster name avoids that conflict
            Path finalMirrorPath = new Path(getDestCluster().getMirrorConsumePath(consumer),
                    tmpPath + "_" + new Long(System.currentTimeMillis()).toString());
            consumerCommitPaths.put(tmpConsumerPath, finalMirrorPath);
        } // for each consumer
    } // for each stream

    if (consumerCommitPaths == null || consumerCommitPaths.size() == 0) {
        LOG.info("consumerCommitPaths is empty for all stream, skipping mirrorCommit");
        missingDirsCommittedPaths.clear();
        return;
    }
    // Do the final mirrorCommit
    LOG.info("Committing [" + consumerCommitPaths.size() + "] paths for " + "mirrored Stream");
    FileSystem fs = FileSystem.get(getDestCluster().getHadoopConf());
    for (Map.Entry<Path, Path> entry : consumerCommitPaths.entrySet()) {
        LOG.info("Renaming [" + entry.getKey() + "] to [" + entry.getValue() + "]");
        fs.mkdirs(entry.getValue().getParent());
        if (fs.rename(entry.getKey(), entry.getValue()) == false) {
            LOG.warn("Failed to Commit for Mirrored Path. Aborting Transaction " + "to avoid DATA LOSS, "
                    + "Partial data replay can happen for merged and mirror stream");
            throw new Exception("Rename failed from [" + entry.getKey() + "] to [" + entry.getValue() + "]");
        }
    }
    missingDirsCommittedPaths.clear();
}

---

From source file:com.inmobi.databus.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    Path src = new Path(key.toString());
    String dest = value.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();

    FileSystem fs = FileSystem.get(context.getConfiguration());
    Path target = getTempPath(context, src, category, collector);
    FileUtil.gzip(src, target, context.getConfiguration());
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    Path destPath = new Path(dest + File.separator + collector + "-" + src.getName() + ".gz");
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);

}

---

From source file:com.inmobi.databus.local.LocalStreamService.java

License:Apache License

private void commit(Map<Path, Path> commitPaths) throws Exception {
    LOG.info("Committing " + commitPaths.size() + " paths.");
    FileSystem fs = FileSystem.get(cluster.getHadoopConf());
    for (Map.Entry<Path, Path> entry : commitPaths.entrySet()) {
        LOG.info("Renaming " + entry.getKey() + " to " + entry.getValue());
        fs.mkdirs(entry.getValue().getParent());
        if (fs.rename(entry.getKey(), entry.getValue()) == false) {
            LOG.warn("Rename failed, aborting transaction COMMIT to avoid "
                    + "dataloss. Partial data replay could happen in next run");
            throw new Exception("Abort transaction Commit. Rename failed from [" + entry.getKey() + "] to ["
                    + entry.getValue() + "]");
        }/*from  w w  w  .  j  a  v a  2  s.  com*/
    }

}

---

From source file:com.inmobi.grill.server.GrillServices.java

License:Apache License

private void persistGrillServiceState() throws IOException {
    if (conf.getBoolean(GrillConfConstants.GRILL_SERVER_RESTART_ENABLED,
            GrillConfConstants.DEFAULT_GRILL_SERVER_RESTART_ENABLED)) {
        FileSystem fs = persistDir.getFileSystem(conf);
        LOG.info("Persisting server state in " + persistDir);

        for (GrillService service : grillServices) {
            LOG.info("Persisting state of service:" + service.getName());
            Path serviceWritePath = new Path(persistDir, service.getName() + ".out");
            ObjectOutputStream out = null;
            try {
                out = new ObjectOutputStream(fs.create(serviceWritePath));
                service.writeExternal(out);
            } finally {
                if (out != null) {
                    out.close();//w  w  w  . j  av  a 2  s.c  o  m
                }
            }
            Path servicePath = getServicePersistPath(service);
            fs.rename(serviceWritePath, servicePath);
            LOG.info("Persisted service " + service.getName() + " to " + servicePath);
        }
    } else {
        LOG.info("Server restart is not enabled. Not persisting the server state");
    }
}

---

From source file:com.intel.hadoop.graphbuilder.demoapps.wikipedia.docwordgraph.TransformToTFIDF.java

License:Open Source License

public static void main(String[] args) throws IOException, NotFoundException, InstantiationException,
        IllegalAccessException, CannotCompileException {
    String numDocs = args[0];//from www  . ja v a2 s . co m
    String input = args[1];
    String output = args[2];

    LOG.info(" ================ Computing TF ===================");
    JobTF job1 = new TransformToTFIDF().new JobTF();
    job1.run(EdgeTransformMR.SOURCE, input + "/edata", output + "/temp");
    JobTFIDF job2 = new TransformToTFIDF().new JobTFIDF();
    LOG.info(" ================== Compute TFIDF =======================");
    job2.addUserOpt("NumDocs", numDocs);
    job2.run(EdgeTransformMR.TARGET, output + "/temp", output + "/edata");
    LOG.info("Done");
    LOG.info("Moving vdata from " + input + " to " + output);
    try {
        FileSystem fs = FileSystem.get(new JobConf(TransformToTFIDF.class));
        fs.rename(new Path(input + "/vdata"), new Path(output + "/vdata"));
    } catch (IOException e) {
        e.printStackTrace();
    }
}

---

From source file:com.liferay.hadoop.action.HadoopJob.java

License:Open Source License

public String doExecute(HttpServletRequest request, HttpServletResponse response) throws Exception {

    response.setContentType(ContentTypes.TEXT_PLAIN_UTF8);

    PrintWriter writer = response.getWriter();

    FileSystem fileSystem = HadoopManager.getFileSystem();

    JobClient jobClient = HadoopManager.getJobClient();

    writer.println("-- Job Status --");

    Path inputPath = new Path("/index/*/*");
    Path outputPath = new Path("/wordcount/results");

    try {/*from www  .j a va2s.  co  m*/
        if (_runningJob == null) {
            writer.println("Creating job");

            if (fileSystem.exists(_jobPath)) {
                fileSystem.delete(_jobPath, false);
            }

            if (!fileSystem.exists(_jobPath)) {
                writer.println("Deploying the job code to cluster");

                FSDataOutputStream outputStream = null;

                try {
                    outputStream = fileSystem.create(_jobPath);

                    ServletContext servletContext = HadoopManager.getServletContext();

                    InputStream inputStream = servletContext.getResourceAsStream("/WEB-INF/lib/hadoop-job.jar");

                    StreamUtil.transfer(inputStream, outputStream, false);
                } finally {
                    StreamUtil.cleanUp(outputStream);
                }

                writer.println("Job code deployed to cluster");
            }

            if (fileSystem.exists(outputPath)) {
                writer.println("A previous job output was found, backing it up");

                fileSystem.rename(outputPath,
                        outputPath.getParent().suffix("/.results-" + System.currentTimeMillis()));
            }

            _jobConf = HadoopManager.createNewJobConf();

            _jobConf.setJobName("Word Count");

            writer.println("Job '" + _jobConf.getJobName() + "' is being configured");

            _jobConf.setJarByClass(Map.class);
            _jobConf.setOutputKeyClass(Text.class);
            _jobConf.setOutputValueClass(IntWritable.class);
            _jobConf.setMapperClass(Map.class);
            _jobConf.setCombinerClass(Reduce.class);
            _jobConf.setReducerClass(Reduce.class);
            _jobConf.setInputFormat(TextInputFormat.class);
            _jobConf.setOutputFormat(TextOutputFormat.class);

            writer.println("Job code deployed to distributed cache's classpath");

            DistributedCache.addArchiveToClassPath(_jobPath, _jobConf, fileSystem);

            FileInputFormat.setInputPaths(_jobConf, inputPath);
            FileOutputFormat.setOutputPath(_jobConf, outputPath);

            writer.println("Submitting job the first time");

            _runningJob = jobClient.submitJob(_jobConf);

            writer.println("Job submitted");
        }

        int jobState = _runningJob.getJobState();

        writer.println(
                "Job status: " + jobState + " (RUNNING = 1, SUCCEEDED = 2, FAILED = 3, PREP = 4, KILLED = 5)");

        if ((jobState != JobStatus.RUNNING) && (jobState != JobStatus.PREP)) {

            writer.println("Re-issuing the job");

            if (fileSystem.exists(outputPath)) {
                writer.println("A previous job output was found, backing it up");

                fileSystem.rename(outputPath,
                        outputPath.getParent().suffix("/.results-" + System.currentTimeMillis()));
            }

            writer.println("Submitting job the first time");

            _runningJob = jobClient.submitJob(_jobConf);

            writer.println("Job submitted");
        }
    } catch (Exception ioe) {
        writer.println("Job error: ");

        ioe.printStackTrace(writer);
    }

    writer.flush();
    writer.close();

    return null;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »