Example usage for org.apache.hadoop.mapreduce Job getConfiguration

List of usage examples for org.apache.hadoop.mapreduce Job getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getConfiguration.

Prototype

public Configuration getConfiguration()

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.inmobi.conduit.distcp.tools.DistCp.java

License:Apache License

/**
 * A hack to get the DistCp job id. New Job API doesn't return the job id
 *
 * @param job - Handle to job//w ww .  j a v a2  s. c  o m
 * @return JobID
 */
private String getJobID(Job job) {
    return new Path(job.getConfiguration().get("mapreduce.job.dir")).getName();
}

From source file:com.inmobi.conduit.distcp.tools.DistCp.java

License:Apache License

/**
 * Setup output format appropriately/*from www.  j  a  v  a  2s  .  c  o  m*/
 *
 * @param job - Job handle
 * @throws IOException - Exception if any
 */
private void configureOutputFormat(Job job) throws IOException {
    final Configuration configuration = job.getConfiguration();
    Path targetPath = inputOptions.getTargetPath();
    targetPath = targetPath.makeQualified(targetPath.getFileSystem(configuration));

    if (inputOptions.shouldAtomicCommit()) {
        Path workDir = inputOptions.getAtomicWorkPath();
        if (workDir == null) {
            workDir = targetPath.getParent();
        }
        workDir = new Path(workDir, WIP_PREFIX + targetPath.getName() + rand.nextInt());
        FileSystem workFS = workDir.getFileSystem(configuration);
        FileSystem targetFS = targetPath.getFileSystem(configuration);
        if (!DistCpUtils.compareFs(targetFS, workFS)) {
            throw new IllegalArgumentException("Work path " + workDir + " and target path " + targetPath
                    + " are in different file system");
        }
        CopyOutputFormat.setWorkingDirectory(job, workDir);
    } else {
        CopyOutputFormat.setWorkingDirectory(job, targetPath);
    }
    CopyOutputFormat.setCommitDirectory(job, targetPath);

    Path counterFilePath = inputOptions.getOutPutDirectory();
    if (counterFilePath == null) {
        LOG.error("Output directory is null for distcp");
    } else {
        LOG.info("DistCp output directory path: " + counterFilePath);
        CopyOutputFormat.setOutputPath(job, counterFilePath);
    }

}

From source file:com.inmobi.conduit.distcp.tools.DistCp.java

License:Apache License

/**
 * Create input listing by invoking an appropriate copy listing
 * implementation. Also add delegation tokens for each path
 * to job's credential store/*from  ww  w  .  jav  a 2  s . com*/
 *
 * @param job - Handle to job
 * @return Returns the path where the copy listing is created
 * @throws IOException - If any
 */
protected Path createInputFileListing(Job job) throws IOException {
    Path fileListingPath = getFileListingPath();
    CopyListing copyListing = CopyListing.getCopyListing(job.getConfiguration(), job.getCredentials(),
            inputOptions);
    copyListing.buildListing(fileListingPath, inputOptions);
    LOG.info("Number of paths considered for copy: " + copyListing.getNumberOfPaths());
    LOG.info("Number of bytes considered for copy: " + copyListing.getBytesToCopy()
            + " (Actual number of bytes copied depends on whether any files are " + "skipped or overwritten.)");
    return fileListingPath;
}

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyOutputFormat.java

License:Apache License

public static void setWorkingDirectory(Job job, Path workingDirectory) {
    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, workingDirectory.toString());
}

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyOutputFormat.java

License:Apache License

public static void setCommitDirectory(Job job, Path commitDirectory) {
    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, commitDirectory.toString());
}

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyOutputFormat.java

License:Apache License

public static Path getWorkingDirectory(Job job) {
    return getWorkingDirectory(job.getConfiguration());
}

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyOutputFormat.java

License:Apache License

public static Path getCommitDirectory(Job job) {
    return getCommitDirectory(job.getConfiguration());
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyOutputFormat.java

License:Apache License

@Test
public void testSetCommitDirectory() {
    try {//  w  w w  . j a v a2  s.c  o  m
        Job job = new Job(new Configuration());
        Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

        job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, "");
        Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

        Path directory = new Path("/tmp/test");
        CopyOutputFormat.setCommitDirectory(job, directory);
        Assert.assertEquals(directory, CopyOutputFormat.getCommitDirectory(job));
        Assert.assertEquals(directory.toString(),
                job.getConfiguration().get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
    } catch (IOException e) {
        LOG.error("Exception encountered while running test", e);
        Assert.fail("Failed while testing for set Commit Directory");
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyOutputFormat.java

License:Apache License

@Test
public void testSetWorkingDirectory() {
    try {//from   www .j  a v a 2s  .  co  m
        Job job = new Job(new Configuration());
        Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

        job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
        Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

        Path directory = new Path("/tmp/test");
        CopyOutputFormat.setWorkingDirectory(job, directory);
        Assert.assertEquals(directory, CopyOutputFormat.getWorkingDirectory(job));
        Assert.assertEquals(directory.toString(),
                job.getConfiguration().get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
    } catch (IOException e) {
        LOG.error("Exception encountered while running test", e);
        Assert.fail("Failed while testing for set Working Directory");
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyOutputFormat.java

License:Apache License

@Test
public void testCheckOutputSpecs() {
    try {//from  w w  w. ja v  a2  s. co m
        OutputFormat outputFormat = new CopyOutputFormat();
        Configuration conf = new Configuration();
        Job job = new Job(conf);
        JobID jobID = new JobID("200707121733", 1);

        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
            Assert.fail("No checking for invalid work/commit path");
        } catch (IllegalStateException ignore) {
        }

        CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
            Assert.fail("No checking for invalid commit path");
        } catch (IllegalStateException ignore) {
        }

        job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
        CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));
        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
            Assert.fail("No checking for invalid work path");
        } catch (IllegalStateException ignore) {
        }

        CopyOutputFormat.setWorkingDirectory(job, new Path("/tmp/work"));
        CopyOutputFormat.setCommitDirectory(job, new Path("/tmp/commit"));
        try {
            JobContext context = Mockito.mock(JobContext.class);
            Mockito.when(context.getConfiguration()).thenReturn(job.getConfiguration());
            Mockito.when(context.getJobID()).thenReturn(jobID);
            outputFormat.checkOutputSpecs(context);
        } catch (IllegalStateException ignore) {
            ignore.printStackTrace();
            Assert.fail("Output spec check failed.");
        }

    } catch (IOException e) {
        LOG.error("Exception encountered while testing checkoutput specs", e);
        Assert.fail("Checkoutput Spec failure");
    } catch (InterruptedException e) {
        LOG.error("Exception encountered while testing checkoutput specs", e);
        Assert.fail("Checkoutput Spec failure");
    }
}