Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:cascading.tap.hadoop.io.StreamedFileSystem.java

License:Open Source License

public static String getMD5SumFor(Configuration conf, String path) {
    return conf.get(path + ".md5");
}

---

From source file:cascading.tap.hadoop.util.Hadoop18TapUtil.java

License:Open Source License

public static synchronized void setupTask(Configuration conf) throws IOException {
    String workpath = conf.get("mapred.work.output.dir");

    if (workpath == null)
        return;/*from  www .j av a2 s  .  c o m*/

    FileSystem fs = getFSSafe(conf, new Path(workpath));

    if (fs == null)
        return;

    String taskId = conf.get("mapred.task.id", conf.get("mapreduce.task.id"));

    LOG.info("setting up task: '{}' - {}", taskId, workpath);

    AtomicInteger integer = pathCounts.get(workpath);

    if (integer == null) {
        integer = new AtomicInteger();
        pathCounts.put(workpath, integer);
    }

    integer.incrementAndGet();
}

---

From source file:cascading.tap.hadoop.util.Hadoop18TapUtil.java

License:Open Source License

public static boolean needsTaskCommit(Configuration conf) throws IOException {
    String workpath = conf.get("mapred.work.output.dir");

    if (workpath == null)
        return false;

    Path taskOutputPath = new Path(workpath);

    if (taskOutputPath != null) {
        FileSystem fs = getFSSafe(conf, taskOutputPath);

        if (fs == null)
            return false;

        if (fs.exists(taskOutputPath))
            return true;
    }/* www. j a  v  a2 s .  c o m*/

    return false;
}

---

From source file:cascading.tap.hadoop.util.Hadoop18TapUtil.java

License:Open Source License

/**
 * copies all files from the taskoutputpath to the outputpath
 *
 * @param conf//from   w  w  w.  j a  v  a  2  s . c  o  m
 */
public static void commitTask(Configuration conf) throws IOException {
    Path taskOutputPath = new Path(conf.get("mapred.work.output.dir"));

    FileSystem fs = getFSSafe(conf, taskOutputPath);

    if (fs == null)
        return;

    AtomicInteger integer = pathCounts.get(taskOutputPath.toString());

    if (integer.decrementAndGet() != 0)
        return;

    String taskId = conf.get("mapred.task.id", conf.get("mapreduce.task.id"));

    LOG.info("committing task: '{}' - {}", taskId, taskOutputPath);

    if (taskOutputPath != null) {
        if (writeDirectlyToWorkingPath(conf, taskOutputPath))
            return;

        if (fs.exists(taskOutputPath)) {
            Path jobOutputPath = taskOutputPath.getParent().getParent();
            // Move the task outputs to their final place
            moveTaskOutputs(conf, fs, jobOutputPath, taskOutputPath);

            // Delete the temporary task-specific output directory
            if (!fs.delete(taskOutputPath, true))
                LOG.info("failed to delete the temporary output directory of task: '{}' - {}", taskId,
                        taskOutputPath);

            LOG.info("saved output of task '{}' to {}", taskId, jobOutputPath);
        }
    }
}

---

From source file:cascading.tuple.hadoop.util.DeserializerComparator.java

License:Open Source License

public static Comparator[] getFieldComparatorsFrom(Configuration conf, String name) {
    String value = conf.get(name);

    if (value == null)
        return new Comparator[conf.getInt(name + ".size", 1)];

    try {/*from  ww w.  j  a  v  a  2  s.c o  m*/
        return HadoopUtil.deserializeBase64(value, conf, Fields.class).getComparators();
    } catch (IOException exception) {
        throw new CascadingException("unable to deserialize comparators for: " + name);
    }
}

---

From source file:cgl.hadoop.apps.runner.RunnerMap.java

License:Open Source License

@Override
public void setup(Context context) throws IOException {
    Configuration conf = context.getConfiguration();
    Path[] local = DistributedCache.getLocalCacheArchives(conf);

    localBlastProgram = local[0].toUri().getPath();
    localDB = local[0].toUri().getPath() + File.separator + conf.get(DataAnalysis.DB_ARCHIVE) + File.separator
            + conf.get(DataAnalysis.DB_NAME);

    /** Write your code here
        get two absolute filepath for localDB and localBlastBinary
    **//*from   w w w. j ava 2s  . com*/
}

---

From source file:cgl.hadoop.apps.runner.RunnerMap.java

License:Open Source License

public void map(String key, String value, Context context) throws IOException, InterruptedException {

    long startTime = System.currentTimeMillis();
    String endTime = "";

    Configuration conf = context.getConfiguration();
    String programDir = conf.get(DataAnalysis.PROGRAM_DIR);
    String execName = conf.get(DataAnalysis.EXECUTABLE);
    String cmdArgs = conf.get(DataAnalysis.PARAMETERS);
    String outputDir = conf.get(DataAnalysis.OUTPUT_DIR);
    String workingDir = conf.get(DataAnalysis.WORKING_DIR);

    System.out.println("the map key : " + key);
    System.out.println("the value path : " + value.toString());
    System.out.println("Local DB : " + this.localDB);

    // We have the full file names in the value.
    String localInputFile = "";
    String outFile = "";
    String stdOutFile = "";
    String stdErrFile = "";
    String fileNameOnly = "";

    fileNameOnly = key;//from  w  w w. j av a2 s.  co m
    localInputFile = workingDir + File.separator + fileNameOnly;
    outFile = workingDir + File.separator + fileNameOnly + ".output";
    stdErrFile = workingDir + File.separator + fileNameOnly + ".error";
    stdOutFile = workingDir + File.separator + fileNameOnly + ".input";
    /**
    Write your code to get localInputFile, outFile,
    stdOutFile and stdErrFile
    **/

    // download the file from HDFS
    Path inputFilePath = new Path(value);
    FileSystem fs = inputFilePath.getFileSystem(conf);
    fs.copyToLocalFile(inputFilePath, new Path(localInputFile));

    // Prepare the arguments to the executable
    String execCommand = cmdArgs.replaceAll("#_INPUTFILE_#", localInputFile);
    if (cmdArgs.indexOf("#_OUTPUTFILE_#") > -1) {
        execCommand = execCommand.replaceAll("#_OUTPUTFILE_#", outFile);
    } else {
        outFile = stdOutFile;
    }

    endTime = Double.toString(((System.currentTimeMillis() - startTime) / 1000.0));
    System.out.println("Before running the executable Finished in " + endTime + " seconds");

    execCommand = this.localBlastProgram + File.separator + execName + " " + execCommand + " -db "
            + this.localDB;
    //Create the external process

    startTime = System.currentTimeMillis();

    Process p = Runtime.getRuntime().exec(execCommand);

    OutputHandler inputStream = new OutputHandler(p.getInputStream(), "INPUT", stdOutFile);
    OutputHandler errorStream = new OutputHandler(p.getErrorStream(), "ERROR", stdErrFile);

    // start the stream threads.
    inputStream.start();
    errorStream.start();

    p.waitFor();
    //end time of this procress
    endTime = Double.toString(((System.currentTimeMillis() - startTime) / 1000.0));
    System.out.println("Program Finished in " + endTime + " seconds");

    //Upload the results to HDFS
    startTime = System.currentTimeMillis();

    Path outputDirPath = new Path(outputDir);
    Path outputFileName = new Path(outputDirPath, fileNameOnly);
    fs.copyFromLocalFile(new Path(outFile), outputFileName);

    endTime = Double.toString(((System.currentTimeMillis() - startTime) / 1000.0));
    System.out.println("Upload Result Finished in " + endTime + " seconds");

}

---

From source file:cgl.hadoop.apps.runner.RunnerMap.java

License:Open Source License

@Override
public void setup(Context context) throws IOException {
    Configuration conf = context.getConfiguration();
    Path[] local = DistributedCache.getLocalCacheArchives(conf);

    //Custom Code
    this.localDB = local[0].toUri().getPath() + File.separator + conf.get(DataAnalysis.DB_ARCHIVE)
            + File.separator + conf.get(DataAnalysis.DB_NAME);
    this.localBlastProgram = local[0].toUri().getPath();
}

---

From source file:cgl.hadoop.apps.runner.RunnerMap.java

License:Open Source License

public void map(String key, String value, Context context) throws IOException, InterruptedException {

    long startTime = System.currentTimeMillis();
    String endTime = "";

    Configuration conf = context.getConfiguration();
    String programDir = conf.get(DataAnalysis.PROGRAM_DIR);
    String execName = conf.get(DataAnalysis.EXECUTABLE);
    String cmdArgs = conf.get(DataAnalysis.PARAMETERS);
    String outputDir = conf.get(DataAnalysis.OUTPUT_DIR);
    String workingDir = conf.get(DataAnalysis.WORKING_DIR);

    System.out.println("the map key : " + key);
    System.out.println("the value path : " + value.toString());
    System.out.println("Local DB : " + this.localDB);

    // We have the full file names in the value.
    String localInputFile = "";
    String outFile = "";
    String stdOutFile = "";
    String stdErrFile = "";
    String fileNameOnly = "";

    //Custom code
    String[] tmp = value.split(File.separator);
    fileNameOnly = tmp[tmp.length - 1];/*from   w w w .  j a v  a  2s  .co  m*/
    localInputFile = workingDir + File.separator + fileNameOnly;
    outFile = workingDir + File.separator + fileNameOnly + ".out";
    stdOutFile = workingDir + File.separator + fileNameOnly + ".stdout";
    stdErrFile = workingDir + File.separator + fileNameOnly + ".stderr";

    // download the file from HDFS
    Path inputFilePath = new Path(value);
    FileSystem fs = inputFilePath.getFileSystem(conf);
    fs.copyToLocalFile(inputFilePath, new Path(localInputFile));

    // Prepare the arguments to the executable
    String execCommand = cmdArgs.replaceAll("#_INPUTFILE_#", localInputFile);
    if (cmdArgs.indexOf("#_OUTPUTFILE_#") > -1) {
        execCommand = execCommand.replaceAll("#_OUTPUTFILE_#", outFile);
    } else {
        outFile = stdOutFile;
    }

    endTime = Double.toString(((System.currentTimeMillis() - startTime) / 1000.0));
    System.out.println("Before running the executable Finished in " + endTime + " seconds");

    execCommand = this.localBlastProgram + File.separator + execName + " " + execCommand + " -db "
            + this.localDB;
    //Create the external process

    startTime = System.currentTimeMillis();

    Process p = Runtime.getRuntime().exec(execCommand);

    OutputHandler inputStream = new OutputHandler(p.getInputStream(), "INPUT", stdOutFile);
    OutputHandler errorStream = new OutputHandler(p.getErrorStream(), "ERROR", stdErrFile);

    // start the stream threads.
    inputStream.start();
    errorStream.start();

    p.waitFor();
    //end time of this procress
    endTime = Double.toString(((System.currentTimeMillis() - startTime) / 1000.0));
    System.out.println("Program Finished in " + endTime + " seconds");

    //Upload the results to HDFS
    startTime = System.currentTimeMillis();

    Path outputDirPath = new Path(outputDir);
    Path outputFileName = new Path(outputDirPath, fileNameOnly);
    fs.copyFromLocalFile(new Path(outFile), outputFileName);

    endTime = Double.toString(((System.currentTimeMillis() - startTime) / 1000.0));
    System.out.println("Upload Result Finished in " + endTime + " seconds");

}

---

From source file:cloudbase.core.client.mapreduce.CloudbaseOutputFormatShim.java

License:Apache License

protected static Instance getInstance(JobContext job) {
    Configuration conf = job.getConfiguration();
    if (isMock(conf)) {
        return new MockInstanceShim(conf.get(INSTANCE_NAME));
    }/*from   w w w  .j a va  2 s . c om*/
    return CloudbaseOutputFormat.getInstance(job);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »