Example usage for org.apache.hadoop.mapred FileInputFormat setInputPaths

List of usage examples for org.apache.hadoop.mapred FileInputFormat setInputPaths

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. FileInputFormat
  6. setInputPaths

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred FileInputFormat setInputPaths.

Prototype

public static void setInputPaths(JobConf conf, Path... inputPaths)

Source Link

Document

Set the array of Path s as the list of inputs for the map-reduce job.

Usage

From source file:org.sf.xrime.algorithms.layout.gfr.RepulsiveForceDisp.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf conf = new JobConf(context, RepulsiveForceDisp.class);
    conf.setJobName("RepulsiveForceDisp");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(LabeledAdjSetVertex.class);
    conf.setMapperClass(MapClass.class);
    // No combiner is permitted, since the logic of reducer depends on the completeness
    // of information.
    conf.setReducerClass(ReduceClass.class);
    // makes the file format suitable for machine processing.
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    // Enable compression.
    conf.setCompressMapOutput(true);//www  .  j  av a 2  s.  c  o  m
    conf.setMapOutputCompressorClass(GzipCodec.class);
    try {
        FileInputFormat.setInputPaths(conf, getSource().getPath());
        FileOutputFormat.setOutputPath(conf, getDestination().getPath());
    } catch (IllegalAccessException e1) {
        throw new ProcessorExecutionException(e1);
    }
    conf.setNumMapTasks(getMapperNum());
    conf.setNumReduceTasks(getReducerNum());

    try {
        this.runningJob = JobClient.runJob(conf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.algorithms.layout.radialtree.RadialTreeStep1.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    try {//from  w  w w  .  jav  a  2  s .  c om
        context.setParameter(continueFileKey, continueFlagFile());

        jobConf = new JobConf(context, RadialTreeStep1.class);
        jobConf.setJobName("RadialTreeVisit");

        FileInputFormat.setInputPaths(jobConf, context.getSource().getPath());

        // Using RadialTreeVisitMapper.class and RadialTreeVisitReducer.class to
        // visit all the graph and get the topoolgy.
        jobConf.setInputFormat(SequenceFileInputFormat.class);
        jobConf.setMapperClass(RadialTreeVisitMapper.class);
        jobConf.setNumMapTasks(getMapperNum());
        jobConf.setMapOutputValueClass(LabeledAdjVertex.class);

        jobConf.setReducerClass(RadialTreeVisitReducer.class);
        jobConf.setNumReduceTasks(getReducerNum());

        jobConf.setOutputKeyClass(Text.class);
        jobConf.setOutputValueClass(LabeledAdjVertex.class);

        FileOutputFormat.setOutputPath(jobConf, context.getDestination().getPath());
        jobConf.setOutputFormat(SequenceFileOutputFormat.class);

        this.runningJob = JobClient.runJob(jobConf);

        if (client == null) {
            client = FileSystem.get(jobConf);
        }
        if (client.exists(new Path(continueFlagFile()))) {
            end = false;
            client.delete(new Path(continueFlagFile()), true);
        } else {
            end = true;
        }
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    }
}

From source file:org.sf.xrime.algorithms.layout.radialtree.RadialTreeStep2.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    try {/*from   w w  w . j av a  2s  .  c o  m*/
        context.setParameter(continueFileKeyWeight, continueFlagFile());

        jobConf = new JobConf(context, RadialTreeStep2.class);
        jobConf.setJobName("RadialTreeWeight");

        FileInputFormat.setInputPaths(jobConf, context.getSource().getPath());

        // Using RadialTreeWeightMapper.class and RadialTreeWeightReducer.class to
        // caculate the every vertex's weight.
        jobConf.setInputFormat(SequenceFileInputFormat.class);
        jobConf.setMapperClass(RadialTreeWeightMapper.class);
        jobConf.setNumMapTasks(getMapperNum());
        jobConf.setMapOutputValueClass(LabeledAdjVertex.class);

        jobConf.setReducerClass(RadialTreeWeightReducer.class);
        jobConf.setNumReduceTasks(getReducerNum());

        jobConf.setOutputKeyClass(Text.class);
        jobConf.setOutputValueClass(LabeledAdjVertex.class);

        FileOutputFormat.setOutputPath(jobConf, context.getDestination().getPath());
        jobConf.setOutputFormat(SequenceFileOutputFormat.class);

        this.runningJob = JobClient.runJob(jobConf);

        if (client == null) {
            client = FileSystem.get(jobConf);
        }
        if (client.exists(new Path(continueFlagFile()))) {
            end = false;
            client.delete(new Path(continueFlagFile()), true);
        } else {
            end = true;
        }
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    }
}

From source file:org.sf.xrime.algorithms.layout.radialtree.RadialTreeStep3.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    try {/*from   ww  w.  j a  v a  2 s .  c o  m*/
        context.setParameter(continueFileKeyWeight, continueFlagFile());

        jobConf = new JobConf(context, RadialTreeStep3.class);
        jobConf.setJobName("RadialTreeLocate");

        FileInputFormat.setInputPaths(jobConf, context.getSource().getPath());

        // Using RadialTreeLocateMapper.class and RadialTreeLocateReducer.class
        // to caculate vertex coordinates.
        jobConf.setInputFormat(SequenceFileInputFormat.class);
        jobConf.setMapperClass(RadialTreeLocateMapper.class);
        jobConf.setNumMapTasks(getMapperNum());
        jobConf.setMapOutputValueClass(LabeledAdjVertex.class);

        jobConf.setReducerClass(RadialTreeLocateReducer.class);
        jobConf.setNumReduceTasks(getReducerNum());

        jobConf.setOutputKeyClass(Text.class);
        jobConf.setOutputValueClass(LabeledAdjVertex.class);

        FileOutputFormat.setOutputPath(jobConf, context.getDestination().getPath());
        jobConf.setOutputFormat(SequenceFileOutputFormat.class);

        this.runningJob = JobClient.runJob(jobConf);

        if (client == null) {
            client = FileSystem.get(jobConf);
        }
        if (client.exists(new Path(continueFlagFile()))) {
            end = false;
            client.delete(new Path(continueFlagFile()), true);
        } else {
            end = true;
        }
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    }
}

From source file:org.sf.xrime.algorithms.layout.radialtree.Raw2OutLabeledAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2OutLabeledAdjVertexTransformer.class);
    jobConf.setJobName("Smth - OutLabeledAdjVertexesList");

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(LabeledAdjVertex.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);// www  .  j a v a2s  .co  m
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.algorithms.MST.MSTStep.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    try {//from  w  w  w .  j  a v a 2s  . co m
        /**
         * Set related properties for MST job
         */
        context.setParameter(terminateFileKey, terminateFlagFile());
        context.setParameter(MSTStep.autoWakeUpListKey, autoWakeUpList);

        jobConf = new JobConf(context, MSTStep.class);
        jobConf.setJobName("MST");

        FileInputFormat.setInputPaths(jobConf, context.getSource().getPath());

        jobConf.setInputFormat(SequenceFileInputFormat.class);
        if (!isEnd())
            jobConf.setMapperClass(MSTMapper.class);
        else
            jobConf.setMapperClass(MSTFinalMapper.class);

        jobConf.setNumMapTasks(getMapperNum());
        jobConf.setMapOutputValueClass(LabeledAdjVertex.class);

        if (!isEnd())
            jobConf.setReducerClass(MSTReducer.class);
        else
            jobConf.setReducerClass(MSTFinalReducer.class);

        jobConf.setNumReduceTasks(getReducerNum());

        jobConf.setOutputKeyClass(Text.class);
        jobConf.setOutputValueClass(LabeledAdjVertex.class);

        FileOutputFormat.setOutputPath(jobConf, context.getDestination().getPath());
        jobConf.setOutputFormat(SequenceFileOutputFormat.class);

        this.runningJob = JobClient.runJob(jobConf);

        if (client == null) {
            client = FileSystem.get(jobConf);
        }
        if (client.exists(new Path(terminateFlagFile()))) {
            end = true;
        } else {
            end = false;
        }
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    }
}

From source file:org.sf.xrime.algorithms.pagerank.PageRankStep.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    try {// w  w  w  . j av a2s .  c o  m
        if (server == null) {
            startServer(); // start the server
        }
        resetZeroOutDegreeVertexInfo(); // reset zeroOutDegreeVertexRank &
                                        // zeroOutDegreeVertexCount

        // {{ algorithm properties
        context.setParameter(continueFileKey, continueFlagFile());
        context.setParameter(rankCollectorHost, ipAddress.getHostName());
        context.setParameter(rankCollectorPort, Integer.toString(ipPort));
        context.setParameter(PageRankAlgorithm.pageRankDampingFactorKey, Double.toString(dampingFactor));
        context.setParameter(PageRankAlgorithm.pageRankStopThresholdKey, Double.toString(stopThreshold));
        // }} algorithm properties

        // {{ Mapper, Reducer configuration
        jobConf = new JobConf(context, PageRankStep.class);
        jobConf.setJobName("BFS");

        FileInputFormat.setInputPaths(jobConf, context.getSource().getPath());

        jobConf.setInputFormat(SequenceFileInputFormat.class);
        jobConf.setMapperClass(PageRankMapper.class);
        jobConf.setNumMapTasks(getMapperNum());
        jobConf.setMapOutputValueClass(ObjectWritable.class);

        jobConf.setReducerClass(PageRankReducer.class);
        jobConf.setNumReduceTasks(getReducerNum());

        jobConf.setOutputKeyClass(Text.class);
        jobConf.setOutputValueClass(LabeledAdjSetVertex.class);

        FileOutputFormat.setOutputPath(jobConf, context.getDestination().getPath());
        jobConf.setOutputFormat(SequenceFileOutputFormat.class);
        // }} Mapper, Reducer configuration

        this.runningJob = JobClient.runJob(jobConf);

        // {{ end indicator
        if (client == null) {
            client = FileSystem.get(jobConf);
        }

        if (client.exists(new Path(continueFlagFile()))) {
            end = false;
            client.delete(new Path(continueFlagFile()), true);
        } else {
            end = true;
        }
        // }} end indicator

        // {{ correction
        if (zeroOutDegreeVertexRank > 0) {
            SequenceTempDirMgr dirmgr = new SequenceTempDirMgr(context.getDestination().getPath().toString());
            Path tmpFile = dirmgr.getTempDir();

            // prop.setProperty(PageRankAlgorithm.pageRankVertexAdjKey,
            // Double.toString(dampingFactor*zeroOutDegreeVertexRank/vertexNumber));
            context.setParameter(PageRankAlgorithm.pageRankVertexAdjKey,
                    Double.toString(dampingFactor * zeroOutDegreeVertexRank));
            System.out.println(
                    "dampingFactor*zeroOutDegreeVertexRank: " + dampingFactor * zeroOutDegreeVertexRank);

            // {{ Mapper, Reducer configuration
            jobConf = new JobConf(context, PageRankStep.class);
            jobConf.setJobName("BFS");

            FileInputFormat.setInputPaths(jobConf, context.getDestination().getPath());

            jobConf.setInputFormat(SequenceFileInputFormat.class);
            jobConf.setMapperClass(PageRankCorrectionMapper.class);
            jobConf.setNumMapTasks(getMapperNum());
            jobConf.setNumReduceTasks(0); // no reducer here

            jobConf.setOutputKeyClass(Text.class);
            jobConf.setOutputValueClass(LabeledAdjSetVertex.class);

            FileOutputFormat.setOutputPath(jobConf, tmpFile);
            jobConf.setOutputFormat(SequenceFileOutputFormat.class);
            // }} Mapper, Reducer configuration

            this.runningJob = JobClient.runJob(jobConf);

            // {{End? We need additional check here since PageRankCorrectionMapper
            // will generate new PR value.
            if (client.exists(new Path(continueFlagFile()))) {
                end = false;
                client.delete(new Path(continueFlagFile()), true);
            } else {
                end = true;
            }
            // }}End? We need additional check here since PageRankCorrectionMapper
            // will generate new PR value.

            client.delete(context.getDestination().getPath(), true);
            client.rename(tmpFile, context.getDestination().getPath());
        }
        // }} correction
    } catch (IOException e) {
        end = true;
        throw new ProcessorExecutionException(e);
    } catch (IllegalAccessException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.algorithms.partitions.connected.bi.EdgeSetExpand.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf conf = new JobConf(context, EdgeSetExpand.class);
    conf.setJobName("EdgeSetExpand");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(EdgeSet.class);
    conf.setMapperClass(MapClass.class);
    // No combiner is permitted. Or we may get wrong with labeling.
    conf.setReducerClass(ReduceClass.class);
    // makes the file format suitable for machine processing.
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    // Two input pathes (link to lower and upper layers), one output pathes.
    try {/*from  w  ww .ja v  a2  s  .co m*/
        Path[] input_paths = new Path[2];
        FileInputFormat.setInputPaths(conf, getSource().getPaths().toArray(input_paths));
        FileOutputFormat.setOutputPath(conf, getDestination().getPath());
    } catch (IllegalAccessException e1) {
        throw new ProcessorExecutionException(e1);
    }
    conf.setNumMapTasks(getMapperNum());
    conf.setNumReduceTasks(getReducerNum());
    conf.setMapOutputCompressorClass(GzipCodec.class);
    conf.setCompressMapOutput(true);

    try {
        this.runningJob = JobClient.runJob(conf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.algorithms.partitions.connected.bi.EdgeSetJoin.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf conf = new JobConf(context, EdgeSetJoin.class);
    conf.setJobName("EdgeSetJoin");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(EdgeSet.class);
    conf.setMapperClass(MapClass.class);
    // Since this is a join operation, combiner is not permitted here.
    conf.setReducerClass(ReduceClass.class);
    // makes the file format suitable for machine processing.
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    try {//from   w ww.j  a  v  a 2s  . co m
        FileInputFormat.setInputPaths(conf, getSource().getPath());
        FileOutputFormat.setOutputPath(conf, getDestination().getPath());
    } catch (IllegalAccessException e1) {
        throw new ProcessorExecutionException(e1);
    }
    conf.setNumMapTasks(getMapperNum());
    conf.setNumReduceTasks(getReducerNum());
    conf.setMapOutputCompressorClass(GzipCodec.class);
    conf.setCompressMapOutput(true);

    try {
        this.runningJob = JobClient.runJob(conf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.algorithms.partitions.connected.bi.EdgeSetMinorExpand.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf conf = new JobConf(context, EdgeSetMinorExpand.class);
    conf.setJobName("EdgeSetMinorExpand");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(EdgeSet.class);
    conf.setMapperClass(MapClass.class);
    conf.setCombinerClass(ReduceClass.class);
    conf.setReducerClass(ReduceClass.class);
    // makes the file format suitable for machine processing.
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    try {//from w ww . j a  va  2s  . co  m
        FileInputFormat.setInputPaths(conf, getSource().getPath());
        FileOutputFormat.setOutputPath(conf, getDestination().getPath());
    } catch (IllegalAccessException e1) {
        throw new ProcessorExecutionException(e1);
    }
    conf.setNumMapTasks(getMapperNum());
    conf.setNumReduceTasks(getReducerNum());
    conf.setCompressMapOutput(true);
    conf.setMapOutputCompressorClass(GzipCodec.class);

    try {
        this.runningJob = JobClient.runJob(conf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}