Example usage for org.apache.hadoop.conf Configuration set

List of usage examples for org.apache.hadoop.conf Configuration set

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:com.cg.mapreduce.fpgrowth.mahout.fpm.PFPGrowth.java

License:Apache License

/**
 * Count the frequencies of various features in parallel using Map/Reduce
 *///from ww w . j a v a 2  s . c om
public static void startParallelCounting(Parameters params, Configuration conf)
        throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(PFP_PARAMETERS, params.toString());

    conf.set("mapred.compress.map.output", "true");
    conf.set("mapred.output.compression.type", "BLOCK");

    String input = params.get(INPUT);
    Job job = new Job(conf, "Parallel Counting Driver running over input: " + input);
    job.setJarByClass(PFPGrowth.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    FileInputFormat.addInputPath(job, new Path(input));
    Path outPath = new Path(params.get(OUTPUT), PARALLEL_COUNTING);
    FileOutputFormat.setOutputPath(job, outPath);

    HadoopUtil.delete(conf, outPath);

    job.setInputFormatClass(TextInputFormat.class);
    job.setMapperClass(ParallelCountingMapper.class);
    job.setCombinerClass(ParallelCountingReducer.class);
    job.setReducerClass(ParallelCountingReducer.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    boolean succeeded = job.waitForCompletion(true);
    if (!succeeded) {
        throw new IllegalStateException("Job failed!");
    }

}

From source file:com.cg.mapreduce.fpgrowth.mahout.fpm.PFPGrowth.java

License:Apache License

/**
 * Run the Parallel FPGrowth Map/Reduce Job to calculate the Top K features of group dependent shards
 *//*from  w w  w  . j  av  a 2  s  . c om*/
public static void startParallelFPGrowth(Parameters params, Configuration conf)
        throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(PFP_PARAMETERS, params.toString());
    conf.set("mapred.compress.map.output", "true");
    conf.set("mapred.output.compression.type", "BLOCK");
    Path input = new Path(params.get(INPUT));
    Job job = new Job(conf, "PFP Growth Driver running over input" + input);
    job.setJarByClass(PFPGrowth.class);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(TransactionTree.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(TopKStringPatterns.class);

    FileInputFormat.addInputPath(job, input);
    Path outPath = new Path(params.get(OUTPUT), FPGROWTH);
    FileOutputFormat.setOutputPath(job, outPath);

    HadoopUtil.delete(conf, outPath);

    job.setInputFormatClass(TextInputFormat.class);
    job.setMapperClass(ParallelFPGrowthMapper.class);
    job.setCombinerClass(ParallelFPGrowthCombiner.class);
    job.setReducerClass(ParallelFPGrowthReducer.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    boolean succeeded = job.waitForCompletion(true);
    if (!succeeded) {
        throw new IllegalStateException("Job failed!");
    }
}

From source file:com.cg.mapreduce.myfpgrowth.PFPGrowth.java

License:Apache License

/**
 * @throws ClassNotFoundException /* w  w w  .  j  a v  a 2s . c  o m*/
* @throws InterruptedException 
* @throws IOException 
* @params
 *    input, output locations, additional parameters like minSupport(3), maxHeapSize(50), numGroups(1000)
 * @conf
 *    initial Hadoop configuration to use.
 * 
 * */
public static void runPFPGrowth(Parameters params)
        throws IOException, InterruptedException, ClassNotFoundException {

    Configuration conf = new Configuration();
    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
            + "org.apache.hadoop.io.serializer.WritableSerialization");
    startParallelCounting(params, conf);

    genGroupList(params, conf);

    startParallelFPGrowth(params, conf);
}

From source file:com.cg.mapreduce.myfpgrowth.PFPGrowth.java

License:Apache License

/**
 * Count the frequencies of various features in parallel using Map/Reduce
 *//*from w w  w  .  j av  a 2  s . c o m*/
public static void startParallelCounting(Parameters params, Configuration conf)
        throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(PFP_PARAMETERS, params.toString());
    conf.set("mapred.compress.map.output", "true");
    conf.set("mapred.output.compression.type", "BLOCK");

    String input = params.get(INPUT);
    Job job = new Job(conf, "Parallel Counting Driver running over input: " + input);
    job.setJarByClass(PFPGrowth.class);

    //    Job job = initJob(conf);  
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    FileInputFormat.addInputPath(job, new Path(input));
    Path outPath = new Path(params.get(OUTPUT), PARALLEL_COUNTING);
    FileOutputFormat.setOutputPath(job, outPath);

    HadoopUtil.delete(conf, outPath);

    job.setInputFormatClass(TextInputFormat.class);
    job.setMapperClass(ParallelCountingMapper.class);
    job.setCombinerClass(ParallelCountingReducer.class);
    job.setReducerClass(ParallelCountingReducer.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    boolean succeeded = job.waitForCompletion(true);
    if (!succeeded) {
        throw new IllegalStateException("Job failed!");
    }

}

From source file:com.cg.mapreduce.myfpgrowth.PFPGrowth.java

License:Apache License

/**
 * Run the Parallel FPGrowth Map/Reduce Job to calculate the Top K features of group dependent shards
 *///  w w w.  j a  va  2 s.co  m
public static void startParallelFPGrowth(Parameters params, Configuration conf)
        throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(PFP_PARAMETERS, params.toString());
    conf.set("mapred.compress.map.output", "true");
    conf.set("mapred.output.compression.type", "BLOCK");

    Path input = new Path(params.get(INPUT));
    Job job = new Job(conf, "PFP Growth Driver running over input" + input);
    job.setJarByClass(PFPGrowth.class);
    //    Job job = initJob(conf);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(ArrayList.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    FileInputFormat.addInputPath(job, input);
    Path outPath = new Path(params.get(OUTPUT), FPGROWTH);
    FileOutputFormat.setOutputPath(job, outPath);

    HadoopUtil.delete(conf, outPath);

    job.setInputFormatClass(TextInputFormat.class);
    job.setMapperClass(ParallelFPGrowthMapper.class);
    //job.setCombinerClass(ParallelFPGrowthCombiner.class);
    job.setReducerClass(ParallelFPGrowthReducer.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    boolean succeeded = job.waitForCompletion(true);
    if (!succeeded) {
        throw new IllegalStateException("Job failed!");
    }
}

From source file:com.cg.mapreduce.myfpgrowth.PFPGrowth.java

License:Apache License

private static Job initJob(Configuration conf) {
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/core-site.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/hdfs-default.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/hdfs-site.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/yarn-default.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/yarn-site.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/mapred-site.xml"));
    conf.set("HADOOP_USER_NAME", "hadoop");
    conf.set("mapred.reduce.tasks", "3");
    Job job = null;//from  w  w  w.j  a v  a  2  s . com
    try {
        File jarFile = EJob.createTempJar("bin");
        EJob.addClasspath("D:/program/hadoop-2.6.0/etc/hadoop/");
        ClassLoader classLoader = EJob.getClassLoader();
        Thread.currentThread().setContextClassLoader(classLoader);
        job = new Job(conf, "PFP");
        ((JobConf) job.getConfiguration()).setJar(jarFile.toString());
    } catch (IOException e) {
        e.printStackTrace();
    }
    return job;
}

From source file:com.checkup.tez.test.DriverTest.java

public static Configuration getTezDecoratedConfiguration() {
    Configuration configuration = new Configuration();
    configuration.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
    return configuration;
}

From source file:com.chinamobile.bcbsp.bspcontroller.JobInProgress.java

License:Apache License

/**
 * clean checkpoint and delete the checkpoint information
 * @return chean result./*from   www . j  av a2  s .  c om*/
 */
private boolean cleanCheckpoint() {
    if (job.getCheckpointType().equals("HBase")) {
        String tableName = null;
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.property.clientPort", job.getConf().get(Constants.ZOOKEPER_CLIENT_PORT));
        conf.set("hbase.zookeeper.quorum", "master");
        conf.set("hbase.master", "master:60000");
        HBaseAdmin admin;
        try {
            admin = new HBaseAdmin(conf);
            for (int i = 0; i < this.attemptIDList.size(); i++) {
                tableName = this.attemptIDList.get(i).toString();
                if (admin.tableExists(tableName)) {
                    admin.disableTable(tableName);
                    admin.deleteTable(tableName);
                }
            }
            admin.close();
            return true;
        } catch (MasterNotRunningException e) {
            LOG.error("Exception has happened and been catched!", e);
            e.printStackTrace();
        } catch (ZooKeeperConnectionException e) {
            LOG.error("Exception has happened and been catched!", e);
            e.printStackTrace();
        } catch (IOException e) {
            LOG.error("Exception has happened and been catched!", e);
            e.printStackTrace();
        }
        return false;
    } else if (job.getCheckpointType().equals("HDFS")) {
        try {
            String uri = conf.get(Constants.BC_BSP_CHECKPOINT_WRITEPATH) + "/" + job.getJobID().toString()
                    + "/";
            // FileSystem fs = FileSystem.get(URI.create(uri), conf);
            BSPFileSystem bspfs = new BSPFileSystemImpl(URI.create(uri), conf);
            // if(fs.exists(new Path(uri))) {
            // fs.delete(new Path(uri), true);
            // }
            if (bspfs.exists(new BSPHdfsImpl().newPath(uri))) {
                bspfs.delete(new BSPHdfsImpl().newPath(uri), true);
            }
            return true;
        } catch (IOException e) {
            LOG.error("Exception has happened and been catched!", e);
            return false;
        }
    }
    return false;
}

From source file:com.chinamobile.bcbsp.fault.storage.Checkpoint.java

License:Apache License

private void createTable(BSPJob job, StaffAttemptID staffId) {
    LOG.info("create hbase table");
    Configuration conf = HBaseConfiguration.create();
    //    conf.addResource(new Path("/usr/local/termite/bc-bsp-1.0/conf/bcbsp-site.xml")); 
    conf.set("hbase.zookeeper.property.clientPort", job.getConf().get(Constants.ZOOKEPER_CLIENT_PORT));
    conf.set("hbase.zookeeper.quorum", "master");
    conf.set("hbase.master", "master:60000");

    String tableName = staffId.toString();
    String columnFamilyName = "BorderNode";

    try {//from   ww w.ja v a 2 s  .co m
        HBaseAdmin admin = new HBaseAdmin(conf);
        if (admin.tableExists(tableName)) {
            admin.disableTable(tableName);
            admin.deleteTable(tableName);
        }
        HTableDescriptor descriptor = new HTableDescriptor(tableName);
        descriptor.addFamily(new HColumnDescriptor(columnFamilyName));
        admin.createTable(descriptor);
        admin.close();
    } catch (MasterNotRunningException e1) {
        e1.printStackTrace();
    } catch (ZooKeeperConnectionException e1) {
        e1.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Add a {@link Path} to the list of inputs for the BC_BSP job.
 *
 * @param job/*www. ja  va  2 s. c  om*/
 *        the current job BSPJob.
 * @param path
 *        {@link Path} to be added to the list of inputs for the BC_BSP job.
 */
public static void addInputPath(BSPJob job, Path path) throws IOException {
    Configuration conf = job.getConf();
    FileSystem fs = FileSystem.get(conf);
    path = path.makeQualified(fs);
    String dirStr = StringUtils.escapeString(path.toString());
    String dirs = conf.get(Constants.USER_BC_BSP_JOB_INPUT_DIR);
    conf.set(Constants.USER_BC_BSP_JOB_INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr);
}