Example usage for org.apache.hadoop.conf Configuration setClass

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setClass.

Prototype

public void setClass(String name, Class<?> theClass, Class<?> xface) 

Source Link

Document

Set the value of the name property to the name of a theClass implementing the given interface xface.

Usage

From source file:org.kiji.mapreduce.gather.KijiGatherJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override//from  ww w . java 2 s.  c om
protected void configureJob(Job job) throws IOException {
    // Construct the gatherer instance.
    if (null == mGathererClass) {
        throw new JobConfigurationException("Must specify a gatherer.");
    }

    final Configuration conf = job.getConfiguration();

    // Serialize the gatherer class name into the job configuration.
    conf.setClass(KijiConfKeys.KIJI_GATHERER_CLASS, mGathererClass, KijiGatherer.class);

    if ((getJobOutput() instanceof HFileMapReduceJobOutput) && (null == mReducerClass)) {
        mReducerClass = IdentityReducer.class;
    }

    final StringBuilder name = new StringBuilder("Kiji gather: " + mGathererClass.getSimpleName());
    if (null != mReducerClass) {
        name.append(" / " + mReducerClass.getSimpleName());
    }
    job.setJobName(name.toString());

    mGatherer = ReflectionUtils.newInstance(mGathererClass, conf);
    mMapper.setConf(conf);
    mDataRequest = mGatherer.getDataRequest();

    // Construct the combiner instance (if specified).
    if (null != mCombinerClass) {
        mCombiner = ReflectionUtils.newInstance(mCombinerClass, conf);
    }

    // Construct the reducer instance (if specified).
    if (null != mReducerClass) {
        mReducer = ReflectionUtils.newInstance(mReducerClass, conf);
    }

    // Configure the table input job (requires mGatherer, mMapper and mReducer to be set):
    super.configureJob(job);

    // Some validation:
    if (getJobOutput() instanceof HFileMapReduceJobOutput) {
        if (mReducer instanceof IdentityReducer) {
            Preconditions.checkState(mGatherer.getOutputKeyClass() == HFileKeyValue.class,
                    String.format("Gatherer '%s' writing HFiles must output HFileKeyValue keys, but got '%s'",
                            mGathererClass.getName(), mGatherer.getOutputKeyClass().getName()));
            Preconditions.checkState(mGatherer.getOutputValueClass() == NullWritable.class,
                    String.format("Gatherer '%s' writing HFiles must output NullWritable values, but got '%s'",
                            mGathererClass.getName(), mGatherer.getOutputValueClass().getName()));
        }
        Preconditions.checkState(mReducer.getOutputKeyClass() == HFileKeyValue.class,
                String.format("Reducer '%s' writing HFiles must output HFileKeyValue keys, but got '%s'",
                        mReducerClass.getName(), mReducer.getOutputKeyClass().getName()));
        Preconditions.checkState(mReducer.getOutputValueClass() == NullWritable.class,
                String.format("Reducer '%s' writing HFiles must output NullWritable values, but got '%s'",
                        mReducerClass.getName(), mReducer.getOutputValueClass().getName()));
    }
}

---

From source file:org.kiji.mapreduce.KijiGatherJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   ww  w . ja  v a 2  s. c  om*/
protected void configureJob(Job job) throws IOException {
    // Construct the gatherer instance.
    if (null == mGathererClass) {
        throw new JobConfigurationException("Must specify a gatherer.");
    }

    final Configuration conf = job.getConfiguration();

    // Serialize the gatherer class name into the job configuration.
    conf.setClass(KijiConfKeys.KIJI_GATHERER_CLASS, mGathererClass, KijiGatherer.class);

    if ((getJobOutput() instanceof HFileMapReduceJobOutput) && (null == mReducerClass)) {
        mReducerClass = IdentityReducer.class;
    }

    mGatherer = ReflectionUtils.newInstance(mGathererClass, conf);
    mMapper.setConf(conf);
    mDataRequest = mGatherer.getDataRequest();

    // Construct the combiner instance (if specified).
    if (null != mCombinerClass) {
        mCombiner = ReflectionUtils.newInstance(mCombinerClass, conf);
    }

    // Construct the reducer instance (if specified).
    if (null != mReducerClass) {
        mReducer = ReflectionUtils.newInstance(mReducerClass, conf);
    }

    final StringBuilder name = new StringBuilder("Kiji gather: " + mGathererClass.getSimpleName());
    if (null != mReducerClass) {
        name.append(" / " + mReducerClass.getSimpleName());
    }
    job.setJobName(name.toString());

    // Configure the table input job.
    super.configureJob(job);

    // Some validation:
    if (getJobOutput() instanceof HFileMapReduceJobOutput) {
        if (mReducer instanceof IdentityReducer) {
            Preconditions.checkState(mGatherer.getOutputKeyClass() == HFileKeyValue.class,
                    String.format("Gatherer '%s' writing HFiles must output HFileKeyValue keys, but got '%s'",
                            mGathererClass.getName(), mGatherer.getOutputKeyClass().getName()));
            Preconditions.checkState(mGatherer.getOutputValueClass() == NullWritable.class,
                    String.format("Gatherer '%s' writing HFiles must output NullWritable values, but got '%s'",
                            mGathererClass.getName(), mGatherer.getOutputValueClass().getName()));
        }
        Preconditions.checkState(mReducer.getOutputKeyClass() == HFileKeyValue.class,
                String.format("Reducer '%s' writing HFiles must output HFileKeyValue keys, but got '%s'",
                        mReducerClass.getName(), mReducer.getOutputKeyClass().getName()));
        Preconditions.checkState(mReducer.getOutputValueClass() == NullWritable.class,
                String.format("Reducer '%s' writing HFiles must output NullWritable values, but got '%s'",
                        mReducerClass.getName(), mReducer.getOutputValueClass().getName()));
    }
}

---

From source file:org.kiji.mapreduce.KijiProduceJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   www.  j  av  a  2 s.co m*/
protected void configureJob(Job job) throws IOException {
    final Configuration conf = job.getConfiguration();

    // Construct the producer instance.
    if (null == mProducerClass) {
        throw new JobConfigurationException("Must specify a producer.");
    }
    mProducer = ReflectionUtils.newInstance(mProducerClass, job.getConfiguration());
    mDataRequest = mProducer.getDataRequest();

    // Serialize the producer class name into the job configuration.
    conf.setClass(KijiConfKeys.KIJI_PRODUCER_CLASS, mProducerClass, KijiProducer.class);

    // Configure the mapper and reducer to use.
    Preconditions.checkState(getJobOutput() instanceof KijiTableMapReduceJobOutput);
    // Write to the table, but make sure the output table is the same as the input table.
    if (!getInputTable().equals(((KijiTableMapReduceJobOutput) getJobOutput()).getTable())) {
        throw new JobConfigurationException("Output table must be the same as the input table.");
    }

    // Producers should output to HFiles.
    mMapper = new ProduceMapper();
    mReducer = new IdentityReducer<Object, Object>();

    job.setJobName("Kiji produce: " + mProducerClass.getSimpleName());

    // Configure the table input job.
    super.configureJob(job);
}

---

From source file:org.kiji.mapreduce.output.DirectKijiTableMapReduceJobOutput.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w  w w.  jav a  2 s.c  o m*/
public void configure(Job job) throws IOException {
    // sets Hadoop output format, Kiji output table and # of reducers:
    super.configure(job);

    final Configuration conf = job.getConfiguration();

    // Kiji table context:
    conf.setClass(KijiConfKeys.KIJI_TABLE_CONTEXT_CLASS, DirectKijiTableWriterContext.class,
            KijiTableContext.class);

    // Since there's no "commit" operation for an entire map task writing to a
    // Kiji table, do not use speculative execution when writing directly to a Kiji table.
    conf.setBoolean("mapred.map.tasks.speculative.execution", false);
}

---

From source file:org.kiji.mapreduce.output.framework.HFileReducerMapReduceJobOutput.java

License:Apache License

/** {@inheritDoc} */
@Override//from w  w w .  j  ava2  s  .  com
public void configure(Job job) throws IOException {
    super.configure(job); // sets the Hadoop output format

    final Configuration conf = job.getConfiguration();
    conf.set(KijiConfKeys.KIJI_OUTPUT_TABLE_URI, mJobOutput.getOutputTableURI().toString());

    // Kiji table context:
    conf.setClass(KijiConfKeys.KIJI_TABLE_CONTEXT_CLASS, HFileWriterContext.class, KijiTableContext.class);

    // Set the output path.
    FileOutputFormat.setOutputPath(job, mJobOutput.getPath());

    job.setNumReduceTasks(mJobOutput.getNumReduceTasks());
}

---

From source file:org.kiji.mapreduce.output.HFileMapReduceJobOutput.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w ww  . j  a va  2s .  c o  m*/
public void configure(Job job) throws IOException {
    // sets Hadoop output format, Kiji output table and # of reducers:
    super.configure(job);

    final Configuration conf = job.getConfiguration();

    // Kiji table context:
    conf.setClass(KijiConfKeys.KIJI_TABLE_CONTEXT_CLASS, HFileWriterContext.class, KijiTableContext.class);

    // Set the output path.
    FileOutputFormat.setOutputPath(job, mPath);

    // Configure the total order partitioner so generated HFile shards are contiguous and sorted.
    configurePartitioner(job, makeTableKeySplit(getTable(), getNumReduceTasks()));

    // Note: the HFile job output requires the reducer of the map/reduce to be IdentityReducer.
    //     This is enforced externally.
}

---

From source file:org.kiji.mapreduce.output.HFileReducerMapReduceJobOutput.java

License:Apache License

/** {@inheritDoc} */
@Override// w ww .  j  ava  2  s.  c  o  m
public void configure(Job job) throws IOException {
    super.configure(job); // sets the Hadoop output format

    final Configuration conf = job.getConfiguration();
    conf.set(KijiConfKeys.KIJI_OUTPUT_TABLE_URI, mJobOutput.getTable().getURI().toString());

    // Kiji table context:
    conf.setClass(KijiConfKeys.KIJI_TABLE_CONTEXT_CLASS, HFileWriterContext.class, KijiTableContext.class);

    // Set the output path.
    FileOutputFormat.setOutputPath(job, mJobOutput.getPath());

    job.setNumReduceTasks(mJobOutput.getNumReduceTasks());
}

---

From source file:org.kiji.mapreduce.pivot.KijiPivotJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override//  w  ww  . ja v  a 2 s.co  m
protected void configureJob(Job job) throws IOException {
    final Configuration conf = job.getConfiguration();

    if (null == mPivoterClass) {
        throw new JobConfigurationException("Must specify a KijiPivoter class.");
    }

    // Serialize the pivoter class name into the job configuration.
    conf.setClass(KijiConfKeys.KIJI_PIVOTER_CLASS, mPivoterClass, KijiPivoter.class);

    // Producers should output to HFiles.
    mMapper = new PivoterMapper();
    mReducer = new IdentityReducer<Object, Object>();

    job.setJobName("KijiPivoter: " + mPivoterClass.getSimpleName());

    mPivoter = ReflectionUtils.newInstance(mPivoterClass, job.getConfiguration());
    mDataRequest = mPivoter.getDataRequest();

    // Configure the table input job.
    super.configureJob(job);
}

---

From source file:org.kiji.mapreduce.produce.KijiProduceJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override//from  w  ww. ja v  a 2 s  .c om
protected void configureJob(Job job) throws IOException {
    final Configuration conf = job.getConfiguration();

    // Construct the producer instance.
    if (null == mProducerClass) {
        throw new JobConfigurationException("Must specify a producer.");
    }

    // Serialize the producer class name into the job configuration.
    conf.setClass(KijiConfKeys.KIJI_PRODUCER_CLASS, mProducerClass, KijiProducer.class);

    // Write to the table, but make sure the output table is the same as the input table.
    if (!getInputTableURI().equals(mJobOutput.getOutputTableURI())) {
        throw new JobConfigurationException("Output table must be the same as the input table.");
    }

    // Producers should output to HFiles.
    mMapper = new ProduceMapper();
    mReducer = new IdentityReducer<Object, Object>();

    job.setJobName("Kiji produce: " + mProducerClass.getSimpleName());

    mProducer = ReflectionUtils.newInstance(mProducerClass, job.getConfiguration());
    mDataRequest = mProducer.getDataRequest();

    // Configure the table input job.
    super.configureJob(job);
}

---

From source file:org.kiji.scoring.batch.ScoreFunctionJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override//from  ww w  . ja  v  a  2 s .  co  m
protected void configureJob(final Job job) throws IOException {
    if (null == mScoreFunctionClass) {
        throw new JobConfigurationException("Must specify a ScoreFunction class.");
    }
    if (null == mClientDataRequest) {
        mClientDataRequest = DEFAULT_CLIENT_REQUEST;
    }
    if (null == mAttachedColumn) {
        throw new JobConfigurationException("Must specified an AttachedColumn.");
    }
    if (null == mParameters) {
        mParameters = DEFAULT_PARAMETERS;
    }

    final Configuration conf = job.getConfiguration();
    conf.setClass(SCORE_FUNCTION_CLASS_CONF_KEY, mScoreFunctionClass, ScoreFunction.class);
    if (!getInputTableURI().equals(mJobOutput.getOutputTableURI())) {
        throw new JobConfigurationException(
                String.format("Output table must be the same as the input" + "table. Got input: %s output: %s",
                        getInputTableURI(), mJobOutput.getOutputTableURI()));
    }
    conf.set(SCORE_FUNCTION_ATTACHED_COLUMN_CONF_KEY, mAttachedColumn.getName());
    conf.set(SCORE_FUNCTION_PARAMETERS_CONF_KEY, GSON.toJson(mParameters, Map.class));
    conf.set(SCORE_FUNCTION_CLIENT_DATA_REQUEST_CONF_KEY,
            Base64.encodeBase64String(SerializationUtils.serialize(mClientDataRequest)));
    mMapper = new ScoreFunctionMapper();
    mReducer = new IdentityReducer<Object, Object>();
    job.setJobName("Kiji ScoreFunction: " + mScoreFunctionClass.getSimpleName());
    mScoreFunction = ReflectionUtils.newInstance(mScoreFunctionClass, conf);
    final FreshenerContext context = InternalFreshenerContext.create(mClientDataRequest, mAttachedColumn,
            mParameters, Maps.<String, String>newHashMap(),
            KeyValueStoreReaderFactory.create(getRequiredStores()));
    mScoreFunctionDataRequest = mScoreFunction.getDataRequest(context);

    super.configureJob(job);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• »