Example usage for org.apache.hadoop.conf Configuration setClass

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setClass.

Prototype

public void setClass(String name, Class<?> theClass, Class<?> xface)

Source Link

Document

Set the value of the name property to the name of a theClass implementing the given interface xface.

Usage

From source file:com.moz.fiji.mapreduce.pivot.FijiPivotJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override//from ww  w . ja  va2  s.  c o  m
protected void configureJob(Job job) throws IOException {
    final Configuration conf = job.getConfiguration();

    if (null == mPivoterClass) {
        throw new JobConfigurationException("Must specify a FijiPivoter class.");
    }

    // Serialize the pivoter class name into the job configuration.
    conf.setClass(FijiConfKeys.FIJI_PIVOTER_CLASS, mPivoterClass, FijiPivoter.class);

    // Producers should output to HFiles.
    mMapper = new PivoterMapper();
    mReducer = new IdentityReducer<Object, Object>();

    job.setJobName("FijiPivoter: " + mPivoterClass.getSimpleName());

    mPivoter = ReflectionUtils.newInstance(mPivoterClass, job.getConfiguration());
    mDataRequest = mPivoter.getDataRequest();

    // Configure the table input job.
    super.configureJob(job);
}

From source file:com.moz.fiji.mapreduce.produce.FijiProduceJobBuilder.java

License:Apache License

/** {@inheritDoc} */
@Override//w  w  w  . j  a  v a2 s  .c o  m
protected void configureJob(Job job) throws IOException {
    final Configuration conf = job.getConfiguration();

    // Construct the producer instance.
    if (null == mProducerClass) {
        throw new JobConfigurationException("Must specify a producer.");
    }

    // Serialize the producer class name into the job configuration.
    conf.setClass(FijiConfKeys.FIJI_PRODUCER_CLASS, mProducerClass, FijiProducer.class);

    // Write to the table, but make sure the output table is the same as the input table.
    if (!getInputTableURI().equals(mJobOutput.getOutputTableURI())) {
        throw new JobConfigurationException("Output table must be the same as the input table.");
    }

    // Producers should output to HFiles.
    mMapper = new ProduceMapper();
    mReducer = new IdentityReducer<Object, Object>();

    job.setJobName("Fiji produce: " + mProducerClass.getSimpleName());

    mProducer = ReflectionUtils.newInstance(mProducerClass, job.getConfiguration());
    mDataRequest = mProducer.getDataRequest();

    // Configure the table input job.
    super.configureJob(job);
}

From source file:com.ricemap.spateDB.core.SpatialSite.java

License:Apache License

public static void setShapeClass(Configuration conf, Class<? extends Shape> klass) {
    conf.setClass(ShapeClass, klass, Shape.class);
}

From source file:com.scaleoutsoftware.soss.hserver.GridOutputFormat.java

License:Apache License

/**
 * Sets the {@link NamedMap} to direct output to.
 *
 * @param job job to modify/*from   w  ww.  jav a2s  .c  o m*/
 * @param map named map to be used for output
 */
public static void setNamedMap(Job job, NamedMap map) {
    Configuration configuration = job.getConfiguration();
    configuration.setBoolean(outputIsNamedMapProperty, true);
    configuration.setStrings(outputNamedMapProperty, map.getMapName());
    CustomSerializer keySerializer = map.getKeySerializer();
    CustomSerializer valueSerializer = map.getValueSerializer();
    SerializationMode serializationMode = map.getSerializationMode();
    AvailabilityMode availabilityMode = map.getAvailabilityMode();
    configuration.setInt(SERIALIZATION_MODE, serializationMode.ordinal());
    configuration.setInt(AVAILABILITY_MODE, availabilityMode.ordinal());
    configuration.setClass(outputNamedMapKeySerializerProperty, keySerializer.getClass(), Object.class);
    configuration.setClass(outputNamedMapValueSerializerProperty, valueSerializer.getClass(), Object.class);
    if (keySerializer.getObjectClass() != null) {
        configuration.setClass(outputNamedMapKeyProperty, keySerializer.getObjectClass(), Object.class);
    }
    if (valueSerializer.getObjectClass() != null) {
        configuration.setClass(outputNamedMapValueProperty, valueSerializer.getObjectClass(), Object.class);
    }
}

From source file:com.scaleoutsoftware.soss.hserver.NamedMapInputFormat.java

License:Apache License

/**
 * Sets {@link com.scaleoutsoftware.soss.client.map.NamedMap} as an input source for the job.
 *
 * @param job job to modify/*from w  w w. j  a v  a 2  s  .  co m*/
 * @param map name of the map to be used as a job input
 * @param <K> the type of the key
 * @param <V> the type of the value
 */
public static <K, V> void setNamedMap(Job job, NamedMap<K, V> map) {
    Configuration configuration = job.getConfiguration();
    configuration.setInt(inputAppIdProperty, map.getMapId());
    CustomSerializer<K> keySerializer = map.getKeySerializer();
    CustomSerializer<V> valueSerializer = map.getValueSerializer();
    SerializationMode serializationMode = map.getSerializationMode();
    AvailabilityMode availabilityMode = map.getAvailabilityMode();
    configuration.setInt(SERIALIZATION_MODE, serializationMode.ordinal());
    configuration.setInt(AVAILABILITY_MODE, availabilityMode.ordinal());
    configuration.setClass(inputNamedMapKeySerializerProperty, keySerializer.getClass(), Object.class);
    configuration.setClass(inputNamedMapValueSerializerProperty, valueSerializer.getClass(), Object.class);
    if (keySerializer.getObjectClass() != null) {
        configuration.setClass(inputNamedMapKeyProperty, keySerializer.getObjectClass(), Object.class);
    }
    if (valueSerializer.getObjectClass() != null) {
        configuration.setClass(inputNamedMapValueProperty, valueSerializer.getObjectClass(), Object.class);
    }

}

From source file:com.splicemachine.derby.stream.spark.SMOutputFormatTest.java

License:Apache License

@Test
public void readExceptionsCauseAbort() throws StandardException, IOException {
    SparkPairDataSet<ExecRow, ExecRow> dataset = new SparkPairDataSet<>(
            SpliceSpark.getContext().parallelizePairs(tenRows).mapToPair(new FailFunction()));
    JavaPairRDD<ExecRow, Either<Exception, ExecRow>> rdd = dataset.wrapExceptions();

    final Configuration conf = new Configuration(HConfiguration.unwrapDelegate());
    TableWriterUtils.serializeInsertTableWriterBuilder(conf, new FakeTableWriterBuilder(false));
    conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, FakeOutputFormat.class, FakeOutputFormat.class);
    File file = File.createTempFile(SMOutputFormatTest.class.getName(), "exception");
    file.delete();//from www  .j  a  va 2s.com
    file.mkdir();
    conf.set("abort.directory", file.getAbsolutePath());
    try {
        rdd.saveAsNewAPIHadoopDataset(conf);
        Assert.fail("Expected exception");
    } catch (Exception se) {
        Assert.assertTrue("Unexpected exception", se instanceof SparkException);
    }
    File[] files = file.listFiles();
    Assert.assertTrue("Abort() not called", files.length > 0);
}

From source file:com.splicemachine.derby.stream.spark.SMOutputFormatTest.java

License:Apache License

@Test
public void writeExceptionsCauseAbort() throws StandardException, IOException {
    SparkPairDataSet<RowLocation, ExecRow> dataset = new SparkPairDataSet<>(
            SpliceSpark.getContext().parallelizePairs(tenRows).mapToPair(new ToRowLocationFunction()));
    JavaPairRDD<RowLocation, Either<Exception, ExecRow>> rdd = dataset.wrapExceptions();

    final Configuration conf = new Configuration(HConfiguration.unwrapDelegate());
    TableWriterUtils.serializeInsertTableWriterBuilder(conf, new FakeTableWriterBuilder(true));
    conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, FakeOutputFormat.class, FakeOutputFormat.class);
    File file = File.createTempFile(SMOutputFormatTest.class.getName(), "exception");
    file.delete();//from w  ww. j ava  2  s .  c  om
    file.mkdir();
    conf.set("abort.directory", file.getAbsolutePath());
    try {
        rdd.saveAsNewAPIHadoopDataset(conf);
        Assert.fail("Expected exception");
    } catch (Exception se) {
        Assert.assertTrue("Unexpected exception", se instanceof SparkException);
    }
    File[] files = file.listFiles();
    Assert.assertTrue("Abort() not called", files.length > 0);
}

From source file:com.splicemachine.derby.stream.spark.SMOutputFormatTest.java

License:Apache License

@Test
public void abortNotCalled() throws StandardException, IOException {
    SparkPairDataSet<RowLocation, ExecRow> dataset = new SparkPairDataSet<>(
            SpliceSpark.getContext().parallelizePairs(tenRows).mapToPair(new ToRowLocationFunction()));
    JavaPairRDD<RowLocation, Either<Exception, ExecRow>> rdd = dataset.wrapExceptions();

    final Configuration conf = new Configuration(HConfiguration.unwrapDelegate());
    TableWriterUtils.serializeInsertTableWriterBuilder(conf, new FakeTableWriterBuilder(false));
    conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, FakeOutputFormat.class, FakeOutputFormat.class);
    File file = File.createTempFile(SMOutputFormatTest.class.getName(), "noException");
    file.delete();/*w  ww .  j  a v a 2s.  c  o  m*/
    file.mkdir();
    conf.set("abort.directory", file.getAbsolutePath());
    rdd.saveAsNewAPIHadoopDataset(conf);
    File[] files = file.listFiles();
    Assert.assertEquals("Abort() was called", 0, files.length);
}

From source file:com.splicemachine.derby.stream.spark.SparkDeleteTableWriterBuilder.java

License:Apache License

@Override
public DataSetWriter build() throws StandardException {
    try {//www.  j  ava 2s  .  c o m
        if (operationContext.getOperation() != null) {
            operationContext.getOperation().fireBeforeStatementTriggers();
        }
        Configuration conf = new Configuration(HConfiguration.unwrapDelegate());
        TableWriterUtils.serializeDeleteTableWriterBuilder(conf, this);
        conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, SMOutputFormat.class, SMOutputFormat.class);
        return new DeleteDataSetWriter<>(rdd, operationContext, conf);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

From source file:com.splicemachine.derby.stream.spark.SparkDirectWriterBuilder.java

License:Apache License

@Override
public DataSetWriter build() throws StandardException {
    try {//from w  ww. j a  va  2 s .c o m
        Configuration conf = new Configuration(HConfiguration.unwrapDelegate());
        TableWriterUtils.serializeHTableWriterBuilder(conf, this);
        conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, HTableOutputFormat.class, HTableOutputFormat.class);
        JavaSparkContext context = SpliceSpark.getContext();
        return new SparkDirectDataSetWriter<>(rdd, context, opCtx, conf, skipIndex, destConglomerate, txn);
    } catch (Exception e) {
        throw Exceptions.parseException(e);
    }
}