Example usage for org.apache.hadoop.conf Configuration clear

List of usage examples for org.apache.hadoop.conf Configuration clear

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. clear

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration clear.

Prototype

public void clear()

Source Link

Document

Clears all keys from the configuration.

Usage

From source file:at.ac.tuwien.infosys.jcloudscale.datastore.driver.hbase.HbaseConfig.java

License:Apache License

/**
 * Create the HBase Configuration for a given datastore
 *
 * @param datastore the given datastore/*  ww w  . j av a2  s .  co m*/
 * @return the HBase Configuration
 */
public static Configuration getConfig(Datastore datastore) {
    Configuration configuration = HBaseConfiguration.create();
    configuration.clear();
    configuration.set("hbase.zookeeper.quorum", datastore.getHost());
    return configuration;
}

From source file:co.cask.cdap.app.runtime.spark.SparkRuntimeContextProvider.java

License:Apache License

private static Configuration createHConf() throws MalformedURLException {
    Configuration hConf = new Configuration();
    hConf.clear();
    hConf.addResource(new File(HCONF_FILE_NAME).toURI().toURL());
    return hConf;
}

From source file:co.cask.cdap.data.stream.StreamInputFormatProvider.java

License:Apache License

/**
 * Sets the {@link StreamEventDecoder} to be used by the InputFormat for the given type. If the
 * {@link StreamBatchReadable} already defined a {@link StreamEventDecoder} or {@link FormatSpecification},
 * this method is a no-op.//from   w  w  w.j av  a 2 s . c  om
 *
 * @param configuration configuration to update
 * @param type type for {@link StreamEventData} to decode to
 * @return the same configuration map as in the argument.
 */
public Map<String, String> setDecoderType(Map<String, String> configuration, Type type) {
    if (streamBatchReadable.getFormatSpecification() == null && streamBatchReadable.getDecoderType() == null) {
        Configuration hConf = new Configuration();
        hConf.clear();
        StreamInputFormat.inferDecoderClass(hConf, type);
        configuration.putAll(ConfigurationUtil.toMap(hConf));
    }
    return configuration;
}

From source file:co.cask.cdap.data.stream.StreamInputFormatProvider.java

License:Apache License

@Override
public Map<String, String> getInputFormatConfiguration() {
    Id.Stream streamId = Id.Stream.from(namespaceId, streamBatchReadable.getStreamName());
    try {// ww  w . j  a  va  2s  .  c o m
        StreamConfig streamConfig = streamAdmin.getConfig(streamId);
        Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(),
                StreamUtils.getGeneration(streamConfig));
        Configuration hConf = new Configuration();
        hConf.clear();

        StreamInputFormat.setTTL(hConf, streamConfig.getTTL());
        StreamInputFormat.setStreamPath(hConf, streamPath.toURI());
        StreamInputFormat.setTimeRange(hConf, streamBatchReadable.getStartTime(),
                streamBatchReadable.getEndTime());
        FormatSpecification formatSpec = streamBatchReadable.getFormatSpecification();
        if (formatSpec != null) {
            StreamInputFormat.setBodyFormatSpecification(hConf, formatSpec);
        } else {
            String decoderType = streamBatchReadable.getDecoderType();
            if (decoderType != null) {
                StreamInputFormat.setDecoderClassName(hConf, decoderType);
            }
        }

        return ConfigurationUtil.toMap(hConf);
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}

From source file:co.cask.cdap.etl.batch.spark.SparkBatchSinkFactory.java

License:Apache License

<K, V> void writeFromRDD(JavaPairRDD<K, V> rdd, JavaSparkExecutionContext sec, String sinkName,
        Class<K> keyClass, Class<V> valueClass) {
    Set<String> outputNames = sinkOutputs.get(sinkName);
    if (outputNames == null || outputNames.size() == 0) {
        // should never happen if validation happened correctly at pipeline configure time
        throw new IllegalArgumentException(
                sinkName + " has no outputs. " + "Please check that the sink calls addOutput at some point.");
    }/*from w  w  w .j  a  v a 2 s .c o m*/

    for (String outputName : outputNames) {
        OutputFormatProvider outputFormatProvider = outputFormatProviders.get(outputName);
        if (outputFormatProvider != null) {
            Configuration hConf = new Configuration();
            hConf.clear();
            for (Map.Entry<String, String> entry : outputFormatProvider.getOutputFormatConfiguration()
                    .entrySet()) {
                hConf.set(entry.getKey(), entry.getValue());
            }
            hConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, outputFormatProvider.getOutputFormatClassName());
            rdd.saveAsNewAPIHadoopDataset(hConf);
        }

        DatasetInfo datasetInfo = datasetInfos.get(outputName);
        if (datasetInfo != null) {
            sec.saveAsDataset(rdd, datasetInfo.getDatasetName(), datasetInfo.getDatasetArgs());
        }
    }
}

From source file:co.cask.cdap.etl.batch.spark.SparkBatchSourceFactory.java

License:Apache License

@SuppressWarnings("unchecked")
public <K, V> JavaPairRDD<K, V> createRDD(JavaSparkExecutionContext sec, JavaSparkContext jsc,
        Class<K> keyClass, Class<V> valueClass) {
    if (streamBatchReadable != null) {
        FormatSpecification formatSpec = streamBatchReadable.getFormatSpecification();
        if (formatSpec != null) {
            return (JavaPairRDD<K, V>) sec.fromStream(streamBatchReadable.getStreamName(), formatSpec,
                    streamBatchReadable.getStartTime(), streamBatchReadable.getEndTime(),
                    StructuredRecord.class);
        }//from   w  w w .j  a va 2 s .c  o m

        String decoderType = streamBatchReadable.getDecoderType();
        if (decoderType == null) {
            return (JavaPairRDD<K, V>) sec.fromStream(streamBatchReadable.getStreamName(),
                    streamBatchReadable.getStartTime(), streamBatchReadable.getEndTime(), valueClass);
        } else {
            try {
                Class<StreamEventDecoder<K, V>> decoderClass = (Class<StreamEventDecoder<K, V>>) Thread
                        .currentThread().getContextClassLoader().loadClass(decoderType);
                return sec.fromStream(streamBatchReadable.getStreamName(), streamBatchReadable.getStartTime(),
                        streamBatchReadable.getEndTime(), decoderClass, keyClass, valueClass);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    }
    if (inputFormatProvider != null) {
        Configuration hConf = new Configuration();
        hConf.clear();
        for (Map.Entry<String, String> entry : inputFormatProvider.getInputFormatConfiguration().entrySet()) {
            hConf.set(entry.getKey(), entry.getValue());
        }
        ClassLoader classLoader = Objects.firstNonNull(currentThread().getContextClassLoader(),
                getClass().getClassLoader());
        try {
            @SuppressWarnings("unchecked")
            Class<InputFormat> inputFormatClass = (Class<InputFormat>) classLoader
                    .loadClass(inputFormatProvider.getInputFormatClassName());
            return jsc.newAPIHadoopRDD(hConf, inputFormatClass, keyClass, valueClass);
        } catch (ClassNotFoundException e) {
            throw Throwables.propagate(e);
        }
    }
    if (datasetInfo != null) {
        return sec.fromDataset(datasetInfo.getDatasetName(), datasetInfo.getDatasetArgs());
    }
    // This should never happen since the constructor is private and it only get calls from static create() methods
    // which make sure one and only one of those source type will be specified.
    throw new IllegalStateException("Unknown source type");
}

From source file:co.cask.cdap.etl.spark.batch.SparkBatchSinkFactory.java

License:Apache License

public <K, V> void writeFromRDD(JavaPairRDD<K, V> rdd, JavaSparkExecutionContext sec, String sinkName,
        Class<K> keyClass, Class<V> valueClass) {
    Set<String> outputNames = sinkOutputs.get(sinkName);
    if (outputNames == null || outputNames.isEmpty()) {
        // should never happen if validation happened correctly at pipeline configure time
        throw new IllegalArgumentException(
                sinkName + " has no outputs. " + "Please check that the sink calls addOutput at some point.");
    }/* w w w .  j a v a  2  s  . c  o  m*/

    for (String outputName : outputNames) {
        OutputFormatProvider outputFormatProvider = outputFormatProviders.get(outputName);
        if (outputFormatProvider != null) {
            Configuration hConf = new Configuration();
            hConf.clear();
            for (Map.Entry<String, String> entry : outputFormatProvider.getOutputFormatConfiguration()
                    .entrySet()) {
                hConf.set(entry.getKey(), entry.getValue());
            }
            hConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, outputFormatProvider.getOutputFormatClassName());
            rdd.saveAsNewAPIHadoopDataset(hConf);
        }

        DatasetInfo datasetInfo = datasetInfos.get(outputName);
        if (datasetInfo != null) {
            sec.saveAsDataset(rdd, datasetInfo.getDatasetName(), datasetInfo.getDatasetArgs());
        }
    }
}

From source file:co.cask.cdap.etl.spark.batch.SparkBatchSourceFactory.java

License:Apache License

@SuppressWarnings("unchecked")
private <K, V> JavaPairRDD<K, V> createInputRDD(JavaSparkExecutionContext sec, JavaSparkContext jsc,
        String inputName, Class<K> keyClass, Class<V> valueClass) {
    if (streams.containsKey(inputName)) {
        Input.StreamInput streamInput = streams.get(inputName);
        FormatSpecification formatSpec = streamInput.getBodyFormatSpec();
        if (formatSpec != null) {
            return (JavaPairRDD<K, V>) sec.fromStream(streamInput.getName(), formatSpec,
                    streamInput.getStartTime(), streamInput.getEndTime(), StructuredRecord.class);
        }//www  . ja v a2  s . c  o m

        String decoderType = streamInput.getDecoderType();
        if (decoderType == null) {
            return (JavaPairRDD<K, V>) sec.fromStream(streamInput.getName(), streamInput.getStartTime(),
                    streamInput.getEndTime(), valueClass);
        } else {
            try {
                Class<StreamEventDecoder<K, V>> decoderClass = (Class<StreamEventDecoder<K, V>>) Thread
                        .currentThread().getContextClassLoader().loadClass(decoderType);
                return sec.fromStream(streamInput.getName(), streamInput.getStartTime(),
                        streamInput.getEndTime(), decoderClass, keyClass, valueClass);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    }

    if (inputFormatProviders.containsKey(inputName)) {
        InputFormatProvider inputFormatProvider = inputFormatProviders.get(inputName);
        Configuration hConf = new Configuration();
        hConf.clear();
        for (Map.Entry<String, String> entry : inputFormatProvider.getInputFormatConfiguration().entrySet()) {
            hConf.set(entry.getKey(), entry.getValue());
        }
        ClassLoader classLoader = Objects.firstNonNull(currentThread().getContextClassLoader(),
                getClass().getClassLoader());
        try {
            @SuppressWarnings("unchecked")
            Class<InputFormat> inputFormatClass = (Class<InputFormat>) classLoader
                    .loadClass(inputFormatProvider.getInputFormatClassName());
            return jsc.newAPIHadoopRDD(hConf, inputFormatClass, keyClass, valueClass);
        } catch (ClassNotFoundException e) {
            throw Throwables.propagate(e);
        }
    }

    if (datasetInfos.containsKey(inputName)) {
        DatasetInfo datasetInfo = datasetInfos.get(inputName);
        return sec.fromDataset(datasetInfo.getDatasetName(), datasetInfo.getDatasetArgs());
    }
    // This should never happen since the constructor is private and it only get calls from static create() methods
    // which make sure one and only one of those source type will be specified.
    throw new IllegalStateException("Unknown source type");
}

From source file:co.cask.cdap.internal.app.runtime.batch.dataset.DatasetInputFormatProvider.java

License:Apache License

private Map<String, String> createBatchReadableConfiguration() {
    List<Split> splits = this.splits;
    if (splits == null) {
        splits = ((BatchReadable<?, ?>) dataset).getSplits();
    }/*from w w  w  . j a v a  2 s . co  m*/
    Configuration hConf = new Configuration();
    hConf.clear();

    try {
        AbstractBatchReadableInputFormat.setDatasetSplits(hConf, datasetName, datasetArgs, splits);
        return ConfigurationUtil.toMap(hConf);
    } catch (IOException e) {
        throw new IllegalArgumentException(e);
    }
}

From source file:co.cask.cdap.internal.app.runtime.batch.dataset.DatasetOutputFormatProvider.java

License:Apache License

private Map<String, String> createDatasetConfiguration(String datasetName, Map<String, String> datasetArgs) {
    Configuration hConf = new Configuration();
    hConf.clear();
    AbstractBatchWritableOutputFormat.setDataset(hConf, datasetName, datasetArgs);
    return ConfigurationUtil.toMap(hConf);
}