Example usage for org.apache.hadoop.conf Configuration get

List of usage examples for org.apache.hadoop.conf Configuration get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    long ttl = conf.getLong(STREAM_TTL, Long.MAX_VALUE);
    long endTime = conf.getLong(EVENT_END_TIME, Long.MAX_VALUE);
    long startTime = Math.max(conf.getLong(EVENT_START_TIME, 0L), getCurrentTime() - ttl);
    long maxSplitSize = conf.getLong(MAX_SPLIT_SIZE, Long.MAX_VALUE);
    long minSplitSize = Math.min(conf.getLong(MIN_SPLIT_SIZE, 1L), maxSplitSize);
    StreamInputSplitFinder<InputSplit> splitFinder = StreamInputSplitFinder
            .builder(URI.create(conf.get(STREAM_PATH))).setStartTime(startTime).setEndTime(endTime)
            .setMinSplitSize(minSplitSize).setMaxSplitSize(maxSplitSize).build(splitFactory);
    return splitFinder.getSplits(conf);
}

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
protected StreamEventDecoder<K, V> createStreamEventDecoder(Configuration conf) {
    Class<? extends StreamEventDecoder> decoderClass = getDecoderClass(conf);
    Preconditions.checkNotNull(decoderClass, "Failed to load stream event decoder %s", conf.get(DECODER_TYPE));
    try {//from  ww w. ja  va2  s .com
        // if this is a FormatStreamEventDecoder, we need to create and initialize the format that will be used
        // to format the stream body.
        if (decoderClass.isAssignableFrom(FormatStreamEventDecoder.class)) {
            try {
                RecordFormat<StreamEvent, V> bodyFormat = getInitializedFormat(conf);
                return (StreamEventDecoder<K, V>) new FormatStreamEventDecoder(bodyFormat);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to get the stream body format.");
            }
        } else {
            return (StreamEventDecoder<K, V>) decoderClass.newInstance();
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

private RecordFormat<StreamEvent, V> getInitializedFormat(Configuration conf) throws UnsupportedTypeException,
        IllegalAccessException, ClassNotFoundException, InstantiationException {
    String formatSpecStr = conf.get(BODY_FORMAT);
    if (formatSpecStr == null || formatSpecStr.isEmpty()) {
        throw new IllegalArgumentException(BODY_FORMAT
                + " must be set in the configuration in order to use a format for the stream body.");
    }/*from w  w  w  . j a v  a2s.c  o  m*/
    FormatSpecification formatSpec = GSON.fromJson(formatSpecStr, FormatSpecification.class);
    return RecordFormats.createInitializedFormat(formatSpec);
}

From source file:co.cask.cdap.etl.batch.mapreduce.MapReduceTransformExecutorFactory.java

License:Apache License

public MapReduceTransformExecutorFactory(MapReduceTaskContext taskContext,
        PipelinePluginInstantiator pluginInstantiator, Metrics metrics,
        Map<String, Map<String, String>> pluginRuntimeArgs) {
    super(pluginInstantiator, metrics);
    this.taskContext = taskContext;
    this.pluginRuntimeArgs = pluginRuntimeArgs;
    JobContext hadoopContext = (JobContext) taskContext.getHadoopContext();
    Configuration hConf = hadoopContext.getConfiguration();
    this.mapOutputKeyClassName = hConf.get(ETLMapReduce.GROUP_KEY_CLASS);
    this.mapOutputValClassName = hConf.get(ETLMapReduce.GROUP_VAL_CLASS);
    this.isMapper = hadoopContext instanceof Mapper.Context;
}

From source file:co.cask.cdap.etl.batch.mapreduce.TransformRunner.java

License:Apache License

public TransformRunner(MapReduceTaskContext<Object, Object> context, Metrics metrics) throws Exception {
    JobContext jobContext = context.getHadoopContext();
    Configuration hConf = jobContext.getConfiguration();

    // figure out whether we are writing to a single output or to multiple outputs
    Map<String, String> properties = context.getSpecification().getProperties();
    BatchPhaseSpec phaseSpec = GSON.fromJson(properties.get(Constants.PIPELINEID), BatchPhaseSpec.class);
    this.outputWriter = getSinkWriter(context, phaseSpec.getPhase(), hConf);

    // instantiate and initialize all transformations and setup the TransformExecutor
    PipelinePluginInstantiator pluginInstantiator = new PipelinePluginInstantiator(context, phaseSpec);
    // stage name -> runtime args for that stage
    Map<String, Map<String, String>> runtimeArgs = GSON.fromJson(hConf.get(ETLMapReduce.RUNTIME_ARGS_KEY),
            ETLMapReduce.RUNTIME_ARGS_TYPE);

    PipelinePhase phase = phaseSpec.getPhase();
    Set<StageInfo> aggregators = phase.getStagesOfType(BatchAggregator.PLUGIN_TYPE);
    if (!aggregators.isEmpty()) {
        String aggregatorName = aggregators.iterator().next().getName();
        // if we're in the mapper, get the part of the pipeline starting from sources and ending at aggregator
        if (jobContext instanceof Mapper.Context) {
            phase = phase.subsetTo(ImmutableSet.of(aggregatorName));
        } else {// w ww .jav  a2 s.  co  m
            // if we're in the reducer, get the part of the pipeline starting from the aggregator and ending at sinks
            phase = phase.subsetFrom(ImmutableSet.of(aggregatorName));
        }
    }
    TransformExecutorFactory<KeyValue<KEY, VALUE>> transformExecutorFactory = new MapReduceTransformExecutorFactory<>(
            context, pluginInstantiator, metrics, runtimeArgs);
    this.transformExecutor = transformExecutorFactory.create(phase);

    // setup error dataset information
    this.transformsWithoutErrorDataset = new HashSet<>();
    this.transformErrorSinkMap = new HashMap<>();
    for (StageInfo transformInfo : phaseSpec.getPhase().getStagesOfType(Transform.PLUGIN_TYPE)) {
        String errorDatasetName = transformInfo.getErrorDatasetName();
        if (errorDatasetName != null) {
            transformErrorSinkMap.put(transformInfo.getName(),
                    new ErrorOutputWriter<>(context, errorDatasetName));
        }
    }
}

From source file:co.cask.cdap.etl.batch.mapreduce.TransformRunner.java

License:Apache License

private OutputWriter<Object, Object> getSinkWriter(MapReduceTaskContext<Object, Object> context,
        PipelinePhase pipelinePhase, Configuration hConf) {
    Set<StageInfo> aggregators = pipelinePhase.getStagesOfType(BatchAggregator.PLUGIN_TYPE);
    if (!aggregators.isEmpty()) {
        String aggregatorName = aggregators.iterator().next().getName();
        if (pipelinePhase.getSinks().contains(aggregatorName)) {
            return new SingleOutputWriter<>(context);
        }// w  ww  .j a  va2 s. c o m
    }

    String sinkOutputsStr = hConf.get(ETLMapReduce.SINK_OUTPUTS_KEY);

    // should never happen, this is set in beforeSubmit
    Preconditions.checkNotNull(sinkOutputsStr, "Sink outputs not found in Hadoop conf.");
    Map<String, SinkOutput> sinkOutputs = GSON.fromJson(sinkOutputsStr, ETLMapReduce.SINK_OUTPUTS_TYPE);
    return hasSingleOutput(pipelinePhase.getStagesOfType(Transform.PLUGIN_TYPE), sinkOutputs)
            ? new SingleOutputWriter<>(context)
            : new MultiOutputWriter<>(context, sinkOutputs);
}

From source file:co.cask.cdap.explore.security.JobHistoryServerTokenUtils.java

License:Apache License

/**
 * Gets a JHS delegation token and stores it in the given Credentials.
 *
 * @return the same Credentials instance as the one given in parameter.
 *//* w  ww  .  j  ava  2 s.com*/
public static Credentials obtainToken(Configuration configuration, Credentials credentials) {
    if (!UserGroupInformation.isSecurityEnabled()) {
        return credentials;
    }

    String historyServerAddress = configuration.get("mapreduce.jobhistory.address");
    HostAndPort hostAndPort = HostAndPort.fromString(historyServerAddress);
    try {
        LOG.info("Obtaining delegation token for JHS");

        ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(new YarnConfiguration(configuration));
        MRClientCache clientCache = new MRClientCache(configuration, resourceMgrDelegate);
        MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
        GetDelegationTokenRequest request = new GetDelegationTokenRequestPBImpl();
        request.setRenewer(YarnUtils.getYarnTokenRenewer(configuration));

        InetSocketAddress address = new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPort());
        Token<TokenIdentifier> token = ConverterUtils
                .convertFromYarn(hsProxy.getDelegationToken(request).getDelegationToken(), address);

        credentials.addToken(new Text(token.getService()), token);
        return credentials;
    } catch (Exception e) {
        LOG.error("Failed to get secure token for JHS at {}.", hostAndPort, e);
        throw Throwables.propagate(e);
    }
}

From source file:co.cask.cdap.explore.service.ExploreServiceUtilsTest.java

License:Apache License

@Test
public void hijackConfFileTest() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set("foo", "bar");
    Assert.assertEquals(1, conf.size());

    File tempDir = tmpFolder.newFolder();

    File confFile = tmpFolder.newFile("hive-site.xml");

    try (FileOutputStream os = new FileOutputStream(confFile)) {
        conf.writeXml(os);/*from w  w  w .  j  a v a 2s.  c o  m*/
    }

    File newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir);

    conf = new Configuration(false);
    conf.addResource(newConfFile.toURI().toURL());

    Assert.assertEquals(3, conf.size());
    Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_USER_CLASSPATH_FIRST));
    Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_CLASSLOADER));
    Assert.assertEquals("bar", conf.get("foo"));

    // check yarn-site changes
    confFile = tmpFolder.newFile("yarn-site.xml");
    conf = new YarnConfiguration();

    try (FileOutputStream os = new FileOutputStream(confFile)) {
        conf.writeXml(os);
    }

    String yarnApplicationClassPath = "$PWD/*," + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));

    newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir);

    conf = new Configuration(false);
    conf.addResource(newConfFile.toURI().toURL());

    Assert.assertEquals(yarnApplicationClassPath, conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH));

    // check mapred-site changes
    confFile = tmpFolder.newFile("mapred-site.xml");
    conf = new YarnConfiguration();

    try (FileOutputStream os = new FileOutputStream(confFile)) {
        conf.writeXml(os);
    }

    String mapredApplicationClassPath = "$PWD/*," + conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
            MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH);

    newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir);

    conf = new Configuration(false);
    conf.addResource(newConfFile.toURI().toURL());

    Assert.assertEquals(mapredApplicationClassPath, conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH));

    // Ensure conf files that are not hive-site.xml/mapred-site.xml/yarn-site.xml are unchanged
    confFile = tmpFolder.newFile("core-site.xml");
    Assert.assertEquals(confFile, ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir));
}

From source file:co.cask.cdap.hive.ConfCodecTest.java

License:Apache License

@Test
public void testCConfCodec() throws Exception {
    // Serialize/*from  ww w  .ja  v a 2  s .  com*/
    CConfiguration conf = CConfiguration.create();
    conf.set("foo", "bar");

    Configuration hconf = HBaseConfiguration.create();
    hconf.set("hfoo", "hbar");

    Map<String, String> confMap = Maps.newHashMap();
    ConfigurationUtil.set(confMap, Constants.Explore.CCONF_KEY, CConfCodec.INSTANCE, conf);
    ConfigurationUtil.set(confMap, Constants.Explore.HCONF_KEY, HConfCodec.INSTANCE, hconf);

    // Deserialize
    CConfiguration newConf = ConfigurationUtil.get(confMap, Constants.Explore.CCONF_KEY, CConfCodec.INSTANCE);
    Assert.assertEquals("bar", newConf.get("foo"));

    Configuration newHconf = ConfigurationUtil.get(confMap, Constants.Explore.HCONF_KEY, HConfCodec.INSTANCE);
    Assert.assertEquals("hbar", newHconf.get("hfoo"));
}

From source file:co.cask.cdap.hive.datasets.DatasetAccessor.java

License:Apache License

public DatasetAccessor(Configuration conf) throws IOException {
    String datasetName = conf.get(Constants.Explore.DATASET_NAME);
    String namespace = conf.get(Constants.Explore.DATASET_NAMESPACE);
    Preconditions.checkArgument(!Strings.isNullOrEmpty(datasetName), "dataset name not present in config");
    Preconditions.checkArgument(!Strings.isNullOrEmpty(namespace), "namespace not present in config");

    this.datasetId = Id.DatasetInstance.from(namespace, datasetName);
    this.context = ContextManager.getContext(conf);
    this.transaction = ConfigurationUtil.get(conf, Constants.Explore.TX_QUERY_KEY, TxnCodec.INSTANCE);
    this.datasetInstantiator = context.createDatasetInstantiator(conf.getClassLoader());
}