List of usage examples for org.apache.hadoop.conf Configuration setIfUnset
public synchronized void setIfUnset(String name, String value)
From source file:org.apache.gora.pig.StorageConfiguration.java
License:Apache License
/** * Updates the parameter configuration with the data in this bean. This method does not overwrite the existing key-values! * @param hadoopConfiguration//from w w w . j av a 2s.c o m */ public void mergeConfiguration(Configuration hadoopConfiguration) { this.configuration.forEach((key, value) -> hadoopConfiguration.setIfUnset(key, value)); }
From source file:org.apache.phoenix.index.PhoenixIndexBuilder.java
License:Apache License
@Override public void setup(RegionCoprocessorEnvironment env) throws IOException { super.setup(env); Configuration conf = env.getConfiguration(); // Install handler that will attempt to disable the index first before killing the region // server/*from w w w . j ava 2 s . c om*/ conf.setIfUnset(IndexWriter.INDEX_FAILURE_POLICY_CONF_KEY, PhoenixIndexFailurePolicy.class.getName()); }
From source file:org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder.java
License:Apache License
/** * Return EdgeProperty that connects two vertices. * * @param from//from w w w. j a v a 2 s .c o m * @param to * @return EdgeProperty * @throws IOException */ private EdgeProperty newEdge(TezOperator from, TezOperator to) throws IOException { TezEdgeDescriptor edge = to.inEdges.get(from.getOperatorKey()); PhysicalPlan combinePlan = edge.combinePlan; InputDescriptor in = InputDescriptor.create(edge.inputClassName); OutputDescriptor out = OutputDescriptor.create(edge.outputClassName); Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties(), false); if (!combinePlan.isEmpty()) { addCombiner(combinePlan, to, conf); } List<POLocalRearrangeTez> lrs = PlanHelper.getPhysicalOperators(from.plan, POLocalRearrangeTez.class); for (POLocalRearrangeTez lr : lrs) { if (lr.getOutputKey().equals(to.getOperatorKey().toString())) { byte keyType = lr.getKeyType(); setIntermediateOutputKeyValue(keyType, conf, to, lr.isConnectedToPackage()); // In case of secondary key sort, main key type is the actual key type conf.set("pig.reduce.key.type", Byte.toString(lr.getMainKeyType())); break; } } conf.setIfUnset(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName()); if (edge.getIntermediateOutputKeyClass() != null) { conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edge.getIntermediateOutputKeyClass()); } if (edge.getIntermediateOutputValueClass() != null) { conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edge.getIntermediateOutputValueClass()); } if (edge.getIntermediateOutputKeyComparatorClass() != null) { conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edge.getIntermediateOutputKeyComparatorClass()); } conf.setBoolean(MRConfiguration.MAPPER_NEW_API, true); conf.set("pig.pigContext", ObjectSerializer.serialize(pc)); conf.set("udf.import.list", ObjectSerializer.serialize(PigContext.getPackageImportList())); if (to.isGlobalSort() || to.isLimitAfterSort()) { conf.set("pig.sortOrder", ObjectSerializer.serialize(to.getSortOrder())); } if (edge.isUseSecondaryKey()) { conf.set("pig.secondarySortOrder", ObjectSerializer.serialize(edge.getSecondarySortOrder())); conf.set(org.apache.hadoop.mapreduce.MRJobConfig.PARTITIONER_CLASS_ATTR, SecondaryKeyPartitioner.class.getName()); // These needs to be on the vertex as well for POShuffleTezLoad to pick it up. // Tez framework also expects this to be per vertex and not edge. IFile.java picks // up keyClass and valueClass from vertex config. TODO - check with Tez folks // In MR - job.setSortComparatorClass() or MRJobConfig.KEY_COMPARATOR conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, PigSecondaryKeyComparator.class.getName()); // In MR - job.setOutputKeyClass() or MRJobConfig.OUTPUT_KEY_CLASS conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, NullableTuple.class.getName()); setGroupingComparator(conf, PigSecondaryKeyGroupComparator.class.getName()); } if (edge.partitionerClass != null) { conf.set(org.apache.hadoop.mapreduce.MRJobConfig.PARTITIONER_CLASS_ATTR, edge.partitionerClass.getName()); } conf.set("udf.import.list", ObjectSerializer.serialize(PigContext.getPackageImportList())); MRToTezHelper.processMRSettings(conf, globalConf); in.setUserPayload(TezUtils.createUserPayloadFromConf(conf)); out.setUserPayload(TezUtils.createUserPayloadFromConf(conf)); if (edge.dataMovementType != DataMovementType.BROADCAST && to.getEstimatedParallelism() != -1 && (to.isGlobalSort() || to.isSkewedJoin())) { // Use custom edge return EdgeProperty.create((EdgeManagerPluginDescriptor) null, edge.dataSourceType, edge.schedulingType, out, in); } return EdgeProperty.create(edge.dataMovementType, edge.dataSourceType, edge.schedulingType, out, in); }
From source file:org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper.java
License:Apache License
/** * Convert MR settings to Tez settings and set on conf. * * @param conf Configuration on which MR equivalent Tez settings should be set * @param mrConf Configuration that contains MR settings *///from ww w . j av a2 s . c o m private static void convertMRToTezRuntimeConf(Configuration conf, Configuration mrConf) { for (Entry<String, String> dep : DeprecatedKeys.getMRToTezRuntimeParamMap().entrySet()) { if (mrConf.get(dep.getKey()) != null) { conf.unset(dep.getKey()); LOG.info("Setting " + dep.getValue() + " to " + mrConf.get(dep.getKey()) + " from MR setting " + dep.getKey()); conf.setIfUnset(dep.getValue(), mrConf.get(dep.getKey())); } } }
From source file:org.apache.sqoop.hive.HiveConfig.java
License:Apache License
/** * Add hive conf to configuration object without overriding already set properties. * @param hiveConf/*from ww w . j av a2 s. c om*/ * @param conf */ public static void addHiveConfigs(Configuration hiveConf, Configuration conf) { for (Map.Entry<String, String> item : hiveConf) { conf.setIfUnset(item.getKey(), item.getValue()); } }
From source file:org.apache.sqoop.mapreduce.PGBulkloadExportJob.java
License:Apache License
@Override protected void propagateOptionsToJob(Job job) { super.propagateOptionsToJob(job); SqoopOptions opts = context.getOptions(); Configuration conf = job.getConfiguration(); conf.setIfUnset("pgbulkload.bin", "pg_bulkload"); if (opts.getNullStringValue() != null) { conf.set("pgbulkload.null.string", opts.getNullStringValue()); }//from w w w .j a va2 s. c o m setDelimiter("pgbulkload.input.field.delim", opts.getInputFieldDelim(), conf); setDelimiter("pgbulkload.input.record.delim", opts.getInputRecordDelim(), conf); setDelimiter("pgbulkload.input.enclosedby", opts.getInputEnclosedBy(), conf); setDelimiter("pgbulkload.input.escapedby", opts.getInputEscapedBy(), conf); conf.setBoolean("pgbulkload.input.encloserequired", opts.isInputEncloseRequired()); conf.setIfUnset("pgbulkload.check.constraints", "YES"); conf.setIfUnset("pgbulkload.parse.errors", "INFINITE"); conf.setIfUnset("pgbulkload.duplicate.errors", "INFINITE"); conf.set("mapred.jar", context.getJarFile()); conf.setBoolean("mapred.map.tasks.speculative.execution", false); conf.setBoolean("mapred.reduce.tasks.speculative.execution", false); conf.setInt("mapred.map.max.attempts", 1); conf.setInt("mapred.reduce.max.attempts", 1); conf.setIfUnset("mapred.reduce.tasks", "1"); if (context.getOptions().doClearStagingTable()) { conf.setBoolean("pgbulkload.clear.staging.table", true); } }
From source file:org.culturegraph.mf.cluster.job.beacon.FileBeacon.java
License:Apache License
@Override protected Configuration prepareConf(final Configuration conf) { addRequiredArguments(ConfigConst.INPUT_PATH, ConfigConst.MORPH_DEF, ConfigConst.OUTPUT_PATH, ConfigConst.STORE_RAW_DATA); conf.setIfUnset(ConfigConst.STORE_RAW_DATA, "false"); return HBaseConfiguration.create(getConf()); }
From source file:org.culturegraph.mf.cluster.job.ingest.BibIngest.java
License:Apache License
@Override protected Configuration prepareConf(final Configuration config) { addRequiredArguments(ConfigConst.OUTPUT_TABLE, ConfigConst.FORMAT, ConfigConst.INGEST_PREFIX, ConfigConst.INPUT_PATH, ConfigConst.MORPH_DEF); addOptionalArguments(ConfigConst.STORE_RAW_DATA, "hbase.zookeeper.quorum"); final Configuration conf = HBaseConfiguration.create(config); conf.setIfUnset(ConfigConst.STORE_RAW_DATA, ConfigConst.FALSE); conf.setIfUnset("mapred.map.tasks.speculative.execution", "false"); setJobName("Ingest " + config.get(ConfigConst.INPUT_PATH) + " -> " + config.get(ConfigConst.OUTPUT_TABLE)); return conf;// w ww. java2 s .c o m }
From source file:org.culturegraph.mf.cluster.job.ingest.PetrusIngest.java
License:Apache License
@Override protected Configuration prepareConf(final Configuration config) { addRequiredArguments(ConfigConst.OUTPUT_TABLE, ConfigConst.INPUT_PATH); setJobName("Ingest " + config.get(ConfigConst.INPUT_PATH) + " -> " + config.get(ConfigConst.OUTPUT_TABLE)); config.setIfUnset("mapred.map.tasks.speculative.execution", "false"); return HBaseConfiguration.create(config); }
From source file:org.culturegraph.mf.cluster.job.ingest.PropIngest.java
License:Apache License
@Override protected Configuration prepareConf(final Configuration config) { addRequiredArguments(ConfigConst.OUTPUT_TABLE, ConfigConst.INPUT_PATH); final Configuration conf = HBaseConfiguration.create(config); config.setIfUnset("mapred.map.tasks.speculative.execution", "false"); setJobName("Ingest " + config.get(ConfigConst.INPUT_PATH) + " -> " + config.get(ConfigConst.OUTPUT_TABLE)); return conf;//from w w w .jav a 2s .c om }