List of usage examples for org.apache.hadoop.conf Configuration iterator
@Override
public Iterator<Map.Entry<String, String>> iterator()
String key-value pairs in the configuration. From source file:org.apache.tez.common.TezUtils.java
License:Apache License
public static String convertToHistoryText(String description, Configuration conf) { // Add a version if this serialization is changed JSONObject jsonObject = new JSONObject(); try {/*www. j a v a 2 s . c om*/ if (description != null && !description.isEmpty()) { jsonObject.put(ATSConstants.DESCRIPTION, description); } if (conf != null) { JSONObject confJson = new JSONObject(); Iterator<Entry<String, String>> iter = conf.iterator(); while (iter.hasNext()) { Entry<String, String> entry = iter.next(); confJson.put(entry.getKey(), conf.get(entry.getKey())); } jsonObject.put(ATSConstants.CONFIG, confJson); } } catch (JSONException e) { throw new TezUncheckedException("Error when trying to convert description/conf to JSON", e); } return jsonObject.toString(); }
From source file:org.apache.tez.dag.api.DAG.java
License:Apache License
@Private public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCredentials, Map<String, LocalResource> tezJarResources, LocalResource binaryConfig, boolean tezLrsAsArchive, Map<String, String> additionalConfigs) { verify(true);//from w w w.j a v a 2s.c om DAGPlan.Builder dagBuilder = DAGPlan.newBuilder(); dagBuilder.setName(this.name); if (this.dagInfo != null && !this.dagInfo.isEmpty()) { dagBuilder.setDagInfo(this.dagInfo); } if (!vertexGroups.isEmpty()) { for (VertexGroup av : vertexGroups) { GroupInfo groupInfo = av.getGroupInfo(); PlanVertexGroupInfo.Builder groupBuilder = PlanVertexGroupInfo.newBuilder(); groupBuilder.setGroupName(groupInfo.getGroupName()); for (Vertex v : groupInfo.getMembers()) { groupBuilder.addGroupMembers(v.getName()); } groupBuilder.addAllOutputs(groupInfo.outputs); for (Map.Entry<String, InputDescriptor> entry : groupInfo.edgeMergedInputs.entrySet()) { groupBuilder.addEdgeMergedInputs( PlanGroupInputEdgeInfo.newBuilder().setDestVertexName(entry.getKey()) .setMergedInput(DagTypeConverters.convertToDAGPlan(entry.getValue()))); } dagBuilder.addVertexGroups(groupBuilder); } } Credentials dagCredentials = new Credentials(); if (extraCredentials != null) { dagCredentials.mergeAll(extraCredentials); } dagCredentials.mergeAll(credentials); if (!commonTaskLocalFiles.isEmpty()) { dagBuilder.addAllLocalResource(DagTypeConverters.convertToDAGPlan(commonTaskLocalFiles)); } Preconditions.checkArgument(topologicalVertexStack.size() == vertices.size(), "size of topologicalVertexStack is:" + topologicalVertexStack.size() + " while size of vertices is:" + vertices.size() + ", make sure they are the same in order to sort the vertices"); while (!topologicalVertexStack.isEmpty()) { Vertex vertex = vertices.get(topologicalVertexStack.pop()); // infer credentials, resources and parallelism from data source Resource vertexTaskResource = vertex.getTaskResource(); if (vertexTaskResource == null) { vertexTaskResource = Resource.newInstance( tezConf.getInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT), tezConf.getInt(TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT)); } Map<String, LocalResource> vertexLRs = Maps.newHashMap(); vertexLRs.putAll(vertex.getTaskLocalFiles()); List<DataSourceDescriptor> dataSources = vertex.getDataSources(); for (DataSourceDescriptor dataSource : dataSources) { if (dataSource.getCredentials() != null) { dagCredentials.addAll(dataSource.getCredentials()); } if (dataSource.getAdditionalLocalFiles() != null) { TezCommonUtils.addAdditionalLocalResources(dataSource.getAdditionalLocalFiles(), vertexLRs, "Vertex " + vertex.getName()); } } if (tezJarResources != null) { TezCommonUtils.addAdditionalLocalResources(tezJarResources, vertexLRs, "Vertex " + vertex.getName()); } if (binaryConfig != null) { vertexLRs.put(TezConstants.TEZ_PB_BINARY_CONF_NAME, binaryConfig); } int vertexParallelism = vertex.getParallelism(); VertexLocationHint vertexLocationHint = vertex.getLocationHint(); if (dataSources.size() == 1) { DataSourceDescriptor dataSource = dataSources.get(0); if (vertexParallelism == -1 && dataSource.getNumberOfShards() > -1) { vertexParallelism = dataSource.getNumberOfShards(); } if (vertexLocationHint == null && dataSource.getLocationHint() != null) { vertexLocationHint = dataSource.getLocationHint(); } } if (vertexParallelism == -1) { Preconditions.checkState(vertexLocationHint == null, "Cannot specify vertex location hint without specifying vertex parallelism. Vertex: " + vertex.getName()); } else if (vertexLocationHint != null) { Preconditions.checkState(vertexParallelism == vertexLocationHint.getTaskLocationHints().size(), "vertex task location hint must equal vertex parallelism. Vertex: " + vertex.getName()); } for (DataSinkDescriptor dataSink : vertex.getDataSinks()) { if (dataSink.getCredentials() != null) { dagCredentials.addAll(dataSink.getCredentials()); } } VertexPlan.Builder vertexBuilder = VertexPlan.newBuilder(); vertexBuilder.setName(vertex.getName()); vertexBuilder.setType(PlanVertexType.NORMAL); // vertex type is implicitly NORMAL until TEZ-46. vertexBuilder .setProcessorDescriptor(DagTypeConverters.convertToDAGPlan(vertex.getProcessorDescriptor())); if (vertex.getInputs().size() > 0) { for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : vertex.getInputs()) { vertexBuilder.addInputs(DagTypeConverters.convertToDAGPlan(input)); } } if (vertex.getOutputs().size() > 0) { for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : vertex .getOutputs()) { vertexBuilder.addOutputs(DagTypeConverters.convertToDAGPlan(output)); } } if (vertex.getConf() != null && vertex.getConf().size() > 0) { ConfigurationProto.Builder confBuilder = ConfigurationProto.newBuilder(); for (Map.Entry<String, String> entry : vertex.getConf().entrySet()) { PlanKeyValuePair.Builder keyValueBuilder = PlanKeyValuePair.newBuilder(); keyValueBuilder.setKey(entry.getKey()); keyValueBuilder.setValue(entry.getValue()); confBuilder.addConfKeyValues(keyValueBuilder); } vertexBuilder.setVertexConf(confBuilder); } //task config PlanTaskConfiguration.Builder taskConfigBuilder = PlanTaskConfiguration.newBuilder(); taskConfigBuilder.setNumTasks(vertexParallelism); taskConfigBuilder.setMemoryMb(vertexTaskResource.getMemory()); taskConfigBuilder.setVirtualCores(vertexTaskResource.getVirtualCores()); taskConfigBuilder.setJavaOpts( TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vertex.getTaskLaunchCmdOpts(), tezConf)); taskConfigBuilder.setTaskModule(vertex.getName()); if (!vertexLRs.isEmpty()) { taskConfigBuilder.addAllLocalResource(DagTypeConverters.convertToDAGPlan(vertexLRs)); } Map<String, String> taskEnv = Maps.newHashMap(vertex.getTaskEnvironment()); TezYARNUtils.setupDefaultEnv(taskEnv, tezConf, TezConfiguration.TEZ_TASK_LAUNCH_ENV, TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); for (Map.Entry<String, String> entry : taskEnv.entrySet()) { PlanKeyValuePair.Builder envSettingBuilder = PlanKeyValuePair.newBuilder(); envSettingBuilder.setKey(entry.getKey()); envSettingBuilder.setValue(entry.getValue()); taskConfigBuilder.addEnvironmentSetting(envSettingBuilder); } if (vertexLocationHint != null) { if (vertexLocationHint.getTaskLocationHints() != null) { for (TaskLocationHint hint : vertexLocationHint.getTaskLocationHints()) { PlanTaskLocationHint.Builder taskLocationHintBuilder = PlanTaskLocationHint.newBuilder(); // we can allow this later on if needed if (hint.getAffinitizedTask() != null) { throw new TezUncheckedException( "Task based affinity may not be specified via the DAG API"); } if (hint.getHosts() != null) { taskLocationHintBuilder.addAllHost(hint.getHosts()); } if (hint.getRacks() != null) { taskLocationHintBuilder.addAllRack(hint.getRacks()); } vertexBuilder.addTaskLocationHint(taskLocationHintBuilder); } } } if (vertex.getVertexManagerPlugin() != null) { vertexBuilder.setVertexManagerPlugin( DagTypeConverters.convertToDAGPlan(vertex.getVertexManagerPlugin())); } for (Edge inEdge : vertex.getInputEdges()) { vertexBuilder.addInEdgeId(inEdge.getId()); } for (Edge outEdge : vertex.getOutputEdges()) { vertexBuilder.addOutEdgeId(outEdge.getId()); } vertexBuilder.setTaskConfig(taskConfigBuilder); dagBuilder.addVertex(vertexBuilder); } for (Edge edge : edges) { EdgePlan.Builder edgeBuilder = EdgePlan.newBuilder(); edgeBuilder.setId(edge.getId()); edgeBuilder.setInputVertexName(edge.getInputVertex().getName()); edgeBuilder.setOutputVertexName(edge.getOutputVertex().getName()); edgeBuilder.setDataMovementType( DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getDataMovementType())); edgeBuilder.setDataSourceType( DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getDataSourceType())); edgeBuilder.setSchedulingType( DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getSchedulingType())); edgeBuilder.setEdgeSource(DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getEdgeSource())); edgeBuilder.setEdgeDestination( DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getEdgeDestination())); if (edge.getEdgeProperty().getDataMovementType() == DataMovementType.CUSTOM) { if (edge.getEdgeProperty().getEdgeManagerDescriptor() != null) { edgeBuilder.setEdgeManager( DagTypeConverters.convertToDAGPlan(edge.getEdgeProperty().getEdgeManagerDescriptor())); } // else the AM will deal with this. } dagBuilder.addEdge(edgeBuilder); } ConfigurationProto.Builder confProtoBuilder = ConfigurationProto.newBuilder(); if (dagAccessControls != null) { Configuration aclConf = new Configuration(false); dagAccessControls.serializeToConfiguration(aclConf); Iterator<Entry<String, String>> aclConfIter = aclConf.iterator(); while (aclConfIter.hasNext()) { Entry<String, String> entry = aclConfIter.next(); PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); kvp.setKey(entry.getKey()); kvp.setValue(entry.getValue()); TezConfiguration.validateProperty(entry.getKey(), Scope.DAG); confProtoBuilder.addConfKeyValues(kvp); } } if (additionalConfigs != null && !additionalConfigs.isEmpty()) { for (Entry<String, String> entry : additionalConfigs.entrySet()) { PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); kvp.setKey(entry.getKey()); kvp.setValue(entry.getValue()); TezConfiguration.validateProperty(entry.getKey(), Scope.DAG); confProtoBuilder.addConfKeyValues(kvp); } } if (this.dagConf != null && !this.dagConf.isEmpty()) { for (Entry<String, String> entry : this.dagConf.entrySet()) { PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); kvp.setKey(entry.getKey()); kvp.setValue(entry.getValue()); confProtoBuilder.addConfKeyValues(kvp); } } dagBuilder.setDagConf(confProtoBuilder); if (dagCredentials != null) { dagBuilder.setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCredentials)); TezCommonUtils.logCredentials(LOG, dagCredentials, "dag"); } return dagBuilder.build(); }
From source file:org.apache.tez.dag.history.utils.DAGUtils.java
License:Apache License
public static Map<String, String> convertConfigurationToATSMap(Configuration conf) { Iterator<Entry<String, String>> iter = conf.iterator(); Map<String, String> atsConf = new TreeMap<String, String>(); while (iter.hasNext()) { Entry<String, String> entry = iter.next(); atsConf.put(entry.getKey(), entry.getValue()); }//from w ww. j ava 2s . c om return atsConf; }
From source file:org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil.java
License:Apache License
@Private static Configuration extractStageConf(Configuration baseConf, String prefix) { Configuration strippedConf = new Configuration(false); Configuration conf = new Configuration(false); Iterator<Entry<String, String>> confEntries = baseConf.iterator(); while (confEntries.hasNext()) { Entry<String, String> entry = confEntries.next(); String key = entry.getKey(); if (key.startsWith(prefix)) { // Ignore keys for other intermediate stages in case of an initial or final stage. if (prefix.equals("") && key.startsWith(MRJobConfig.MRR_INTERMEDIATE_STAGE_PREFIX)) { continue; }/*from ww w .j av a2 s. com*/ String newKey = key.replace(prefix, ""); strippedConf.set(newKey, entry.getValue()); } else { // Ignore keys for other intermediate stages. if (key.startsWith(MRJobConfig.MRR_INTERMEDIATE_STAGE_PREFIX)) { continue; } // Set all base keys in the new conf conf.set(key, entry.getValue()); } } // Replace values from strippedConf into the finalConf. Override values // which may have been copied over from the baseConf root level. Iterator<Entry<String, String>> entries = strippedConf.iterator(); while (entries.hasNext()) { Entry<String, String> entry = entries.next(); if (!Configuration.isDeprecated(entry.getKey())) { conf.set(entry.getKey(), entry.getValue()); } } return conf; }
From source file:org.apache.tez.mapreduce.task.MRRuntimeTask.java
License:Apache License
public static void copyTezConfigParameters(Configuration conf, Configuration tezTaskConf) { Iterator<Entry<String, String>> iter = tezTaskConf.iterator(); while (iter.hasNext()) { Entry<String, String> entry = iter.next(); if (conf.get(entry.getKey()) == null) { conf.set(entry.getKey(), tezTaskConf.get(entry.getKey())); }/*from w w w . jav a 2s . c o m*/ } }
From source file:org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil.java
License:Apache License
public static org.apache.commons.configuration.Configuration makeApacheConfiguration( final Configuration hadoopConfiguration) { final BaseConfiguration apacheConfiguration = new BaseConfiguration(); apacheConfiguration.setDelimiterParsingDisabled(true); hadoopConfiguration.iterator() .forEachRemaining(e -> apacheConfiguration.setProperty(e.getKey(), e.getValue())); return apacheConfiguration; }
From source file:org.commonvox.hbase_column_manager.TestConfiguration.java
License:Apache License
@Test public void testConfigurationLoading() throws Exception { ClasspathSearcher.confirmFileInClasspath("config", MConnectionFactory.COLUMN_MANAGER_CONFIG_FILE); try (Connection mConnection = MConnectionFactory.createConnection(); Admin mAdmin = mConnection.getAdmin()) { mAdmin.getClusterStatus(); // assure valid mConnection established (otherwise throws Exception) // Assure that every property in the COLUMN_MANAGER_CONFIG_FILE // appears as expected in the mConnection's configuration. Configuration sessionConf = mConnection.getConfiguration(); Configuration colManagerConf = new Configuration(); colManagerConf.addResource(MConnectionFactory.COLUMN_MANAGER_CONFIG_FILE); Iterator<Entry<String, String>> parmIterator = colManagerConf.iterator(); while (parmIterator.hasNext()) { Entry<String, String> colManagerParameter = parmIterator.next(); if (colManagerParameter.getKey().startsWith(Repository.HBASE_CONFIG_PARM_KEY_PREFIX)) { assertEquals(/*w ww.j av a 2s. co m*/ CONFIGURATION_FAILURE + colManagerParameter.getKey() + " parameter not found with expected value in connection configuration.", colManagerParameter.getValue(), sessionConf.get(colManagerParameter.getKey())); } } } catch (Exception e) { e.printStackTrace(); fail("Exception thrown by MConnectionFactory#createConnection: " + e.getMessage()); } }
From source file:org.godhuli.rhipe.RHMRHelper.java
License:Apache License
void addJobConfToEnvironment(Configuration conf, Properties env) { Iterator it = conf.iterator(); while (it.hasNext()) { Map.Entry en = (Map.Entry) it.next(); String name = (String) en.getKey(); if (name.equals("mapred.input.dir") || name.equals("rhipe_input_folder")) continue; String value = null;//from w w w . j a v a 2s . c o m if (!(name.equals("LD_LIBRARY_PATH") || name.equals("PATH"))) { value = conf.get(name); // does variable expansion } else { value = conf.getRaw(name); } env.put(name, value); } }
From source file:org.huahinframework.manager.util.JobUtils.java
License:Apache License
/** * @param trackingURL job tracking URL//from w w w. j a va 2 s . co m * @param jobId job ID * @return job configuration map * @throws IOException * @throws HttpException * @throws DocumentException */ public static Map<String, String> getJobConfiguration(final String jobFile, JobConf conf) throws HttpException, IOException { final Map<String, String> m = new HashMap<String, String>(); URI uri = URI.create(jobFile); String path = uri.getPath(); FileSystem fs = FileSystem.get(conf); FileStatus fstatus = fs.getFileStatus(new Path(path)); if (fstatus == null) { return null; } InputStream is = fs.open(fstatus.getPath()); Configuration jobConf = new JobConf(false); jobConf.addResource(is); Iterator<Entry<String, String>> ite = jobConf.iterator(); while (ite.hasNext()) { Entry<String, String> entry = ite.next(); m.put(entry.getKey(), entry.getValue()); } is.close(); return m; }
From source file:org.lilyproject.client.impl.HBaseConnections.java
License:Apache License
private Map<String, String> toMap(Configuration conf) { Map<String, String> result = new HashMap<String, String>(); Iterator<Map.Entry<String, String>> it = conf.iterator(); while (it.hasNext()) { Map.Entry<String, String> entry = it.next(); result.put(entry.getKey(), entry.getValue()); }/* w ww. j a v a2s.c om*/ return result; }