List of usage examples for org.apache.hadoop.conf Configuration getClass
public Class<?> getClass(String name, Class<?> defaultValue)
name property as a Class. From source file:org.apache.oozie.action.hadoop.JavaMain.java
License:Apache License
@Override protected void run(String[] args) throws Exception { Configuration actionConf = loadActionConf(); setYarnTag(actionConf);/* w ww .ja v a2 s.co m*/ LauncherMainHadoopUtils.killChildYarnJobs(actionConf); Class<?> klass = actionConf.getClass(JAVA_MAIN_CLASS, Object.class); System.out.println("Main class : " + klass.getName()); System.out.println("Arguments :"); for (String arg : args) { System.out.println(" " + arg); } System.out.println(); Method mainMethod = klass.getMethod("main", String[].class); try { mainMethod.invoke(null, (Object) args); } catch (InvocationTargetException ex) { // Get rid of the InvocationTargetException and wrap the Throwable throw new JavaMainException(ex.getCause()); } }
From source file:org.apache.oozie.service.ConfigurationService.java
License:Apache License
public static Class<?> getClass(Configuration conf, String name) { return conf.getClass(name, Object.class); }
From source file:org.apache.oozie.service.PartitionDependencyManagerService.java
License:Apache License
private void init(Configuration conf) throws ServiceException { Class<?> defaultClass = conf.getClass(CACHE_MANAGER_IMPL, null); dependencyCache = (defaultClass == null) ? new SimpleHCatDependencyCache() : (HCatDependencyCache) ReflectionUtils.newInstance(defaultClass, null); dependencyCache.init(conf);// w w w .j a v a 2s . co m LOG.info("PartitionDependencyManagerService initialized. Dependency cache is {0} ", dependencyCache.getClass().getName()); purgeEnabled = Services.get().get(JobsConcurrencyService.class).isHighlyAvailableMode(); if (purgeEnabled) { Runnable purgeThread = new CachePurgeWorker(dependencyCache); // schedule runnable by default every 10 min Services.get().get(SchedulerService.class).schedule(purgeThread, 10, Services.get().getConf().getInt(CACHE_PURGE_INTERVAL, 600), SchedulerService.Unit.SEC); registeredCoordActionMap = new ConcurrentHashMap<String, Long>(); } }
From source file:org.apache.oozie.service.URIHandlerService.java
License:Apache License
private void init(Configuration conf) throws ClassNotFoundException { cache = new HashMap<String, URIHandler>(); String[] classes = ConfigurationService.getStrings(conf, URI_HANDLERS); for (String classname : classes) { Class<?> clazz = Class.forName(classname.trim()); URIHandler uriHandler = (URIHandler) ReflectionUtils.newInstance(clazz, null); uriHandler.init(conf);//w w w . j av a 2 s . com for (String scheme : uriHandler.getSupportedSchemes()) { cache.put(scheme, uriHandler); } } Class<?> defaultClass = conf.getClass(URI_HANDLER_DEFAULT, null); defaultHandler = (defaultClass == null) ? new FSURIHandler() : (URIHandler) ReflectionUtils.newInstance(defaultClass, null); defaultHandler.init(conf); for (String scheme : defaultHandler.getSupportedSchemes()) { cache.put(scheme, defaultHandler); } initLauncherClassesToShip(); initLauncherURIHandlerConf(); LOG.info("Loaded urihandlers {0}", Arrays.toString(classes)); LOG.info("Loaded default urihandler {0}", defaultHandler.getClass().getName()); }
From source file:org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.java
License:Apache License
public static Class<?> getInputClass(final Configuration configuration) { return configuration.getClass(INPUT_CLASS, NullDBWritable.class); }
From source file:org.apache.slider.server.appmaster.rpc.RpcBinder.java
License:Apache License
/** * Verify that the conf is set up for protobuf transport of Slider RPC * @param conf configuration/*from ww w. j ava2 s . c o m*/ * @param sliderClusterAPIClass class for the API * @return */ public static boolean verifyBondedToProtobuf(Configuration conf, Class<SliderClusterProtocolPB> sliderClusterAPIClass) { return conf.getClass("rpc.engine." + sliderClusterAPIClass.getName(), RpcEngine.class) .equals(ProtobufRpcEngine.class); }
From source file:org.apache.tez.engine.common.sort.impl.ExternalSorter.java
License:Apache License
public void initialize(Configuration conf, Master master) throws IOException, InterruptedException { this.job = conf; LOG.info("TEZ_ENGINE_TASK_ATTEMPT_ID: " + job.get(Constants.TEZ_ENGINE_TASK_ATTEMPT_ID)); partitions = task.getOutputSpecList().get(0).getNumOutputs(); // partitions = // job.getInt( // TezJobConfig.TEZ_ENGINE_TASK_OUTDEGREE, // TezJobConfig.DEFAULT_TEZ_ENGINE_TASK_OUTDEGREE); rfs = ((LocalFileSystem) FileSystem.getLocal(job)).getRaw(); // sorter/* w w w . jav a 2 s . c o m*/ sorter = ReflectionUtils.newInstance( job.getClass(TezJobConfig.TEZ_ENGINE_INTERNAL_SORTER_CLASS, QuickSort.class, IndexedSorter.class), job); comparator = ConfigUtils.getIntermediateOutputKeyComparator(job); // k/v serialization keyClass = ConfigUtils.getIntermediateOutputKeyClass(job); valClass = ConfigUtils.getIntermediateOutputValueClass(job); serializationFactory = new SerializationFactory(job); keySerializer = serializationFactory.getSerializer(keyClass); valSerializer = serializationFactory.getSerializer(valClass); // counters mapOutputByteCounter = runningTaskContext.getTaskReporter().getCounter(TaskCounter.MAP_OUTPUT_BYTES); mapOutputRecordCounter = runningTaskContext.getTaskReporter().getCounter(TaskCounter.MAP_OUTPUT_RECORDS); fileOutputByteCounter = runningTaskContext.getTaskReporter() .getCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES); spilledRecordsCounter = runningTaskContext.getTaskReporter().getCounter(TaskCounter.SPILLED_RECORDS); // compression if (ConfigUtils.shouldCompressIntermediateOutput(job)) { Class<? extends CompressionCodec> codecClass = ConfigUtils.getIntermediateOutputCompressorClass(job, DefaultCodec.class); codec = ReflectionUtils.newInstance(codecClass, job); } else { codec = null; } // Task outputs mapOutputFile = (TezTaskOutput) ReflectionUtils.newInstance( conf.getClass(Constants.TEZ_ENGINE_TASK_OUTPUT_MANAGER, TezTaskOutputFiles.class), conf); // sortPhase sortPhase = runningTaskContext.getProgress().addPhase("sort"); }
From source file:org.apache.tez.mapreduce.partition.MRPartitioner.java
License:Apache License
public MRPartitioner(Configuration conf) { this.useNewApi = ConfigUtils.useNewApi(conf); int partitions = conf.getInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS, 1); if (useNewApi) { oldPartitioner = null;//from w w w . j a v a 2 s.co m if (partitions > 1) { newPartitioner = (org.apache.hadoop.mapreduce.Partitioner) ReflectionUtils .newInstance((Class<? extends org.apache.hadoop.mapreduce.Partitioner<?, ?>>) conf.getClass( MRJobConfig.PARTITIONER_CLASS_ATTR, org.apache.hadoop.mapreduce.lib.partition.HashPartitioner.class), conf); } else { newPartitioner = new org.apache.hadoop.mapreduce.Partitioner() { @Override public int getPartition(Object key, Object value, int numPartitions) { return numPartitions - 1; } }; } } else { newPartitioner = null; if (partitions > 1) { oldPartitioner = (org.apache.hadoop.mapred.Partitioner) ReflectionUtils.newInstance( (Class<? extends org.apache.hadoop.mapred.Partitioner>) conf.getClass( "mapred.partitioner.class", org.apache.hadoop.mapred.lib.HashPartitioner.class), new JobConf(conf)); } else { oldPartitioner = new org.apache.hadoop.mapred.Partitioner() { @Override public void configure(JobConf job) { } @Override public int getPartition(Object key, Object value, int numPartitions) { return numPartitions - 1; } }; } } }
From source file:org.apache.tez.runtime.library.common.TezRuntimeUtils.java
License:Apache License
public static TezTaskOutput instantiateTaskOutputManager(Configuration conf, OutputContext outputContext) { Class<?> clazz = conf.getClass(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER, TezTaskOutputFiles.class); try {//from w w w . j ava 2 s . c o m Constructor<?> ctor = clazz.getConstructor(Configuration.class, String.class); ctor.setAccessible(true); TezTaskOutput instance = (TezTaskOutput) ctor.newInstance(conf, outputContext.getUniqueIdentifier()); return instance; } catch (Exception e) { throw new TezUncheckedException("Unable to instantiate configured TezOutputFileManager: " + conf.get(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER, TezTaskOutputFiles.class.getName()), e); } }
From source file:org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer.java
License:Apache License
@Override public Future<ComputerResult> submit() { if (this.executed) throw Exceptions.computerHasAlreadyBeenSubmittedAVertexProgram(); else/*w ww. j a v a 2 s. com*/ this.executed = true; // it is not possible execute a computer if it has no vertex program nor mapreducers if (null == this.vertexProgram && this.mapReducers.isEmpty()) throw GraphComputer.Exceptions.computerHasNoVertexProgramNorMapReducers(); // it is possible to run mapreducers without a vertex program if (null != this.vertexProgram) { GraphComputerHelper.validateProgramOnComputer(this, vertexProgram); this.mapReducers.addAll(this.vertexProgram.getMapReducers()); } // determine persistence and result graph options if (!this.persist.isPresent()) this.persist = Optional .of(null == this.vertexProgram ? Persist.NOTHING : this.vertexProgram.getPreferredPersist()); if (!this.resultGraph.isPresent()) this.resultGraph = Optional.of(null == this.vertexProgram ? ResultGraph.ORIGINAL : this.vertexProgram.getPreferredResultGraph()); if (this.resultGraph.get().equals(ResultGraph.ORIGINAL)) if (!this.persist.get().equals(Persist.NOTHING)) throw GraphComputer.Exceptions.resultGraphPersistCombinationNotSupported(this.resultGraph.get(), this.persist.get()); // apache and hadoop configurations that are used throughout final org.apache.commons.configuration.Configuration apacheConfiguration = new HadoopConfiguration( this.hadoopGraph.configuration()); apacheConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.get().equals(Persist.EDGES)); final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(apacheConfiguration); return CompletableFuture.<ComputerResult>supplyAsync(() -> { final long startTime = System.currentTimeMillis(); SparkMemory memory = null; SparkExecutor.deleteOutputLocation(hadoopConfiguration); // wire up a spark context final SparkConf sparkConfiguration = new SparkConf(); sparkConfiguration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX + (null == this.vertexProgram ? "No VertexProgram" : this.vertexProgram) + "[" + this.mapReducers + "]"); hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue())); if (FileInputFormat.class.isAssignableFrom( hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) hadoopConfiguration.set(Constants.MAPRED_INPUT_DIR, SparkExecutor.getInputLocation(hadoopConfiguration)); // necessary for Spark and newAPIHadoopRDD // execute the vertex program and map reducers and if there is a failure, auto-close the spark context try (final JavaSparkContext sparkContext = new JavaSparkContext(sparkConfiguration)) { // add the project jars to the cluster SparkGraphComputer.loadJars(sparkContext, hadoopConfiguration); // create a message-passing friendly rdd from the hadoop input format JavaPairRDD<Object, SparkPayload<Object>> graphRDD = sparkContext .newAPIHadoopRDD(hadoopConfiguration, (Class<InputFormat<NullWritable, VertexWritable>>) hadoopConfiguration .getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class), NullWritable.class, VertexWritable.class) .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), (SparkPayload<Object>) new SparkVertexPayload<>(tuple._2().get()))) .reduceByKey((a, b) -> a); // partition the graph across the cluster // todo: cache? //////////////////////////////// // process the vertex program // //////////////////////////////// if (null != this.vertexProgram) { // set up the vertex program and wire up configurations memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext); this.vertexProgram.setup(memory); memory.broadcastMemory(sparkContext); final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration(); this.vertexProgram.storeState(vertexProgramConfiguration); ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration); ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration); // execute the vertex program while (true) { memory.setInTask(true); graphRDD = SparkExecutor.executeVertexProgramIteration(graphRDD, memory, vertexProgramConfiguration); memory.setInTask(false); if (this.vertexProgram.terminate(memory)) break; else { memory.incrIteration(); memory.broadcastMemory(sparkContext); } } // write the output graph back to disk if (!this.persist.get().equals(Persist.NOTHING)) SparkExecutor.saveGraphRDD(graphRDD, hadoopConfiguration); } final Memory.Admin finalMemory = null == memory ? new MapMemory() : new MapMemory(memory); ////////////////////////////// // process the map reducers // ////////////////////////////// if (!this.mapReducers.isEmpty()) { // drop all edges and messages in the graphRDD as they are no longer needed for the map reduce jobs graphRDD = graphRDD.mapValues(vertex -> { vertex.getMessages().clear(); vertex.asVertexPayload().getOutgoingMessages().clear(); vertex.asVertexPayload().getVertex().edges(Direction.BOTH).forEachRemaining(Edge::remove); return vertex; }); // todo: cache()? for (final MapReduce mapReduce : this.mapReducers) { // execute the map reduce job final HadoopConfiguration newApacheConfiguration = new HadoopConfiguration( apacheConfiguration); mapReduce.storeState(newApacheConfiguration); // map final JavaPairRDD mapRDD = SparkExecutor.executeMap((JavaPairRDD) graphRDD, mapReduce, newApacheConfiguration); // combine TODO? is this really needed // reduce final JavaPairRDD reduceRDD = (mapReduce.doStage(MapReduce.Stage.REDUCE)) ? SparkExecutor.executeReduce(mapRDD, mapReduce, newApacheConfiguration) : null; // write the map reduce output back to disk (memory) SparkExecutor.saveMapReduceRDD(null == reduceRDD ? mapRDD : reduceRDD, mapReduce, finalMemory, hadoopConfiguration); } } // close the context or else bad things happen // TODO: does this happen automatically cause of the try(resource) {} block? sparkContext.close(); // update runtime and return the newly computed graph finalMemory.setRuntime(System.currentTimeMillis() - startTime); return new DefaultComputerResult( HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph.get(), this.persist.get()), finalMemory.asImmutable()); } }); }