List of usage examples for org.apache.hadoop.conf Configuration getClass
public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface)
name property as a Class implementing the interface specified by xface. From source file:edu.uci.ics.pregelix.api.util.BspUtils.java
License:Apache License
/** * Get the user's subclassed {@link GlobalAggregator}. * /*from w w w . java 2s . c o m*/ * @param conf * Configuration to check * @return User's vertex combiner class */ @SuppressWarnings({ "rawtypes", "unchecked" }) public static <I extends WritableComparable, V extends Writable, E extends Writable, M extends Writable, P extends Writable, F extends Writable> Class<? extends GlobalAggregator<I, V, E, M, P, F>> getGlobalAggregatorClass( Configuration conf) { return (Class<? extends GlobalAggregator<I, V, E, M, P, F>>) conf .getClass(PregelixJob.GLOBAL_AGGREGATOR_CLASS, GlobalCountAggregator.class, GlobalAggregator.class); }
From source file:edu.uci.ics.pregelix.api.util.BspUtils.java
License:Apache License
/** * Get the user's subclassed Vertex./*from w w w .j av a 2s. c o m*/ * * @param conf * Configuration to check * @return User's vertex class */ @SuppressWarnings({ "rawtypes", "unchecked" }) public static <I extends WritableComparable, V extends Writable, E extends Writable, M extends Writable> Class<? extends Vertex<I, V, E, M>> getVertexClass( Configuration conf) { return (Class<? extends Vertex<I, V, E, M>>) conf.getClass(PregelixJob.VERTEX_CLASS, null, Vertex.class); }
From source file:edu.umn.cs.spatialHadoop.mapreduce.SpatialInputFormat3.java
License:Open Source License
@Override protected List<FileStatus> listStatus(JobContext job) throws IOException { try {//from w w w .j a v a2s . c o m Configuration jobConf = job.getConfiguration(); // The block filter associated with this job BlockFilter blockFilter = null; if (jobConf.get(InputQueryRange) != null) { // This job requires a range query blockFilter = new RangeFilter(OperationsParams.getShape(jobConf, InputQueryRange)); } // Retrieve the BlockFilter set by the developers in the JobConf Class<? extends BlockFilter> blockFilterClass = jobConf.getClass(SpatialSite.FilterClass, null, BlockFilter.class); if (blockFilterClass != null) { BlockFilter userBlockFilter = blockFilterClass.newInstance(); blockFilter = blockFilter == null ? userBlockFilter : new CombineBlockFilter(blockFilter, userBlockFilter); } if (blockFilter == null) { // No block filter specified by user LOG.info("No block filter specified"); return super.listStatus(job); } // Get all blocks the user wants to process blockFilter.configure(jobConf); // Filter files based on user specified filter function List<FileStatus> result = new ArrayList<FileStatus>(); Path[] inputDirs = getInputPaths(job); for (Path dir : inputDirs) { FileSystem fs = dir.getFileSystem(jobConf); listStatus(fs, dir, result, blockFilter); } LOG.info("Spatial filter function matched with " + result.size() + " cells"); return result; } catch (InstantiationException e) { LOG.warn(e); return super.listStatus(job); } catch (IllegalAccessException e) { LOG.warn(e); return super.listStatus(job); } }
From source file:edu.umn.cs.spatialHadoop.visualization.Plotter.java
License:Open Source License
public static Plotter getPlotter(Configuration job) { try {// www .j a v a 2s . c o m Class<? extends Plotter> plotterClass = job.getClass(PlotterClass, null, Plotter.class); if (plotterClass == null) throw new RuntimeException("Plotter class not set in job"); Plotter plotter = plotterClass.newInstance(); plotter.configure(job); return plotter; } catch (InstantiationException e) { throw new RuntimeException("Error creating plotter", e); } catch (IllegalAccessException e) { throw new RuntimeException("Error constructing plotter", e); } }
From source file:edu.umn.cs.spatialHadoop.visualization.Rasterizer.java
License:Open Source License
public static Rasterizer getRasterizer(Configuration job) { try {//from w w w . j a va 2 s . c o m Class<? extends Rasterizer> rasterizerClass = job.getClass(RasterizerClass, null, Rasterizer.class); if (rasterizerClass == null) throw new RuntimeException("Rasterizer class not set in job"); Rasterizer rasterizer = rasterizerClass.newInstance(); rasterizer.configure(job); return rasterizer; } catch (InstantiationException e) { throw new RuntimeException("Error creating rasterizer", e); } catch (IllegalAccessException e) { throw new RuntimeException("Error constructing rasterizer", e); } }
From source file:gobblin.restli.throttling.LocalStressTest.java
License:Apache License
public static void main(String[] args) throws Exception { CommandLine cli = StressTestUtils.parseCommandLine(OPTIONS, args); int stressorThreads = Integer.parseInt( cli.getOptionValue(STRESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_STRESSOR_THREADS))); int processorThreads = Integer.parseInt( cli.getOptionValue(PROCESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_PROCESSOR_THREADS))); int artificialLatency = Integer.parseInt( cli.getOptionValue(ARTIFICIAL_LATENCY.getOpt(), Integer.toString(DEFAULT_ARTIFICIAL_LATENCY))); long targetQps = Integer.parseInt(cli.getOptionValue(QPS.getOpt(), Integer.toString(DEFAULT_TARGET_QPS))); Configuration configuration = new Configuration(); StressTestUtils.populateConfigFromCli(configuration, cli); String resourceLimited = LocalStressTest.class.getSimpleName(); Map<String, String> configMap = Maps.newHashMap(); ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory(); SharedLimiterKey res1key = new SharedLimiterKey(resourceLimited); configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), QPSPolicy.FACTORY_ALIAS); configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, QPSPolicy.QPS), Long.toString(targetQps)); ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig(); guiceServletConfig.initialize(ConfigFactory.parseMap(configMap)); LimiterServerResource limiterServer = guiceServletConfig.getInjector() .getInstance(LimiterServerResource.class); RateComputingLimiterContainer limiterContainer = new RateComputingLimiterContainer(); Class<? extends Stressor> stressorClass = configuration.getClass(StressTestUtils.STRESSOR_CLASS, StressTestUtils.DEFAULT_STRESSOR_CLASS, Stressor.class); ExecutorService executorService = Executors.newFixedThreadPool(stressorThreads); SharedResourcesBroker broker = guiceServletConfig.getInjector().getInstance( Key.get(SharedResourcesBroker.class, Names.named(LimiterServerResource.BROKER_INJECT_NAME))); ThrottlingPolicy policy = (ThrottlingPolicy) broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceLimited)); ScheduledExecutorService reportingThread = Executors.newSingleThreadScheduledExecutor(); reportingThread.scheduleAtFixedRate(new Reporter(limiterContainer, policy), 0, 15, TimeUnit.SECONDS); Queue<Future<?>> futures = new LinkedList<>(); MockRequester requester = new MockRequester(limiterServer, artificialLatency, processorThreads); requester.start();/* w w w . j av a2s . c om*/ for (int i = 0; i < stressorThreads; i++) { RestliServiceBasedLimiter restliLimiter = RestliServiceBasedLimiter.builder() .resourceLimited(resourceLimited).requestSender(requester).serviceIdentifier("stressor" + i) .build(); Stressor stressor = stressorClass.newInstance(); stressor.configure(configuration); futures.add(executorService .submit(new StressorRunner(limiterContainer.decorateLimiter(restliLimiter), stressor))); } int stressorFailures = 0; for (Future<?> future : futures) { try { future.get(); } catch (ExecutionException ee) { stressorFailures++; } } requester.stop(); executorService.shutdownNow(); if (stressorFailures > 0) { log.error("There were " + stressorFailures + " failed stressor threads."); } System.exit(stressorFailures); }
From source file:it.crs4.pydoop.mapreduce.pipes.PipesNonJavaInputFormat.java
License:Apache License
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); return ReflectionUtils .newInstance(conf.getClass(Submitter.INPUT_FORMAT, TextInputFormat.class, InputFormat.class), conf) .getSplits(context);/*from ww w . j a v a 2 s .co m*/ }
From source file:it.crs4.pydoop.mapreduce.pipes.CommandLineParser.java
License:Apache License
/** * Get the user's original partitioner.//www . j ava2 s .c o m * @param conf the configuration to look in * @return the class that the user submitted */ static Class<? extends Partitioner> getJavaPartitioner(Configuration conf) { return conf.getClass(Submitter.PARTITIONER, HashPartitioner.class, Partitioner.class); }
From source file:mvm.rya.indexing.accumulo.ConfigUtils.java
License:Apache License
public static Tokenizer getFreeTextTokenizer(Configuration conf) { Class<? extends Tokenizer> c = conf.getClass(TOKENIZER_CLASS, LuceneTokenizer.class, Tokenizer.class); return ReflectionUtils.newInstance(c, conf); }
From source file:org.apache.ambari.servicemonitor.utils.MonitorUtils.java
License:Apache License
/** * Create a reporter/*from w w w.ja v a 2 s. c o m*/ * @param conf the configuration to drive this * @return an inited reporter * @throws ExitMainException if the creation failed */ public static Reporter createReporter(Configuration conf) throws ExitMainException { Class<? extends Reporter> reporterClass = conf.getClass(MonitorKeys.MONITOR_REPORTER, LogReporter.class, Reporter.class); try { Reporter reporter = reporterClass.newInstance(); reporter.init(conf); return reporter; } catch (Exception e) { LOG.error("Instantiation of " + reporterClass + " failed, cause=" + e, e); throw new ExitMainException(Exit.EXIT_ERROR, "Failed to create an instance of " + reporterClass + ": " + e, e); } }