List of usage examples for org.apache.hadoop.conf Configuration setClass
public void setClass(String name, Class<?> theClass, Class<?> xface)
name property to the name of a theClass implementing the given interface xface. From source file:org.mrgeo.buildpyramid.BuildPyramidDriver.java
License:Apache License
private static boolean buildlevelto(final String pyramidName, final int fromlevel, final int tolevel, final Aggregator aggregator, final Configuration config, final Progress progress, final JobListener jobListener, final Properties providerProperties) throws IOException, JobCancelledException, JobFailedException { final MrsImageDataProvider provider = DataProviderFactory.getMrsImageDataProvider(pyramidName, AccessMode.READ, providerProperties); final MrsImagePyramidMetadata metadata = provider.getMetadataReader().read(); deletelevels(fromlevel, tolevel, metadata, provider); final Job job = MapReduceUtils .createTiledJob("BuildPyramid-" + metadata.getPyramid() + "-" + fromlevel + "-" + tolevel, config); Configuration conf = job.getConfiguration(); conf.setInt(TO_LEVEL, tolevel);/*from www . j a va 2 s.c o m*/ conf.setInt(FROM_LEVEL, fromlevel); conf.setClass(AGGREGATOR, aggregator.getClass(), Aggregator.class); final int tilesize = metadata.getTilesize(); HadoopUtils.setMetadata(job, metadata); job.setMapperClass(BuildPyramidMapper.class); job.setReducerClass(BuildPyramidReducer.class); job.setMapOutputKeyClass(TileIdZoomWritable.class); job.setMapOutputValueClass(RasterWritable.class); MrsImageDataProvider.setupMrsPyramidInputFormat(job, metadata.getPyramid(), fromlevel, metadata.getTilesize(), providerProperties); final Bounds bounds = metadata.getBounds(); // need to set up a empty dummy directory so hadoop can use the multiple output stuff, we're // using the adhoc provider because it provides the exact functionality for creating a temporary // directory AdHocDataProvider dummy = DataProviderFactory.createAdHocDataProvider(providerProperties); // mimic FileOutputFormat.setOutputPath(job, path); conf.set("mapred.output.dir", dummy.getResourceName()); final AdHocDataProvider[] statsProviders = new AdHocDataProvider[fromlevel - tolevel]; final MrsImageOutputFormatProvider[] outputProviders = new MrsImageOutputFormatProvider[fromlevel - tolevel]; MrsImageDataProvider.setupMrsPyramidMultipleOutputJob(job); for (int level = tolevel; level < fromlevel; level++) { int lo = level - tolevel; statsProviders[lo] = DataProviderFactory.createAdHocDataProvider(providerProperties); conf.set(ADHOC_PROVIDER + level, statsProviders[lo].getResourceName()); final String l = Integer.toString(level); outputProviders[lo] = MrsImageDataProvider.addMrsPyramidMultipleOutputFormat(job, metadata.getPyramid(), l, bounds, level, tilesize, metadata.getTileType(), metadata.getBands(), metadata.getProtectionLevel(), providerProperties); } HadoopUtils.setJar(job, BuildPyramidDriver.class); final boolean success = MapReduceUtils.runJob(job, progress, jobListener); // remove the dummy directory output dummy.delete(); if (success) { // update the resampling method used to build pyramids metadata.setResamplingMethod( AggregatorRegistry.aggregatorRegistry.inverse().get(aggregator.getClass())); for (int level = tolevel; level < fromlevel; level++) { int lo = level - tolevel; outputProviders[lo].teardown(job); final TMSUtils.TileBounds tb = TMSUtils.boundsToTile( new TMSUtils.Bounds(bounds.getMinX(), bounds.getMinY(), bounds.getMaxX(), bounds.getMaxY()), level, tilesize); final LongRectangle b = new LongRectangle(tb.w, tb.s, tb.e, tb.n); final TMSUtils.Pixel psw = TMSUtils.latLonToPixels(bounds.getMinY(), bounds.getMinX(), level, tilesize); final TMSUtils.Pixel pne = TMSUtils.latLonToPixels(bounds.getMaxY(), bounds.getMaxX(), level, tilesize); // need to update the metadata metadata.setPixelBounds(level, new LongRectangle(0, 0, pne.px - psw.px, pne.py - psw.py)); metadata.setTileBounds(level, b); metadata.setName(level); // update the pyramid level stats final ImageStats[] levelStats = ImageStats.readStats(statsProviders[lo]); statsProviders[lo].delete(); metadata.setImageStats(level, levelStats); provider.getMetadataWriter().write(); } } return success; }
From source file:org.mrgeo.buildpyramid.BuildPyramidReducerTest.java
License:Apache License
@Test @Category(UnitTest.class) public void reduce() { Configuration config = new Configuration(); config.setInt(BuildPyramidDriver.TO_LEVEL, zoom); config.setInt(BuildPyramidDriver.FROM_LEVEL, zoom + 1); // manually set up the configs DiretoryMultipleOutputs String namedOutput = "5"; config.set(base, config.get(base, "") + " " + namedOutput); config.setClass(MO_PREFIX + namedOutput + FORMAT, HdfsMrsPyramidOutputFormat.class, OutputFormat.class); config.setClass(MO_PREFIX + namedOutput + KEY, TileIdWritable.class, Object.class); config.setClass(MO_PREFIX + namedOutput + VALUE, RasterWritable.class, Object.class); config.set(MO_PREFIX + namedOutput + HDFS_PATH, Defs.OUTPUT_HDFS); MrsImagePyramidMetadata metadata = new MrsImagePyramidMetadata(); metadata.setTilesize(size);/*from w ww.j ava 2 s. c om*/ metadata.setBands(b + 1); metadata.setMaxZoomLevel(zoom); metadata.setDefaultValues(new double[] { Double.NaN }); try { HadoopUtils.setMetadata(config, metadata); } catch (IOException e) { e.printStackTrace(); Assert.fail("Failed to set metadata to Hadoop configuration."); } // Tile Id for to Tile childTile = TMSUtils.latLonToTile(lat, lon, zoom, size); TileIdZoomWritable key = new TileIdZoomWritable(TMSUtils.tileid(childTile.tx, childTile.ty, zoom), zoom); // Tile Ids for froms TileIdZoomWritable parentKeyUL = new TileIdZoomWritable( TMSUtils.tileid(2 * childTile.tx, 2 * childTile.ty + 1, zoom + 1), zoom + 1); TileIdZoomWritable parentKeyUR = new TileIdZoomWritable( TMSUtils.tileid(2 * childTile.tx + 1, 2 * childTile.ty + 1, zoom + 1), zoom + 1); TileIdZoomWritable parentKeyLL = new TileIdZoomWritable( TMSUtils.tileid(2 * childTile.tx, 2 * childTile.ty, zoom + 1), zoom + 1); TileIdZoomWritable parentKeyLR = new TileIdZoomWritable( TMSUtils.tileid(2 * childTile.tx + 1, 2 * childTile.ty, zoom + 1), zoom + 1); List<RasterWritable> value; try { value = new ArrayList<RasterWritable>(); value.add(RasterWritable.toWritable(rasterUL, parentKeyUL)); value.add(RasterWritable.toWritable(rasterUR, parentKeyUR)); value.add(RasterWritable.toWritable(rasterLL, parentKeyLL)); value.add(RasterWritable.toWritable(rasterLR, parentKeyLR)); ReduceDriver<TileIdZoomWritable, RasterWritable, TileIdWritable, RasterWritable> driver = new ReduceDriver<TileIdZoomWritable, RasterWritable, TileIdWritable, RasterWritable>() .withConfiguration(config).withReducer(new BuildPyramidReducer()).withInputKey(key) .withInputValues(value); java.util.List<Pair<TileIdWritable, RasterWritable>> results = driver.run(); // Test the results Assert.assertEquals("Bad number of reduces returned", 1, results.size()); java.util.ListIterator<Pair<TileIdWritable, RasterWritable>> iter = results.listIterator(); Assert.assertTrue("Reduce iterator doesn't have a next item", iter.hasNext()); Pair<TileIdWritable, RasterWritable> item = iter.next(); // Assert that output child key equals input key Assert.assertEquals("Input tileid doesn't match output", key.get(), item.getFirst().get()); TestUtils.compareRasters(childRaster, RasterWritable.toRaster(item.getSecond())); // TODO: Test the stats calculations // test the counters, why is the plus one for the child included? Assert.assertEquals("Tile count (counter) incorrect.", 4 + 1, driver.getCounters().findCounter("BuildPyramid Reducer", "Parent Tiles Processed").getValue()); } catch (NullPointerException ex) { log.warn("Could not write temporary stats file, which is normal in the unit test."); } catch (Exception e) { e.printStackTrace(); Assert.fail("Reduce test threw an exception: " + e.getMessage()); } }
From source file:org.mrgeo.format.FilteredFeatureInputFormat.java
License:Apache License
/** * Sets the input format that this feature filter will use. * // ww w .j av a 2 s.co m * @param conf * @param theClass */ public static void setInputFormat(Configuration conf, Class<? extends InputFormat<LongWritable, Geometry>> theClass) { conf.setClass(FilteredFeatureInputFormat.class.getName() + "input.format", theClass, InputFormat.class); }
From source file:org.mrgeo.hadoop.multipleoutputs.DirectoryMultipleOutputs.java
License:Apache License
/** * Adds a named output for the job.//w w w. j a v a2s . c o m * <p/> * * @param job * job to add the named output * @param namedOutput * named output name, it has to be a word, letters and numbers only, cannot be the word * 'part' as that is reserved for the default output. * * The named output is a key used internally that references the hdfsPath give as the * next argument * * @param Path * to output in HDFS * @param outputFormatClass * OutputFormat class. * @param keyClass * key class * @param valueClass * value class */ @SuppressWarnings("rawtypes") public static void addNamedOutput(final Job job, final String namedOutput, final Path hdfsPath, final Class<? extends OutputFormat> outputFormatClass, final Class<?> keyClass, final Class<?> valueClass) { checkNamedOutputName(job, namedOutput, true); final Configuration conf = job.getConfiguration(); conf.set(base, conf.get(base, "") + " " + namedOutput); conf.setClass(MO_PREFIX + namedOutput + FORMAT, outputFormatClass, OutputFormat.class); conf.setClass(MO_PREFIX + namedOutput + KEY, keyClass, Object.class); conf.setClass(MO_PREFIX + namedOutput + VALUE, valueClass, Object.class); conf.set(MO_PREFIX + namedOutput + HDFS_PATH, hdfsPath.toString()); }
From source file:org.springframework.hadoop.mapreduce.AutowiringPartitionerTests.java
License:Apache License
@Test public void testPartitioner() throws Exception { Configuration configuration = new Configuration(); configuration.setClass(DefaultContextLoader.SPRING_CONFIG_LOCATION, TestConfiguration.class, Object.class); HadoopApplicationContextUtils.getBean(configuration, String.class); AutowiringPartitioner<Integer, String> partitioner = new AutowiringPartitioner<Integer, String>(); partitioner.setConf(configuration);/*ww w . j a va2s.co m*/ assertEquals(new Integer(123).hashCode(), partitioner.getPartition(123, "bar", 10)); }
From source file:org.springframework.yarn.examples.grid.yarn.YarnManagedContainerGroupsTests.java
License:Apache License
private YarnManagedContainerGroups createYmcgResolveToDefaultWithRack() throws Exception { Configuration configuration = new Configuration(); configuration.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, TestRackResolver.class, DNSToSwitchMapping.class); // For some weird reason hadoop's rack resolver init // is static so we need to tweak field via reflection // order to re-init it. // TODO: this may break other tests if rack resolver is initialized after this test Field initCalledField = RackResolver.class.getDeclaredField("initCalled"); initCalledField.setAccessible(true); initCalledField.setBoolean(null, false); RackResolver.init(configuration);// w w w . j av a 2 s .c o m GenericContainerGroupResolver groupResolver = new GenericContainerGroupResolver(); groupResolver.setConfiguration(configuration); groupResolver.setResolveRacks(true); Map<String, List<String>> resolves = new Hashtable<String, List<String>>(); resolves.put(YarnManagedContainerGroups.DEFAULT_GROUP, Arrays.asList(new String[] { "*" })); resolves.put(RACK_GROUP, Arrays.asList(new String[] { "/" + RACK3 })); groupResolver.setResolves(resolves); groupResolver.afterPropertiesSet(); YarnManagedContainerGroups managedGroups = new YarnManagedContainerGroups(true); managedGroups.setResolver(groupResolver); // default group size to 1 Map<String, Integer> groupSizes = new Hashtable<String, Integer>(); groupSizes.put(YarnManagedContainerGroups.DEFAULT_GROUP, 2); groupSizes.put(RACK_GROUP, 1); managedGroups.setGroupSizes(groupSizes); return managedGroups; }
From source file:org.warcbase.mapreduce.lib.Chain.java
License:Apache License
/** * Adds a Mapper class to the chain job. * * <p/>/* w ww . j a v a 2 s . c o m*/ * The configuration properties of the chain job have precedence over the * configuration properties of the Mapper. * * @param isMap * indicates if the Chain is for a Mapper or for a Reducer. * @param job * chain job. * @param klass * the Mapper class to add. * @param inputKeyClass * mapper input key class. * @param inputValueClass * mapper input value class. * @param outputKeyClass * mapper output key class. * @param outputValueClass * mapper output value class. * @param mapperConf * a configuration for the Mapper class. It is recommended to use a * Configuration without default values using the * <code>Configuration(boolean loadDefaults)</code> constructor with * FALSE. */ @SuppressWarnings("unchecked") protected static void addMapper(Job job, Class<? extends Mapper> klass, Class<?> inputKeyClass, Class<?> inputValueClass, Class<?> outputKeyClass, Class<?> outputValueClass, Configuration mapperConf) { String prefix = getPrefix(); Configuration jobConf = job.getConfiguration(); // set the mapper class int index = getIndex(jobConf, prefix); jobConf.setClass(prefix + CHAIN_MAPPER_CLASS + index, klass, Mapper.class); validateKeyValueTypes(jobConf, inputKeyClass, inputValueClass, outputKeyClass, outputValueClass, index, prefix); setMapperConf(jobConf, inputKeyClass, inputValueClass, outputKeyClass, outputValueClass, mapperConf, index, prefix); }
From source file:org.warcbase.mapreduce.lib.Chain.java
License:Apache License
protected static void setMapperConf(Configuration jobConf, Class<?> inputKeyClass, Class<?> inputValueClass, Class<?> outputKeyClass, Class<?> outputValueClass, Configuration mapperConf, int index, String prefix) {//w w w . ja v a 2 s. c om // if the Mapper does not have a configuration, create an empty one if (mapperConf == null) { // using a Configuration without defaults to make it lightweight. // still the chain's conf may have all defaults and this conf is // overlapped to the chain configuration one. mapperConf = new Configuration(true); } // store the input/output classes of the mapper in the mapper conf mapperConf.setClass(MAPPER_INPUT_KEY_CLASS, inputKeyClass, Object.class); mapperConf.setClass(MAPPER_INPUT_VALUE_CLASS, inputValueClass, Object.class); mapperConf.setClass(MAPPER_OUTPUT_KEY_CLASS, outputKeyClass, Object.class); mapperConf.setClass(MAPPER_OUTPUT_VALUE_CLASS, outputValueClass, Object.class); // serialize the mapper configuration in the chain configuration. Stringifier<Configuration> stringifier = new DefaultStringifier<Configuration>(jobConf, Configuration.class); try { jobConf.set(prefix + CHAIN_MAPPER_CONFIG + index, stringifier.toString(new Configuration(mapperConf))); } catch (IOException ioEx) { throw new RuntimeException(ioEx); } // increment the chain counter jobConf.setInt(prefix + CHAIN_MAPPER_SIZE, index + 1); }
From source file:org.zuinnote.hadoop.office.example.MapReduceExcelInputIntegrationTest.java
License:Apache License
@BeforeAll public static void oneTimeSetUp() throws IOException { // Create temporary directory for HDFS base and shutdownhook // create temp directory tmpPath = Files.createTempDirectory(tmpPrefix); // create shutdown hook to remove temp files (=HDFS MiniCluster) after shutdown, may need to rethink to avoid many threads are created Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override// w w w. j ava 2 s .com public void run() { try { Files.walkFileTree(tmpPath, new SimpleFileVisitor<java.nio.file.Path>() { @Override public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) throws IOException { Files.delete(file); return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(java.nio.file.Path dir, IOException e) throws IOException { if (e == null) { Files.delete(dir); return FileVisitResult.CONTINUE; } throw e; } }); } catch (IOException e) { throw new RuntimeException( "Error temporary files in following path could not be deleted " + tmpPath, e); } } })); // Create Configuration Configuration conf = new Configuration(); // create HDFS cluster File baseDir = new File(tmpPath.toString()).getAbsoluteFile(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); dfsCluster = builder.numDataNodes(NOOFDATANODES).build(); // create Yarn cluster YarnConfiguration clusterConf = new YarnConfiguration(conf); conf.set("fs.defaultFS", dfsCluster.getFileSystem().getUri().toString()); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); miniCluster = new MiniMRYarnCluster(CLUSTERNAME, NOOFNODEMANAGERS, STARTTIMELINESERVER); miniCluster.init(conf); miniCluster.start(); }