List of usage examples for org.apache.hadoop.conf Configuration setClass
public void setClass(String name, Class<?> theClass, Class<?> xface)
name property to the name of a theClass implementing the given interface xface. From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java
License:Apache License
public static void setInputFormat(final Configuration conf, final Class<? extends InputFormat> val) { conf.setClass(MongoConfigUtil.JOB_INPUT_FORMAT, val, InputFormat.class); }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: MaxTemperatureWithMapOutputCompression " + "<input path> <output path>"); System.exit(-1);/*from w ww . j a v a 2 s .c om*/ } // vv MaxTemperatureWithMapOutputCompression Configuration conf = new Configuration(); conf.setBoolean("mapred.compress.map.output", true); conf.setClass("mapred.map.output.compression.codec", GzipCodec.class, CompressionCodec.class); Job job = new Job(conf); // ^^ MaxTemperatureWithMapOutputCompression job.setJarByClass(MaxTemperature.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setMapperClass(MaxTemperatureMapper.class); job.setCombinerClass(MaxTemperatureReducer.class); job.setReducerClass(MaxTemperatureReducer.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsMultipleOutputs.java
License:Apache License
/** * Adds a named output for the job.//from w w w . ja va 2 s.c o m * <p/> * * @param job job to add the named output * @param namedOutput named output name, it has to be a word, letters and * numbers only, cannot be the word 'part' as that is reserved for the * default output. * @param outputFormatClass OutputFormat class. * @param keyClass key class * @param valueClass value class */ @SuppressWarnings("unchecked") public static void addNamedOutput(Job job, String namedOutput, Class<? extends OutputFormat> outputFormatClass, Class<?> keyClass, Class<?> valueClass) { checkNamedOutputName(job, namedOutput, true); Configuration conf = job.getConfiguration(); conf.set(MULTIPLE_OUTPUTS, conf.get(MULTIPLE_OUTPUTS, "") + " " + namedOutput); conf.setClass(MO_PREFIX + namedOutput + FORMAT, outputFormatClass, OutputFormat.class); conf.setClass(MO_PREFIX + namedOutput + KEY, keyClass, Object.class); conf.setClass(MO_PREFIX + namedOutput + VALUE, valueClass, Object.class); }
From source file:edu.indiana.d2i.htrc.skmeans.StreamingKMeansAdapterTest.java
License:Apache License
@Test public static void testCluster() { int dimension = 500; // construct data samplers centered on the corners of a unit cube Matrix mean = new DenseMatrix(8, dimension); List<MultiNormal> rowSamplers = Lists.newArrayList(); for (int i = 0; i < 8; i++) { // mean.viewRow(i).assign( // new double[] { 0.25 * (i & 4), 0.5 * (i & 2), i & 1 }); double[] random = new double[dimension]; for (int j = 0; j < random.length; j++) { random[j] = Math.random(); }/*from w w w. jav a2 s . c o m*/ mean.viewRow(i).assign(random); rowSamplers.add(new MultiNormal(0.01, mean.viewRow(i))); } // sample a bunch of data points Matrix data = new DenseMatrix(10000, dimension); for (MatrixSlice row : data) { row.vector().assign(rowSamplers.get(row.index() % 8).sample()); } // cluster the data long t0 = System.currentTimeMillis(); double cutoff = StreamingKMeansAdapter.estimateCutoff(data, 100); Configuration conf = new Configuration(); conf.setInt(StreamingKMeansConfigKeys.MAXCLUSTER, 1000); conf.setFloat(StreamingKMeansConfigKeys.CUTOFF, (float) cutoff); conf.setClass(StreamingKMeansConfigKeys.DIST_MEASUREMENT, EuclideanDistanceMeasure.class, DistanceMeasure.class); conf.setInt(StreamingKMeansConfigKeys.VECTOR_DIMENSION, dimension); StreamingKMeansAdapter skmeans = new StreamingKMeansAdapter(conf); // for (MatrixSlice row : Iterables.skip(data, 1)) { // skmeans.cluster(row.vector()); // } for (MatrixSlice row : data) { skmeans.cluster(row.vector()); } // validate Searcher r = skmeans.getCentroids(); // StreamingKMeansAdapter skmeans = new StreamingKMeansAdapter(); // Searcher r = skmeans.cluster(data, 1000, centroidFactory); long t1 = System.currentTimeMillis(); assertEquals("Total weight not preserved", totalWeight(data), totalWeight(r), 1e-9); // and verify that each corner of the cube has a centroid very nearby for (MatrixSlice row : mean) { WeightedVector v = r.search(row.vector(), 1).get(0); assertTrue(v.getWeight() < 0.05); } System.out.printf("%.2f for clustering\n%.1f us per row\n", (t1 - t0) / 1000.0, (t1 - t0) / 1000.0 / data.rowSize() * 1e6); System.out.println("Done??"); }
From source file:edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection.java
License:Apache License
public YarnClientRMConnection(YarnConfiguration config) { this.config = config; InetSocketAddress remoteAddress = NetUtils .createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); Configuration appsManagerServerConf = new Configuration(config); appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER, ClientRMSecurityInfo.class, SecurityInfo.class); YarnRPC rpc = YarnRPC.create(appsManagerServerConf); crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf)); }
From source file:edu.umn.cs.spatialHadoop.core.Partitioner.java
License:Open Source License
/** * Sets the class and value of a partitioner in the given job * @param conf/* w w w . j ava 2 s. co m*/ * @param partitioner * @throws IOException */ public static void setPartitioner(Configuration conf, Partitioner partitioner) throws IOException { conf.setClass(PartitionerClass, partitioner.getClass(), Partitioner.class); Path tempFile; FileSystem fs = FileSystem.get(conf); do { tempFile = new Path("cells_" + (int) (Math.random() * 1000000) + ".partitions"); } while (fs.exists(tempFile)); FSDataOutputStream out = fs.create(tempFile); partitioner.write(out); out.close(); fs.deleteOnExit(tempFile); DistributedCache.addCacheFile(tempFile.toUri(), conf); conf.set(PartitionerValue, tempFile.getName()); }
From source file:edu.umn.cs.spatialHadoop.core.SpatialSite.java
License:Open Source License
/** * It sets the given class in the configuration and, in addition, it sets * the jar of that class to the class path of this job which allows it to * run correctly in a distributed mode./* w w w. j a va 2 s .co m*/ * @param conf - Configuration to set the key * @param key - the key to set * @param klass - the class to use as a value * @param xface - the interface that the provided class should implement */ public static void setClass(Configuration conf, String key, Class<?> klass, Class<?> xface) { conf.setClass(key, klass, xface); addClassToPath(conf, klass); }
From source file:edu.umn.cs.spatialHadoop.indexing.Indexer.java
License:Open Source License
/** * Set the local indexer for the given job configuration. * @param job/* www .j a va2 s. c om*/ * @param sindex */ private static void setLocalIndexer(Configuration conf, String sindex) { Class<? extends LocalIndexer> localIndexerClass = LocalIndexes.get(sindex); if (localIndexerClass != null) conf.setClass(LocalIndexer.LocalIndexerClass, localIndexerClass, LocalIndexer.class); }
From source file:edu.umn.cs.spatialHadoop.nasa.HDFToText.java
License:Open Source License
/** * Performs an HDF to text operation as a MapReduce job and returns total * number of points generated.//from ww w .ja v a 2 s . com * @param inPath * @param outPath * @param datasetName * @param skipFillValue * @return * @throws IOException * @throws ClassNotFoundException * @throws InterruptedException */ public static long HDFToTextMapReduce(Path inPath, Path outPath, String datasetName, boolean skipFillValue, OperationsParams params) throws IOException, InterruptedException, ClassNotFoundException { Job job = new Job(params, "HDFToText"); Configuration conf = job.getConfiguration(); job.setJarByClass(HDFToText.class); job.setJobName("HDFToText"); // Set Map function details job.setMapperClass(HDFToTextMap.class); job.setNumReduceTasks(0); // Set input information job.setInputFormatClass(SpatialInputFormat3.class); SpatialInputFormat3.setInputPaths(job, inPath); if (conf.get("shape") == null) conf.setClass("shape", NASAPoint.class, Shape.class); conf.set("dataset", datasetName); conf.setBoolean("skipfillvalue", skipFillValue); // Set output information job.setOutputFormatClass(TextOutputFormat3.class); TextOutputFormat3.setOutputPath(job, outPath); // Run the job boolean verbose = conf.getBoolean("verbose", false); job.waitForCompletion(verbose); Counters counters = job.getCounters(); Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS); final long resultCount = outputRecordCounter.getValue(); return resultCount; }
From source file:edu.umn.cs.spatialHadoop.visualization.Plotter.java
License:Open Source License
public static void setPlotter(Configuration job, Class<? extends Plotter> plotterClass) { job.setClass(PlotterClass, plotterClass, Plotter.class); }