List of usage examples for org.apache.hadoop.conf Configuration getClassByName
public Class<?> getClassByName(String name) throws ClassNotFoundException
From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java
License:Apache License
/** * Get the {@link CompressionCodec} for compressing the job outputs. * * @param job the {@link Job} to look in * @param defaultValue the {@link CompressionCodec} to return if not set * @return the {@link CompressionCodec} to be used to compress the job * outputs// w ww . ja v a2 s. c om * @throws IllegalArgumentException if the class was specified, but not * found */ public static Class<? extends CompressionCodec> getOutputCompressorClass(JobContext job, Class<? extends CompressionCodec> defaultValue) { Class<? extends CompressionCodec> codecClass = defaultValue; Configuration conf = job.getConfiguration(); String name = conf.get("edu.arizona.cs.hadoop.fs.irods.mapred.output.compression.codec"); if (name != null) { try { codecClass = conf.getClassByName(name).asSubclass(CompressionCodec.class); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Compression codec " + name + " was not found.", e); } } return codecClass; }
From source file:edu.cmu.lemurproject.WarcFileRecordReader.java
License:Open Source License
public WarcFileRecordReader(Configuration conf, InputSplit split) throws IOException { this.fs = FileSystem.get(conf); this.conf = conf; if (split instanceof FileSplit) { this.filePathList = new Path[1]; this.filePathList[0] = ((FileSplit) split).getPath(); } else if (split instanceof MultiFileSplit) { this.filePathList = ((MultiFileSplit) split).getPaths(); } else {//from ww w . j a v a 2s .co m throw new IOException("InputSplit is not a file split or a multi-file split - aborting"); } // get the total file sizes for (int i = 0; i < filePathList.length; i++) { totalFileSize += fs.getFileStatus(filePathList[i]).getLen(); } Class<? extends CompressionCodec> codecClass = null; try { codecClass = conf.getClassByName("org.apache.hadoop.io.compress.GzipCodec") .asSubclass(CompressionCodec.class); compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); } catch (ClassNotFoundException cnfEx) { compressionCodec = null; LOG.info("!!! ClassNotFoun Exception thrown setting Gzip codec"); } openNextFile(); }
From source file:edu.umn.cs.spatialHadoop.OperationsParams.java
License:Open Source License
public static TextSerializable getTextSerializable(Configuration conf, String key, TextSerializable defaultValue) { String shapeType = conf.get(key); if (shapeType == null) return defaultValue; int separatorIndex = shapeType.indexOf(ShapeValueSeparator); Text shapeValue = null;/*from w ww .j a v a 2s .c o m*/ if (separatorIndex != -1) { shapeValue = new Text(shapeType.substring(separatorIndex + ShapeValueSeparator.length())); shapeType = shapeType.substring(0, separatorIndex); } TextSerializable shape; try { Class<? extends TextSerializable> shapeClass = conf.getClassByName(shapeType) .asSubclass(TextSerializable.class); shape = shapeClass.newInstance(); } catch (Exception e) { // shapeClass is not an explicit class name String shapeTypeI = shapeType.toLowerCase(); if (shapeTypeI.startsWith("rect")) { shape = new Rectangle(); } else if (shapeTypeI.startsWith("point")) { shape = new Point(); } else if (shapeTypeI.startsWith("tiger")) { shape = new TigerShape(); } else if (shapeTypeI.startsWith("osm")) { shape = new OSMPolygon(); } else if (shapeTypeI.startsWith("poly")) { shape = new Polygon(); } else if (shapeTypeI.startsWith("ogc")) { shape = new OGCESRIShape(); } else if (shapeTypeI.startsWith("wkt")) { shape = new OGCJTSShape(); } else if (shapeTypeI.startsWith("nasapoint")) { shape = new NASAPoint(); } else if (shapeTypeI.startsWith("nasarect")) { shape = new NASARectangle(); } else if (shapeTypeI.startsWith("text")) { shape = new Text2(); } else { // Couldn't detect shape from short name or full class name // May be it's an actual value that we can parse if (shapeType.split(",").length == 2) { // A point shape = new Point(); shape.fromText(new Text((String) conf.get(key))); } else if (shapeType.split(",").length == 4) { // A rectangle shape = new Rectangle(); shape.fromText(new Text((String) conf.get(key))); } else { LOG.warn("unknown shape type: '" + conf.get(key) + "'"); return null; } } } if (shapeValue != null) shape.fromText(shapeValue); // Special case for CSVOGC shape, specify the column if possible if (shape instanceof CSVOGC) { CSVOGC csvShape = (CSVOGC) shape; String strColumnIndex = conf.get("column"); if (strColumnIndex != null) csvShape.setColumn(Integer.parseInt(strColumnIndex)); String strSeparator = conf.get("separator"); if (strSeparator != null) csvShape.setSeparator(strSeparator.charAt(0)); } return shape; }
From source file:io.hops.erasure_coding.Codec.java
License:Apache License
public ErasureCode createErasureCode(Configuration conf) { // Create the scheduler Class<?> erasureCode = null; try {//from w ww .j ava 2s. c om erasureCode = conf.getClass(ERASURE_CODE_KEY_PREFIX + this.id, conf.getClassByName(this.erasureCodeClass)); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } ErasureCode code = (ErasureCode) ReflectionUtils.newInstance(erasureCode, conf); code.init(this); return code; }
From source file:it.crs4.pydoop.mapreduce.pipes.CommandLineParser.java
License:Apache License
private static <InterfaceType> Class<? extends InterfaceType> getClass(CommandLine cl, String key, Configuration conf, Class<InterfaceType> cls) throws ClassNotFoundException { return conf.getClassByName(cl.getOptionValue(key)).asSubclass(cls); }
From source file:org.apache.hama.bsp.BSPJobClient.java
License:Apache License
/** * Get the {@link CompressionCodec} for compressing the job outputs. * //from w w w.j av a 2 s . c o m * @param job the {@link BSPJob} to look in * @param defaultValue the {@link CompressionCodec} to return if not set * @return the {@link CompressionCodec} to be used to compress the job outputs * @throws IllegalArgumentException if the class was specified, but not found */ static Class<? extends CompressionCodec> getOutputCompressorClass(BSPJob job, Class<? extends CompressionCodec> defaultValue) { Class<? extends CompressionCodec> codecClass = defaultValue; Configuration conf = job.getConfiguration(); String name = conf.get("bsp.partitioning.compression.codec"); if (name != null) { try { codecClass = conf.getClassByName(name).asSubclass(CompressionCodec.class); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Compression codec " + name + " was not found.", e); } } return codecClass; }
From source file:org.apache.hama.bsp.message.compress.BSPMessageCompressorFactory.java
License:Apache License
/** * Returns a compressor via reflection based on what was configured. * /*from w ww .j av a 2 s .com*/ * @param conf * @return a compressor that was configured. */ @SuppressWarnings("unchecked") public BSPMessageCompressor<M> getCompressor(Configuration conf) { if (conf.get(COMPRESSION_CODEC_CLASS) != null) { try { return (BSPMessageCompressor<M>) ReflectionUtils.newInstance(conf.getClassByName( conf.get(COMPRESSION_CODEC_CLASS, Bzip2Compressor.class.getCanonicalName())), conf); } catch (ClassNotFoundException e) { e.printStackTrace(); } } return null; }
From source file:org.apache.hama.bsp.message.MessageManagerFactory.java
License:Apache License
/** * Returns a messenger via reflection based on what was configured. * // w ww . j a v a 2 s .c om * @param conf * @return a messenger that was configured. */ @SuppressWarnings("unchecked") public static <M extends Writable> MessageManager<M> getMessageManager(Configuration conf) throws ClassNotFoundException { return (MessageManager<M>) ReflectionUtils .newInstance( conf.getClassByName(conf.get(MESSAGE_MANAGER_CLASS, org.apache.hama.bsp.message.HamaMessageManagerImpl.class.getCanonicalName())), conf); }
From source file:org.apache.hama.bsp.SequenceFileRecordWriter.java
License:Apache License
public SequenceFileRecordWriter(FileSystem fs, BSPJob job, String name) throws IOException, ClassNotFoundException { Configuration conf = job.getConfiguration(); writer = new SequenceFile.Writer(fs, conf, new Path(conf.get("bsp.output.dir"), name), conf.getClassByName(conf.get("bsp.output.key.class")), conf.getClassByName(conf.get("bsp.output.value.class"))); }
From source file:org.apache.hama.bsp.sync.SyncServiceFactory.java
License:Apache License
/** * Returns a sync client via reflection based on what was configured. *//*from w ww.j a v a 2 s .c o m*/ public static PeerSyncClient getPeerSyncClient(Configuration conf) throws ClassNotFoundException { return (PeerSyncClient) ReflectionUtils.newInstance( conf.getClassByName(conf.get(SYNC_CLIENT_CLASS, ZooKeeperSyncClientImpl.class.getName())), conf); }