List of usage examples for org.apache.hadoop.conf Configuration getClassByName
public Class<?> getClassByName(String name) throws ClassNotFoundException
From source file:com.cloudera.sqoop.mapreduce.MergeJob.java
License:Apache License
public boolean runMergeJob() throws IOException { Configuration conf = options.getConf(); Job job = new Job(conf); String userClassName = options.getClassName(); if (null == userClassName) { // Shouldn't get here. throw new IOException("Record class name not specified with " + "--class-name."); }//w w w .ja va2 s .co m // Set the external jar to use for the job. String existingJar = options.getExistingJarName(); if (existingJar != null) { // User explicitly identified a jar path. LOG.debug("Setting job jar to user-specified jar: " + existingJar); job.getConfiguration().set("mapred.jar", existingJar); } else { // Infer it from the location of the specified class, if it's on the // classpath. try { Class<? extends Object> userClass = conf.getClassByName(userClassName); if (null != userClass) { String userJar = Jars.getJarPathForClass(userClass); LOG.debug("Setting job jar based on user class " + userClassName + ": " + userJar); job.getConfiguration().set("mapred.jar", userJar); } else { LOG.warn("Specified class " + userClassName + " is not in a jar. " + "MapReduce may not find the class"); } } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe); } } try { Path oldPath = new Path(options.getMergeOldPath()); Path newPath = new Path(options.getMergeNewPath()); Configuration jobConf = job.getConfiguration(); FileSystem fs = FileSystem.get(jobConf); oldPath = oldPath.makeQualified(fs); newPath = newPath.makeQualified(fs); FileInputFormat.addInputPath(job, oldPath); FileInputFormat.addInputPath(job, newPath); jobConf.set(MERGE_OLD_PATH_KEY, oldPath.toString()); jobConf.set(MERGE_NEW_PATH_KEY, newPath.toString()); jobConf.set(MERGE_KEY_COL_KEY, options.getMergeKeyCol()); jobConf.set(MERGE_SQOOP_RECORD_KEY, userClassName); FileOutputFormat.setOutputPath(job, new Path(options.getTargetDir())); if (ExportJobBase.isSequenceFiles(jobConf, newPath)) { job.setInputFormatClass(SequenceFileInputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(MergeRecordMapper.class); } else { job.setMapperClass(MergeTextMapper.class); job.setOutputFormatClass(RawKeyTextOutputFormat.class); } jobConf.set("mapred.output.key.class", userClassName); job.setOutputValueClass(NullWritable.class); job.setReducerClass(MergeReducer.class); // Set the intermediate data types. job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(MergeRecord.class); // Make sure Sqoop and anything else we need is on the classpath. cacheJars(job, null); return this.runJob(job); } catch (InterruptedException ie) { throw new IOException(ie); } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe); } }
From source file:com.digitalpebble.behemoth.io.warc.WarcFileRecordReader.java
License:Open Source License
public WarcFileRecordReader(Configuration conf, InputSplit split) throws IOException { this.fs = FileSystem.get(conf); if (split instanceof FileSplit) { this.filePathList = new Path[1]; this.filePathList[0] = ((FileSplit) split).getPath(); } else if (split instanceof MultiFileSplit) { this.filePathList = ((MultiFileSplit) split).getPaths(); } else {//from w w w.j a v a 2 s . co m throw new IOException("InputSplit is not a file split or a multi-file split - aborting"); } // get the total file sizes for (int i = 0; i < filePathList.length; i++) { totalFileSize += fs.getFileStatus(filePathList[i]).getLen(); } Class<? extends CompressionCodec> codecClass = null; try { codecClass = conf.getClassByName("org.apache.hadoop.io.compress.GzipCodec") .asSubclass(CompressionCodec.class); compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); } catch (ClassNotFoundException cnfEx) { compressionCodec = null; LOG.info("!!! ClassNotFoun Exception thrown setting Gzip codec"); } openNextFile(); }
From source file:com.ebay.nest.io.sede.thrift.ThriftDeserializer.java
License:Apache License
@Override public void initialize(Configuration job, Properties tbl) throws SerDeException { try {//w ww . j a va 2s. co m // both the classname and the protocol name are Table properties // the only hardwired assumption is that records are fixed on a // per Table basis String className = tbl.getProperty(serdeConstants.SERIALIZATION_CLASS); Class<?> recordClass = job.getClassByName(className); String protoName = tbl.getProperty(serdeConstants.SERIALIZATION_FORMAT); if (protoName == null) { protoName = "TBinaryProtocol"; } // For backward compatibility protoName = protoName.replace("com.facebook.thrift.protocol", "org.apache.thrift.protocol"); TProtocolFactory tp = TReflectionUtils.getProtocolFactoryByName(protoName); tsd = new ThriftByteStreamTypedSerDe(recordClass, tp, tp); } catch (Exception e) { throw new SerDeException(e); } }
From source file:com.ery.server.util.ReflectionUtils.java
License:Apache License
/** * This code is to support backward compatibility and break the compile time * dependency of core on mapred. This should be made deprecated along with * the mapred package HADOOP-1230. Should be removed when mapred package is * removed./* w w w. j ava 2 s .c om*/ */ private static void setJobConf(Object theObject, Configuration conf) { // If JobConf and JobConfigurable are in classpath, AND // theObject is of type JobConfigurable AND // conf is of type JobConf then // invoke configure on theObject try { Class<?> jobConfClass = conf.getClassByName("org.apache.hadoop.mapred.JobConf"); Class<?> jobConfigurableClass = conf.getClassByName("org.apache.hadoop.mapred.JobConfigurable"); if (jobConfClass.isAssignableFrom(conf.getClass()) && jobConfigurableClass.isAssignableFrom(theObject.getClass())) { Method configureMethod = jobConfigurableClass.getMethod("configure", jobConfClass); configureMethod.invoke(theObject, conf); } } catch (ClassNotFoundException e) { // JobConf/JobConfigurable not in classpath. no need to configure } catch (Exception e) { throw new RuntimeException("Error in configuring object", e); } }
From source file:com.facebook.presto.hive.s3.PrestoS3ClientFactory.java
License:Apache License
private static AWSCredentialsProvider getCustomAWSCredentialsProvider(Configuration conf, String providerClass) {//from w w w. ja v a 2 s . c o m try { return conf.getClassByName(providerClass).asSubclass(AWSCredentialsProvider.class) .getConstructor(URI.class, Configuration.class).newInstance(null, conf); } catch (ReflectiveOperationException e) { throw new RuntimeException(format("Error creating an instance of %s", providerClass), e); } }
From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java
License:Apache License
private static AWSCredentialsProvider getCustomAWSCredentialsProvider(URI uri, Configuration conf, String providerClass) {//w ww. ja v a 2 s . com try { log.debug("Using AWS credential provider %s for URI %s", providerClass, uri); return conf.getClassByName(providerClass).asSubclass(AWSCredentialsProvider.class) .getConstructor(URI.class, Configuration.class).newInstance(uri, conf); } catch (ReflectiveOperationException e) { throw new RuntimeException(format("Error creating an instance of %s for URI %s", providerClass, uri), e); } }
From source file:com.fliaping.trip.warc.lemurproject.WarcFileRecordReader.java
License:Open Source License
public WarcFileRecordReader(Configuration conf, InputSplit split) throws IOException { if (split instanceof FileSplit) { this.filePathList = new Path[1]; this.filePathList[0] = ((FileSplit) split).getPath(); } else if (split instanceof MultiFileSplit) { this.filePathList = ((MultiFileSplit) split).getPaths(); } else {/*from ww w .j ava2s . c om*/ throw new IOException("InputSplit is not a file split or a multi-file split - aborting"); } fs = this.filePathList[0].getFileSystem(conf); // get the total file sizes for (int i = 0; i < filePathList.length; i++) { totalFileSize += fs.getFileStatus(filePathList[i]).getLen(); } Class<? extends CompressionCodec> codecClass = null; try { codecClass = conf.getClassByName("org.apache.hadoop.io.compress.GzipCodec") .asSubclass(CompressionCodec.class); compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); } catch (ClassNotFoundException cnfEx) { compressionCodec = null; LOG.info("!!! ClassNotFoun Exception thrown setting Gzip codec"); } openNextFile(); }
From source file:com.github.dryangkun.hbase.tidx.hive.HBaseSerDeParameters.java
License:Apache License
private static Class<?> loadClass(String className, @Nullable Configuration configuration) throws Exception { if (configuration != null) { return configuration.getClassByName(className); }//from w w w . j av a 2s. c om return JavaUtils.loadClass(className); }
From source file:com.github.dryangkun.hbase.tidx.hive.struct.AvroHBaseValueFactory.java
License:Apache License
@Override public void init(HBaseSerDeParameters hbaseParams, Configuration conf, Properties properties) throws SerDeException { super.init(hbaseParams, conf, properties); String avroSchemaRetClass = properties.getProperty(AvroSerdeUtils.SCHEMA_RETRIEVER); if (avroSchemaRetClass != null) { Class<?> avroSchemaRetrieverClass = null; try {/*from w w w. j ava 2s. c om*/ avroSchemaRetrieverClass = conf.getClassByName(avroSchemaRetClass); } catch (ClassNotFoundException e) { throw new SerDeException(e); } initAvroSchemaRetriever(avroSchemaRetrieverClass, conf, properties); } }
From source file:com.google.appengine.tools.mapreduce.SerializationUtil.java
License:Apache License
/** * Deserialize an object from a byte array. This uses {@code conf}'s * serialization preferences to support arbitrary serialization mechanisms * using {@link SerializationFactory}.// www.java 2 s .com * * @param conf the configuration to use for serialization preferences * @param expectedClass a type token to set the return type * @param className the name of the class to deserialize to * @param toDeserialize the serialized object as a byte array * @param initialState an object with initial state. Some deserializers may * throw this away. You can pass {@code null} to signify that there is no * initial state. */ @SuppressWarnings("unchecked") public static <T> T deserializeFromByteArray(Configuration conf, Class<T> expectedClass, String className, byte[] toDeserialize, T initialState) { log.fine("Trying to deserialize: " + className); SerializationFactory serializationFactory = new SerializationFactory(conf); try { Class<?> deserializationClass = conf.getClassByName(className); if (!expectedClass.isAssignableFrom(deserializationClass)) { throw new ClassCastException("Attempted to deserialize a " + deserializationClass.getCanonicalName() + " but expected a " + expectedClass.getCanonicalName()); } Deserializer<T> deserializer = serializationFactory.getDeserializer((Class<T>) deserializationClass); ByteArrayInputStream inputStream = new ByteArrayInputStream(toDeserialize); deserializer.open(inputStream); return deserializer.deserialize(initialState); } catch (ClassNotFoundException e) { // If we're deserializing, then we should have already seen this class // and this is strictly a programming error. Hence the RuntimeException. throw new RuntimeException("Couldn't get class for deserializing " + className, e); } catch (UnsupportedEncodingException e) { throw new RuntimeException("JDK doesn't understand UTF8", e); } catch (IOException e) { throw new RuntimeException("Got an IOException from a ByteArrayInputStream. This should never happen.", e); } }