Example usage for org.apache.hadoop.conf Configuration getClassLoader

List of usage examples for org.apache.hadoop.conf Configuration getClassLoader

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getClassLoader

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getClassLoader.

Prototype

public ClassLoader getClassLoader()

Source Link

Document

Get the ClassLoader for this job.

Usage

From source file:com.asakusafw.vanilla.client.VanillaLauncher.java

License:Apache License

/**
 * Program entry./*from w  ww. ja  va 2 s  . c om*/
 * @param hadoop the context Hadoop configuration
 * @param args launching configurations
 * @return the exit code
 * @throws LaunchConfigurationException if launching configuration is something wrong
 */
public static int exec(Configuration hadoop, String... args) throws LaunchConfigurationException {
    ClassLoader loader = hadoop.getClassLoader();
    RuntimeContext.set(RuntimeContext.DEFAULT.apply(System.getenv()));
    RuntimeContext.get().verifyApplication(loader);

    LaunchConfiguration conf = LaunchConfiguration.parse(loader, Arrays.asList(args));
    VanillaLauncher launcher = new VanillaLauncher(conf, hadoop);
    return launcher.exec();
}

From source file:com.facebook.hiveio.mapreduce.output.WritingTool.java

License:Apache License

/**
 * set hive configuration/*from  w  ww. j  a v a  2s  .  co  m*/
 *
 * @param conf Configuration
 */
private void adjustConfigurationForHive(Configuration conf) {
    // when output partitions are used, workers register them to the
    // metastore at cleanup stage, and on HiveConf's initialization, it
    // looks for hive-site.xml.
    addToStringCollection(conf, "tmpfiles", conf.getClassLoader().getResource("hive-site.xml").toString());

    // Or, more effectively, we can provide all the jars client needed to
    // the workers as well
    String[] hadoopJars = System.getenv("HADOOP_CLASSPATH").split(File.pathSeparator);
    List<String> hadoopJarURLs = Lists.newArrayList();
    for (String jarPath : hadoopJars) {
        File file = new File(jarPath);
        if (file.exists() && file.isFile()) {
            String jarURL = file.toURI().toString();
            hadoopJarURLs.add(jarURL);
        }
    }
    addToStringCollection(conf, "tmpjars", hadoopJarURLs);
}

From source file:com.facebook.presto.hive.rcfile.RcFilePageSourceFactory.java

License:Apache License

@Override
public Optional<? extends ConnectorPageSource> createPageSource(Configuration configuration,
        ConnectorSession session, Path path, long start, long length, Properties schema,
        List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> effectivePredicate,
        DateTimeZone hiveStorageTimeZone) {
    if (!isRcfileOptimizedReaderEnabled(session)) {
        return Optional.empty();
    }/* w  w w .  j  a  v a  2 s  .c  o m*/

    RcFileEncoding rcFileEncoding;
    String deserializerClassName = getDeserializerClassName(schema);
    if (deserializerClassName.equals(LazyBinaryColumnarSerDe.class.getName())) {
        rcFileEncoding = new BinaryRcFileEncoding();
    } else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) {
        rcFileEncoding = createTextVectorEncoding(schema, hiveStorageTimeZone);
    } else {
        return Optional.empty();
    }

    long size;
    FSDataInputStream inputStream;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration);
        size = fileSystem.getFileStatus(path).getLen();
        inputStream = fileSystem.open(path);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }

    try {
        ImmutableMap.Builder<Integer, Type> readColumns = ImmutableMap.builder();
        for (HiveColumnHandle column : columns) {
            readColumns.put(column.getHiveColumnIndex(), column.getHiveType().getType(typeManager));
        }

        RcFileReader rcFileReader = new RcFileReader(
                new HdfsRcFileDataSource(path.toString(), inputStream, size), rcFileEncoding,
                readColumns.build(),
                new AircompressorCodecFactory(new HadoopCodecFactory(configuration.getClassLoader())), start,
                length, new DataSize(1, Unit.MEGABYTE));

        return Optional.of(new RcFilePageSource(rcFileReader, columns, hiveStorageTimeZone, typeManager));
    } catch (Throwable e) {
        try {
            inputStream.close();
        } catch (IOException ignored) {
        }
        throw Throwables.propagate(e);
    }
}

From source file:com.marklogic.contentpump.ContentPump.java

License:Apache License

/**
 * Set class loader for current thread and for Confifguration based on 
 * Hadoop home./*from  ww w. j a  va  2 s. co m*/
 * 
 * @param hdConfDir Hadoop home directory
 * @param conf Hadoop configuration
 * @throws MalformedURLException
 */
private static void setClassLoader(File hdConfDir, Configuration conf) throws Exception {
    ClassLoader parent = conf.getClassLoader();
    URL url = hdConfDir.toURI().toURL();
    URL[] urls = new URL[1];
    urls[0] = url;
    ClassLoader classLoader = new URLClassLoader(urls, parent);
    Thread.currentThread().setContextClassLoader(classLoader);
    conf.setClassLoader(classLoader);
}

From source file:com.thinkbiganalytics.kylo.catalog.spark.sources.spark.DeletePathsListener.java

License:Apache License

/**
 * Constructs a {@code DeletePathsListener} for the specified paths.
 *
 * @param paths paths to be deleted/*w w  w  .java  2  s .c  om*/
 * @param flag  indicates when the files may be deleted
 * @param conf  Hadoop configuration
 */
public DeletePathsListener(@Nonnull final List<String> paths, @Nonnull final Accumulable<Boolean, ?> flag,
        @Nonnull final Configuration conf) {
    this.paths = paths;
    this.flag = flag;
    this.conf = conf;
    loader = (conf.getClassLoader() instanceof DataSourceResourceLoader)
            ? (DataSourceResourceLoader) conf.getClassLoader()
            : null;
}

From source file:com.twitter.pig.backend.hadoop.executionengine.tez.TezExecutionEngine.java

License:Apache License

@SuppressWarnings("deprecation")
private void init(Properties properties) throws ExecException {
    //First set the ssh socket factory
    setSSHFactory();// ww w.  j  a  va  2 s . com

    String cluster = null;
    String nameNode = null;

    // We need to build a configuration object first in the manner described below
    // and then get back a properties object to inspect the JOB_TRACKER_LOCATION
    // and FILE_SYSTEM_LOCATION. The reason to do this is if we looked only at
    // the existing properties object, we may not get the right settings. So we want
    // to read the configurations in the order specified below and only then look
    // for JOB_TRACKER_LOCATION and FILE_SYSTEM_LOCATION.

    // Hadoop by default specifies two resources, loaded in-order from the classpath:
    // 1. hadoop-default.xml : Read-only defaults for hadoop.
    // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation.
    // Now add the settings from "properties" object to override any existing properties
    // All of the above is accomplished in the method call below

    JobConf jc = null;
    if (this.pigContext.getExecType() == ExecType.TEZ) {
        // Check existence of user provided configs
        String isHadoopConfigsOverriden = properties.getProperty("pig.use.overriden.hadoop.configs");
        if (isHadoopConfigsOverriden != null && isHadoopConfigsOverriden.equals("true")) {
            jc = new JobConf(ConfigurationUtil.toConfiguration(properties));
        } else {
            // Check existence of hadoop-site.xml or core-site.xml in classpath
            // if user provided confs are not being used
            Configuration testConf = new Configuration();
            ClassLoader cl = testConf.getClassLoader();
            URL hadoop_site = cl.getResource(HADOOP_SITE);
            URL core_site = cl.getResource(CORE_SITE);

            if (hadoop_site == null && core_site == null) {
                throw new ExecException(
                        "Cannot find hadoop configurations in classpath (neither hadoop-site.xml nor core-site.xml was found in the classpath)."
                                + " If you plan to use local mode, please put -x local option in command line",
                        4010);
            }
            jc = new JobConf();
        }
        jc.addResource("pig-cluster-hadoop-site.xml");
        jc.addResource(YARN_SITE);

        /*
        // Trick to invoke static initializer of DistributedFileSystem to add hdfs-default.xml 
        // into configuration
        new DistributedFileSystem();
        */
        //the method below alters the properties object by overriding the
        //hadoop properties with the values from properties and recomputing
        //the properties
        recomputeProperties(jc, properties);
    } else {
        // If we are running in local mode we dont read the hadoop conf file
        if (properties.getProperty("mapreduce.framework.name") == null) {
            properties.setProperty("mapreduce.framework.name", "local");
        }
        properties.setProperty(JOB_TRACKER_LOCATION, LOCAL);
        properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
        properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");

        jc = new JobConf(false);
        jc.addResource("core-default.xml");
        jc.addResource("mapred-default.xml");
        jc.addResource("yarn-default.xml");
        recomputeProperties(jc, properties);
    }

    cluster = jc.get(JOB_TRACKER_LOCATION);
    nameNode = jc.get(FILE_SYSTEM_LOCATION);
    if (nameNode == null)
        nameNode = (String) pigContext.getProperties().get(ALTERNATIVE_FILE_SYSTEM_LOCATION);

    if (cluster != null && cluster.length() > 0) {
        if (!cluster.contains(":") && !cluster.equalsIgnoreCase(LOCAL)) {
            cluster = cluster + ":50020";
        }
        properties.setProperty(JOB_TRACKER_LOCATION, cluster);
    }

    if (nameNode != null && nameNode.length() > 0) {
        if (!nameNode.contains(":") && !nameNode.equalsIgnoreCase(LOCAL)) {
            nameNode = nameNode + ":8020";
        }
        properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
    }

    log.info("Connecting to hadoop file system at: " + (nameNode == null ? LOCAL : nameNode));
    // constructor sets DEFAULT_REPLICATION_FACTOR_KEY
    ds = new HDataStorage(properties);

    if (cluster != null && !cluster.equalsIgnoreCase(LOCAL)) {
        log.info("Connecting to map-reduce job tracker at: " + jc.get(JOB_TRACKER_LOCATION));
    }

    // Set job-specific configuration knobs
    jobConf = jc;
}

From source file:eu.stratosphere.yarn.Utils.java

License:Apache License

private static void addPathToConfig(Configuration conf, File path) {
    // chain-in a new classloader
    URL fileUrl = null;/* w w  w  .  ja  va  2s.  co m*/
    try {
        fileUrl = path.toURL();
    } catch (MalformedURLException e) {
        throw new RuntimeException("Erroneous config file path", e);
    }
    URL[] urls = { fileUrl };
    ClassLoader cl = new URLClassLoader(urls, conf.getClassLoader());
    conf.setClassLoader(cl);
}

From source file:io.apigee.lembos.utils.RunnerUtils.java

License:Apache License

/**
 * Adds the "-libjars" entries, if any, to the {@link ClassLoader}.
 *
 * @param conf the Hadoop configuration//  www  .  j  a v a 2s.c  o m
 *
 * @throws IOException if there is an issue creating the new ClassLoader
 */
public static void addLibJarsToClassLoader(final Configuration conf) throws IOException {
    final URL[] libJars = GenericOptionsParser.getLibJars(conf);

    if (libJars != null && libJars.length > 0) {
        final ClassLoader loader = new URLClassLoader(libJars, conf.getClassLoader());

        Thread.currentThread().setContextClassLoader(loader);

        conf.setClassLoader(loader);
    }
}

From source file:io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory.java

License:Apache License

@Override
public Optional<? extends ConnectorPageSource> createPageSource(Configuration configuration,
        ConnectorSession session, Path path, long start, long length, long fileSize, Properties schema,
        List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> effectivePredicate,
        DateTimeZone hiveStorageTimeZone) {
    RcFileEncoding rcFileEncoding;//from  w  w  w.j  a v a2 s .co  m
    String deserializerClassName = getDeserializerClassName(schema);
    if (deserializerClassName.equals(LazyBinaryColumnarSerDe.class.getName())) {
        rcFileEncoding = new BinaryRcFileEncoding();
    } else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) {
        rcFileEncoding = createTextVectorEncoding(schema, hiveStorageTimeZone);
    } else {
        return Optional.empty();
    }

    if (fileSize == 0) {
        throw new PrestoException(HIVE_BAD_DATA, "RCFile is empty: " + path);
    }

    FSDataInputStream inputStream;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration);
        inputStream = fileSystem.open(path);
    } catch (Exception e) {
        if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed")
                || e instanceof FileNotFoundException) {
            throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e);
        }
        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, splitError(e, path, start, length), e);
    }

    try {
        ImmutableMap.Builder<Integer, Type> readColumns = ImmutableMap.builder();
        for (HiveColumnHandle column : columns) {
            readColumns.put(column.getHiveColumnIndex(), column.getHiveType().getType(typeManager));
        }

        RcFileReader rcFileReader = new RcFileReader(
                new HdfsRcFileDataSource(path.toString(), inputStream, fileSize, stats), rcFileEncoding,
                readColumns.build(),
                new AircompressorCodecFactory(new HadoopCodecFactory(configuration.getClassLoader())), start,
                length, new DataSize(8, Unit.MEGABYTE));

        return Optional.of(new RcFilePageSource(rcFileReader, columns, hiveStorageTimeZone, typeManager));
    } catch (Throwable e) {
        try {
            inputStream.close();
        } catch (IOException ignored) {
        }
        if (e instanceof PrestoException) {
            throw (PrestoException) e;
        }
        String message = splitError(e, path, start, length);
        if (e instanceof RcFileCorruptionException) {
            throw new PrestoException(HIVE_BAD_DATA, message, e);
        }
        if (e instanceof BlockMissingException) {
            throw new PrestoException(HIVE_MISSING_DATA, message, e);
        }
        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e);
    }
}

From source file:ml.shifu.guagua.hadoop.io.GuaguaOptionsParser.java

License:Apache License

/**
 * Modify configuration according user-specified generic options
 * //  w ww .  jav a  2  s.  com
 * @param conf
 *            Configuration to be modified
 * @param line
 *            User-specified generic options
 */
private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException {
    if (line.hasOption("fs")) {
        FileSystem.setDefaultUri(conf, line.getOptionValue("fs"));
    }

    if (line.hasOption("jt")) {
        conf.set("mapred.job.tracker", line.getOptionValue("jt"));
    }
    if (line.hasOption("conf")) {
        String[] values = line.getOptionValues("conf");
        for (String value : values) {
            conf.addResource(new Path(value));
        }
    }
    if (line.hasOption("libjars")) {
        conf.set("tmpjars", validateFiles(line.getOptionValue("libjars"), conf));
        // setting libjars in client classpath
        URL[] libjars = getLibJars(conf);
        if (libjars != null && libjars.length > 0) {
            conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
            Thread.currentThread().setContextClassLoader(
                    new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
        }
    }
    if (line.hasOption("files")) {
        conf.set("tmpfiles", validateFiles(line.getOptionValue("files"), conf));
    }
    if (line.hasOption("archives")) {
        conf.set("tmparchives", validateFiles(line.getOptionValue("archives"), conf));
    }
    if (line.hasOption('D')) {
        String[] property = line.getOptionValues('D');
        for (String prop : property) {
            String[] keyval = prop.split("=", 2);
            if (keyval.length == 2) {
                conf.set(keyval[0], keyval[1]);
            }
        }
    }
    conf.setBoolean("mapred.used.genericoptionsparser", true);

    // tokensFile
    if (line.hasOption("tokenCacheFile")) {
        String fileName = line.getOptionValue("tokenCacheFile");
        // check if the local file exists
        try {
            FileSystem localFs = FileSystem.getLocal(conf);
            Path p = new Path(fileName);
            if (!localFs.exists(p)) {
                throw new FileNotFoundException("File " + fileName + " does not exist.");
            }

            LOG.debug("setting conf tokensFile: {}", fileName);
            conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p).toString());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}