Example usage for org.apache.hadoop.conf Configuration getClassLoader

List of usage examples for org.apache.hadoop.conf Configuration getClassLoader

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getClassLoader

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getClassLoader.

Prototype

public ClassLoader getClassLoader()

Source Link

Document

Get the ClassLoader for this job.

Usage

From source file:com.asakusafw.lang.compiler.extension.directio.DirectFileIoPortProcessorTest.java

License:Apache License

private Map<Integer, String> collect(InputFormatInfo info, String args)
        throws IOException, InterruptedException, ClassNotFoundException {
    assertThat(info.getKeyClass(), is(classOf(NullWritable.class)));
    assertThat(info.getValueClass(), is(classOf(MockData.class)));
    Configuration conf = directio.newConfiguration();
    ConfigurationEditor.merge(conf, info.getExtraConfiguration());
    ConfigurationEditor.putStageInfo(conf, new StageInfo("u", "b", "f", "s", "e", args));
    InputFormatTester tester = new InputFormatTester(conf,
            info.getFormatClass().resolve(conf.getClassLoader()));
    Map<Integer, String> results = new LinkedHashMap<>();
    tester.collect((MockData object) -> results.put(object.getKey(), object.getValue()));
    return results;
}

From source file:com.asakusafw.lang.compiler.mapreduce.testing.MapReduceRunner.java

License:Apache License

/**
 * Executes the client class./*from  w w w.  ja  v  a  2  s . c om*/
 * @param conf the current configuration
 * @param clientClass the target client class
 * @param executionId the execution ID
 * @param batchArguments the batch arguments
 * @param libraries additional libraries
 * @return the exit status code
 * @throws Exception if exception occurred while executing the stage
 */
public static int execute(Configuration conf, ClassDescription clientClass, String executionId,
        Map<String, String> batchArguments, File... libraries) throws Exception {
    if (libraries.length == 0) {
        return execute0(conf, conf.getClassLoader(), clientClass, executionId, batchArguments);
    } else {
        URL[] urls = new URL[libraries.length];
        for (int i = 0; i < libraries.length; i++) {
            urls[i] = libraries[i].toURI().toURL();
        }
        try (URLClassLoader classLoader = URLClassLoader.newInstance(urls, conf.getClassLoader())) {
            return execute0(conf, classLoader, clientClass, executionId, batchArguments);
        }
    }
}

From source file:com.asakusafw.lang.compiler.mapreduce.testing.MapReduceRunner.java

License:Apache License

/**
 * Executes the client class.//w  w w. ja v  a 2  s. c  o  m
 * @param conf the current configuration
 * @param clientClass the target client class
 * @param executionId the execution ID
 * @param batchArguments the batch arguments
 * @return the exit status code
 * @throws Exception if exception occurred while executing the stage
 */
public static int execute(Configuration conf, ClassDescription clientClass, String executionId,
        Map<String, String> batchArguments) throws Exception {
    return execute0(conf, conf.getClassLoader(), clientClass, executionId, batchArguments);
}

From source file:com.asakusafw.lang.compiler.mapreduce.testing.MapReduceRunner.java

License:Apache License

private static Tool resolveClient(Configuration conf, ClassDescription client) {
    try {//from  w  w  w  .  j a  v a  2  s.co  m
        Class<?> aClass = client.resolve(conf.getClassLoader());
        if (Tool.class.isAssignableFrom(aClass) == false) {
            throw new IllegalArgumentException(MessageFormat.format(
                    "MapReduce client class must implement Tool interface: {0}", client.getClassName()));
        }
        Tool tool = ReflectionUtils.newInstance(aClass.asSubclass(Tool.class), conf);
        return tool;
    } catch (ReflectiveOperationException e) {
        throw new IllegalArgumentException(
                MessageFormat.format("failed to resolve MapReduce client class: {0}", client.getClassName()));
    }
}

From source file:com.asakusafw.m3bp.client.M3bpLauncher.java

License:Apache License

M3bpLauncher(LaunchInfo configuration, Configuration hadoop) {
    Arguments.requireNonNull(configuration);
    Arguments.requireNonNull(hadoop);/* w ww . j  a v a  2  s.c  o  m*/
    this.configuration = configuration;
    this.hadoop = hadoop;
    this.applicationLoader = hadoop.getClassLoader();
}

From source file:com.asakusafw.runtime.stage.launcher.ApplicationLauncher.java

License:Apache License

private static int launch(Configuration conf, Tool tool, String[] args) throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Launching application: {0}{1}", //$NON-NLS-1$
                tool.getClass().getName(), Arrays.toString(args)));
    }// w  w w .  j  a v a  2 s  .  c  om
    ClassLoader contextClassLoader = switchContextClassLoader(conf.getClassLoader());
    try {
        return ToolRunner.run(conf, tool, args);
    } finally {
        switchContextClassLoader(contextClassLoader);
    }
}

From source file:com.asakusafw.runtime.stage.optimizer.LibraryCopySuppressionConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    if (conf.getBoolean(KEY_ENABLED, DEFAULT_ENABLED) == false) {
        return;/*  w w w .  j  av a 2 s  .  com*/
    }
    // activates only if application launcher is used
    if (conf.getBoolean(ApplicationLauncher.KEY_LAUNCHER_USED, false) == false) {
        return;
    }
    if (JobCompatibility.isLocalMode(job) == false) {
        return;
    }
    String libraries = conf.get(KEY_CONF_LIBRARIES);
    if (libraries == null || libraries.isEmpty()) {
        return;
    }
    Set<String> loaded = new HashSet<>();
    ClassLoader loader = conf.getClassLoader();
    if (loader instanceof URLClassLoader) {
        for (URL url : ((URLClassLoader) loader).getURLs()) {
            try {
                loaded.add(url.toURI().toString());
            } catch (URISyntaxException e) {
                LOG.warn(MessageFormat.format("Failed to analyze classpath: {0}", url));
            }
        }
    }
    if (loaded.isEmpty()) {
        return;
    }
    StringBuilder result = new StringBuilder();
    for (String library : libraries.split(",")) { //$NON-NLS-1$
        if (loaded.contains(library)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Keep library: {0}", library)); //$NON-NLS-1$
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Suppress library: {0}", library)); //$NON-NLS-1$
            }
            if (result.length() != 0) {
                result.append(',');
            }
            result.append(library);
        }
    }
    if (result.length() > 0) {
        conf.set(KEY_CONF_LIBRARIES, result.toString());
    } else {
        if (CONFIGURATION_UNSET != null) {
            try {
                CONFIGURATION_UNSET.invoke(conf, KEY_CONF_LIBRARIES);
                return;
            } catch (Exception e) {
                LOG.warn(MessageFormat.format("Failed to invoke {0}", CONFIGURATION_UNSET), e);
            }
        }
        String newLibraries = selectLibraries(libraries);
        conf.set(KEY_CONF_LIBRARIES, newLibraries);
    }
}

From source file:com.asakusafw.runtime.util.hadoop.ConfigurationProviderTest.java

License:Apache License

/**
 * reuses class loaders./*from   ww w . j  av  a  2  s  . com*/
 * @throws Exception if failed
 */
@Test
public void newInstance_classloader_reuse() throws Exception {
    File file = putConf("conf/core-site.xml");
    URL dir = file.getParentFile().toURI().toURL();

    Configuration c1 = new ConfigurationProvider(dir).newInstance();
    assertThat(isLoaded(c1), is(true));

    Configuration c2 = new ConfigurationProvider(dir).newInstance();
    assertThat(c2.getClassLoader(), is(sameInstance(c1.getClassLoader())));
}

From source file:com.asakusafw.testdriver.OperatorTestEnvironment.java

License:Apache License

/**
 * Returns a new configuration object for {@link RuntimeResourceManager}.
 * @return the created configuration object
 *///from  w  w  w.  j a  v a 2 s.  c  o m
protected Configuration createConfig() {
    Configuration conf = ConfigurationFactory.getDefault().newInstance();
    URL resource = conf.getClassLoader().getResource(configurationPath);
    if (resource == null && explicitConfigurationPath == false) {
        // if implicit configuration file is not found, we use the embedded default configuration file
        resource = OperatorTestEnvironment.class.getResource(DEFAULT_CONFIGURATION_PATH);
    }
    if (resource == null) {
        throw new IllegalStateException(MessageFormat.format(
                Messages.getString("OperatorTestEnvironment.errorMissingConfigurationFile"), //$NON-NLS-1$
                configurationPath));
    }
    for (Map.Entry<String, String> entry : extraConfigurations.entrySet()) {
        conf.set(entry.getKey(), entry.getValue());
    }
    conf.addResource(resource);
    return conf;
}

From source file:com.asakusafw.vanilla.client.VanillaLauncher.java

License:Apache License

VanillaLauncher(LaunchInfo configuration, Configuration hadoop) {
    Arguments.requireNonNull(configuration);
    Arguments.requireNonNull(hadoop);/*from   ww  w  .j  a v a 2s. c om*/
    this.configuration = configuration;
    this.hadoop = hadoop;
    this.applicationLoader = hadoop.getClassLoader();
}