Java tutorial
/*********************************************************************************************************************** * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu) * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. **********************************************************************************************************************/ package eu.stratosphere.yarn; import java.io.File; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Enumeration; import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarFile; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.ConverterUtils; import eu.stratosphere.configuration.ConfigConstants; import eu.stratosphere.configuration.GlobalConfiguration; public class Utils { private static final Log LOG = LogFactory.getLog(Utils.class); public static void copyJarContents(String prefix, String pathToJar) throws IOException { LOG.info("Copying jar (location: " + pathToJar + ") to prefix " + prefix); JarFile jar = null; jar = new JarFile(pathToJar); Enumeration<JarEntry> enumr = jar.entries(); byte[] bytes = new byte[1024]; while (enumr.hasMoreElements()) { JarEntry entry = enumr.nextElement(); if (entry.getName().startsWith(prefix)) { if (entry.isDirectory()) { File cr = new File(entry.getName()); cr.mkdirs(); continue; } InputStream inStream = jar.getInputStream(entry); File outFile = new File(entry.getName()); if (outFile.exists()) { throw new RuntimeException("File unexpectedly exists"); } FileOutputStream outputStream = new FileOutputStream(outFile); int read = 0; while ((read = inStream.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } inStream.close(); outputStream.close(); } } jar.close(); } public static void getStratosphereConfiguration(String confDir) { GlobalConfiguration.loadConfiguration(confDir); } private static void addPathToConfig(Configuration conf, File path) { // chain-in a new classloader URL fileUrl = null; try { fileUrl = path.toURL(); } catch (MalformedURLException e) { throw new RuntimeException("Erroneous config file path", e); } URL[] urls = { fileUrl }; ClassLoader cl = new URLClassLoader(urls, conf.getClassLoader()); conf.setClassLoader(cl); } private static void setDefaultConfValues(Configuration conf) { if (conf.get("fs.hdfs.impl", null) == null) { conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); } if (conf.get("fs.file.impl", null) == null) { conf.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); } } public static Configuration initializeYarnConfiguration() { Configuration conf = new YarnConfiguration(); String configuredHadoopConfig = GlobalConfiguration.getString(ConfigConstants.PATH_HADOOP_CONFIG, null); if (configuredHadoopConfig != null) { LOG.info("Using hadoop configuration path from " + ConfigConstants.PATH_HADOOP_CONFIG + " setting."); addPathToConfig(conf, new File(configuredHadoopConfig)); setDefaultConfValues(conf); return conf; } String[] envs = { "YARN_CONF_DIR", "HADOOP_CONF_DIR", "HADOOP_CONF_PATH" }; for (int i = 0; i < envs.length; ++i) { String confPath = System.getenv(envs[i]); if (confPath != null) { LOG.info("Found " + envs[i] + ", adding it to configuration"); addPathToConfig(conf, new File(confPath)); setDefaultConfValues(conf); return conf; } } LOG.info("Could not find HADOOP_CONF_PATH, using HADOOP_HOME."); String hadoopHome = null; try { hadoopHome = Shell.getHadoopHome(); } catch (IOException e) { LOG.fatal("Unable to get hadoop home. Please set HADOOP_HOME variable!", e); System.exit(1); } File tryConf = new File(hadoopHome + "/etc/hadoop"); if (tryConf.exists()) { LOG.info("Found configuration using hadoop home."); addPathToConfig(conf, tryConf); } else { tryConf = new File(hadoopHome + "/conf"); if (tryConf.exists()) { addPathToConfig(conf, tryConf); } } setDefaultConfValues(conf); return conf; } public static void setupEnv(Configuration conf, Map<String, String> appMasterEnv) { for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim()); } Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*"); } /** * * @return Path to remote file (usually hdfs) * @throws IOException */ public static Path setupLocalResource(Configuration conf, FileSystem fs, String appId, Path localRsrcPath, LocalResource appMasterJar, Path homedir) throws IOException { // copy to HDFS String suffix = ".stratosphere/" + appId + "/" + localRsrcPath.getName(); Path dst = new Path(homedir, suffix); LOG.info("Copying from " + localRsrcPath + " to " + dst); fs.copyFromLocalFile(localRsrcPath, dst); registerLocalResource(fs, dst, appMasterJar); return dst; } public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource) throws IOException { FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); } public static void setTokensFor(ContainerLaunchContext amContainer, Path[] paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS TokenCache.obtainTokensForNamenodes(credentials, paths, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); } public static void logFilesInCurrentDirectory(final Log logger) { new File(".").list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { logger.info(dir.getAbsolutePath() + "/" + name); return true; } }); } }