Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package edu.uci.ics.asterix.aoya; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Vector; import java.util.regex.Pattern; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import com.google.common.collect.ImmutableMap; import edu.uci.ics.asterix.common.configuration.AsterixConfiguration; import edu.uci.ics.asterix.common.configuration.Coredump; import edu.uci.ics.asterix.common.configuration.Store; import edu.uci.ics.asterix.common.configuration.TransactionLogDir; import edu.uci.ics.asterix.event.schema.yarnCluster.Cluster; import edu.uci.ics.asterix.event.schema.yarnCluster.Node; @InterfaceAudience.Public @InterfaceStability.Unstable public class AsterixYARNClient { public static enum Mode { INSTALL("install"), START("start"), STOP("stop"), KILL("kill"), DESTROY("destroy"), ALTER( "alter"), LIBINSTALL("libinstall"), DESCRIBE("describe"), BACKUP( "backup"), LSBACKUP("lsbackups"), RMBACKUP("rmbackup"), RESTORE("restore"), NOOP(""); public final String alias; Mode(String alias) { this.alias = alias; } public static Mode fromAlias(String a) { return STRING_TO_MODE.get(a.toLowerCase()); } } public static final Map<String, AsterixYARNClient.Mode> STRING_TO_MODE = ImmutableMap.<String, AsterixYARNClient.Mode>builder() .put(Mode.INSTALL.alias, Mode.INSTALL).put(Mode.START.alias, Mode.START).put(Mode.STOP.alias, Mode.STOP) .put(Mode.KILL.alias, Mode.KILL).put(Mode.DESTROY.alias, Mode.DESTROY).put(Mode.ALTER.alias, Mode.ALTER) .put(Mode.LIBINSTALL.alias, Mode.LIBINSTALL).put(Mode.DESCRIBE.alias, Mode.DESCRIBE) .put(Mode.BACKUP.alias, Mode.BACKUP).put(Mode.LSBACKUP.alias, Mode.LSBACKUP) .put(Mode.RMBACKUP.alias, Mode.RMBACKUP).put(Mode.RESTORE.alias, Mode.RESTORE).build(); private static final Log LOG = LogFactory.getLog(AsterixYARNClient.class); public static final String CONF_DIR_REL = ".asterix" + File.separator; private static final String instanceLock = "instance"; public static final String CONFIG_DEFAULT_NAME = "cluster-config.xml"; public static final String PARAMS_DEFAULT_NAME = "asterix-configuration.xml"; private static String DEFAULT_PARAMETERS_PATH = "conf" + File.separator + "base-asterix-configuration.xml"; private static String MERGED_PARAMETERS_PATH = "conf" + File.separator + PARAMS_DEFAULT_NAME; private static final String JAVA_HOME = System.getProperty("java.home"); public static final String NC_JAVA_OPTS_KEY = "nc.java.opts"; public static final String CC_JAVA_OPTS_KEY = "cc.java.opts"; public static final String CC_REST_PORT_KEY = "api.port"; private Mode mode = Mode.NOOP; // Hadoop Configuration private Configuration conf; private YarnClient yarnClient; // Application master specific info to register a new Application with // RM/ASM private String appName = ""; // App master priority private int amPriority = 0; // Queue for App master private String amQueue = ""; // Amt. of memory resource to request for to run the App Master private int amMemory = 1000; // Main class to invoke application master private final String appMasterMainClass = "edu.uci.ics.asterix.aoya.AsterixApplicationMaster"; //instance name private String instanceName = ""; //location of distributable AsterixDB zip private String asterixZip = ""; // Location of cluster configuration private String asterixConf = ""; // Location of optional external libraries private String extLibs = ""; private String instanceFolder = ""; // log4j.properties file // if available, add to local resources and set into classpath private String log4jPropFile = ""; // Debug flag boolean debugFlag = false; private boolean refresh = false; private boolean force = false; // Command line options private Options opts; private String libDataverse; private String snapName = ""; private String baseConfig = "."; private String ccJavaOpts = ""; private String ncJavaOpts = ""; //Ports private int ccRestPort = 19002; /** * @param args * Command line arguments */ public static void main(String[] args) { try { AsterixYARNClient client = new AsterixYARNClient(); try { client.init(args); AsterixYARNClient.execute(client); } catch (ParseException | ApplicationNotFoundException e) { LOG.fatal(e); client.printUsage(); System.exit(-1); } } catch (Exception e) { LOG.fatal("Error running client", e); System.exit(1); } LOG.info("Command executed successfully."); System.exit(0); } public static void execute(AsterixYARNClient client) throws IOException, YarnException { YarnClientApplication app; List<DFSResourceCoordinate> res; System.out.println("JAVA HOME: " + JAVA_HOME); switch (client.mode) { case START: startAction(client); break; case STOP: try { client.stopInstance(); } catch (ApplicationNotFoundException e) { LOG.info(e); System.out.println("Asterix instance by that name already exited or was never started"); client.deleteLockFile(); } break; case KILL: if (client.isRunning() && Utils.confirmAction( "Are you sure you want to kill this instance? In-progress tasks will be aborted")) { try { AsterixYARNClient.killApplication(client.getLockFile(), client.yarnClient); } catch (ApplicationNotFoundException e) { LOG.info(e); System.out.println("Asterix instance by that name already exited or was never started"); client.deleteLockFile(); } } else if (!client.isRunning()) { System.out.println("Asterix instance by that name already exited or was never started"); client.deleteLockFile(); } break; case DESCRIBE: Utils.listInstances(client.conf, CONF_DIR_REL); break; case INSTALL: installAction(client); break; case LIBINSTALL: client.installExtLibs(); break; case ALTER: client.writeAsterixConfig(Utils.parseYarnClusterConfig(client.asterixConf)); client.installAsterixConfig(true); System.out.println("Configuration successfully modified"); break; case DESTROY: try { if (client.force || Utils.confirmAction( "Are you really sure you want to obliterate this instance? This action cannot be undone!")) { app = client.makeApplicationContext(); res = client.deployConfig(); res.addAll(client.distributeBinaries()); client.removeInstance(app, res); } } catch (YarnException | IOException e) { LOG.error("Asterix failed to deploy on to cluster"); throw e; } break; case BACKUP: if (client.force || Utils.confirmAction("Performing a backup will stop a running instance.")) { app = client.makeApplicationContext(); res = client.deployConfig(); res.addAll(client.distributeBinaries()); client.backupInstance(app, res); } break; case LSBACKUP: Utils.listBackups(client.conf, CONF_DIR_REL, client.instanceName); break; case RMBACKUP: Utils.rmBackup(client.conf, CONF_DIR_REL, client.instanceName, Long.parseLong(client.snapName)); break; case RESTORE: if (client.force || Utils.confirmAction("Performing a restore will stop a running instance.")) { app = client.makeApplicationContext(); res = client.deployConfig(); res.addAll(client.distributeBinaries()); client.restoreInstance(app, res); } break; default: LOG.fatal( "Unknown mode. Known client modes are: start, stop, install, describe, kill, destroy, describe, backup, restore, lsbackup, rmbackup"); client.printUsage(); System.exit(-1); } } private static void startAction(AsterixYARNClient client) throws YarnException { YarnClientApplication app; List<DFSResourceCoordinate> res; ApplicationId appId; try { app = client.makeApplicationContext(); res = client.deployConfig(); res.addAll(client.distributeBinaries()); appId = client.deployAM(app, res, client.mode); LOG.info("Asterix started up with Application ID: " + appId.toString()); if (Utils.waitForLiveness(appId, "Waiting for AsterixDB instance to resume ", client.yarnClient, client.instanceName, client.conf, client.ccRestPort)) { System.out.println("Asterix successfully deployed and is now running."); } else { LOG.fatal("AsterixDB appears to have failed to install and start"); throw new YarnException("AsterixDB appears to have failed to install and start"); } } catch (IOException e) { throw new YarnException(e); } } private static void installAction(AsterixYARNClient client) throws YarnException { YarnClientApplication app; List<DFSResourceCoordinate> res; ApplicationId appId; try { app = client.makeApplicationContext(); client.installConfig(); client.writeAsterixConfig(Utils.parseYarnClusterConfig(client.asterixConf)); client.installAsterixConfig(false); res = client.deployConfig(); res.addAll(client.distributeBinaries()); appId = client.deployAM(app, res, client.mode); LOG.info("Asterix started up with Application ID: " + appId.toString()); if (Utils.waitForLiveness(appId, "Waiting for new AsterixDB Instance to start ", client.yarnClient, client.instanceName, client.conf, client.ccRestPort)) { System.out.println("Asterix successfully deployed and is now running."); } else { LOG.fatal("AsterixDB appears to have failed to install and start"); throw new YarnException("AsterixDB appears to have failed to install and start"); } } catch (IOException e) { LOG.fatal("Asterix failed to deploy on to cluster"); throw new YarnException(e); } } public AsterixYARNClient(Configuration conf) throws Exception { this.conf = conf; yarnClient = YarnClient.createYarnClient(); //If the HDFS jars aren't on the classpath this won't be set if (conf.get("fs.hdfs.impl", null) == conf.get("fs.file.impl", null)) { //only would happen if both are null conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); conf.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); } yarnClient.init(conf); opts = parseConf(conf); } private static Options parseConf(Configuration conf) { Options opts = new Options(); opts.addOption(new Option("appname", true, "Application Name. Default value - Asterix")); opts.addOption(new Option("priority", true, "Application Priority. Default 0")); opts.addOption(new Option("queue", true, "RM Queue in which this application is to be submitted")); opts.addOption(new Option("master_memory", true, "Amount of memory in MB to be requested to run the application master")); opts.addOption(new Option("log_properties", true, "log4j.properties file")); opts.addOption(new Option("n", "name", true, "Asterix instance name (required)")); opts.addOption(new Option("zip", "asterixZip", true, "zip file with AsterixDB inside- if in non-default location")); opts.addOption(new Option("bc", "baseConfig", true, "base Asterix parameters configuration file if not in default position")); opts.addOption(new Option("c", "asterixConf", true, "Asterix cluster config (required on install)")); opts.addOption(new Option("l", "externalLibs", true, "Libraries to deploy along with Asterix instance")); opts.addOption(new Option("ld", "libDataverse", true, "Dataverse to deploy external libraries to")); opts.addOption(new Option("r", "refresh", false, "If starting an existing instance, this will replace them with the local copy on startup")); opts.addOption( new Option("appId", true, "ApplicationID to monitor if running client in status monitor mode")); opts.addOption(new Option("masterLibsDir", true, "Directory that contains the JARs needed to run the AM")); opts.addOption(new Option("s", "snapshot", true, "Backup timestamp for arguments requiring a specific backup (rm, restore)")); opts.addOption(new Option("v", "debug", false, "Dump out debug information")); opts.addOption(new Option("help", false, "Print usage")); opts.addOption(new Option("f", "force", false, "Execute this command as fully as possible, disregarding any caution")); return opts; } /** */ public AsterixYARNClient() throws Exception { this(new YarnConfiguration()); } /** * Helper function to print out usage */ private void printUsage() { new HelpFormatter().printHelp("Asterix YARN client. Usage: asterix [options] [mode]", opts); } /** * Initialize the client's arguments and parameters before execution. * * @param args * - Standard command-line arguments. * @throws ParseException */ public void init(String[] args) throws ParseException { CommandLine cliParser = new GnuParser().parse(opts, args); if (cliParser.hasOption("help")) { printUsage(); return; } //initialize most things debugFlag = cliParser.hasOption("debug"); force = cliParser.hasOption("force"); baseConfig = cliParser.getOptionValue("baseConfig"); extLibs = cliParser.getOptionValue("externalLibs"); libDataverse = cliParser.getOptionValue("libDataverse"); appName = cliParser.getOptionValue("appname", "AsterixDB"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); amQueue = cliParser.getOptionValue("queue", "default"); amMemory = Integer.parseInt(cliParser.getOptionValue("master_memory", "10")); instanceName = cliParser.getOptionValue("name"); instanceFolder = instanceName + '/'; appName = appName + ": " + instanceName; asterixConf = cliParser.getOptionValue("asterixConf"); log4jPropFile = cliParser.getOptionValue("log_properties", ""); //see if the given argument values are sane in general checkConfSanity(args, cliParser); //intialize the mode, see if it is a valid one. initMode(args, cliParser); //now check the validity of the arguments given the mode checkModeSanity(args, cliParser); //if we are going to refresh the binaries, find that out refresh = cliParser.hasOption("refresh"); //same goes for snapshot restoration/removal snapName = cliParser.getOptionValue("snapshot"); if (!cliParser.hasOption("asterixZip") && (mode == Mode.INSTALL || mode == Mode.ALTER || mode == Mode.DESTROY || mode == Mode.BACKUP)) { asterixZip = cliParser.getOptionValue("asterixZip", getAsterixDistributableLocation().getAbsolutePath()); } else { asterixZip = cliParser.getOptionValue("asterixZip"); } } /** * Cursory sanity checks for argument sanity, without considering the mode of the client * * @param args * @param cliParser * The parsed arguments. * @throws ParseException */ private void checkConfSanity(String[] args, CommandLine cliParser) throws ParseException { String message = null; //Sanity check for no args if (args.length == 0) { message = "No args specified for client to initialize"; } //AM memory should be a sane value else if (amMemory < 0) { message = "Invalid memory specified for application master, exiting." + " Specified memory=" + amMemory; } //we're good! else { return; } //default: throw new ParseException(message); } /** * Initialize the mode of the client from the arguments. * * @param args * @param cliParser * @throws ParseException */ private void initMode(String[] args, CommandLine cliParser) throws ParseException { @SuppressWarnings("unchecked") List<String> clientVerb = cliParser.getArgList(); String message = null; //Now check if there is a mode if (clientVerb == null || clientVerb.size() < 1) { message = "You must specify an action."; } //But there can only be one mode... else if (clientVerb.size() > 1) { message = "Trailing arguments, or too many arguments. Only one action may be performed at a time."; } if (message != null) { throw new ParseException(message); } //Now we can initialize the mode and check it against parameters mode = Mode.fromAlias(clientVerb.get(0)); if (mode == null) { mode = Mode.NOOP; } } /** * Determine if the command line arguments are sufficient for the requested client mode. * * @param args * The command line arguments. * @param cliParser * Parsed command line arguments. * @throws ParseException */ private void checkModeSanity(String[] args, CommandLine cliParser) throws ParseException { String message = null; //The only time you can use the client without specifiying an instance, is to list all of the instances it sees. if (!cliParser.hasOption("name") && mode != Mode.DESCRIBE) { message = "You must give a name for the instance to be acted upon"; } else if (mode == Mode.INSTALL && !cliParser.hasOption("asterixConf")) { message = "No Configuration XML given. Please specify a config for cluster installation"; } else if (mode != Mode.START && cliParser.hasOption("refresh")) { message = "Cannot specify refresh in any mode besides start, mode is: " + mode; } else if (cliParser.hasOption("snapshot") && !(mode == Mode.RESTORE || mode == Mode.RMBACKUP)) { message = "Cannot specify a snapshot to restore in any mode besides restore or rmbackup, mode is: " + mode; } else if ((mode == Mode.ALTER || mode == Mode.INSTALL) && baseConfig == null && !(new File(DEFAULT_PARAMETERS_PATH).exists())) { message = "Default asterix parameters file is not in the default location, and no custom location is specified"; } //nothing is wrong, so exit else { return; } //otherwise, something is bad. throw new ParseException(message); } /** * Find the distributable asterix bundle, be it in the default location or in a user-specified location. * * @return */ private File getAsterixDistributableLocation() { //Look in the PWD for the "asterix" folder File tarDir = new File("asterix"); if (!tarDir.exists()) { throw new IllegalArgumentException( "Default directory structure not in use- please specify an asterix zip and base config file to distribute"); } FileFilter tarFilter = new WildcardFileFilter("asterix-server*.zip"); File[] tarFiles = tarDir.listFiles(tarFilter); if (tarFiles.length != 1) { throw new IllegalArgumentException( "There is more than one canonically named asterix distributable in the default directory. Please leave only one there."); } return tarFiles[0]; } /** * Initialize and register the application attempt with the YARN ResourceManager. * * @return * @throws IOException * @throws YarnException */ public YarnClientApplication makeApplicationContext() throws IOException, YarnException { //first check to see if an instance already exists. FileSystem fs = FileSystem.get(conf); Path lock = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); LOG.info("Running Deployment"); yarnClient.start(); if (fs.exists(lock)) { ApplicationId lockAppId = getLockFile(); try { ApplicationReport previousAppReport = yarnClient.getApplicationReport(lockAppId); YarnApplicationState prevStatus = previousAppReport.getYarnApplicationState(); if (!(prevStatus == YarnApplicationState.FAILED || prevStatus == YarnApplicationState.KILLED || prevStatus == YarnApplicationState.FINISHED) && mode != Mode.DESTROY && mode != Mode.BACKUP && mode != Mode.RESTORE) { throw new IllegalStateException("Instance is already running in: " + lockAppId); } else if (mode != Mode.DESTROY && mode != Mode.BACKUP && mode != Mode.RESTORE) { //stale lock file LOG.warn("Stale lockfile detected. Instance attempt " + lockAppId + " may have exited abnormally"); deleteLockFile(); } } catch (YarnException e) { LOG.warn( "Stale lockfile detected, but the RM has no record of this application's last run. This is normal if the cluster was restarted."); deleteLockFile(); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); return app; } /** * Upload the Asterix cluster description on to the DFS. This will persist the state of the instance. * * @return * @throws YarnException * @throws IOException */ private List<DFSResourceCoordinate> deployConfig() throws YarnException, IOException { FileSystem fs = FileSystem.get(conf); List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); FileStatus destStatus; try { destStatus = fs.getFileStatus(dstConf); } catch (IOException e) { throw new YarnException("Asterix instance by that name does not appear to exist in DFS"); } LocalResource asterixConfLoc = Records.newRecord(LocalResource.class); asterixConfLoc.setType(LocalResourceType.FILE); asterixConfLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixConfLoc.setResource(ConverterUtils.getYarnUrlFromPath(dstConf)); asterixConfLoc.setTimestamp(destStatus.getModificationTime()); DFSResourceCoordinate conf = new DFSResourceCoordinate(); conf.envs.put(dstConf.toUri().toString(), AConstants.CONFLOCATION); conf.envs.put(Long.toString(asterixConfLoc.getSize()), AConstants.CONFLEN); conf.envs.put(Long.toString(asterixConfLoc.getTimestamp()), AConstants.CONFTIMESTAMP); conf.name = CONFIG_DEFAULT_NAME; conf.res = asterixConfLoc; resources.add(conf); return resources; } /** * Install the current Asterix parameters to the DFS. This can be modified via alter. * * @throws YarnException * @throws IOException */ private void installConfig() throws YarnException, IOException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); try { fs.getFileStatus(dstConf); if (mode == Mode.INSTALL) { throw new IllegalStateException("Instance with this name already exists."); } } catch (FileNotFoundException e) { if (mode == Mode.START) { throw new IllegalStateException("Instance does not exist for this user", e); } } if (mode == Mode.INSTALL) { Path src = new Path(asterixConf); fs.copyFromLocalFile(false, true, src, dstConf); } } /** * Upload External libraries and functions to HDFS for an instance to use when started * @throws IllegalStateException * @throws IOException */ private void installExtLibs() throws IllegalStateException, IOException { FileSystem fs = FileSystem.get(conf); if (!instanceExists()) { throw new IllegalStateException("No instance by name " + instanceName + " found."); } if (isRunning()) { throw new IllegalStateException( "Instance " + instanceName + " is running. Please stop it before installing any libraries."); } String libPathSuffix = CONF_DIR_REL + instanceFolder + "library" + Path.SEPARATOR + libDataverse + Path.SEPARATOR; Path src = new Path(extLibs); String fullLibPath = libPathSuffix + src.getName(); Path libFilePath = new Path(fs.getHomeDirectory(), fullLibPath); LOG.info("Copying Asterix external library to DFS"); fs.copyFromLocalFile(false, true, src, libFilePath); } /** * Finds the minimal classes and JARs needed to start the AM only. * @return Resources the AM needs to start on the initial container. * @throws IllegalStateException * @throws IOException */ private List<DFSResourceCoordinate> installAmLibs() throws IllegalStateException, IOException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); FileSystem fs = FileSystem.get(conf); String fullLibPath = CONF_DIR_REL + instanceFolder + "am_jars" + Path.SEPARATOR; String[] cp = System.getProperty("java.class.path").split(System.getProperty("path.separator")); String asterixJarPattern = "^(asterix).*(jar)$"; //starts with asterix,ends with jar String commonsJarPattern = "^(commons).*(jar)$"; String surefireJarPattern = "^(surefire).*(jar)$"; //for maven tests String jUnitTestPattern = "^(asterix-yarn" + File.separator + "target)$"; LOG.info(File.separator); for (String j : cp) { String[] pathComponents = j.split(Pattern.quote(File.separator)); LOG.info(j); LOG.info(pathComponents[pathComponents.length - 1]); if (pathComponents[pathComponents.length - 1].matches(asterixJarPattern) || pathComponents[pathComponents.length - 1].matches(commonsJarPattern) || pathComponents[pathComponents.length - 1].matches(surefireJarPattern) || pathComponents[pathComponents.length - 1].matches(jUnitTestPattern)) { LOG.info("Loading JAR/classpath: " + j); File f = new File(j); Path dst = new Path(fs.getHomeDirectory(), fullLibPath + f.getName()); if (!fs.exists(dst) || refresh) { fs.copyFromLocalFile(false, true, new Path(f.getAbsolutePath()), dst); } FileStatus dstSt = fs.getFileStatus(dst); LocalResource amLib = Records.newRecord(LocalResource.class); amLib.setType(LocalResourceType.FILE); amLib.setVisibility(LocalResourceVisibility.PRIVATE); amLib.setResource(ConverterUtils.getYarnUrlFromPath(dst)); amLib.setTimestamp(dstSt.getModificationTime()); amLib.setSize(dstSt.getLen()); DFSResourceCoordinate amLibCoord = new DFSResourceCoordinate(); amLibCoord.res = amLib; amLibCoord.name = f.getName(); if (f.getName().contains("asterix-yarn") || f.getName().contains("surefire")) { amLibCoord.envs.put(dst.toUri().toString(), AConstants.APPLICATIONMASTERJARLOCATION); amLibCoord.envs.put(Long.toString(dstSt.getLen()), AConstants.APPLICATIONMASTERJARLEN); amLibCoord.envs.put(Long.toString(dstSt.getModificationTime()), AConstants.APPLICATIONMASTERJARTIMESTAMP); } resources.add(amLibCoord); } } if (resources.size() == 0) { throw new IOException("Required JARs are missing. Please check your directory structure"); } return resources; } /** * Uploads a AsterixDB cluster configuration to HDFS for the AM to use. * @param overwrite Overwrite existing configurations by the same name. * @throws IllegalStateException * @throws IOException */ private void installAsterixConfig(boolean overwrite) throws IllegalStateException, IOException { FileSystem fs = FileSystem.get(conf); File srcfile = new File(MERGED_PARAMETERS_PATH); Path src = new Path(srcfile.getCanonicalPath()); String pathSuffix = CONF_DIR_REL + instanceFolder + File.separator + PARAMS_DEFAULT_NAME; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); if (fs.exists(dst) && !overwrite) { throw new IllegalStateException( "Instance exists. Please delete an existing instance before trying to overwrite"); } fs.copyFromLocalFile(false, true, src, dst); } /** * Uploads binary resources to HDFS for use by the AM * @return * @throws IOException * @throws YarnException */ public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src, dst; FileStatus destStatus; String pathSuffix; // adding info so we can add the jar to the App master container path // Add the asterix tarfile to HDFS for easy distribution // Keep it all archived for now so add it as a file... pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip"; dst = new Path(fs.getHomeDirectory(), pathSuffix); if (refresh) { if (fs.exists(dst)) { fs.delete(dst, false); } } if (!fs.exists(dst)) { src = new Path(asterixZip); LOG.info("Copying Asterix distributable to DFS"); fs.copyFromLocalFile(false, true, src, dst); } destStatus = fs.getFileStatus(dst); LocalResource asterixTarLoc = Records.newRecord(LocalResource.class); asterixTarLoc.setType(LocalResourceType.ARCHIVE); asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); asterixTarLoc.setTimestamp(destStatus.getModificationTime()); // adding info so we can add the tarball to the App master container path DFSResourceCoordinate tar = new DFSResourceCoordinate(); tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION); tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN); tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP); tar.res = asterixTarLoc; tar.name = "asterix-server.zip"; resources.add(tar); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); DFSResourceCoordinate l4j = new DFSResourceCoordinate(); tar.res = log4jRsrc; tar.name = "log4j.properties"; resources.add(l4j); } resources.addAll(installAmLibs()); return resources; } /** * Submits the request to start the AsterixApplicationMaster to the YARN ResourceManager. * * @param app * The application attempt handle. * @param resources * Resources to be distributed as part of the container launch * @param mode * The mode of the ApplicationMaster * @return The application ID of the new Asterix instance. * @throws IOException * @throws YarnException */ public ApplicationId deployAM(YarnClientApplication app, List<DFSResourceCoordinate> resources, Mode mode) throws IOException, YarnException { // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); // Set local resource info into app master container launch context Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); for (DFSResourceCoordinate res : resources) { localResources.put(res.name, res.res); } amContainer.setLocalResources(localResources); // Set the env variables to be setup in the env where the application // master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // using the env info, the application master will create the correct // local resource for the // eventual containers that will be launched to execute the shell // scripts for (DFSResourceCoordinate res : resources) { if (res.envs == null) { //some entries may not have environment variables. continue; } for (Map.Entry<String, String> e : res.envs.entrySet()) { env.put(e.getValue(), e.getKey()); } } //this is needed for when the RM address isn't known from the environment of the AM env.put(AConstants.RMADDRESS, conf.get("yarn.resourcemanager.address")); env.put(AConstants.RMSCHEDULERADDRESS, conf.get("yarn.resourcemanager.scheduler.address")); ///add miscellaneous environment variables. env.put(AConstants.INSTANCESTORE, CONF_DIR_REL + instanceFolder); env.put(AConstants.DFS_BASE, FileSystem.get(conf).getHomeDirectory().toUri().toString()); env.put(AConstants.CC_JAVA_OPTS, ccJavaOpts); env.put(AConstants.NC_JAVA_OPTS, ncJavaOpts); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder("").append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("." + File.separator + "log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { LOG.info("In YARN MiniCluster"); classPathEnv.append(System.getProperty("path.separator")); classPathEnv.append(System.getProperty("java.class.path")); env.put("HADOOP_CONF_DIR", System.getProperty("user.dir") + File.separator + "target" + File.separator); } LOG.info("AM Classpath:" + classPathEnv.toString()); env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(JAVA_HOME + File.separator + "bin" + File.separator + "java"); // Set class name vargs.add(appMasterMainClass); //Set params for Application Master if (debugFlag) { vargs.add("-debug"); } if (mode == Mode.DESTROY) { vargs.add("-obliterate"); } else if (mode == Mode.BACKUP) { vargs.add("-backup"); } else if (mode == Mode.RESTORE) { vargs.add("-restore " + snapName); } else if (mode == Mode.INSTALL) { vargs.add("-initial "); } if (refresh) { vargs.add("-refresh"); } //vargs.add("/bin/ls -alh asterix-server.zip/repo"); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + "AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + "AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // The following are not required for launching an application master // amContainer.setContainerId(containerId); appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = // applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on // success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); //now write the instance lock if (mode == Mode.INSTALL || mode == Mode.START) { FileSystem fs = FileSystem.get(conf); Path lock = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); if (fs.exists(lock)) { throw new IllegalStateException("Somehow, this instance has been launched twice. "); } BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(lock, true))); try { out.write(app.getApplicationSubmissionContext().getApplicationId().toString()); out.close(); } finally { out.close(); } } return app.getApplicationSubmissionContext().getApplicationId(); } /** * Asks YARN to kill a given application by appId * @param appId The application to kill. * @param yarnClient The YARN client object that is connected to the RM. * @throws YarnException * @throws IOException */ public static void killApplication(ApplicationId appId, YarnClient yarnClient) throws YarnException, IOException { if (appId == null) { throw new YarnException("No Application given to kill"); } if (yarnClient.isInState(STATE.INITED)) { yarnClient.start(); } YarnApplicationState st; ApplicationReport rep = yarnClient.getApplicationReport(appId); st = rep.getYarnApplicationState(); if (st == YarnApplicationState.FINISHED || st == YarnApplicationState.KILLED || st == YarnApplicationState.FAILED) { LOG.info("Application " + appId + " already exited."); return; } LOG.info("Killing applicaiton with ID: " + appId); yarnClient.killApplication(appId); } /** * Tries to stop a running AsterixDB instance gracefully. * @throws IOException * @throws YarnException */ private void stopInstanceIfRunning() throws IOException, YarnException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); //if the instance is up, fix that if (isRunning()) { try { this.stopInstance(); } catch (IOException e) { throw new YarnException(e); } } else if (!fs.exists(dstConf)) { throw new YarnException("No instance configured with that name exists"); } } /** * Start a YARN job to delete local AsterixDB resources of an extant instance * @param app The Client connection * @param resources AM resources * @throws IOException * @throws YarnException */ private void removeInstance(YarnClientApplication app, List<DFSResourceCoordinate> resources) throws IOException, YarnException { FileSystem fs = FileSystem.get(conf); //if the instance is up, fix that stopInstanceIfRunning(); //now try deleting all of the on-disk artifacts on the cluster ApplicationId deleter = deployAM(app, resources, Mode.DESTROY); boolean delete_start = Utils.waitForApplication(deleter, yarnClient, "Waiting for deletion to start", ccRestPort); if (!delete_start) { if (force) { fs.delete(new Path(CONF_DIR_REL + instanceFolder), true); LOG.error("Forcing deletion of HDFS resources"); } LOG.fatal(" of on-disk persistient resources on individual nodes failed."); throw new YarnException(); } boolean deleted = waitForCompletion(deleter, "Deletion in progress"); if (!(deleted || force)) { LOG.fatal("Cleanup of on-disk persistent resources failed."); return; } else { fs.delete(new Path(CONF_DIR_REL + instanceFolder), true); } System.out.println("Deletion of instance succeeded."); } /** * Start a YARN job to copy all data-containing resources of an AsterixDB instance to HDFS * @param app * @param resources * @throws IOException * @throws YarnException */ private void backupInstance(YarnClientApplication app, List<DFSResourceCoordinate> resources) throws IOException, YarnException { stopInstanceIfRunning(); ApplicationId backerUpper = deployAM(app, resources, Mode.BACKUP); boolean backupStart; backupStart = Utils.waitForApplication(backerUpper, yarnClient, "Waiting for backup " + backerUpper.toString() + "to start", ccRestPort); if (!backupStart) { LOG.fatal("Backup failed to start"); throw new YarnException(); } boolean complete; complete = waitForCompletion(backerUpper, "Backup in progress"); if (!complete) { LOG.fatal("Backup failed- timeout waiting for completion"); return; } System.out.println("Backup of instance succeeded."); } /** * Start a YARN job to copy a set of resources from backupInstance to restore the state of an extant AsterixDB instance * @param app * @param resources * @throws IOException * @throws YarnException */ private void restoreInstance(YarnClientApplication app, List<DFSResourceCoordinate> resources) throws IOException, YarnException { stopInstanceIfRunning(); ApplicationId restorer = deployAM(app, resources, Mode.RESTORE); boolean restoreStart = Utils.waitForApplication(restorer, yarnClient, "Waiting for restore to start", ccRestPort); if (!restoreStart) { LOG.fatal("Restore failed to start"); throw new YarnException(); } boolean complete = waitForCompletion(restorer, "Restore in progress"); if (!complete) { LOG.fatal("Restore failed- timeout waiting for completion"); return; } System.out.println("Restoration of instance succeeded."); } /** * Stops the instance and remove the lockfile to allow a restart. * * @throws IOException * @throws JAXBException * @throws YarnException */ private void stopInstance() throws IOException, YarnException { ApplicationId appId = getLockFile(); //get CC rest API port if it is nonstandard readConfigParams(locateConfig()); if (yarnClient.isInState(STATE.INITED)) { yarnClient.start(); } System.out.println("Stopping instance " + instanceName); if (!isRunning()) { LOG.fatal("AsterixDB instance by that name is stopped already"); return; } try { String ccIp = Utils.getCCHostname(instanceName, conf); Utils.sendShutdownCall(ccIp, ccRestPort); } catch (IOException e) { LOG.error("Error while trying to issue safe shutdown:", e); } //now make sure it is actually gone and not "stuck" String message = "Waiting for AsterixDB to shut down"; boolean completed = waitForCompletion(appId, message); if (!completed && force) { LOG.warn("Instance failed to stop gracefully, now killing it"); try { AsterixYARNClient.killApplication(appId, yarnClient); completed = true; } catch (YarnException e1) { LOG.fatal("Could not stop nor kill instance gracefully.", e1); return; } } if (completed) { deleteLockFile(); } } private void deleteLockFile() throws IOException { if (instanceName == null || instanceName == "") { return; } FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceName + '/' + instanceLock); if (fs.exists(lockPath)) { fs.delete(lockPath, false); } } private boolean instanceExists() throws IOException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); return fs.exists(dstConf); } private boolean isRunning() throws IOException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); if (fs.exists(dstConf)) { Path lock = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); return fs.exists(lock); } else { return false; } } private ApplicationId getLockFile() throws IOException, YarnException { if (instanceFolder == "") { throw new IllegalStateException("Instance name not given."); } FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); if (!fs.exists(lockPath)) { throw new YarnException("Instance appears to not be running. If you know it is, try using kill"); } BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(lockPath))); String lockAppId = br.readLine(); br.close(); return ConverterUtils.toApplicationId(lockAppId); } public static ApplicationId getLockFile(String instanceName, Configuration conf) throws IOException { if (instanceName == "") { throw new IllegalStateException("Instance name not given."); } FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceName + '/' + instanceLock); if (!fs.exists(lockPath)) { return null; } BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(lockPath))); String lockAppId = br.readLine(); br.close(); return ConverterUtils.toApplicationId(lockAppId); } /** * Locate the Asterix parameters file. * @return * @throws FileNotFoundException * @throws IOException */ private AsterixConfiguration locateConfig() throws FileNotFoundException, IOException { AsterixConfiguration configuration; String configPathBase = MERGED_PARAMETERS_PATH; if (baseConfig != null) { configuration = Utils.loadAsterixConfig(baseConfig); configPathBase = new File(baseConfig).getParentFile().getAbsolutePath() + File.separator + PARAMS_DEFAULT_NAME; MERGED_PARAMETERS_PATH = configPathBase; } else { configuration = Utils.loadAsterixConfig(DEFAULT_PARAMETERS_PATH); } return configuration; } /** * */ private void readConfigParams(AsterixConfiguration configuration) { //this is the "base" config that is inside the zip, we start here for (edu.uci.ics.asterix.common.configuration.Property property : configuration.getProperty()) { if (property.getName().equalsIgnoreCase(CC_JAVA_OPTS_KEY)) { ccJavaOpts = property.getValue(); } else if (property.getName().equalsIgnoreCase(NC_JAVA_OPTS_KEY)) { ncJavaOpts = property.getValue(); } else if (property.getName().equalsIgnoreCase(CC_REST_PORT_KEY)) { ccRestPort = Integer.parseInt(property.getValue()); } } } /** * Retrieves necessary information from the cluster configuration and splices it into the Asterix configuration parameters * @param cluster * @throws FileNotFoundException * @throws IOException */ private void writeAsterixConfig(Cluster cluster) throws FileNotFoundException, IOException { String metadataNodeId = Utils.getMetadataNode(cluster).getId(); String asterixInstanceName = instanceName; AsterixConfiguration configuration = locateConfig(); readConfigParams(configuration); String version = Utils.getAsterixVersionFromClasspath(); configuration.setVersion(version); configuration.setInstanceName(asterixInstanceName); String storeDir = null; List<Store> stores = new ArrayList<Store>(); for (Node node : cluster.getNode()) { storeDir = node.getStore() == null ? cluster.getStore() : node.getStore(); stores.add(new Store(node.getId(), storeDir)); } configuration.setStore(stores); List<Coredump> coredump = new ArrayList<Coredump>(); String coredumpDir = null; List<TransactionLogDir> txnLogDirs = new ArrayList<TransactionLogDir>(); String txnLogDir = null; for (Node node : cluster.getNode()) { coredumpDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir(); coredump.add(new Coredump(node.getId(), coredumpDir + "coredump" + File.separator)); txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir(); //node or cluster-wide txnLogDirs.add(new TransactionLogDir(node.getId(), txnLogDir + (txnLogDir.charAt(txnLogDir.length() - 1) == File.separatorChar ? File.separator : "") + "txnLogs" //if the string doesn't have a trailing / add one + File.separator)); } configuration.setMetadataNode(metadataNodeId); configuration.setCoredump(coredump); configuration.setTransactionLogDir(txnLogDirs); FileOutputStream os = new FileOutputStream(MERGED_PARAMETERS_PATH); try { JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class); Marshaller marshaller = ctx.createMarshaller(); marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); marshaller.marshal(configuration, os); } catch (JAXBException e) { throw new IOException(e); } finally { os.close(); } } private boolean waitForCompletion(ApplicationId appId, String message) throws YarnException, IOException { return Utils.waitForApplication(appId, yarnClient, message, ccRestPort); } private class DFSResourceCoordinate { String name; LocalResource res; Map<String, String> envs; public DFSResourceCoordinate() { envs = new HashMap<String, String>(3); } } }