Java tutorial
/* * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ /* * WekaPackageManager.java * Copyright (C) 2009-2013 University of Waikato, Hamilton, New Zealand */ package weka.core; import weka.core.converters.ConverterUtils; import weka.core.logging.Logger; import weka.core.packageManagement.DefaultPackageManager; import weka.core.packageManagement.Dependency; import weka.core.packageManagement.Package; import weka.core.packageManagement.PackageConstraint; import weka.core.packageManagement.PackageManager; import weka.core.packageManagement.VersionPackageConstraint; import weka.gui.ConverterFileChooser; import weka.gui.GenericObjectEditor; import weka.gui.GenericPropertiesCreator; import weka.gui.beans.BeansProperties; import weka.gui.beans.KnowledgeFlowApp; import weka.gui.explorer.ExplorerDefaults; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.net.URLConnection; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.StringTokenizer; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; /** * Class providing package management and manipulation routines. Also provides a * command line interface for package management. * * @author Mark Hall (mhall{[at]}pentaho{[dot]}com) * @version $Revision$ */ public class WekaPackageManager { /** Package metadata key for dependency injection */ public static final String INJECT_DEPENDENCY_KEY = "InjectDependencies"; /** Package metadata key for version info */ public static final String VERSION_KEY = "Version"; /** Package metadata key for package disablement */ public static final String DISABLE_KEY = "Disable"; /** Package metadata key for package disablement */ public static final String DISABLED_KEY = "Disabled"; /** Package metadata key for package preclusion */ public static final String PRECLUDES_KEY = "Precludes"; /** * Package metadata key for OS name. Entries in this list are checked against * the java property os.name using a String.contains() operation - any match * in the list is taken as passing the test. */ public static final String OS_NAME_KEY = "OSName"; /** * Package metadata key for OS architecture. Entries in this list are checked * against the java property os.arch using a String.equalsIgnoreCase() * operation, with the exception of entries that are either "64" or "32" in * which case the os.arch is checked to see if it contains these numbers. Any * match in the list is considered as passing the test. */ public static final String OS_ARCH_KEY = "OSArch"; /** * Package metadata key for JVM version. Values can be prefixed by a less-than * or greater-than sign. Otherwise, equality is assumed. */ public static final String VM_VERSION_KEY = "JVMVersion"; /** * Package metadata key for preventing load if an environment variable is not * set */ public static final String DO_NOT_LOAD_IF_ENV_VAR_NOT_SET_KEY = "DoNotLoadIfEnvVarNotSet"; /** * Package metadata key for preventing load if an environment variable is not * set */ public static final String DO_NOT_LOAD_IF_ENV_VAR_NOT_SET_MESSAGE_KEY = "DoNotLoadIfEnvVarNotSetMessage"; /** Package metadata key for preventing load if a class is not available */ public static final String DO_NOT_LOAD_IF_CLASS_NOT_PRESENT_KEY = "DoNotLoadIfClassNotPresent"; /** Package metadata key for preventing load if a class is not available */ public static final String DO_NOT_LOAD_IF_CLASS_NOT_PRESENT_MESSAGE_KEY = "DoNotLoadIfClassNotPresentMessage"; /** Package metadata key for preventing load if a file is not present */ public static final String DO_NOT_LOAD_IF_FILE_NOT_PRESENT_KEY = "DoNotLoadIfFileNotPresent"; /** Package metadata key for preventing load if a file is not present */ public static final String DO_NOT_LOAD_IF_FILE_NOT_PRESENT_MESSAGE_KEY = "DoNotLoadIfFileNotPresentMessage"; /** Package metadata key for a message to display on installation */ public static final String MESSAGE_TO_DISPLAY_ON_INSTALLATION_KEY = "MessageToDisplayOnInstallation"; /** Package metadata key for setting system properties */ public static final String SET_SYSTEM_PROPERTIES_KEY = "SetSystemProperties"; /** The default folder name for Weka bits and bobs */ private static String WEKAFILES_DIR_NAME = "wekafiles"; /** Default path to where Weka's configuration and packages are stored */ public static File WEKA_HOME = new File(System.getProperty("user.home") + File.separator + WEKAFILES_DIR_NAME); /** The default packages directory */ public static File PACKAGES_DIR = new File(WEKA_HOME.toString() + File.separator + "packages"); /** The default props dir name */ private static String PROPERTIES_DIR_NAME = "props"; /** The default properties directory */ public static File PROPERTIES_DIR = new File(WEKA_HOME.toString() + File.separator + PROPERTIES_DIR_NAME); public static String NATIVE_LIBS_DIR_NAME = "native"; /** The default native libraries directory */ public static File NATIVE_LIBS_DIR = new File(WEKA_HOME.toString() + File.separator + NATIVE_LIBS_DIR_NAME); /** The underlying package manager */ private static PackageManager PACKAGE_MANAGER = PackageManager.create(); /** Current repository URL to use */ private static URL REP_URL; /** Location of the repository cache */ private static URL CACHE_URL; /** True if a cache build is required */ private static boolean INITIAL_CACHE_BUILD_NEEDED = false; /** * The name of the file that contains the list of packages in the repository */ private static String PACKAGE_LIST_FILENAME = "packageListWithVersion.txt"; /** Primary repository */ private static String PRIMARY_REPOSITORY = "https://weka.sourceforge.io/packageMetaData"; /** Backup mirror of the repository */ private static String REP_MIRROR; /** * True if the user has specified a custom repository via a property in * PackageManager.props */ private static boolean USER_SET_REPO = false; /** The package manager's property file */ private static String PACKAGE_MANAGER_PROPS_FILE_NAME = "PackageManager.props"; /** Operating offline? */ public static boolean m_offline; /** Load packages? */ private static boolean m_loadPackages = true; /** Established WEKA_HOME successfully? */ protected static boolean m_wekaHomeEstablished; /** Packages loaded OK? */ protected static boolean m_packagesLoaded; /** File to check against server for new/updated packages */ protected static final String PACKAGE_LIST_WITH_VERSION_FILE = "packageListWithVersion.txt"; /** File to check against server equivalent for forced refresh */ protected static final String FORCED_REFRESH_COUNT_FILE = "forcedRefreshCount.txt"; /** Package loading in progress? */ public static boolean m_initialPackageLoadingInProcess = false; /* True if an initial cache build is needed and working offline */ public static boolean m_noPackageMetaDataAvailable; /** The set of packages that the user has requested not to load */ public static Set<String> m_doNotLoadList; /** * Holds how many times we've failed to establish the repository backup mirror * URL */ protected static int m_mirrorFailureCount; static { establishWekaHome(); // make sure MTJ classes are injected to the root classloader // so that MTJ native library loading can see them WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().injectMTJCoreClasses(); } /** * Establish WEKA_HOME if needed * * @return true if WEKA_HOME was successfully established */ protected static boolean establishWekaHome() { if (m_wekaHomeEstablished) { return true; } // process core PluginManager.props before any from packages. // This is to have some control over the order of certain plugins try { PluginManager.addFromProperties( WekaPackageManager.class.getClassLoader().getResourceAsStream("weka/PluginManager.props"), true); } catch (Exception ex) { log(Logger.Level.WARNING, "[WekaPackageManager] unable to read weka/PluginManager.props"); } // look to see if WEKA_HOME has been defined as an environment // variable Environment env = Environment.getSystemWide(); String wh = env.getVariableValue("WEKA_HOME"); if (wh != null) { WEKA_HOME = new File(wh); PACKAGES_DIR = new File(wh + File.separator + "packages"); PROPERTIES_DIR = new File(wh + File.separator + PROPERTIES_DIR_NAME); NATIVE_LIBS_DIR = new File(wh + File.separator + NATIVE_LIBS_DIR_NAME); } else { env.addVariableSystemWide("WEKA_HOME", WEKA_HOME.toString()); } String nativePath = System.getProperty("java.library.path", ""); if (!nativePath.contains(NATIVE_LIBS_DIR.toString())) { nativePath += File.pathSeparator + NATIVE_LIBS_DIR.toString(); System.setProperty("java.library.path", nativePath); } boolean ok = true; if (!WEKA_HOME.exists()) { // create it for the user if (!WEKA_HOME.mkdir()) { System.err.println("Unable to create WEKA_HOME (" + WEKA_HOME.getAbsolutePath() + ")"); ok = false; } } if (!PACKAGES_DIR.exists()) { // create the packages dir if (!PACKAGES_DIR.mkdir()) { System.err.println("Unable to create packages directory (" + PACKAGES_DIR.getAbsolutePath() + ")"); ok = false; } } if (!NATIVE_LIBS_DIR.exists()) { if (!NATIVE_LIBS_DIR.mkdir()) { System.err.println( "Unable to create native libs directory (" + NATIVE_LIBS_DIR.getAbsolutePath() + ")"); } } m_wekaHomeEstablished = ok; PACKAGE_MANAGER.setPackageHome(PACKAGES_DIR); m_doNotLoadList = getDoNotLoadList(); try { // setup the backup mirror first // establishMirror(); // user-supplied repository URL takes precedence over anything else String repURL = env.getVariableValue("weka.core.wekaPackageRepositoryURL"); if (repURL == null || repURL.length() == 0) { // See if there is a URL named in // $WEKA_HOME/props/PackageRepository.props File repPropsFile = new File( PROPERTIES_DIR.toString() + File.separator + "PackageRepository.props"); if (repPropsFile.exists()) { Properties repProps = new Properties(); repProps.load(new FileInputStream(repPropsFile)); repURL = repProps.getProperty("weka.core.wekaPackageRepositoryURL"); } } if (repURL == null || repURL.length() == 0) { repURL = PRIMARY_REPOSITORY; } else { log(weka.core.logging.Logger.Level.INFO, "[WekaPackageManager] weka.core.WekaPackageRepositoryURL = " + repURL); // System.err.println("[WekaPackageManager] // weka.core.WekaPackageRepositoryURL = " // + repURL); USER_SET_REPO = true; } REP_URL = new URL(repURL); PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); } catch (MalformedURLException ex) { ex.printStackTrace(); } catch (IOException ex) { ex.printStackTrace(); } PACKAGE_MANAGER.setBaseSystemName("weka"); PACKAGE_MANAGER.setBaseSystemVersion(weka.core.Version.VERSION); // Now check the cache and establish it if necessary File cacheDir = new File(WEKA_HOME.toString() + File.separator + "repCache"); try { String tempCacheString = "file://" + cacheDir.toString(); // sanitize URI and fix slashes (for Windows) tempCacheString = tempCacheString.replace(" ", "%20"); tempCacheString = tempCacheString.replace('\\', '/'); if (tempCacheString.startsWith("file://") && !tempCacheString.startsWith("file:///")) { tempCacheString = tempCacheString.substring(7); tempCacheString = "file:///" + tempCacheString; } URI tempURI = new URI(tempCacheString); // System.err.println(" -- " + tempURI.toString()); CACHE_URL = tempURI.toURL(); } catch (Exception e) { e.printStackTrace(); } File packagesList = new File(cacheDir.getAbsolutePath() + File.separator + PACKAGE_LIST_FILENAME); if (!cacheDir.exists()) { if (!cacheDir.mkdir()) { System.err.println( "Unable to create repository cache directory (" + cacheDir.getAbsolutePath() + ")"); log(weka.core.logging.Logger.Level.WARNING, "Unable to create repository cache directory (" + cacheDir.getAbsolutePath() + ")"); CACHE_URL = null; } else { // refreshCache(); INITIAL_CACHE_BUILD_NEEDED = true; } } if (!packagesList.exists()) { INITIAL_CACHE_BUILD_NEEDED = true; } // Package manager general properties // Set via system props first String offline = env.getVariableValue("weka.packageManager.offline"); if (offline != null) { m_offline = offline.equalsIgnoreCase("true"); } String loadPackages = env.getVariableValue("weka.packageManager.loadPackages"); if (loadPackages == null) { // try legacy loadPackages = env.getVariableValue("weka.core.loadPackages"); } if (loadPackages != null) { m_loadPackages = loadPackages.equalsIgnoreCase("true"); } // load any general package manager properties from props file File generalProps = new File(PROPERTIES_DIR.toString() + File.separator + PACKAGE_MANAGER_PROPS_FILE_NAME); if (generalProps.exists()) { Properties gProps = new Properties(); try { gProps.load(new FileInputStream(generalProps)); // this one takes precedence over the legacy one String repURL = gProps.getProperty("weka.core.wekaPackageRepositoryURL"); if (repURL != null && repURL.length() > 0) { REP_URL = new URL(repURL); PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); } offline = gProps.getProperty("weka.packageManager.offline"); if (offline != null && offline.length() > 0) { m_offline = offline.equalsIgnoreCase("true"); } loadPackages = gProps.getProperty("weka.packageManager.loadPackages"); if (loadPackages == null) { // try legacy loadPackages = env.getVariableValue("weka.core.loadPackages"); } if (loadPackages != null) { m_loadPackages = loadPackages.equalsIgnoreCase("true"); } String pluginManagerDisableList = gProps.getProperty("weka.pluginManager.disable"); if (pluginManagerDisableList != null && pluginManagerDisableList.length() > 0) { List<String> disable = new ArrayList<String>(); String[] parts = pluginManagerDisableList.split(","); for (String s : parts) { disable.add(s.trim()); } PluginManager.addToDisabledList(disable); } } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } if (INITIAL_CACHE_BUILD_NEEDED && m_offline) { m_noPackageMetaDataAvailable = true; } // Pass a valid http URL to the setProxyAuthentication() // method to ensure that a proxy or direct connection // is configured initially. This will prevent this method from // trying to configure the ProxySelector and proxy // via the file-based CACHE_URL somewhere down the track PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); PACKAGE_MANAGER.setProxyAuthentication(REP_URL); return ok; } /** * Establish the location of a mirror */ protected static void establishMirror() { if (m_offline) { return; } try { String mirrorListURL = "https://www.cs.waikato.ac.nz/ml/weka/packageMetaDataMirror.txt"; URLConnection conn = null; URL connURL = new URL(mirrorListURL); if (PACKAGE_MANAGER.setProxyAuthentication(connURL)) { conn = connURL.openConnection(PACKAGE_MANAGER.getProxy()); } else { conn = connURL.openConnection(); } conn.setConnectTimeout(10000); // timeout after 10 seconds conn.setReadTimeout(10000); BufferedReader bi = new BufferedReader(new InputStreamReader(conn.getInputStream())); REP_MIRROR = bi.readLine(); bi.close(); if (REP_MIRROR != null && REP_MIRROR.length() > 0) { // use the mirror if it is different from the primary repo // and the user hasn't specified an explicit repo via the // property if (!REP_MIRROR.equals(PRIMARY_REPOSITORY) && !USER_SET_REPO) { log(weka.core.logging.Logger.Level.INFO, "[WekaPackageManager] Package manager using repository mirror: " + REP_MIRROR); REP_URL = new URL(REP_MIRROR); } } } catch (Exception ex) { log(weka.core.logging.Logger.Level.WARNING, "[WekaPackageManager] The repository meta data mirror file seems " + "to be unavailable (" + ex.getMessage() + ")"); m_mirrorFailureCount++; } } /** * Write to the weka log * * @param level logging level * @param message message to write */ protected static void log(weka.core.logging.Logger.Level level, String message) { try { File logFile = new File(WEKA_HOME.toString() + File.separator + "weka.log"); BufferedWriter writer = new BufferedWriter(new FileWriter(logFile, true)); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); String linefeed = System.getProperty("line.separator"); String m = format.format(new Date()) + " " + level + ": " + message + linefeed; writer.write(m); writer.flush(); writer.close(); } catch (Exception ex) { } } /** * Remove any ExplorerDefaults properties specified in the supplied package * * @param installedPackageName the package specifying properties that should * be removed from ExplorerDefaults */ public static void removeExplorerProps(String installedPackageName) { try { Properties expProps = new Properties(); String explorerProps = getPackageHome().getAbsolutePath() + File.separator + installedPackageName + File.separator + "Explorer.props"; BufferedInputStream bi = new BufferedInputStream(new FileInputStream(explorerProps)); expProps.load(bi); bi.close(); bi = null; Set<Object> keys = expProps.keySet(); Iterator<Object> keysI = keys.iterator(); while (keysI.hasNext()) { String key = (String) keysI.next(); if (!key.endsWith("Policy")) { // See if this key is in the Explorer props String existingVal = ExplorerDefaults.get(key, ""); String toRemove = expProps.getProperty(key); if (existingVal.length() > 0) { // cover the case when the value to remove is at the start // or middle of a list existingVal = existingVal.replace(toRemove + ",", ""); // the case when it's at the end existingVal = existingVal.replace("," + toRemove, ""); ExplorerDefaults.set(key, existingVal); } } } } catch (Exception ex) { } } /** * Process a package's GenericPropertiesCreator.props file * * @param propsFile the props file to process */ protected static void processGenericPropertiesCreatorProps(File propsFile) { try { Properties expProps = new Properties(); BufferedInputStream bi = new BufferedInputStream(new FileInputStream(propsFile)); expProps.load(bi); bi.close(); bi = null; Properties GPCInputProps = GenericPropertiesCreator.getGlobalInputProperties(); Set<Object> keys = expProps.keySet(); Iterator<Object> keysI = keys.iterator(); while (keysI.hasNext()) { String key = (String) keysI.next(); // see if this key is in the GPC input props String existingVal = GPCInputProps.getProperty(key, ""); if (existingVal.length() > 0) { // append String newVal = expProps.getProperty(key); // only append if this value is not already there!! if (existingVal.indexOf(newVal) < 0) { newVal = existingVal + "," + newVal; GPCInputProps.put(key, newVal); } } else { // simply add this new key/value combo String newVal = expProps.getProperty(key); GPCInputProps.put(key, newVal); } } } catch (Exception ex) { // ignore } } /** * Process a package's Explorer.props file * * @param propsFile the properties file to process */ protected static void processExplorerProps(File propsFile) { try { Properties expProps = new Properties(); BufferedInputStream bi = new BufferedInputStream(new FileInputStream(propsFile)); expProps.load(bi); bi.close(); bi = null; Set<Object> keys = expProps.keySet(); Iterator<Object> keysI = keys.iterator(); while (keysI.hasNext()) { String key = (String) keysI.next(); if (!key.endsWith("Policy")) { // See if this key is in the Explorer props String existingVal = ExplorerDefaults.get(key, ""); if (existingVal.length() > 0) { // get the replacement policy (if any) String replacePolicy = expProps.getProperty(key + "Policy"); if (replacePolicy != null && replacePolicy.length() > 0) { if (replacePolicy.equalsIgnoreCase("replace")) { String newVal = expProps.getProperty(key); ExplorerDefaults.set(key, newVal); } else { // default to append String newVal = expProps.getProperty(key); // only append if this value is not already there!! if (existingVal.indexOf(newVal) < 0) { newVal = existingVal + "," + newVal; ExplorerDefaults.set(key, newVal); } } } else { // default to append String newVal = expProps.getProperty(key); // only append if this value is not already there!! if (existingVal.indexOf(newVal) < 0) { newVal = existingVal + "," + newVal; ExplorerDefaults.set(key, newVal); } } } else { // simply add this new key/value combo String newVal = expProps.getProperty(key); ExplorerDefaults.set(key, newVal); } } } } catch (Exception ex) { // ignore } } /** * Process a package's GUIEditors.props file * * @param propsFile the properties file to process * @param verbose true to output more info */ protected static void processGUIEditorsProps(File propsFile, boolean verbose) { GenericObjectEditor.registerEditors(); try { Properties editorProps = new Properties(); BufferedInputStream bi = new BufferedInputStream(new FileInputStream(propsFile)); editorProps.load(bi); bi.close(); bi = null; Enumeration<?> enm = editorProps.propertyNames(); while (enm.hasMoreElements()) { String name = enm.nextElement().toString(); String value = editorProps.getProperty(name, ""); if (verbose) { System.err.println("Registering " + name + " " + value); } GenericObjectEditor.registerEditor(name, value); } } catch (Exception ex) { // ignore } } /** * Process a package's PluginManager.props file * * @param packageName the name of the package that owns this PluginManager * props file * @param propsFile the properties file to process */ protected static void processPluginManagerProps(String packageName, File propsFile) { try { PluginManager.addFromProperties(packageName, propsFile); } catch (Exception ex) { ex.printStackTrace(); } } /** * Process properties in a package directory * * @param directory the package directory to process * @param verbose true for verbose output * @param goePropsFiles store any GenericObjectEditor.props files for * post-processing after all jars have been loaded * @throws Exception if a problem occurs */ protected static void processPackageDirectory(File directory, boolean verbose, List<File> goePropsFiles, boolean avoidTriggeringFullClassDiscovery) throws Exception { File[] contents = directory.listFiles(); if (contents != null) { // make sure that jar files and lib directory get processed first /* * for (File content : contents) { if (content.isFile() && * content.getPath().endsWith(".jar")) { if (verbose) { * System.out.println("[WekaPackageManager] loading " + * content.getPath()); } ClassloaderUtil.addFile(content.getPath()); * * } else if (content.isDirectory() && * content.getName().equalsIgnoreCase("lib")) { // add any jar files in * the lib directory to the classpath processPackageDirectory(content, * verbose, goePropsFiles, avoidTriggeringFullClassDiscovery); } } */ // now any auxilliary files for (File content : contents) { if (content.isFile() && content.getPath().endsWith("Beans.props")) { // KnowledgeFlow plugin -- add the Beans.props file to the list of // bean plugin props BeansProperties.addToPluginBeanProps(content); if (!avoidTriggeringFullClassDiscovery) { KnowledgeFlowApp.disposeSingleton(); } } else if (content.isFile() && content.getPath().endsWith("Explorer.props") && !avoidTriggeringFullClassDiscovery) { // Explorer tabs plugin // process the keys in the properties file and append/add values processExplorerProps(content); } else if (content.isFile() && content.getPath().endsWith("GUIEditors.props") && !avoidTriggeringFullClassDiscovery) { // Editor for a particular component processGUIEditorsProps(content, verbose); } else if (content.isFile() && content.getPath().endsWith("GenericPropertiesCreator.props") && !avoidTriggeringFullClassDiscovery) { if (goePropsFiles != null) { goePropsFiles.add(content); } else { processGenericPropertiesCreatorProps(content); } } else if (content.isFile() && content.getPath().endsWith("PluginManager.props")) { processPluginManagerProps(directory.getName(), content); } } } } /** * Check to see if the named package has been loaded successfully * * @param toCheck the name of the package to check for * @return true if the named package has been loaded successfully */ public static boolean hasBeenLoaded(Package toCheck) { // if it loaded successfully, passed all integrity checks etc., then there // will be package classloader for it return WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .getPackageClassLoader(toCheck.getName()) != null; } /** * Check whether a package should be loaded or not. Checks for missing * classes, unset environment variables, missing dependencies etc. * * @param toLoad the package to check * @param packageRoot the root directory of the package * @param progress for reporting loading progress * @return true if the package is good to load */ public static boolean loadCheck(Package toLoad, File packageRoot, PrintStream... progress) { // first check against the base version of the system boolean load; try { load = toLoad.isCompatibleBaseSystem(); } catch (Exception ex) { ex.printStackTrace(); return false; } if (!load) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Skipping package " + toLoad.getName() + " because it is not compatible with Weka " + PACKAGE_MANAGER.getBaseSystemVersion().toString()); } return false; } // check to see if this package has been disabled for all users try { Package repoP = getRepositoryPackageInfo(toLoad.getName(), toLoad.getPackageMetaDataElement(VERSION_KEY).toString()); if (repoP != null) { Object disabled = repoP.getPackageMetaDataElement(DISABLED_KEY); if (disabled == null) { disabled = repoP.getPackageMetaDataElement(DISABLE_KEY); } if (disabled != null && disabled.toString().equalsIgnoreCase("true")) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Skipping package " + toLoad.getName() + " because it has been marked as disabled at the repository"); } return false; } } } catch (Exception ex) { // Ignore - we will get an exception when checking for an unofficial // package // return true; } if (!osAndArchCheck(toLoad, progress)) { return false; } if (!vmVersionCheck(toLoad, progress)) { return false; } // check for precludes Object precludes = toLoad.getPackageMetaDataElement(PRECLUDES_KEY); if (precludes != null) { try { List<Package> installed = getInstalledPackages(); List<Package> preclusions = toLoad.getPrecludedPackages(installed); if (preclusions.size() > 0) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Skipping package " + toLoad.getName() + " because it precludes one or more packages that are " + "already installed: "); for (Package prec : preclusions) { p.println("\t" + prec); } } return false; } } catch (Exception ex) { ex.printStackTrace(); } } load = !m_doNotLoadList.contains(toLoad.getName()); if (!load) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Skipping package " + toLoad.getName() + " because it is has been marked as do not load"); } return false; } if (m_offline) { return true; } // now check for missing dependencies try { List<Dependency> missing = toLoad.getMissingDependencies(); if (missing.size() > 0) { for (PrintStream p : progress) { p.println("[WekaPackageManager] " + toLoad.getName() + " can't be loaded because the following" + " packages are missing:"); for (Dependency d : missing) { p.println(d.getTarget()); } } return false; } } catch (Exception ex) { ex.printStackTrace(); return false; } // now check for buggered dependencies try { List<Dependency> depends = toLoad.getDependencies(); for (Dependency d : depends) { if (d.getTarget().getPackage().isInstalled()) { if (!loadCheck(d.getTarget().getPackage(), packageRoot, progress)) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Can't load " + toLoad.getName() + " because " + d.getTarget() + " can't be loaded."); } return false; } // check that the version of installed dependency is OK Package installedD = getInstalledPackageInfo(d.getTarget().getPackage().getName()); if (!d.getTarget().checkConstraint(installedD)) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Can't load " + toLoad.getName() + " because the installed " + d.getTarget().getPackage().getName() + " is not compatible (requires: " + d.getTarget() + ")"); } return false; } } } } catch (Exception ex) { ex.printStackTrace(); return false; } return true; } /** * Checks the supplied package against the JVM version running Weka. Packages * that don't specify a JVM version are assumed to be OK. The entry in the * JVMVersion key are expected to be a floating point number, optionally * prefixed by either a greater-than or less-than symbol. Absence of either * of these symbols imply equality as the test. * * @param toLoad the package to check * @param progress PrintStream for progress info * @return true if the supplied package passes the JVM version test. */ public static boolean vmVersionCheck(Package toLoad, PrintStream... progress) { String thisVMVersion = System.getProperty("java.specification.version"); if (thisVMVersion != null && thisVMVersion.length() > 0) { double actualVM = -1; try { actualVM = Double.parseDouble(thisVMVersion); } catch (NumberFormatException e) { e.printStackTrace(); return true; } try { Object reqVMS = toLoad.getPackageMetaDataElement(VM_VERSION_KEY); if (reqVMS != null) { String vm = reqVMS.toString(); String op = "="; if (vm.charAt(0) == '>') { op = ">"; vm = vm.substring(1); } else if (vm.charAt(0) == '<') { op = "<"; vm = vm.substring(1); } double requestedVM = -1; try { requestedVM = Double.parseDouble(vm); } catch (NumberFormatException e) { e.printStackTrace(); return true; } boolean result = true; String failureMessage = ""; if (op.equals("=")) { result = requestedVM == actualVM; if (!result) { failureMessage = "[WekaPackageManager] Unable to load '" + toLoad.getName() + "' because " + "JVM " + actualVM + " does not match requested JVM " + requestedVM; } } else if (op.equals("<")) { result = actualVM < requestedVM; if (!result) { failureMessage = "[WekaPackageManager] Unable to load '" + toLoad.getName() + "' because " + "JVM " + actualVM + " is not < requested JVM " + requestedVM; } } else { result = actualVM > requestedVM; if (!result) { failureMessage = "[WekaPackageManager] Unable to laod '" + toLoad.getName() + "' because " + "JVM " + actualVM + " is not > requested JVM " + requestedVM; } } if (!result) { for (PrintStream p : progress) { p.println(failureMessage); } } return result; } } catch (Exception ex) { ex.printStackTrace(); } } return true; } /** * Checks the supplied package against the current OS and architecture. * Packages that don't specify OS and (optionally) architecture constraints * are assumed to be OK. OS names in the OSName entry of the package's * Description.props are checked against System.getProperty("os.name") via a * String.contains() comparison. Any single match results in a pass. If * supplied, the package's OSArch entries are checked against * System.getProperty("os.arch") using a String.equalsIgnoreCase() comparison, * with the exception of the values "64" and "32" which are checked for with * String.contains(). * * @param toLoad the package to check * @param progress PrintStream for progress info * @return true if the supplied package passes OS/arch constraints. */ public static boolean osAndArchCheck(Package toLoad, PrintStream... progress) { // check for OS restrictions try { Object osNames = toLoad.getPackageMetaDataElement(OS_NAME_KEY); Object archNames = toLoad.getPackageMetaDataElement(OS_ARCH_KEY); if (osNames != null) { boolean osCheck = false; String[] osElements = osNames.toString().split(","); String thisOs = System.getProperty("os.name"); if (thisOs != null) { for (String osEntry : osElements) { if (thisOs.toLowerCase().contains(osEntry.trim().toLowerCase())) { osCheck = true; break; } } String thisArch = System.getProperty("os.arch"); if (osCheck && archNames != null) { String[] archElements = archNames.toString().split(","); if (thisArch != null) { boolean archCheck = false; for (String archEntry : archElements) { if (archEntry.trim().equalsIgnoreCase("32") || archEntry.trim().equalsIgnoreCase("64")) { if (thisArch.toLowerCase().contains(archEntry.trim())) { archCheck = true; break; } } else { if (thisArch.trim().equalsIgnoreCase(archEntry.trim())) { archCheck = true; break; } } } osCheck = archCheck; } } if (!osCheck) { for (PrintStream p : progress) { p.println("[WekaPackageManager] Skipping package " + toLoad.getName() + " because the OS/arch (" + thisOs + " " + (thisArch != null ? thisArch : "") + ") does not meet package OS/arch constraints: " + osNames.toString() + " " + (archNames != null ? archNames.toString() : "")); } return false; } } } } catch (Exception e) { e.printStackTrace(); } return true; } /** * Checks to see if there are any missing files/directories for a given * package. If there are missing files, then the package can't be loaded. An * example would be a connector package that, for whatever reason, can't * include a necessary third-party jar file in its lib folder, and requires * the user to download and install this jar file manually. * * @param toLoad the package to check * @param packageRoot the root directory of the package * @return true if good to go */ public static boolean checkForMissingFiles(Package toLoad, File packageRoot, PrintStream... progress) { boolean result = true; Object doNotLoadIfFileMissing = toLoad.getPackageMetaDataElement(DO_NOT_LOAD_IF_FILE_NOT_PRESENT_KEY); String packageRootPath = packageRoot.getPath() + File.separator; if (doNotLoadIfFileMissing != null && doNotLoadIfFileMissing.toString().length() > 0) { StringTokenizer tok = new StringTokenizer(doNotLoadIfFileMissing.toString(), ","); while (tok.hasMoreTokens()) { String nextT = tok.nextToken().trim(); File toCheck = new File(packageRootPath + nextT); if (!toCheck.exists()) { for (PrintStream p : progress) { p.println("[WekaPackageManager] " + toLoad.getName() + " can't be loaded because " + toCheck.getPath() + " appears to be missing."); } result = false; break; } } } if (!result) { // grab the message to print to the log (if any) Object doNotLoadMessage = toLoad.getPackageMetaDataElement(DO_NOT_LOAD_IF_FILE_NOT_PRESENT_MESSAGE_KEY); if (doNotLoadMessage != null && doNotLoadMessage.toString().length() > 0) { String dnlM = doNotLoadMessage.toString(); try { dnlM = Environment.getSystemWide().substitute(dnlM); } catch (Exception ex) { // quietly ignore } for (PrintStream p : progress) { p.println("[WekaPackageManager] " + dnlM); } } } return result; } /** * Reads the doNotLoad list (if it exists) from the packages directory * * @return a set of package names that should not be loaded. This will be * empty if the doNotLoadList does not exist on disk. */ @SuppressWarnings("unchecked") protected static Set<String> getDoNotLoadList() { Set<String> doNotLoad = new HashSet<String>(); File doNotLoadList = new File(PACKAGES_DIR.toString() + File.separator + "doNotLoad.ser"); if (doNotLoadList.exists() && doNotLoadList.isFile()) { ObjectInputStream ois = null; try { ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(doNotLoadList))); doNotLoad = (Set<String>) ois.readObject(); } catch (FileNotFoundException ex) { } catch (IOException e) { System.err.println("An error occurred while reading the doNotLoad list: " + e.getMessage()); } catch (ClassNotFoundException e) { System.err.println("An error occurred while reading the doNotLoad list: " + e.getMessage()); } finally { if (ois != null) { try { ois.close(); } catch (IOException ex) { ex.printStackTrace(); } } } } return doNotLoad; } /** * Toggle the load status of the supplied list of package names * * @param packageNames the packages to toggle the load status for * @return a list of unknown packages (i.e. any supplied package names that * don't appear to be installed) * @throws Exception if a problem occurs */ public static List<String> toggleLoadStatus(List<String> packageNames) throws Exception { List<String> unknownPackages = new ArrayList<String>(); boolean changeMade = false; for (String s : packageNames) { Package p = PACKAGE_MANAGER.getInstalledPackageInfo(s); if (p == null) { unknownPackages.add(s); } else { if (m_doNotLoadList.contains(s)) { m_doNotLoadList.remove(s); } else { // only mark as do not load if a package is loadable if (loadCheck(p, new File(WekaPackageManager.getPackageHome().toString() + File.separator + s))) { m_doNotLoadList.add(s); } } changeMade = true; } } if (changeMade) { // write the list back to disk File doNotLoadList = new File(PACKAGES_DIR.toString() + File.separator + "doNotLoad.ser"); ObjectOutputStream oos = null; try { oos = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(doNotLoadList))); oos.writeObject(m_doNotLoadList); } finally { if (oos != null) { oos.flush(); oos.close(); } } } return unknownPackages; } /** * Load all packages * * @param verbose true if loading progress should be output */ public static synchronized void loadPackages(boolean verbose) { loadPackages(verbose, false, true); } /** * Load all packages * * @param verbose true if loading progress should be output * @param avoidTriggeringFullClassDiscovery true if we should avoid processing * any properties files that might cause a full class discovery run, * and may involve instantiating GUI classes. * @param refreshGOEProperties true if the GOE properties should be refreshed * after loading (i.e. a re-run of the class discovery mechanism, * re-initialization of the Knowledge Flow etc.) */ public static synchronized void loadPackages(boolean verbose, boolean avoidTriggeringFullClassDiscovery, boolean refreshGOEProperties) { if (!verbose) { // debug property overrides String debug = System.getProperty("weka.core.classloader.debug", "false"); verbose = debug.equalsIgnoreCase("true"); } else { System.setProperty("weka.core.classloader.debug", "true"); } List<File> goePropsFiles = new ArrayList<File>(); if (!m_loadPackages) { return; } if (m_packagesLoaded) { return; } m_packagesLoaded = true; m_initialPackageLoadingInProcess = true; if (establishWekaHome()) { // try and load any jar files and add to the classpath File[] contents = PACKAGES_DIR.listFiles(); // if we have a non-empty packages dir then check // the integrity of our cache first if (contents.length > 0) { // establishCacheIfNeeded(System.out); startupCheck(false, System.out); } // dynamic injection of dependencies between packages Map<String, List<String>> injectDependencies = new HashMap<>(); for (File content : contents) { if (content.isDirectory()) { try { Package toLoad = getInstalledPackageInfo(content.getName()); boolean load; // Only perform the check against the current version of Weka if // there exists // a Description.props file if (toLoad != null) { load = loadCheck(toLoad, content, System.err); if (load) { if (verbose) { System.out.println("[WekaPackageManager] loading package " + content.getName()); } checkForInjectDependencies(toLoad, injectDependencies); WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .addPackageToClassLoader(content); } } } catch (Exception ex) { ex.printStackTrace(); System.err.println("[WekaPackageManager] Problem loading package " + content.getName() + " skipping..."); } } } // now inject any package dependencies injectPackageDependencies(injectDependencies); // now check overall integrity WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().performIntegrityCheck(); // now process the various properties files in the packages for (File content : contents) { try { if (content.isDirectory() && WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .getPackageClassLoader(content.getName()) != null) { processPackageDirectory(content, verbose, goePropsFiles, avoidTriggeringFullClassDiscovery); } } catch (Exception ex) { ex.printStackTrace(); System.err.println( "[WekaPackageManager] Problem loading package " + content.getName() + " skipping..."); } } } m_initialPackageLoadingInProcess = false; // it is best to process all of these after all jars have been // inserted into the classpath since the dynamic class discovery // mechanism will load classes during the process of determining // all implementations of base types, and this can cause problems // if processed at the time of package loading and there are // dependencies between packages if (!avoidTriggeringFullClassDiscovery) { for (File f : goePropsFiles) { processGenericPropertiesCreatorProps(f); } } // do we need to regenerate the list of available schemes for // the GUIs (this is not necessary when executing stuff from // the command line) if (refreshGOEProperties) { if (verbose) { System.err.println("Refreshing GOE props..."); } refreshGOEProperties(); } } /** * Checks a package for the presence of any specified package dependencies to * inject at load time * * @param toLoad the package to check * @param injectDeps a map (keyed by source package name) containing lists of * target packages. Each source package will have the entries in its * target package list injected (via the WekaLibIsolatingClassLoader) * as dependencies. */ protected static void checkForInjectDependencies(Package toLoad, Map<String, List<String>> injectDeps) { Object injectList = toLoad.getPackageMetaDataElement(INJECT_DEPENDENCY_KEY); if (injectList != null) { String[] deps = injectList.toString().split(","); for (String dep : deps) { String[] sourceAndTarget = dep.split(":"); if (sourceAndTarget.length == 2) { String depender = sourceAndTarget[0].trim(); String dependee = sourceAndTarget[1].trim(); List<String> depList = injectDeps.get(depender); if (depList == null) { depList = new ArrayList<>(); injectDeps.put(depender, depList); } depList.add(dependee); } } } } /** * Processes a map of dependencies to inject. Each source package (key) will * have the packages named in the associated list (value) injected as * dependencies at load time. * * @param injectDependencies a map of source to target dependencies. */ protected static void injectPackageDependencies(Map<String, List<String>> injectDependencies) { for (Map.Entry<String, List<String>> e : injectDependencies.entrySet()) { String source = e.getKey(); List<String> targets = e.getValue(); WekaPackageLibIsolatingClassLoader sourceLoader = WekaPackageClassLoaderManager .getWekaPackageClassLoaderManager().getPackageClassLoader(source); if (sourceLoader != null) { String debugS = System.getProperty("weka.core.classloader.debug", "false"); boolean debug = debugS.equalsIgnoreCase("true"); for (String targetPackage : targets) { if (WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .getPackageClassLoader(targetPackage) != null) { if (debug) { System.out.println("[WekaPackageManager] Added a dependency between " + source + " and " + targetPackage); } sourceLoader.addPackageDependency(targetPackage); } } } } } /** * Refresh the generic object editor properties via re-running of the dynamic * class discovery process. */ public static void refreshGOEProperties() { ClassDiscovery.clearClassCache(); GenericPropertiesCreator.regenerateGlobalOutputProperties(); GenericObjectEditor.determineClasses(); ConverterUtils.initialize(); ConverterFileChooser.initDefaultFilters(); KnowledgeFlowApp.disposeSingleton(); KnowledgeFlowApp.reInitialize(); } /** * Get the underlying package manager implementation * * @return the underlying concrete package management implementation. */ public static PackageManager getUnderlyingPackageManager() { return PACKAGE_MANAGER; } /** * Retrieves the size (in KB) of the repository zip archive stored on the * server. * * @return the size of the repository zip archive in KB. */ public static int repoZipArchiveSize() { int size = -1; try { PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); String numPackagesS = PACKAGE_MANAGER.getPackageRepositoryURL().toString() + "/repoSize.txt"; URLConnection conn = null; URL connURL = new URL(numPackagesS); if (PACKAGE_MANAGER.setProxyAuthentication(connURL)) { conn = connURL.openConnection(PACKAGE_MANAGER.getProxy()); } else { conn = connURL.openConnection(); } conn.setConnectTimeout(30000); // timeout after 30 seconds BufferedReader bi = new BufferedReader(new InputStreamReader(conn.getInputStream())); String n = bi.readLine(); try { size = Integer.parseInt(n); } catch (NumberFormatException ne) { System.err.println("[WekaPackageManager] problem parsing the size " + "of repository zip archive from the server."); } bi.close(); } catch (Exception ex) { ex.printStackTrace(); } return size; } /** * Get the number of packages that are available at the repository. * * @return the number of packages that are available (or -1 if this can't be * determined for some reason. */ public static int numRepositoryPackages() { if (m_offline) { return -1; } int numPackages = -1; try { PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); String numPackagesS = PACKAGE_MANAGER.getPackageRepositoryURL().toString() + "/numPackages.txt"; URLConnection conn = null; URL connURL = new URL(numPackagesS); if (PACKAGE_MANAGER.setProxyAuthentication(connURL)) { conn = connURL.openConnection(PACKAGE_MANAGER.getProxy()); } else { conn = connURL.openConnection(); } conn.setConnectTimeout(30000); // timeout after 30 seconds BufferedReader bi = new BufferedReader(new InputStreamReader(conn.getInputStream())); String n = bi.readLine(); try { numPackages = Integer.parseInt(n); } catch (NumberFormatException ne) { System.err.println("[WekaPackageManager] problem parsing number " + "of packages from server."); } bi.close(); } catch (Exception ex) { ex.printStackTrace(); } return numPackages; } /** * Just get a list of the package names. This is faster than calling * getAllPackages(), especially if fetching from the online repository, since * the full meta data for each package doesn't have to be read. * * @param local true if the local package list in the cache should be read * rather than the online repository * * @return a Map<String, String> of all the package names available either * locally or at the repository */ public static Map<String, String> getPackageList(boolean local) { Map<String, String> result = new HashMap<String, String>(); try { useCacheOrOnlineRepository(); if (!local) { PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); } String packageListS = PACKAGE_MANAGER.getPackageRepositoryURL().toString() + "/" + PACKAGE_LIST_WITH_VERSION_FILE; URLConnection conn = null; URL connURL = new URL(packageListS); if (PACKAGE_MANAGER.setProxyAuthentication(connURL)) { conn = connURL.openConnection(PACKAGE_MANAGER.getProxy()); } else { conn = connURL.openConnection(); } conn.setConnectTimeout(30000); // timeout after 30 seconds BufferedReader bi = new BufferedReader(new InputStreamReader(conn.getInputStream())); String l = null; while ((l = bi.readLine()) != null) { String[] parts = l.split(":"); if (parts.length == 2) { result.put(parts[0], parts[1]); } } bi.close(); } catch (Exception ex) { // ex.printStackTrace(); result = null; } return result; } /** * Establish the local copy of the package meta data if needed * * @param progress for reporting progress * @return any Exception raised or null if all is good */ public static Exception establishCacheIfNeeded(PrintStream... progress) { if (m_offline) { return null; } Exception problem = null; if (INITIAL_CACHE_BUILD_NEEDED) { for (PrintStream p : progress) { p.println("Caching repository metadata, please wait..."); } problem = refreshCache(progress); INITIAL_CACHE_BUILD_NEEDED = false; } return problem; } protected static boolean checkForForcedCacheRefresh() throws MalformedURLException { int refreshCountServer = getForcedRefreshCount(false); if (refreshCountServer > 0) { // now check local version of this file... int refreshCountLocal = getForcedRefreshCount(true); return refreshCountServer > refreshCountLocal; } return false; } protected static int getForcedRefreshCount(boolean local) throws MalformedURLException { useCacheOrOnlineRepository(); if (!local) { PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); } String refreshCountS = PACKAGE_MANAGER.getPackageRepositoryURL().toString() + "/" + FORCED_REFRESH_COUNT_FILE; int refreshCount = -1; URLConnection conn = null; URL connURL = new URL(refreshCountS); try { if (PACKAGE_MANAGER.setProxyAuthentication(connURL)) { conn = connURL.openConnection(PACKAGE_MANAGER.getProxy()); } else { conn = connURL.openConnection(); } conn.setConnectTimeout(30000); // timeout after 30 seconds BufferedReader bi = new BufferedReader(new InputStreamReader(conn.getInputStream())); String countS = bi.readLine(); if (countS != null && countS.length() > 0) { try { refreshCount = Integer.parseInt(countS); } catch (NumberFormatException ne) { // ignore } } } catch (IOException ex) { // ignore } return refreshCount; } /** * Check for new packages on the server and refresh the local cache if needed * * @param progress to report progress to * @return any Exception raised or null if all is good */ public static Exception checkForNewPackages(PrintStream... progress) { if (m_offline) { return null; } Exception problem = null; Map<String, String> localPackageNameList = getPackageList(true); if (localPackageNameList == null) { System.err.println("Local package list is missing, trying a " + "cache refresh to restore..."); refreshCache(progress); localPackageNameList = getPackageList(true); if (localPackageNameList == null) { // quietly return and see if we can continue anyway return null; } } Map<String, String> repositoryPackageNameList = getPackageList(false); if (repositoryPackageNameList == null) { // quietly return and see if we can continue anyway return null; } if (repositoryPackageNameList.keySet().size() != localPackageNameList.keySet().size()) { // package(s) have disappeared from the repository. // Force a cache refresh... if (repositoryPackageNameList.keySet().size() < localPackageNameList.keySet().size()) { for (PrintStream p : progress) { p.println("Some packages no longer exist at the repository. " + "Refreshing cache..."); } } else { for (PrintStream p : progress) { p.println("There are new packages at the repository. " + "Refreshing cache..."); } } problem = refreshCache(progress); } else { // check for new versions of packages boolean refresh = false; for (String localPackage : localPackageNameList.keySet()) { String localVersion = localPackageNameList.get(localPackage); String repoVersion = repositoryPackageNameList.get(localPackage); if (repoVersion == null) { continue; } // a difference here indicates a newer version on the server if (!localVersion.equals(repoVersion)) { refresh = true; break; } } if (refresh) { for (PrintStream p : progress) { p.println("There are newer versions of existing packages " + "at the repository. Refreshing cache..."); } problem = refreshCache(progress); } } return problem; } /** * Deletes the contents of $WEKA_HOME/repCache */ protected static void cleanRepCacheDir() { String cacheDir = WEKA_HOME.toString() + File.separator + "repCache"; File cacheDirF = new File(cacheDir); if (cacheDirF.exists()) { File[] contents = cacheDirF.listFiles(); if (contents != null) { for (File f : contents) { if (f.isDirectory()) { File[] packageMetaDirContents = f.listFiles(); if (packageMetaDirContents != null) { for (File pf : packageMetaDirContents) { pf.delete(); } } } f.delete(); } } } } /** * Refresh the local copy of the package meta data * * @param progress to report progress to * @return any Exception raised or null if all is successful */ public static Exception refreshCache(PrintStream... progress) { Exception problem = null; if (CACHE_URL == null) { return null; } PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); String cacheDir = WEKA_HOME.toString() + File.separator + "repCache"; try { for (PrintStream p : progress) { p.println("Refresh in progress. Please wait..."); } byte[] zip = PACKAGE_MANAGER.getRepositoryPackageMetaDataOnlyAsZip(progress); // only blow away the repCache if we successfully get a new zip! cleanRepCacheDir(); ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(zip)); ZipEntry ze; final byte[] buff = new byte[1024]; while ((ze = zis.getNextEntry()) != null) { // System.out.println("Cache: inflating " + ze.getName()); if (ze.isDirectory()) { new File(cacheDir, ze.getName()).mkdir(); continue; } BufferedOutputStream bo = new BufferedOutputStream( new FileOutputStream(new File(cacheDir, ze.getName()))); while (true) { int amountRead = zis.read(buff); if (amountRead == -1) { break; } // write the data here bo.write(buff, 0, amountRead); } bo.close(); } } catch (Exception e) { e.printStackTrace(); // OK, we have a problem with the repository cache - use // the repository itself instead and delete repCache CACHE_URL = null; try { DefaultPackageManager.deleteDir(new File(cacheDir), System.out); } catch (Exception e1) { e1.printStackTrace(); } return e; } return problem; } /** * Check if a named resource exists in an installed package * * @param packageName the name of the package in question * @param resourceName the name of the resource to check for * @return true if the resource exists in the package */ public static boolean installedPackageResourceExists(String packageName, String resourceName) { String fullResourcePath = getPackageHome().toString() + File.separator + packageName + File.separator + resourceName; return new File(fullResourcePath).exists(); } /** * Determines whether to use the local cache or online repository for meta * data */ private static void useCacheOrOnlineRepository() { if (REP_MIRROR == null && m_mirrorFailureCount < 3) { establishMirror(); if (REP_MIRROR == null) { m_mirrorFailureCount++; } } if (CACHE_URL != null) { PACKAGE_MANAGER.setPackageRepositoryURL(CACHE_URL); } else if (REP_URL != null) { PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL); } } public static File getPackageHome() { return PACKAGE_MANAGER.getPackageHome(); } /** * Find the most recent version of the package encapsulated in the supplied * PackageConstraint argument that satisfies the constraint * * @param toCheck the PackageConstraint containing the package in question * @return the most recent version of the package satisfying the constraint * @throws Exception if a version can't be found that satisfies the constraint * or an error occurs while communicating with the respository */ public static Package mostRecentVersionWithRespectToConstraint(PackageConstraint toCheck) throws Exception { Package target = toCheck.getPackage(); Package result = null; List<Object> availableVersions = PACKAGE_MANAGER.getRepositoryPackageVersions(target.getName()); // version numbers will be in descending sorted order from the repository // we want the most recent version that meets the target constraint for (Object version : availableVersions) { Package candidate = PACKAGE_MANAGER.getRepositoryPackageInfo(target.getName(), version); if (toCheck.checkConstraint(candidate) && candidate.isCompatibleBaseSystem()) { result = candidate; break; } } if (result == null) { throw new Exception("[WekaPackageManager] unable to find a version of " + "package " + target.getName() + " that meets constraint " + toCheck.toString()); } return result; } /** * Install the supplied list of packages * * @param toInstall packages to install * @param progress to report progress to * @return true if successful * @throws Exception if a problem occurs */ public static boolean installPackages(List<Package> toInstall, PrintStream... progress) throws Exception { useCacheOrOnlineRepository(); List<Boolean> upgrades = new ArrayList<Boolean>(); for (Package p : toInstall) { if (p.isInstalled()) { upgrades.add(new Boolean(true)); } else { upgrades.add(new Boolean(false)); } } PACKAGE_MANAGER.installPackages(toInstall, progress); boolean atLeastOneUpgrade = false; List<File> gpcFiles = new ArrayList<File>(); int i = 0; Map<String, List<String>> injectDependencies = new HashMap<>(); for (Package p : toInstall) { boolean isAnUpgrade = upgrades.get(i++); if (isAnUpgrade) { atLeastOneUpgrade = true; } String packageName = p.getName(); File packageDir = new File(PACKAGE_MANAGER.getPackageHome().toString() + File.separator + packageName); checkForInjectDependencies(p, injectDependencies); boolean loadIt = loadCheck(p, packageDir, progress); if (loadIt & !isAnUpgrade) { processPackageDirectory(packageDir, false, gpcFiles, false); } } // inject dependencies before triggering any class discovery injectPackageDependencies(injectDependencies); WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().performIntegrityCheck(); for (File f : gpcFiles) { processGenericPropertiesCreatorProps(f); } return atLeastOneUpgrade; } /** * Get the versions of the supplied package available on the server * * @param packageName the package name to get available versions for * @return a list of available versions * @throws Exception if a problem occurs */ public static List<Object> getRepositoryPackageVersions(String packageName) throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getRepositoryPackageVersions(packageName); } /** * Get the package repository URL * * @return the package repository URL */ public static URL getPackageRepositoryURL() { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getPackageRepositoryURL(); } /** * Get a list of all packages * * @return a list of all packages * @throws Exception if a problem occurs */ public static List<Package> getAllPackages() throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getAllPackages(); } /** * Get a list of all available packages (i.e. those not yet installed(. * * @return a list of all available packages * @throws Exception if a problem occurs */ public static List<Package> getAvailablePackages() throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getAvailablePackages(); } /** * Get a list of the most recent version of all available packages (i.e. those * not yet installed or there is a higher version in the repository) that are * compatible with the version of Weka that is installed. * * @return a list of packages that are compatible with the installed version * of Weka * @throws Exception if a problem occurs */ public static List<Package> getAvailableCompatiblePackages() throws Exception { // List<Package> allAvail = getAvailablePackages(); List<Package> allAvail = getAllPackages(); List<Package> compatible = new ArrayList<Package>(); for (Package p : allAvail) { List<Object> availableVersions = PACKAGE_MANAGER.getRepositoryPackageVersions(p.getName()); // version numbers will be in descending sorted order from the repository // we want the most recent version that is compatible with the base weka // version for (Object version : availableVersions) { Package versionedPackage = getRepositoryPackageInfo(p.getName(), version.toString()); if (versionedPackage.isCompatibleBaseSystem()) { if (p.isInstalled()) { // see if the latest compatible version is newer than the installed // version Package installed = getInstalledPackageInfo(p.getName()); String installedV = installed .getPackageMetaDataElement(VersionPackageConstraint.VERSION_KEY).toString(); String versionedV = versionedPackage .getPackageMetaDataElement(VersionPackageConstraint.VERSION_KEY).toString(); VersionPackageConstraint.VersionComparison v = VersionPackageConstraint.compare(versionedV, installedV); if (v == VersionPackageConstraint.VersionComparison.GREATERTHAN) { compatible.add(versionedPackage); } } else { compatible.add(versionedPackage); } break; } } } return compatible; } /** * Get the latest version of the named package that is compatible with the * base version of Weka being used. * * @param packageName the name of the package to get the latest compatible * version of * @return the latest compatible version or null if there is no compatible * package * @throws Exception if a problem occurs */ public static Package getLatestCompatibleVersion(String packageName) throws Exception { Package latest = null; List<Object> availableVersions = PACKAGE_MANAGER.getRepositoryPackageVersions(packageName); for (Object version : availableVersions) { Package versionedPackage = getRepositoryPackageInfo(packageName, version.toString()); if (versionedPackage.isCompatibleBaseSystem()) { latest = versionedPackage; break; } } return latest; } /** * Get a list of installed packages * * @return a list of installed packages * @throws Exception if a problem occurs */ public static List<Package> getInstalledPackages() throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getInstalledPackages(); } /** * Get a list of dependencies for a given package * * @param target the package to get the dependencies for * @param conflicts will hold any conflicts * @return a list of dependencies for the target package * @throws Exception if a problem occurs */ public static List<Dependency> getAllDependenciesForPackage(Package target, Map<String, List<Dependency>> conflicts) throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getAllDependenciesForPackage(target, conflicts); } /** * Extract meta data from a package archive * * @param packageArchivePath the path to the package archive * @return the meta data for the package * @throws Exception if a problem occurs */ public static Package getPackageArchiveInfo(String packageArchivePath) throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getPackageArchiveInfo(packageArchivePath); } /** * Get meta data for an installed package * * @param packageName the name of the package * @return the meta data for the package * @throws Exception if a problem occurs */ public static Package getInstalledPackageInfo(String packageName) throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getInstalledPackageInfo(packageName); } /** * Get meta data for the latest version of a package from the repository * * @param packageName the name of the package * @return the meta data for the package * @throws Exception if a problem occurs */ public static Package getRepositoryPackageInfo(String packageName) throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getRepositoryPackageInfo(packageName); } /** * Get meta data for a specific version of a package from the repository * * @param packageName the name of the package * @param version the version to get meta data for * @return the meta data for the package * @throws Exception if a problem occurs */ public static Package getRepositoryPackageInfo(String packageName, String version) throws Exception { useCacheOrOnlineRepository(); return PACKAGE_MANAGER.getRepositoryPackageInfo(packageName, version); } /** * Install a named package by retrieving the location of the archive from the * meta data stored in the repository * * @param packageName the name of the package to install * @param version the version of the package to install * @param progress for reporting progress * @return true if the package was installed successfully * @throws Exception if a problem occurs */ public static boolean installPackageFromRepository(String packageName, String version, PrintStream... progress) throws Exception { useCacheOrOnlineRepository(); Package toLoad = getRepositoryPackageInfo(packageName); // check to see if a version is already installed. If so, we // wont load the updated version into the classpath immediately in // order to avoid conflicts, class not found exceptions etc. The // user is told to restart Weka for the changes to come into affect // anyway, so there is no point in loading the updated package now. boolean isAnUpgrade = toLoad.isInstalled(); Object specialInstallMessage = toLoad.getPackageMetaDataElement(MESSAGE_TO_DISPLAY_ON_INSTALLATION_KEY); if (specialInstallMessage != null && specialInstallMessage.toString().length() > 0) { String siM = specialInstallMessage.toString(); try { siM = Environment.getSystemWide().substitute(siM); } catch (Exception ex) { // quietly ignore } String message = "**** Special installation message ****\n" + siM + "\n**** Special installation message ****"; for (PrintStream p : progress) { p.println(message); } } PACKAGE_MANAGER.installPackageFromRepository(packageName, version, progress); File packageDir = new File(PACKAGE_MANAGER.getPackageHome().toString() + File.separator + packageName); if (loadCheck(getInstalledPackageInfo(packageName), packageDir, progress)) { WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().addPackageToClassLoader(packageDir); } Map<String, List<String>> injectDependencies = new HashMap<>(); checkForInjectDependencies(toLoad, injectDependencies); // inject dependencies before triggering any class discovery injectPackageDependencies(injectDependencies); // check that all dependencies are available, there are no missing classes // and files etc. WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().performIntegrityCheck(); // If the classloader for the package is still in play then // process all props files if (WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .getPackageClassLoader(packageName) != null) { processPackageDirectory(packageDir, false, null, false); } return isAnUpgrade; } /** * Install a package from an archive (unofficial package install route) * * @param packageArchivePath the path to the package archive file to install * @param progress for reporting progress * @return true if the package was installed successfully * @throws Exception if a problem occurs */ public static String installPackageFromArchive(String packageArchivePath, PrintStream... progress) throws Exception { useCacheOrOnlineRepository(); Package toInstall = PACKAGE_MANAGER.getPackageArchiveInfo(packageArchivePath); Object specialInstallMessage = toInstall.getPackageMetaDataElement(MESSAGE_TO_DISPLAY_ON_INSTALLATION_KEY); if (specialInstallMessage != null && specialInstallMessage.toString().length() > 0) { String siM = specialInstallMessage.toString(); try { siM = Environment.getSystemWide().substitute(siM); } catch (Exception ex) { // quietly ignore } String message = "**** Special installation message ****\n" + siM + "\n**** Special installation message ****"; for (PrintStream p : progress) { p.println(message); } } PACKAGE_MANAGER.installPackageFromArchive(packageArchivePath, progress); return initializeAndLoadUnofficialPackage(toInstall, progress); } private static String initializeAndLoadUnofficialPackage(Package toInstall, PrintStream... progress) throws Exception { File packageDir = new File(PACKAGE_MANAGER.getPackageHome() + File.separator + toInstall.getName()); Package toLoad = getInstalledPackageInfo(toInstall.getName()); // no load check here as those checks involve the central repository (and // this is an unofficial package) WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().addPackageToClassLoader(packageDir); Map<String, List<String>> injectDependencies = new HashMap<>(); checkForInjectDependencies(toLoad, injectDependencies); // inject dependencies before triggering any class discovery injectPackageDependencies(injectDependencies); // check that all dependencies are available, there are no missing classes // and files etc. WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager().performIntegrityCheck(); // If the classloader for the package is still in play then // process all props files if (WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .getPackageClassLoader(toInstall.getName()) != null) { processPackageDirectory(packageDir, false, null, false); } return toInstall.getName(); } /** * Install a package from the supplied URL * * @param packageURL the URL to the package archive to install * @param progress for reporting progress * @return true if the package was installed successfully * @throws Exception if a problem occurs */ public static String installPackageFromURL(URL packageURL, PrintStream... progress) throws Exception { useCacheOrOnlineRepository(); String packageName = PACKAGE_MANAGER.installPackageFromURL(packageURL, progress); Package installed = PACKAGE_MANAGER.getInstalledPackageInfo(packageName); Object specialInstallMessage = installed.getPackageMetaDataElement(MESSAGE_TO_DISPLAY_ON_INSTALLATION_KEY); if (specialInstallMessage != null && specialInstallMessage.toString().length() > 0) { String message = "**** Special installation message ****\n" + specialInstallMessage.toString() + "\n**** Special installation message ****"; for (PrintStream p : progress) { p.println(message); } } return initializeAndLoadUnofficialPackage(installed, progress); } /** * Uninstall a named package * * @param packageName the name of the package to remove * @param updateKnowledgeFlow true if any Knowledge Flow beans provided by the * package should be deregistered from the KnoweledgeFlow * @param progress for reporting progress * @throws Exception if a problem occurs */ public static void uninstallPackage(String packageName, boolean updateKnowledgeFlow, PrintStream... progress) throws Exception { // check to see if this is a KnowledgeFlow package (presence of Beans.props // file) if (updateKnowledgeFlow) { File packageToDel = new File( PACKAGE_MANAGER.getPackageHome().toString() + File.separator + packageName); if (packageToDel.exists() && packageToDel.isDirectory()) { File[] contents = packageToDel.listFiles(); if (contents != null) { for (File content : contents) { if (content.isFile() && content.getPath().endsWith("Beans.props")) { // KnowledgeFlow plugin -- remove this properties file from the // list // of // bean plugin props KnowledgeFlowApp.removeFromPluginBeanProps(content); KnowledgeFlowApp.disposeSingleton(); break; } } } WekaPackageClassLoaderManager.getWekaPackageClassLoaderManager() .removeClassLoaderForPackage(packageName); } } PACKAGE_MANAGER.uninstallPackage(packageName, progress); } private static void printPackageInfo(Map<?, ?> packageProps) { Set<?> keys = packageProps.keySet(); Iterator<?> i = keys.iterator(); while (i.hasNext()) { Object key = i.next(); Object value = packageProps.get(key); System.out.println(Utils.padLeft(key.toString(), 11) + ":\t" + value.toString()); } } /** * Print meta data on a package * * @param packagePath the path to the package to print meta data for * @throws Exception if a problem occurs */ protected static void printPackageArchiveInfo(String packagePath) throws Exception { Map<?, ?> packageProps = getPackageArchiveInfo(packagePath).getPackageMetaData(); printPackageInfo(packageProps); } /** * Print meta data for an installed package * * @param packageName the name of the package to print meta data for * @throws Exception if a problem occurs */ protected static void printInstalledPackageInfo(String packageName) throws Exception { Map<?, ?> packageProps = getInstalledPackageInfo(packageName).getPackageMetaData(); printPackageInfo(packageProps); } /** * Print meta data for a package listed in the repository * * @param packageName the name of the package to print meta data for * @param version the version of the package * @throws Exception if a problem occurs */ protected static void printRepositoryPackageInfo(String packageName, String version) throws Exception { Map<?, ?> packageProps = getRepositoryPackageInfo(packageName, version).getPackageMetaData(); printPackageInfo(packageProps); } private static String queryUser() { java.io.BufferedReader br = new java.io.BufferedReader(new java.io.InputStreamReader(System.in)); String result = null; try { result = br.readLine(); } catch (java.io.IOException ex) { // ignore } return result; } private static void removeInstalledPackage(String packageName, boolean force, PrintStream... progress) throws Exception { List<Package> compromised = new ArrayList<Package>(); // Now check to see which other installed packages depend on this one List<Package> installedPackages = null; if (!force) { installedPackages = getInstalledPackages(); for (Package p : installedPackages) { List<Dependency> tempDeps = p.getDependencies(); for (Dependency d : tempDeps) { if (d.getTarget().getPackage().getName().equals(packageName)) { // add this installed package to the list compromised.add(p); break; } } } if (compromised.size() > 0) { System.out.println("The following installed packages depend on " + packageName + " :\n"); for (Package p : compromised) { System.out.println("\t" + p.getName()); } System.out.println("\nDo you wish to proceed [y/n]?"); String response = queryUser(); if (response != null && (response.equalsIgnoreCase("n") || response.equalsIgnoreCase("no"))) { return; // bail out here } } } if (force) { System.out.println("Forced uninstall."); } compromised = null; installedPackages = null; uninstallPackage(packageName, false, progress); } private static void installPackageFromRepository(String packageName, String version, boolean force) throws Exception { if (version.equals("Latest")) { // if no version/latest has been specified by the user then // look for the latest version of the package that is compatible // with the installed version of Weka. version = "none"; List<Object> availableVersions = PACKAGE_MANAGER.getRepositoryPackageVersions(packageName); // version numbers will be in descending sorted order from the // repository. We want the most recent version that is compatible // with the base weka install for (Object v : availableVersions) { Package versionedPackage = getRepositoryPackageInfo(packageName, v.toString()); if (versionedPackage.isCompatibleBaseSystem()) { version = versionedPackage.getPackageMetaDataElement(VersionPackageConstraint.VERSION_KEY) .toString(); break; } } if (version.equals("none")) { throw new Exception("Was unable to find a version of '" + packageName + "' that is compatible with Weka " + Version.VERSION); } } Package toInstall = null; try { toInstall = getRepositoryPackageInfo(packageName, version); } catch (Exception ex) { System.err.println("[WekaPackageManager] Package " + packageName + " at version " + version + " doesn't seem to exist!"); // System.exit(1); return; } // First check to see if this package is compatible with the base system if (!force) { // check to see if it's disabled Object disabled = toInstall.getPackageMetaDataElement(DISABLE_KEY); if (disabled == null) { disabled = toInstall.getPackageMetaDataElement(DISABLED_KEY); } if (disabled != null && disabled.toString().equalsIgnoreCase("true")) { System.err.println("Can't install package " + packageName + " because it " + "has been disabled at the repository."); return; } boolean ok = toInstall.isCompatibleBaseSystem(); if (!ok) { List<Dependency> baseSysDep = toInstall.getBaseSystemDependency(); StringBuffer depList = new StringBuffer(); for (Dependency bd : baseSysDep) { depList.append(bd.getTarget().toString() + " "); } System.err.println( "Can't install package " + packageName + " because it requires " + depList.toString()); return; } if (!osAndArchCheck(toInstall, System.out)) { return; // bail out here } if (!vmVersionCheck(toInstall, System.out)) { return; // bail out } if (toInstall.isInstalled()) { Package installedVersion = getInstalledPackageInfo(packageName); if (!toInstall.equals(installedVersion)) { System.out.println("Package " + packageName + "[" + installedVersion + "] is already installed. Replace with " + toInstall + " [y/n]?"); String response = queryUser(); if (response != null && (response.equalsIgnoreCase("n") || response.equalsIgnoreCase("no"))) { return; // bail out here } } else { System.out.println("Package " + packageName + "[" + installedVersion + "] is already installed. Install again [y/n]?"); String response = queryUser(); if (response != null && (response.equalsIgnoreCase("n") || response.equalsIgnoreCase("no"))) { return; // bail out here } } } // Now get a full list of dependencies for this package and // check for any conflicts Map<String, List<Dependency>> conflicts = new HashMap<String, List<Dependency>>(); List<Dependency> dependencies = getAllDependenciesForPackage(toInstall, conflicts); if (conflicts.size() > 0) { System.err.println("Package " + packageName + " requires the following packages:\n"); Iterator<Dependency> depI = dependencies.iterator(); while (depI.hasNext()) { Dependency d = depI.next(); System.err.println("\t" + d); } System.err.println("\nThere are conflicting dependencies:\n"); Set<String> pNames = conflicts.keySet(); Iterator<String> pNameI = pNames.iterator(); while (pNameI.hasNext()) { String pName = pNameI.next(); System.err.println("Conflicts for " + pName); List<Dependency> confsForPackage = conflicts.get(pName); Iterator<Dependency> confs = confsForPackage.iterator(); while (confs.hasNext()) { Dependency problem = confs.next(); System.err.println("\t" + problem); } } System.err.println("Unable to continue with installation."); return; // bail out here. } // Next check all dependencies against what is installed and // inform the user about which installed packages will be altered. Also // build the list of only those packages that need to be installed or // upgraded (excluding those that are already installed and are OK). List<PackageConstraint> needsUpgrade = new ArrayList<PackageConstraint>(); List<Package> finalListToInstall = new ArrayList<Package>(); Iterator<Dependency> depI = dependencies.iterator(); while (depI.hasNext()) { Dependency toCheck = depI.next(); if (toCheck.getTarget().getPackage().isInstalled()) { String toCheckName = toCheck.getTarget().getPackage().getPackageMetaDataElement("PackageName") .toString(); Package installedVersion = PACKAGE_MANAGER.getInstalledPackageInfo(toCheckName); if (!toCheck.getTarget().checkConstraint(installedVersion)) { needsUpgrade.add(toCheck.getTarget()); Package mostRecent = toCheck.getTarget().getPackage(); if (toCheck.getTarget() instanceof weka.core.packageManagement.VersionPackageConstraint) { mostRecent = WekaPackageManager .mostRecentVersionWithRespectToConstraint(toCheck.getTarget()); } finalListToInstall.add(mostRecent); } } else { Package mostRecent = toCheck.getTarget().getPackage(); if (toCheck.getTarget() instanceof weka.core.packageManagement.VersionPackageConstraint) { mostRecent = WekaPackageManager .mostRecentVersionWithRespectToConstraint(toCheck.getTarget()); } finalListToInstall.add(mostRecent); } } // now check for precludes - first add compile all installed packages and // then potentially overwrite with what's in the finalListToInstall if (toInstall.getPackageMetaDataElement(PRECLUDES_KEY) != null) { List<Package> installed = getInstalledPackages(); Map<String, Package> packageMap = new HashMap<>(); for (Package p : installed) { packageMap.put(p.getName(), p); } for (Package p : finalListToInstall) { packageMap.put(p.getName(), p); } List<Package> precluded = toInstall .getPrecludedPackages(new ArrayList<Package>(packageMap.values())); if (precluded.size() > 0) { List<Package> finalPrecluded = new ArrayList<>(); Set<String> doNotLoadList = getDoNotLoadList(); for (Package p : precluded) { if (!doNotLoadList.contains(p.getName())) { finalPrecluded.add(p); } } if (finalPrecluded.size() > 0) { System.out.println("\nPackage " + toInstall.getName() + " cannot be " + "installed because it precludes the following packages:\n"); for (Package p : finalPrecluded) { System.out.println("\n\t" + p.toString()); } System.out.println("Either uninstall or disable these packages before " + "continuing."); return; // bail out here } } } if (needsUpgrade.size() > 0) { System.out.println("The following packages will be upgraded in order to install " + packageName); Iterator<PackageConstraint> upI = needsUpgrade.iterator(); while (upI.hasNext()) { PackageConstraint tempC = upI.next(); System.out.println("\t" + tempC); } System.out.print("\nOK to continue [y/n]? > "); String response = queryUser(); if (response != null && (response.equalsIgnoreCase("n") || response.equalsIgnoreCase("no"))) { return; // bail out here } // now take a look at the other installed packages and see if // any would have a problem when these ones are upgraded boolean conflictsAfterUpgrade = false; List<Package> installed = getInstalledPackages(); List<Package> toUpgrade = new ArrayList<Package>(); upI = needsUpgrade.iterator(); while (upI.hasNext()) { toUpgrade.add(upI.next().getPackage()); } // add the actual package the user is wanting to install if it // is going to be an up/downgrade rather than a first install since // other installed packages may depend on the currently installed // version // and thus could be affected after the up/downgrade toUpgrade.add(toInstall); for (int i = 0; i < installed.size(); i++) { Package tempP = installed.get(i); String tempPName = tempP.getName(); boolean checkIt = true; for (int j = 0; j < needsUpgrade.size(); j++) { if (tempPName.equals(needsUpgrade.get(j).getPackage().getName())) { checkIt = false; break; } } if (checkIt) { List<Dependency> problem = tempP.getIncompatibleDependencies(toUpgrade); if (problem.size() > 0) { conflictsAfterUpgrade = true; System.err.println("Package " + tempP.getName() + " will have a compatibility" + "problem with the following packages after upgrading them:"); Iterator<Dependency> dI = problem.iterator(); while (dI.hasNext()) { System.err.println("\t" + dI.next().getTarget().getPackage()); } } } } if (conflictsAfterUpgrade) { System.err.println("Unable to continue with installation."); return; // bail out here } } if (finalListToInstall.size() > 0) { System.out.println("To install " + packageName + " the following packages will" + " be installed/upgraded:\n\n"); Iterator<Package> i = finalListToInstall.iterator(); while (i.hasNext()) { System.out.println("\t" + i.next()); } System.out.print("\nOK to proceed [y/n]? > "); String response = queryUser(); if (response != null && (response.equalsIgnoreCase("n") || response.equalsIgnoreCase("no"))) { return; // bail out here } } // OK, now we can download and install everything // First install the final list of dependencies installPackages(finalListToInstall, System.out); // Now install the package itself installPackageFromRepository(packageName, version, System.out); } else { // just install this package without checking/downloading dependencies // etc. installPackageFromRepository(packageName, version, System.out); } } private static void listPackages(String arg) throws Exception { if (m_offline && (arg.equalsIgnoreCase("all") || arg.equalsIgnoreCase("available"))) { System.out.println("Running offline - unable to display " + "available or all package information"); return; } List<Package> packageList = null; useCacheOrOnlineRepository(); if (arg.equalsIgnoreCase("all")) { packageList = PACKAGE_MANAGER.getAllPackages(); } else if (arg.equals("installed")) { packageList = PACKAGE_MANAGER.getInstalledPackages(); } else if (arg.equals("available")) { // packageList = PACKAGE_MANAGER.getAvailablePackages(); packageList = getAvailableCompatiblePackages(); } else { System.err.println("[WekaPackageManager] Unknown argument " + arg); printUsage(); // System.exit(1); return; } StringBuffer result = new StringBuffer(); result.append("Installed\tRepository\tLoaded\tPackage\n"); result.append("=========\t==========\t======\t=======\n"); boolean userOptedNoLoad = false; Iterator<Package> i = packageList.iterator(); while (i.hasNext()) { Package p = i.next(); String installedV = "----- "; String repositoryV = "----- "; String loaded = "No"; if (p.isInstalled()) { Package installedP = getInstalledPackageInfo(p.getName()); if (loadCheck(installedP, new File(WekaPackageManager.getPackageHome().toString() + File.separator + p.getName()))) { loaded = "Yes"; } else { if (m_doNotLoadList.contains(installedP.getName())) { loaded = "No*"; userOptedNoLoad = true; } } installedV = installedP.getPackageMetaDataElement(VERSION_KEY).toString() + " "; try { if (!m_offline) { Package repP = getRepositoryPackageInfo(p.getName()); repositoryV = repP.getPackageMetaDataElement(VERSION_KEY).toString() + " "; } } catch (Exception ex) { // not at the repository } } else { repositoryV = p.getPackageMetaDataElement(VERSION_KEY).toString() + " "; } String title = p.getPackageMetaDataElement("Title").toString(); result.append( installedV + "\t" + repositoryV + "\t" + loaded + "\t" + p.getName() + ": " + title + "\n"); } if (userOptedNoLoad) { result.append("* User flagged as no load\n"); } System.out.println(result.toString()); } public static Exception startupCheck(boolean force, PrintStream... progress) { Exception problem = null; BufferedReader br = null; PrintWriter pw = null; File newPackageLastTimeCheckFile = new File( WEKA_HOME.toString() + File.separator + "new_package_check.txt"); try { // first check against last time that new packages were checked for boolean doChecks = false; long currentTime = System.currentTimeMillis(); if (!newPackageLastTimeCheckFile.exists()) { doChecks = true; } else if (!force) { br = new BufferedReader(new FileReader(newPackageLastTimeCheckFile)); String t = br.readLine(); long lastTime = Long.parseLong(t); doChecks = (currentTime - lastTime > 720L * 60L * 1000L); } if (doChecks || force) { if (REP_MIRROR == null) { establishMirror(); } establishCacheIfNeeded(progress); boolean forcedCacheRefresh = false; try { if (forcedCacheRefresh = checkForForcedCacheRefresh()) { for (PrintStream p : progress) { p.println("Forced repository metadata refresh, please wait..."); } problem = refreshCache(progress); } } catch (MalformedURLException ex) { problem = ex; } if (!forcedCacheRefresh) { checkForNewPackages(progress); } pw = new PrintWriter(new FileWriter(newPackageLastTimeCheckFile)); pw.println(currentTime); pw.flush(); } } catch (IOException e) { e.printStackTrace(); } finally { if (br != null) { try { br.close(); } catch (IOException e) { e.printStackTrace(); } } if (pw != null) { pw.close(); } } return problem; } private static void printUsage() { System.out.println("Usage: weka.core.WekaPackageManager [-offline] [option]"); System.out.println("Options:\n" + "\t-list-packages <all | installed | available>\n" + "\t-package-info <repository | installed | archive> " + "<packageName | packageZip>\n\t-install-package <packageName | packageZip | URL> [version]\n" + "\t-uninstall-package packageName\n" + "\t-toggle-load-status packageName [packageName packageName ...]\n" + "\t-refresh-cache"); } /** * Main method for using the package manager from the command line * * @param args command line arguments */ public static void main(String[] args) { weka.core.logging.Logger.log(weka.core.logging.Logger.Level.INFO, "Logging started"); try { // scan for -offline for (int i = 0; i < args.length; i++) { if (args[i].equals("-offline")) { m_offline = true; String[] temp = new String[args.length - 1]; if (i > 0) { System.arraycopy(args, 0, temp, 0, i); } System.arraycopy(args, i + 1, temp, i, args.length - (i + 1)); args = temp; } } // establishCacheIfNeeded(System.out); // checkForNewPackages(System.out); startupCheck(true, System.out); if (args.length == 0 || args[0].equalsIgnoreCase("-h") || args[0].equalsIgnoreCase("-help")) { printUsage(); // System.exit(1); return; } if (args[0].equals("-package-info")) { if (args.length < 3) { printUsage(); return; // System.exit(1); } if (args[1].equals("archive")) { printPackageArchiveInfo(args[2]); } else if (args[1].equals("installed")) { printInstalledPackageInfo(args[2]); } else if (args[1].equals("repository")) { String version = "Latest"; if (args.length == 4) { version = args[3]; } try { printRepositoryPackageInfo(args[2], version); } catch (Exception ex) { // problem with getting info on package from repository? // Must not be an "official" repository package System.out.println("[WekaPackageManager] Nothing known about package " + args[2] + " at the repository!"); } } else { System.err.println("[WekaPackageManager] Unknown argument " + args[2]); printUsage(); return; // System.exit(1); } } else if (args[0].equals("-install-package")) { String targetLowerCase = args[1].toLowerCase(); if (targetLowerCase.startsWith("http://") || targetLowerCase.startsWith("https://")) { URL packageURL = new URL(args[1]); installPackageFromURL(packageURL, System.out); } else if (targetLowerCase.endsWith(".zip")) { installPackageFromArchive(args[1], System.out); } else { // assume a named package at the central repository String version = "Latest"; if (args.length == 3) { version = args[2]; } installPackageFromRepository(args[1], version, false); } System.exit(0); } else if (args[0].equals("-uninstall-package")) { if (args.length < 2) { printUsage(); return; // System.exit(1); } boolean force = false; if (args.length == 3) { if (args[2].equals("-force")) { force = true; } } removeInstalledPackage(args[1], force, System.out); // System.exit(0); return; } else if (args[0].equals("-list-packages")) { if (args.length < 2) { printUsage(); // System.exit(1); return; } listPackages(args[1]); } else if (args[0].equals("-toggle-load-status")) { if (args.length == 1) { printUsage(); return; } List<String> toToggle = new ArrayList<String>(); for (int i = 1; i < args.length; i++) { toToggle.add(args[i].trim()); } if (toToggle.size() >= 1) { toggleLoadStatus(toToggle); } } else if (args[0].equals("-refresh-cache")) { refreshCache(System.out); } else { System.err.println("Unknown option: " + args[0]); printUsage(); } // System.exit(0); } catch (Exception ex) { ex.printStackTrace(); } } }