Java tutorial
// yacy.java // ----------------------- // (C) by Michael Peter Christen; mc@yacy.net // first published on http://www.yacy.net // Frankfurt, Germany, 2004, 2005 // // $LastChangedDate$ // $LastChangedRevision$ // $LastChangedBy$ // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA package net.yacy; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import org.apache.http.Header; import org.apache.http.HttpStatus; import org.apache.http.entity.mime.content.ContentBody; import com.google.common.io.Files; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.sorting.Array; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.retrieval.Response; import net.yacy.data.TransactionManager; import net.yacy.data.Translator; import net.yacy.document.Document; import net.yacy.gui.YaCyApp; import net.yacy.gui.framework.Browser; import net.yacy.http.Jetty9HttpServerImpl; import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.OS; import net.yacy.peers.Seed; import net.yacy.peers.operation.yacyBuildProperties; import net.yacy.peers.operation.yacyRelease; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverSwitch; import net.yacy.utils.translation.TranslatorXliff; /** * This is the main class of YaCy. Several threads are started from here: * <ul> * <li>one single instance of the plasmaSwitchboard is generated, which itself * starts a thread with a plasmaHTMLCache object. This object simply counts * files sizes in the cache and terminates them. It also generates a * plasmaCrawlerLoader object, which may itself start some more httpc-calling * threads to load web pages. They terminate automatically when a page has * loaded. * <li>one serverCore - thread is started, which implements a multi-threaded * server. The process may start itself many more processes that handle * connections.lo * <li>finally, all idle-dependent processes are written in a queue in * plasmaSwitchboard which are worked off inside an idle-sensitive loop of the * main process. (here) * </ul> * * On termination, the following must be done: * <ul> * <li>stop feeding of the crawling process because it otherwise fills the * indexing queue. * <li>say goodbye to connected peers and disable new connections. Don't wait for * success. * <li>first terminate the serverCore thread. This prevents that new cache * objects are queued. * <li>wait that the plasmaHTMLCache terminates (it should be normal that this * process already has terminated). * <li>then wait for termination of all loader process of the * plasmaCrawlerLoader. * <li>work off the indexing and cache storage queue. These values are inside a * RAM cache and would be lost otherwise. * <li>write all settings. * <li>terminate. * </ul> */ public final class yacy { // static objects public static final String vString = yacyBuildProperties.getVersion(); public static final String vDATE = yacyBuildProperties.getBuildDate(); public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]"; public static final String hline = "-------------------------------------------------------------------------------"; public static final Semaphore shutdownSemaphore = new Semaphore(0); public static File htDocsPath = null; public static File shareDefaultPath = null; public static File shareDumpDefaultPath = null; /** * a reference to the {@link Switchboard} created by the * {@link yacy#startup(String, long, long)} method. */ private static Switchboard sb = null; /** * Starts up the whole application. Sets up all datastructures and starts * the main threads. * * @param homePath Root-path where all information is to be found. * @param startupFree free memory at startup time, to be used later for statistics */ private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) { String tmpdir = null; try { // start up System.out.println(copyright); System.out.println(hline); // ensure that there is a DATA directory, if not, create one and if that fails warn and die mkdirsIfNeseccary(dataHome); mkdirsIfNeseccary(appHome); File f = new File(dataHome, "DATA/"); mkdirsIfNeseccary(f); if (!(f.exists())) { System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate."); System.exit(-1); } // set jvm tmpdir to a subdir for easy cleanup (as extensive use file.deleteonexit waists memory during long runs, as todelete files names are collected and never cleaned up during runtime) // keep this as earlier as possible, as any other class can use the "java.io.tmpdir" property, even the log manager, when the log file pattern uses "%t" as an alias for the tmp directory try { tmpdir = java.nio.file.Files.createTempDirectory("yacy-tmp-").toString(); // creates sub dir in jvm's temp (see System.property "java.io.tempdir") System.setProperty("java.io.tmpdir", tmpdir); } catch (IOException ex) { } // setting up logging f = new File(dataHome, "DATA/LOG/"); mkdirsIfNeseccary(f); f = new File(dataHome, "DATA/LOG/yacy.logging"); final File f0 = new File(appHome, "defaults/yacy.logging"); if (!f.exists() || f0.lastModified() > f.lastModified()) try { Files.copy(f0, f); } catch (final IOException e) { System.out.println("could not copy yacy.logging"); } try { ConcurrentLog.configureLogging(dataHome, new File(dataHome, "DATA/LOG/yacy.logging")); } catch (final IOException e) { System.out.println("could not find logging properties in homePath=" + dataHome); ConcurrentLog.logException(e); } ConcurrentLog.config("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision()); ConcurrentLog.config("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version")); ConcurrentLog.config("STARTUP", "Operation system: " + System.getProperty("os.name", "unknown")); ConcurrentLog.config("STARTUP", "Application root-path: " + appHome); ConcurrentLog.config("STARTUP", "Data root-path: " + dataHome); ConcurrentLog.config("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis()); ConcurrentLog.config("STARTUP", "Maximum file system path length: " + OS.maxPathLength); f = new File(dataHome, "DATA/yacy.running"); final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator); if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!"); try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID f.deleteOnExit(); FileChannel channel = null; FileLock lock = null; try { channel = new RandomAccessFile(f, "rw").getChannel(); lock = channel.tryLock(); // lock yacy.running } catch (final Exception e) { } try { sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf); } catch (final RuntimeException e) { ConcurrentLog.severe("STARTUP", "YaCy cannot start: " + e.getMessage(), e); System.exit(-1); } //sbSync.V(); // signal that the sb reference was set // switch the memory strategy MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true)); // save information about available memory at startup time sb.setConfig("memoryFreeAfterStartup", startupMemFree); sb.setConfig("memoryTotalAfterStartup", startupMemTotal); // start gui if wanted if (gui) YaCyApp.start("localhost", sb.getLocalPort()); // hardcoded, forced, temporary value-migration sb.setConfig("htTemplatePath", "htroot/env/templates"); double oldVer; try { String tmpversion = sb.getConfig(Seed.VERSION, ""); if (tmpversion.isEmpty()) { // before 1.83009737 only the svnRevision nr was in config (like 9737) tmpversion = yacyBuildProperties.getVersion(); int oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0")); if (oldRev > 1) { oldVer = Double.parseDouble(tmpversion) + oldRev / 100000000.0; } else { oldVer = Double.parseDouble(yacyBuildProperties.getLongVersion()); // failsafe (assume current version = no migration) } } else { oldVer = Double.parseDouble(tmpversion); } } catch (final NumberFormatException e) { oldVer = 0.0d; } final double newRev = Double.parseDouble(yacyBuildProperties.getLongVersion()); sb.setConfig(Seed.VERSION, yacyBuildProperties.getLongVersion()); sb.setConfig("applicationRoot", appHome.toString()); sb.setConfig("dataRoot", dataHome.toString()); // create some directories final File htRootPath = new File(appHome, sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT)); mkdirIfNeseccary(htRootPath); htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT); mkdirIfNeseccary(htDocsPath); //final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs")); // copy the donate iframe (better to copy this once here instead of doing this in an actual iframe in the search result) importDonationIFrame(sb, htDocsPath); // create default notifier picture File notifierFile = new File(htDocsPath, "notifier.gif"); if (!notifierFile.exists()) try { Files.copy(new File(htRootPath, "env/grafics/empty.gif"), notifierFile); } catch (final IOException e) { } final File htdocsReadme = new File(htDocsPath, "readme.txt"); if (!(htdocsReadme.exists())) try { FileUtils.copy(("This is your root directory for individual Web Content\r\n" + "\r\n" + "Please place your html files into the www subdirectory.\r\n" + "The URL of that path is either\r\n" + "http://www.<your-peer-name>.yacy or\r\n" + "http://<your-ip>:<your-port>/www\r\n" + "\r\n" + "Other subdirectories may be created; they map to corresponding sub-domains.\r\n" + "This directory shares it's content with the applications htroot path, so you\r\n" + "may access your yacy search page with\r\n" + "http://<your-peer-name>.yacy/\r\n" + "\r\n").getBytes(), htdocsReadme); } catch (final IOException e) { System.out.println("Error creating htdocs readme: " + e.getMessage()); } shareDefaultPath = new File(htDocsPath, "share"); mkdirIfNeseccary(shareDefaultPath); shareDumpDefaultPath = new File(shareDefaultPath, "dump"); mkdirIfNeseccary(shareDumpDefaultPath); migration.migrate(sb, oldVer, newRev); // delete old release files final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30); yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays); // set user-agent HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent); // start main threads final int port = sb.getLocalPort(); try { // start http server YaCyHttpServer httpServer; httpServer = new Jetty9HttpServerImpl(port); httpServer.startupServer(); sb.setHttpServer(httpServer); // TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used) ConnectionInfo.setServerMaxcount(sb.getConfigInt("connectionsMax", ConnectionInfo.getMaxcount())); ConcurrentLog.info("STARTUP", httpServer.getVersion()); // open the browser window final boolean browserPopUpTrigger = sb .getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true"); if (browserPopUpTrigger) try { final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); //boolean properPW = (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT, "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0); //if (!properPW) browserPopUpPage = "ConfigBasic.html"; /* YaCy main startup process must not hang because browser opening is long or fails. * Let's open try opening the browser in a separate thread */ new Thread("Browser opening") { @Override public void run() { Browser.openBrowser(("http://localhost:" + port) + "/" + browserPopUpPage); } }.start(); // Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage); } catch (final Throwable e) { // cannot open browser. This may be normal in headless environments //Log.logException(e); } // enable browser popup, http server is ready now sb.tray.setReady(); //regenerate Locales from Translationlist, if needed final File locale_source = sb.getAppPath("locale.source", "locales"); final String lang = sb.getConfig("locale.language", ""); // on lang=browser all active translation should be checked (because any could be requested by client) List<String> langlist; if (lang.endsWith("browser")) langlist = Translator.activeTranslations(); // get all translated languages else { langlist = new ArrayList<String>(); langlist.add(lang); } for (String tmplang : langlist) { if (!tmplang.equals("") && !tmplang.equals("default") && !tmplang.equals("browser")) { //locale is used String currentRev = null; BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream( new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang + "/version")))); currentRev = br.readLine(); // may return null } catch (final IOException e) { //Error } finally { try { br.close(); } catch (IOException ioe) { ConcurrentLog.warn("STARTUP", "Could not close " + tmplang + " version file"); } } if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) { try { //is this another version?! final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT)); final File destDir = new File( sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang); if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, tmplang + ".lng"), "html,template,inc", "locale")) { //translate it //write the new Versionnumber final BufferedWriter bw = new BufferedWriter( new PrintWriter(new FileWriter(new File(destDir, "version")))); bw.write(sb.getConfig(Seed.VERSION, "Error getting Version")); bw.close(); } } catch (final IOException e) { } } } } // initialize number formatter with this locale if (!lang.equals("browser")) // "default" is handled by .setLocale() Formatter.setLocale(lang); // registering shutdown hook ConcurrentLog.config("STARTUP", "Registering Shutdown Hook"); final Runtime run = Runtime.getRuntime(); run.addShutdownHook(new shutdownHookThread(sb, shutdownSemaphore)); // save information about available memory after all initializations //try { sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free()); sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total()); System.gc(); sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free()); sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total()); //} catch (final ConcurrentModificationException e) {} // wait for server shutdown try { sb.waitForShutdown(); } catch (final Exception e) { ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(), e); } // shut down Array.terminate(); ConcurrentLog.config("SHUTDOWN", "caught termination signal"); httpServer.stop(); ConcurrentLog.config("SHUTDOWN", "server has terminated"); sb.close(); } catch (final Exception e) { ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(), e); //System.exit(1); } if (lock != null) lock.release(); if (channel != null) channel.close(); } catch (final Exception ee) { ConcurrentLog.severe("STARTUP", "FATAL ERROR: " + ee.getMessage(), ee); } finally { } if (tmpdir != null) FileUtils.deletedelete(new File(tmpdir)); // clean up temp dir (set at startup as subdir of system.propery "java.io.tmpdir") ConcurrentLog.config("SHUTDOWN", "goodbye. (this is the last line)"); ConcurrentLog.shutdown(); shutdownSemaphore.release(1000); try { System.exit(0); } catch (final Exception e) { } // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790) } /** * Concurrently import the donation iframe content to serve it directly from this peer. * @param switchBoard the SwitchBoard instance. Must not be null. * @param htDocsDirectory the custom htdocs directory. Must not be null. */ private static void importDonationIFrame(final Switchboard switchBoard, final File htDocsDirectory) { final File wwwEnvPath = new File(htDocsDirectory, "env"); mkdirIfNeseccary(wwwEnvPath); final String iframesource = switchBoard.getConfig("donation.iframesource", ""); final String iframetarget = switchBoard.getConfig("donation.iframetarget", ""); final File iframefile = new File(htDocsDirectory, iframetarget); if (!iframefile.exists()) new Thread("yacy.importDonationIFrame") { @Override public void run() { final ClientIdentification.Agent agent = ClientIdentification .getAgent(ClientIdentification.yacyInternetCrawlerAgentName); Response documentResponse; try { /* Load the donation html frame content */ documentResponse = switchBoard.loader == null ? null : switchBoard.loader.load( switchBoard.loader.request(new DigestURL(iframesource), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); if (documentResponse != null) { Document[] documents = documentResponse.parse(); if (documents != null && documents.length > 0 && documents[0] != null) { Document donateDocument = documents[0]; String donateDocContent = new String(documentResponse.getContent(), donateDocument.getCharset()); /* Load image resources contained in the page */ if (donateDocument.getImages() != null) { for (DigestURL imgURL : donateDocument.getImages().keySet()) { try { Response response = switchBoard.loader.load( switchBoard.loader.request(imgURL, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); if (response != null) { String imgFileName = imgURL.getFileName(); /* Store each image in the same directory as the iframe target file */ FileUtils.copy(response.getContent(), new File(iframefile.getParentFile(), imgFileName)); /* Transform the original image URL to a relative one */ donateDocContent = donateDocContent .replace(imgURL.getURL().toString(), imgFileName); } } catch (IOException e) { /* Failing to load one image should not stop the whole task */ ConcurrentLog.warn("STARTUP", "Donation frame retrieval : could not get an image resource.", e); } } } FileUtils.copy(donateDocContent.getBytes(donateDocument.getCharset()), iframefile); } } } catch (Exception e) { ConcurrentLog.warn("STARTUP", "Could not retrieve donation frame content.", e); } } }.start(); } /** * @param f */ private static void delete(final File f) { if (!f.delete()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!"); } /** * @see File#mkdir() * @param path */ private static void mkdirIfNeseccary(final File path) { if (!(path.exists())) if (!path.mkdir()) ConcurrentLog.warn("STARTUP", "could not create directory " + path.toString()); } /** * @see File#mkdirs() * @param path */ public static void mkdirsIfNeseccary(final File path) { if (!(path.exists())) if (!path.mkdirs()) ConcurrentLog.warn("STARTUP", "could not create directories " + path.toString()); } /** * Loads the configuration from the data-folder. * FIXME: Why is this called over and over again from every method, instead * of setting the configurationdata once for this class in main? * * @param mes Where are we called from, so that the errormessages can be * more descriptive. * @param homePath Root-path where all the information is to be found. * @return Properties read from the configurationfile. */ private static Properties configuration(final String mes, final File homePath) { ConcurrentLog.config(mes, "Application Root Path: " + homePath.toString()); // read data folder final File dataFolder = new File(homePath, "DATA"); if (!(dataFolder.exists())) { ConcurrentLog.severe(mes, "Application was never started or root path wrong."); System.exit(-1); } final Properties config = new Properties(); FileInputStream fis = null; try { fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf")); config.load(fis); } catch (final FileNotFoundException e) { ConcurrentLog.severe(mes, "could not find configuration file."); System.exit(-1); } catch (final IOException e) { ConcurrentLog.severe(mes, "could not read configuration file."); System.exit(-1); } finally { if (fis != null) { try { fis.close(); } catch (final IOException e) { ConcurrentLog.logException(e); } } } return config; } /** * Call the shutdown-page of YaCy to tell it to shut down. This method is * called if you start yacy with the argument -shutdown. * * @param homePath Root-path where all the information is to be found. */ public static void shutdown(final File homePath) { // start up System.out.println(copyright); System.out.println(hline); final LinkedHashMap<String, ContentBody> post = new LinkedHashMap<String, ContentBody>(); post.put("shutdown", UTF8.StringBody("")); submitPostURL(homePath, "Steering.html", "Terminate YaCy", post); } public static void update(final File homePath) { // start up System.out.println(copyright); System.out.println(hline); submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version"); } /** * Submits post data to the local peer URL, authenticating as administrator * @param homePath directory containing YaCy DATA folder * @param path url relative path part * @param processdescription description of the operation for logging purpose * @param post data to post */ private static void submitPostURL(final File homePath, final String path, final String processdescription, final Map<String, ContentBody> post) { final Properties config = configuration("COMMAND-STEERING", homePath); // read port final int port = Integer.parseInt(config.getProperty(SwitchboardConstants.SERVER_PORT, "8090")); // read password final String encodedPassword = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""); final String adminUser = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"); // send 'wget' to web interface final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); // con.setHeader(requestHeader.entrySet()); try { /* First get a valid transaction token using HTTP GET */ con.GETbytes("http://localhost:" + port + "/" + path, adminUser, encodedPassword, false); if (con.getStatusCode() != HttpStatus.SC_OK) { throw new IOException("Error response from YACY socket: " + con.getHttpResponse().getStatusLine()); } final Header transactionTokenHeader = con.getHttpResponse() .getFirstHeader(HeaderFramework.X_YACY_TRANSACTION_TOKEN); if (transactionTokenHeader == null) { throw new IOException("Could not retrieve a valid transaction token"); } /* Then POST the request */ post.put(TransactionManager.TRANSACTION_TOKEN_PARAM, UTF8.StringBody(transactionTokenHeader.getValue())); con.POSTbytes(new MultiProtocolURL("http://localhost:" + port + "/" + path), null, post, adminUser, encodedPassword, false, false); if (con.getStatusCode() >= HttpStatus.SC_OK && con.getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES) { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription); } else { ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); try { HTTPClient.closeConnectionManager(); } catch (final InterruptedException e1) { e1.printStackTrace(); } RemoteInstance.closeConnectionManager(); System.exit(-1); } } catch (final IOException e) { ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage()); try { HTTPClient.closeConnectionManager(); } catch (final InterruptedException e1) { e1.printStackTrace(); } RemoteInstance.closeConnectionManager(); System.exit(-1); } try { HTTPClient.closeConnectionManager(); } catch (final InterruptedException e) { e.printStackTrace(); } RemoteInstance.closeConnectionManager(); // finished ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription); } private static void submitURL(final File homePath, final String path, final String processdescription) { final Properties config = configuration("COMMAND-STEERING", homePath); // read port final int port = Integer.parseInt(config.getProperty(SwitchboardConstants.SERVER_PORT, "8090")); // read password String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5); if (encodedPassword == null) encodedPassword = ""; // not defined // send 'wget' to web interface final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); // con.setHeader(requestHeader.entrySet()); try { con.GETbytes("http://localhost:" + port + "/" + path, config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), encodedPassword, false); if (con.getStatusCode() > 199 && con.getStatusCode() < 300) { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription); } else { ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); System.exit(-1); } } catch (final IOException e) { ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage()); System.exit(-1); } try { HTTPClient.closeConnectionManager(); } catch (final InterruptedException e) { e.printStackTrace(); } RemoteInstance.closeConnectionManager(); // finished ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription); } /** * read saved config file and perform action which need to be done before main task starts * - like check on system alrady running etc. * * @param dataHome data directory */ static private void preReadSavedConfigandInit(File dataHome) { File lockFile = new File(dataHome, "DATA/yacy.running"); final String conf = "DATA/SETTINGS/yacy.conf"; // If YaCy is actually running, then we check if the server port is open. // If yes, then we consider that a restart is a user mistake and then we just respond // as the user expects and tell the browser to open the start page. // That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply // open the web interface. (They don't think of 'servers' they just want to get to the search page). // We need to parse the configuration file for that to get the host port File configFile = new File(dataHome, conf); if (configFile.exists()) { Properties p = new Properties(); FileInputStream fis = null; try { fis = new FileInputStream(configFile); p.load(fis); // test for yacy already running if (lockFile.exists()) { // another instance running? VM crash? User will have to care about this //standard log system not up yet - use simply stdout // prevents also creation of a log file while just opening browser System.out.println("WARNING: the file " + lockFile + " exists, this usually means that a YaCy instance is still running. If you want to restart YaCy, try first ./stopYACY.sh, then ./startYACY.sh. If ./stopYACY.sh fails, try ./killYACY.sh"); int port = Integer.parseInt(p.getProperty(SwitchboardConstants.SERVER_PORT, "8090")); if (TimeoutRequest.ping("127.0.0.1", port, 1000)) { Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html")); // Thats it; YaCy was running, the user is happy, we can stop now. System.out.println( "WARNING: YaCy instance was still running; just opening the browser and exit."); System.exit(0); } else { // YaCy is not running; thus delete the file an go on as nothing was wrong. System.err.println( "INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown!"); delete(lockFile); } } } catch (IOException ex) { System.err.println("ERROR: config file seems to be corrupt"); System.err.println("ERROR: if problem persists, delete file"); System.err.println(configFile.getAbsolutePath()); ConcurrentLog.logException(ex); ConcurrentLog.severe("Startup", "cannot read " + configFile.toString() + ", please delete the corrupted file if problem persits"); } finally { try { fis.close(); } catch (IOException e) { ConcurrentLog.warn("Startup", "Could not close file " + configFile); } } } } /** * Main-method which is started by java. Checks for special arguments or * starts up the application. * * @param args * Given arguments from the command line. */ public static void main(String args[]) { try { // check assertion status //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true); boolean assertionenabled = false; assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement" if (assertionenabled) System.out.println("Asserts are enabled"); // check memory amount System.gc(); final long startupMemFree = MemoryControl.free(); final long startupMemTotal = MemoryControl.total(); // maybe go into headless awt mode: we have three cases depending on OS and one exception: // windows : better do not go into headless mode // mac : go into headless mode because an application is shown in gui which may not be wanted // linux : go into headless mode because this does not need any head operation // exception : if the -gui option is used then do not go into headless mode since that uses a gui boolean headless = true; if (OS.isWindows) headless = false; if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false; System.setProperty("java.awt.headless", headless ? "true" : "false"); String s = ""; for (final String a : args) s += a + " "; yacyRelease.startParameter = s.trim(); File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/')); File dataRoot = applicationRoot; //System.out.println("args.length=" + args.length); //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]"); if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) { // normal start-up of yacy if (args.length > 1) { if (args[1].startsWith(File.separator)) { /* data root folder provided as an absolute path */ dataRoot = new File(args[1]); } else { /* data root folder provided as a path relative to the user home folder */ dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); } } preReadSavedConfigandInit(dataRoot); startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); } else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) { // start-up of yacy with gui if (args.length > 1) { if (args[1].startsWith(File.separator)) { /* data root folder provided as an absolute path */ dataRoot = new File(args[1]); } else { /* data root folder provided as a path relative to the user home folder */ dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]); } } preReadSavedConfigandInit(dataRoot); startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true); } else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) { // normal shutdown of yacy if (args.length == 2) applicationRoot = new File(args[1]); shutdown(applicationRoot); } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) { // aut-update yacy if (args.length == 2) applicationRoot = new File(args[1]); update(applicationRoot); } else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) { // show yacy version System.out.println(copyright); } else if ((args.length > 1) && (args[0].toLowerCase(Locale.ROOT).equals("-config"))) { // set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password) // on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043") File f = new File(dataRoot, "DATA/SETTINGS/"); if (!f.exists()) { mkdirsIfNeseccary(f); } else { if (new File(dataRoot, "DATA/yacy.running").exists()) { System.out.println("please restart YaCy"); } } // use serverSwitch to read config properties (including init values from yacy.init serverSwitch ss = new serverSwitch(dataRoot, applicationRoot, "defaults/yacy.init", "DATA/SETTINGS/yacy.conf"); for (int icnt = 1; icnt < args.length; icnt++) { String cfg = args[icnt]; int pos = cfg.indexOf('='); if (pos > 0) { String cmd = cfg.substring(0, pos); String val = cfg.substring(pos + 1); if (!val.isEmpty()) { if (cmd.equalsIgnoreCase(SwitchboardConstants.ADMIN_ACCOUNT)) { // special command to set adminusername and md5-pwd int cpos = val.indexOf(':'); //format adminAccount=adminname:adminpwd if (cpos >= 0) { String username = val.substring(0, cpos); String pwdtxt = val.substring(cpos + 1); if (!username.isEmpty()) { ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, username); System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME + " = " + username); } else { username = ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"); } ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "MD5:" + Digest.encodeMD5Hex(username + ":" + ss.getConfig(SwitchboardConstants.ADMIN_REALM, "YaCy") + ":" + pwdtxt)); System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_B64MD5 + " = " + ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } else { ss.setConfig(cmd, val); System.out.println("Set property " + cmd + " = " + val); } } } else { System.out.println( "skip parameter " + cfg + " (equal sign missing, put parameter in doublequotes)"); } System.out.println(); } } else { if (args.length == 1) { applicationRoot = new File(args[0]); } preReadSavedConfigandInit(dataRoot); startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false); } } finally { ConcurrentLog.shutdown(); } } } /** * This class is a helper class whose instance is started, when the java virtual * machine shuts down. Signals the plasmaSwitchboard to shut down. */ class shutdownHookThread extends Thread { private final Switchboard sb; private final Semaphore shutdownSemaphore; public shutdownHookThread(final Switchboard sb, final Semaphore shutdownSemaphore) { super("yacy.shutdownHookThread"); this.sb = sb; this.shutdownSemaphore = shutdownSemaphore; } @Override public void run() { try { if (!this.sb.isTerminated()) { ConcurrentLog.config("SHUTDOWN", "Shutdown via shutdown hook."); /* Print also to the standard output, as the LogManager is likely to have * been concurrently reset by its own shutdown hook thread */ System.out.println("SHUTDOWN Starting shutdown via shutdown hook."); // sending the yacy main thread a shutdown signal ConcurrentLog.fine("SHUTDOWN", "Signaling shutdown to the switchboard."); this.sb.terminate("shutdown hook"); // waiting for the yacy thread to finish execution ConcurrentLog.fine("SHUTDOWN", "Waiting for main thread to finish."); /* Main thread will release the shutdownSemaphore once completely terminated. * We do not wait indefinitely as the application is supposed here to quickly terminate */ this.shutdownSemaphore.tryAcquire(30, TimeUnit.SECONDS); } } catch (final Exception e) { ConcurrentLog.severe("SHUTDOWN", "Unexpected error. " + e.getClass().getName(), e); } } }