List of usage examples for java.lang Thread setName
public final synchronized void setName(String name)
From source file:org.soaplab.gowlab.GowlabJob.java
/************************************************************************** * Create (but not start it yet) and return a thread that will do * the data fetching. It is rarely overwritten - but still it is * possible to create here your own thread. Make sure that your * thread also notifies other threads at its end (as it is done * here).//ww w. j a v a 2s. c o m * * The subclasses usually just overwrite method 'fetch' (which is * called from this thread) in order to do their fetching job. **************************************************************************/ protected Thread getExecuteThread() throws SoaplabException { Thread executeThread = new Thread() { public void run() { try { try { fetch(); reporter.getState().set(JobState.COMPLETED); } catch (SoaplabException e) { error(e.getMessage()); reporter.getState().set(JobState.TERMINATED_BY_ERROR); } } catch (RuntimeException e) { reporter.getState().set(JobState.TERMINATED_BY_ERROR); SoaplabException.formatAndLog(e, log); } finally { try { reporter.setDetailedStatusResult(); reporter.setReportResult(); } catch (SoaplabException e) { log.error("Setting special results failed. " + e.getMessage()); } // release connection if (httpMethod != null) httpMethod.releaseConnection(); // inform other threads waiting for the termination that // it has been done synchronized (reporter) { reporter.notifyAll(); } } } }; executeThread.setName(executeThread.getName().replace("Thread", "GowlabFetcher")); return executeThread; }
From source file:com.cyberway.issue.crawler.framework.CrawlController.java
/** * Operator requested crawl begin/* ww w .j av a 2 s. c om*/ */ public void requestCrawlStart() { runProcessorInitialTasks(); sendCrawlStateChangeEvent(STARTED, CrawlJob.STATUS_PENDING); String jobState; state = RUNNING; jobState = CrawlJob.STATUS_RUNNING; sendCrawlStateChangeEvent(this.state, jobState); // A proper exit will change this value. this.sExit = CrawlJob.STATUS_FINISHED_ABNORMAL; Thread statLogger = new Thread(statistics); statLogger.setName("StatLogger"); statLogger.start(); frontier.start(); }
From source file:JNLPAppletLauncher.java
/** * This method is called by the static initializer to create / initialize * the temp root directory that will hold the temp directories for this * instance of the JVM. This is done as follows: * * 1. Synchronize on a global lock. Note that for this purpose we will * use System.out in the absence of a true global lock facility. * We are careful not to hold this lock too long. * * 2. Check for the existence of the "jnlp.applet.launcher.tmproot" * system property./*from w ww . ja v a 2s .c o m*/ * * a. If set, then some other thread in a different ClassLoader has * already created the tmprootdir, so we just need to * use it. The remaining steps are skipped. * * b. If not set, then we are the first thread in this JVM to run, * and we need to create the the tmprootdir. * * 3. Create the tmprootdir, along with the appropriate locks. * Note that we perform the operations in the following order, * prior to creating tmprootdir itself, to work around the fact that * the file creation and file lock steps are not atomic, and we need * to ensure that a newly-created tmprootdir isn't reaped by a * concurrently running JVM. * * create jlnNNNN.tmp using File.createTempFile() * lock jlnNNNN.tmp * create jlnNNNN.lck while holding the lock on the .tmp file * lock jlnNNNN.lck * * Since the Reaper thread will enumerate the list of *.lck files * before starting, we can guarantee that if there exists a *.lck file * for an active process, then the corresponding *.tmp file is locked * by that active process. This guarantee lets us avoid reaping an * active process' files. * * 4. Set the "jnlp.applet.launcher.tmproot" system property. * * 5. Add a shutdown hook to cleanup jlnNNNN.lck and jlnNNNN.tmp. We * don't actually expect that this shutdown hook will ever be called, * but the act of doing this, ensures that the locks never get * garbage-collected, which is necessary for correct behavior when * the first ClassLoader is later unloaded, while subsequent Applets * are still running. * * 6. Start the Reaper thread to cleanup old installations. */ private static void initTmpRoot() throws IOException { if (VERBOSE) { System.err.println("---------------------------------------------------"); } synchronized (System.out) { // Get the name of the tmpbase directory. String tmpBaseName = System.getProperty("java.io.tmpdir") + File.separator + "jnlp-applet"; tmpBaseDir = new File(tmpBaseName); // Get the value of the tmproot system property final String tmpRootPropName = "jnlp.applet.launcher.tmproot"; tmpRootPropValue = System.getProperty(tmpRootPropName); if (tmpRootPropValue == null) { // Create the tmpbase directory if it doesn't already exist tmpBaseDir.mkdir(); if (!tmpBaseDir.isDirectory()) { throw new IOException("Cannot create directory " + tmpBaseDir); } // Create ${tmpbase}/jlnNNNN.tmp then lock the file File tmpFile = File.createTempFile("jln", ".tmp", tmpBaseDir); if (VERBOSE) { System.err.println("tmpFile = " + tmpFile.getAbsolutePath()); } final FileOutputStream tmpOut = new FileOutputStream(tmpFile); final FileChannel tmpChannel = tmpOut.getChannel(); final FileLock tmpLock = tmpChannel.lock(); // Strip off the ".tmp" to get the name of the tmprootdir String tmpFileName = tmpFile.getAbsolutePath(); String tmpRootName = tmpFileName.substring(0, tmpFileName.lastIndexOf(".tmp")); // create ${tmpbase}/jlnNNNN.lck then lock the file String lckFileName = tmpRootName + ".lck"; File lckFile = new File(lckFileName); if (VERBOSE) { System.err.println("lckFile = " + lckFile.getAbsolutePath()); } lckFile.createNewFile(); final FileOutputStream lckOut = new FileOutputStream(lckFile); final FileChannel lckChannel = lckOut.getChannel(); final FileLock lckLock = lckChannel.lock(); // Create tmprootdir tmpRootDir = new File(tmpRootName); if (DEBUG) { System.err.println("tmpRootDir = " + tmpRootDir.getAbsolutePath()); } if (!tmpRootDir.mkdir()) { throw new IOException("Cannot create " + tmpRootDir); } // Add shutdown hook to cleanup the OutputStream, FileChannel, // and FileLock for the jlnNNNN.lck and jlnNNNN.lck files. // We do this so that the locks never get garbage-collected. Runtime.getRuntime().addShutdownHook(new Thread() { /* @Override */ public void run() { // NOTE: we don't really expect that this code will ever // be called. If it does, we will close the output // stream, which will in turn close the channel. // We will then release the lock. try { tmpOut.close(); tmpLock.release(); lckOut.close(); lckLock.release(); } catch (IOException ex) { // Do nothing } } }); // Set the system property... tmpRootPropValue = tmpRootName.substring(tmpRootName.lastIndexOf(File.separator) + 1); System.setProperty(tmpRootPropName, tmpRootPropValue); if (VERBOSE) { System.err.println("Setting " + tmpRootPropName + "=" + tmpRootPropValue); } // Start a new Reaper thread to do stuff... Thread reaperThread = new Thread() { /* @Override */ public void run() { deleteOldTempDirs(); } }; reaperThread.setName("AppletLauncher-Reaper"); reaperThread.start(); } else { // Make sure that the property is not set to an illegal value if (tmpRootPropValue.indexOf('/') >= 0 || tmpRootPropValue.indexOf(File.separatorChar) >= 0) { throw new IOException("Illegal value of: " + tmpRootPropName); } // Set tmpRootDir = ${tmpbase}/${jnlp.applet.launcher.tmproot} if (VERBOSE) { System.err.println("Using existing value of: " + tmpRootPropName + "=" + tmpRootPropValue); } tmpRootDir = new File(tmpBaseDir, tmpRootPropValue); if (DEBUG) { System.err.println("tmpRootDir = " + tmpRootDir.getAbsolutePath()); } if (!tmpRootDir.isDirectory()) { throw new IOException("Cannot access " + tmpRootDir); } } } }
From source file:ffx.ui.ModelingPanel.java
/** * Launch the active command on the active system in the specified * directory.//from w w w . j a v a 2 s . c o m * * @param command The command to be excuted. * @param dir The directory to execute the command in. * @return a {@link ffx.ui.FFXExec} object. */ public FFXExec launch(String command, String dir) { logger.log(Level.INFO, "Command: {0}\nDirectory: {1}", new Object[] { command, dir }); synchronized (this) { // Check that the TINKER *.exe exists in TINKER/bin String path = MainPanel.ffxDir.getAbsolutePath(); File exe = new File(path + File.separator + activeCommand.toLowerCase()); if (!exe.exists()) { exe = new File(exe.getAbsolutePath() + ".exe"); if (!exe.exists()) { String message = "The " + activeCommand + " executable was not found in " + path + ". Please use the 'Set TINKER...' dialog to change the TINKER directory."; JOptionPane.showMessageDialog(null, message, "Could not launch " + activeCommand, JOptionPane.ERROR_MESSAGE); return null; } } // Check that the directory to execute the command in is valid File dirf = new File(dir); if (!dirf.exists()) { logger.log(Level.WARNING, "Directory doesn''t exist: {0}", dirf.getAbsolutePath()); return null; } // Check if we need a key file if (!commandFileTypes.contains(FileType.ANY)) { if (activeSystem == null) { return null; } activeFileType = FileType.XYZ; // Check that the TINKER command executes on this file type if (!commandFileTypes.contains(activeFileType)) { String message = activeCommand.toUpperCase() + " does not execute on " + activeFileType + " files."; JOptionPane.showMessageDialog(null, message, "Could not launch " + activeCommand, JOptionPane.ERROR_MESSAGE); return null; } // Check that a key file exists or prompt to create one. if (activeSystem.getKeyFile() == null) { mainPanel.createKeyFile(activeSystem); // Give up if the key file is null. if (activeSystem.getKeyFile() == null) { return null; } } } else { // Determine names to use for the output of Protein/Nucleic command = createCommandInput(); String structureName = commandTextArea.getText().trim(); if (!structureName.equalsIgnoreCase("")) { structureName = (structureName.split("\n"))[0]; if (structureName != null) { structureName = structureName.trim(); int dot = structureName.lastIndexOf("."); if (dot > 0) { structureName = structureName.substring(0, dot); } } } // If the above fails, just use the name of the executable // (protein or nulceic) if (structureName == null) { structureName = activeCommand.toLowerCase(); } File file = new File(dir + File.separator + structureName + ".xyz"); file = SystemFilter.version(file); activeSystem = new FFXSystem(file, null, Keyword.loadProperties(file)); File logFile = new File(file.getParent() + File.separator + structureName + ".log"); activeSystem.setLogFile(logFile); loadLogSettings(); activeFileType = FileType.ANY; // Need to have a parameter file chosen. mainPanel.openKey(activeSystem, true); if (activeSystem.getKeyFile() == null) { return null; } } // Decide on a Log file if (((String) logSettings.getSelectedItem()).startsWith("Create")) { File newLog = SystemFilter.version(activeSystem.getLogFile()); activeSystem.setLogFile(newLog); } String logName = activeSystem.getLogFile().getAbsolutePath(); // Determine the command string command = createCommandInput(); // If a new structure file will be created, determine what the name // will be. File newFile = null; if (commandActions.toUpperCase().contains("LOAD")) { File oldFile = activeSystem.getFile(); if (commandActions.toUpperCase().contains("LOADXYZ")) { String fileName = oldFile.getAbsolutePath(); int dot = fileName.lastIndexOf("."); if (dot > 0) { fileName = fileName.substring(0, dot) + ".xyz"; } oldFile = new File(fileName); } else if (commandActions.toUpperCase().contains("LOADINT")) { String fileName = oldFile.getAbsolutePath(); int dot = fileName.lastIndexOf("."); if (dot > 0) { fileName = fileName.substring(0, dot) + ".int"; } oldFile = new File(fileName); } else if (commandActions.toUpperCase().contains("LOADPDB")) { String fileName = oldFile.getAbsolutePath(); int dot = fileName.lastIndexOf("."); if (dot > 0) { fileName = fileName.substring(0, dot) + ".pdb"; } oldFile = new File(fileName); } newFile = SystemFilter.version(oldFile); } // Save any changes that have been made to the key file mainPanel.getKeywordPanel().saveChanges(); // Remove any TINKER *.END files removeEnd(); // Create the input file String commandInput = commandTextArea.getText(); if (commandInput != null && !commandInput.trim().equalsIgnoreCase("")) { File inputFile = new File(dir + File.separator + activeCommand.toLowerCase() + ".in"); inputFile.deleteOnExit(); try { FileWriter fw = new FileWriter(inputFile); fw.write(commandInput); fw.close(); } catch (Exception e) { logger.info(e.toString()); return null; } } // If the job progressively modifies coordinates, open a copy of it. boolean openOnto = false; if (commandActions.toUpperCase().contains("CONNECT")) { // If a version file is created, open it onto the structure used // to // display the job. if (newFile != null) { openOnto = true; } mainPanel.open(activeSystem.getFile(), activeCommand); try { while (mainPanel.isOpening()) { wait(10); } } catch (Exception e) { logger.info(e.toString()); return null; } activeSystem = mainPanel.getHierarchy().getActive(); } // Finally, create and execute the command in a new thread. FFXExec tinkerExec = new FFXExec(activeSystem, logName, command, dir, mainPanel, newFile, openOnto); Thread tinkerThread = new Thread(tinkerExec); tinkerThread.setPriority(Thread.NORM_PRIORITY); tinkerThread.setName(logName); // If the job progressively modifies coordinates, connect to it. if (commandActions.toUpperCase().contains("CONNECT")) { mainPanel.connectToTINKER(activeSystem, tinkerThread); } else { tinkerThread.start(); } // If some action should be taken when the job finishes, // add it to the Modeling Jobs Vector if (!commandActions.equalsIgnoreCase("NONE")) { executingCommands.add(tinkerThread); // mainPanel.getLogPanel().refreshStatus(); } return tinkerExec; } }
From source file:org.dawnsci.commandserver.core.producer.AliveConsumer.java
protected void startNotifications() throws Exception { if (uri == null) throw new NullPointerException("Please set the URI before starting notifications!"); this.cbean = new ConsumerBean(); cbean.setStatus(ConsumerStatus.STARTING); cbean.setName(getName());/*from www .j a v a2 s .com*/ cbean.setConsumerId(consumerId); cbean.setVersion(consumerVersion); cbean.setStartTime(System.currentTimeMillis()); try { cbean.setHostName(InetAddress.getLocalHost().getHostName()); } catch (UnknownHostException e) { // Not fatal but would be nice... e.printStackTrace(); } System.out.println("Running events on topic " + Constants.ALIVE_TOPIC + " to notify of '" + getName() + "' service being available."); final Thread aliveThread = new Thread(new Runnable() { public void run() { try { ConnectionFactory connectionFactory = ConnectionFactoryFacade.createConnectionFactory(uri); aliveConnection = connectionFactory.createConnection(); aliveConnection.start(); final Session session = aliveConnection.createSession(false, Session.AUTO_ACKNOWLEDGE); final Topic topic = session.createTopic(Constants.ALIVE_TOPIC); final MessageProducer producer = session.createProducer(topic); final ObjectMapper mapper = new ObjectMapper(); // Here we are sending the message out to the topic while (isActive()) { try { Thread.sleep(Constants.NOTIFICATION_FREQUENCY); TextMessage temp = session.createTextMessage(mapper.writeValueAsString(cbean)); producer.send(temp, DeliveryMode.NON_PERSISTENT, 1, 5000); if (ConsumerStatus.STARTING.equals(cbean.getStatus())) { cbean.setStatus(ConsumerStatus.RUNNING); } } catch (InterruptedException ne) { break; } catch (Exception neOther) { neOther.printStackTrace(); } } } catch (Exception ne) { ne.printStackTrace(); setActive(false); } } }); aliveThread.setName("Alive Notification Topic"); aliveThread.setDaemon(true); aliveThread.setPriority(Thread.MIN_PRIORITY); aliveThread.start(); }
From source file:net.sf.profiler4j.console.util.task.LongTaskExecutorDialog.java
public void runTask(final LongTask task) { label.setText("Executing long-running task..."); task.setDialog(this); Thread t = new Thread("PROFILER4J_TASK") { public void run() { log.debug("TASK STARTED"); try { task.executeInBackground(); } catch (final Exception e) { SwingUtilities.invokeLater(new Runnable() { public void run() { log.error("Caught task error", e); error = e;/* w w w .j ava 2s .co m*/ }; }); } finally { SwingUtilities.invokeLater(new Runnable() { public void run() { log.debug("TASK COMPLETED"); if (error != null) { setDefaultCloseOperation(DISPOSE_ON_CLOSE); progressBar.setIndeterminate(false); setSize(486, 379); setVisible(true); toFront(); ((JComponent) statusTextArea.getParent()).revalidate(); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); error.printStackTrace(pw); statusTextArea.setText(sw.toString()); statusTextArea.setCaretPosition(0); task.setError(error); } else { setVisible(false); dispose(); } } }); } }; }; t.setName("LongTaskRunner"); t.setPriority(Thread.MIN_PRIORITY); t.setDaemon(false); t.start(); setLocation(getLocation().x, getLocation().y - 120); setVisible(true); }
From source file:com.blackberry.logdriver.admin.LogMaintenance.java
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); // If run by Oozie, then load the Oozie conf too if (System.getProperty("oozie.action.conf.xml") != null) { conf.addResource(new URL("file://" + System.getProperty("oozie.action.conf.xml"))); }/* ww w . ja v a2 s . c om*/ // For some reason, Oozie needs some options to be set in system instead of // in the confiuration. So copy the configs over. { Iterator<Entry<String, String>> i = conf.iterator(); while (i.hasNext()) { Entry<String, String> next = i.next(); System.setProperty(next.getKey(), next.getValue()); } } if (args.length < 3) { printUsage(); return 1; } String userName = args[0]; String dcNumber = args[1]; String service = args[2]; String date = null; String hour = null; if (args.length >= 4) { date = args[3]; } if (args.length >= 5) { hour = args[4]; } // Set from environment variables String mergeJobPropertiesFile = getConfOrEnv(conf, "MERGEJOB_CONF"); String filterJobPropertiesFile = getConfOrEnv(conf, "FILTERJOB_CONF"); String daysBeforeArchive = getConfOrEnv(conf, "DAYS_BEFORE_ARCHIVE"); String daysBeforeDelete = getConfOrEnv(conf, "DAYS_BEFORE_DELETE"); String maxConcurrentMR = getConfOrEnv(conf, "MAX_CONCURRENT_MR", "-1"); String zkConnectString = getConfOrEnv(conf, "ZK_CONNECT_STRING"); String logdir = getConfOrEnv(conf, "logdriver.logdir.name"); boolean resetOrphanedJobs = Boolean.parseBoolean(getConfOrEnv(conf, "reset.orphaned.jobs", "true")); String rootDir = getConfOrEnv(conf, "service.root.dir"); String maxTotalMR = getConfOrEnv(conf, "MAX_TOTAL_MR", "-1"); boolean doMerge = true; boolean doArchive = true; boolean doDelete = true; if (zkConnectString == null) { LOG.error("ZK_CONNECT_STRING is not set. Exiting."); return 1; } if (mergeJobPropertiesFile == null) { LOG.info("MERGEJOB_CONF is not set. Not merging."); doMerge = false; } if (filterJobPropertiesFile == null) { LOG.info("FILTERJOB_CONF is not set. Not archiving."); doArchive = false; } if (daysBeforeArchive == null) { LOG.info("DAYS_BEFORE_ARCHIVE is not set. Not archiving."); doArchive = false; } if (doArchive && Integer.parseInt(daysBeforeArchive) < 0) { LOG.info("DAYS_BEFORE_ARCHIVE is negative. Not archiving."); doArchive = false; } if (daysBeforeDelete == null) { LOG.info("DAYS_BEFORE_DELETE is not set. Not deleting."); doDelete = false; } if (doDelete && Integer.parseInt(daysBeforeDelete) < 0) { LOG.info("DAYS_BEFORE_DELETE is negative. Not deleting."); doDelete = false; } if (logdir == null) { LOG.info("LOGDRIVER_LOGDIR_NAME is not set. Using default value of 'logs'."); logdir = "logs"; } if (rootDir == null) { LOG.info("SERVICE_ROOT_DIR is not set. Using default value of 'service'."); rootDir = "/service"; } // We can hang if this fails. So make sure we abort if it fails. fs = null; try { fs = FileSystem.get(conf); fs.exists(new Path("/")); // Test if it works. } catch (IOException e) { LOG.error("Error getting filesystem.", e); return 1; } // Create the LockUtil instance lockUtil = new LockUtil(zkConnectString); // Now it's safe to create our Job Runner JobRunner jobRunner = new JobRunner(Integer.parseInt(maxConcurrentMR), Integer.parseInt(maxTotalMR)); Thread jobRunnerThread = new Thread(jobRunner); jobRunnerThread.setName("JobRunner"); jobRunnerThread.setDaemon(false); jobRunnerThread.start(); // Figure out what date we start filters on. String filterCutoffDate = ""; if (doArchive) { Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, Integer.parseInt("-" + daysBeforeArchive)); filterCutoffDate = String.format("%04d%02d%02d%02d", cal.get(Calendar.YEAR), (cal.get(Calendar.MONTH) + 1), cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY)); LOG.info("Archiving logs from before {}", filterCutoffDate); } String deleteCutoffDate = ""; if (doDelete) { Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, Integer.parseInt("-" + daysBeforeDelete)); deleteCutoffDate = String.format("%04d%02d%02d%02d", cal.get(Calendar.YEAR), (cal.get(Calendar.MONTH) + 1), cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY)); LOG.info("Deleting logs from before {}", deleteCutoffDate); } long now = System.currentTimeMillis(); // Various exceptions have been popping up here. So make sure I catch them // all. try { // Patterns to recognize hour, day and incoming directories, so that they // can be processed. Pattern datePathPattern; Pattern hourPathPattern; Pattern incomingPathPattern; Pattern dataPathPattern; Pattern archivePathPattern; Pattern workingPathPattern; if (hour != null) { datePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")"); hourPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")"); incomingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/incoming"); dataPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/data"); archivePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/archive"); workingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/working/([^/]+)_(\\d+)"); } else if (date != null) { datePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")"); hourPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})"); incomingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/incoming"); dataPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/data"); archivePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/archive"); workingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/working/([^/]+)_(\\d+)"); } else { datePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})"); hourPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})"); incomingPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/incoming"); dataPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/data"); archivePathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/archive"); workingPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/working/([^/]+)_(\\d+)"); } // Do a depth first search of the directory, processing anything that // looks // interesting along the way Deque<Path> paths = new ArrayDeque<Path>(); Path rootPath = new Path(rootDir + "/" + dcNumber + "/" + service + "/" + logdir + "/"); paths.push(rootPath); while (paths.size() > 0) { Path p = paths.pop(); LOG.debug("{}", p.toString()); if (!fs.exists(p)) { continue; } FileStatus dirStatus = fs.getFileStatus(p); FileStatus[] children = fs.listStatus(p); boolean addChildren = true; boolean old = dirStatus.getModificationTime() < now - WAIT_TIME; LOG.debug(" Was last modified {}ms ago", now - dirStatus.getModificationTime()); if (!old) { LOG.debug(" Skipping, since it's not old enough."); } else if ((!rootPath.equals(p)) && (children.length == 0 || (children.length == 1 && children[0].getPath().getName().equals(READY_MARKER)))) { // old and no children? Delete! LOG.info(" Deleting empty directory {}", p.toString()); fs.delete(p, true); } else { Matcher matcher = datePathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.debug("Checking date directory"); // If this is already done, then skip it. So only process if it // doesn't exist. if (fs.exists(new Path(p, READY_MARKER)) == false) { // Check each subdirectory. If they all have ready markers, then I // guess we're ready. boolean ready = true; for (FileStatus c : children) { if (c.isDirectory() && fs.exists(new Path(c.getPath(), READY_MARKER)) == false) { ready = false; break; } } if (ready) { fs.createNewFile(new Path(p, READY_MARKER)); } } } matcher = hourPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.debug("Checking hour directory"); // If this is already done, then skip it. So only process if it // doesn't exist. if (fs.exists(new Path(p, READY_MARKER)) == false) { // Check each subdirectory. If they all have ready markers, then I // guess we're ready. boolean ready = true; for (FileStatus c : children) { if (c.isDirectory() && fs.exists(new Path(c.getPath(), READY_MARKER)) == false) { ready = false; break; } } if (ready) { fs.createNewFile(new Path(p, READY_MARKER)); } } } // Check to see if we have to run a merge matcher = incomingPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.debug("Checking incoming directory"); String matchDate = matcher.group(1); String matchHour = matcher.group(2); String matchComponent = matcher.group(3); String timestamp = matchDate + matchHour; if (doDelete && timestamp.compareTo(deleteCutoffDate) < 0) { LOG.info("Deleting old directory: {}", p); fs.delete(p, true); addChildren = false; } else if (doMerge) { // old, looks right, and has children? Run it! boolean hasMatchingChildren = false; boolean subdirTooYoung = false; for (FileStatus child : children) { if (!hasMatchingChildren) { FileStatus[] grandchildren = fs.listStatus(child.getPath()); for (FileStatus gc : grandchildren) { if (VALID_FILE.matcher(gc.getPath().getName()).matches()) { hasMatchingChildren = true; break; } } } if (!subdirTooYoung) { if (child.getModificationTime() >= now - WAIT_TIME) { subdirTooYoung = true; LOG.debug(" Subdir {} is too young.", child.getPath()); } } } if (!hasMatchingChildren) { LOG.debug(" No files match the expected pattern ({})", VALID_FILE.pattern()); } if (hasMatchingChildren && !subdirTooYoung) { LOG.info(" Run Merge job {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); Properties jobProps = new Properties(); jobProps.load(new FileInputStream(mergeJobPropertiesFile)); jobProps.setProperty("jobType", "merge"); jobProps.setProperty("rootDir", rootDir); jobProps.setProperty("dcNumber", dcNumber); jobProps.setProperty("service", service); jobProps.setProperty("date", matchDate); jobProps.setProperty("hour", matchHour); jobProps.setProperty("component", matchComponent); jobProps.setProperty("user.name", userName); jobProps.setProperty("logdir", logdir); jobRunner.submit(jobProps); addChildren = false; } } } // Check to see if we need to run a filter and archive matcher = dataPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { String matchDate = matcher.group(1); String matchHour = matcher.group(2); String matchComponent = matcher.group(3); String timestamp = matchDate + matchHour; if (doDelete && timestamp.compareTo(deleteCutoffDate) < 0) { LOG.info("Deleting old directory: {}", p); fs.delete(p, true); addChildren = false; } else if (doArchive && timestamp.compareTo(filterCutoffDate) < 0) { Properties jobProps = new Properties(); jobProps.load(new FileInputStream(filterJobPropertiesFile)); jobProps.setProperty("jobType", "filter"); jobProps.setProperty("rootDir", rootDir); jobProps.setProperty("dcNumber", dcNumber); jobProps.setProperty("service", service); jobProps.setProperty("date", matchDate); jobProps.setProperty("hour", matchHour); jobProps.setProperty("component", matchComponent); jobProps.setProperty("user.name", userName); jobProps.setProperty("logdir", logdir); // Check to see if we should just keep all or delete all here. // The filter file should be here String appPath = jobProps.getProperty("oozie.wf.application.path"); appPath = appPath.replaceFirst("\\$\\{.*?\\}", ""); Path filterFile = new Path( appPath + "/" + conf.get("filter.definition.file", service + ".yaml")); LOG.info("Filter file is {}", filterFile); if (fs.exists(filterFile)) { List<BoomFilterMapper.Filter> filters = BoomFilterMapper.loadFilters(matchComponent, fs.open(filterFile)); if (filters == null) { LOG.warn( " Got null when getting filters. Not processing. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); } else if (filters.size() == 0) { LOG.warn(" Got no filters. Not processing. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); } else if (filters.size() == 1 && filters.get(0) instanceof BoomFilterMapper.KeepAllFilter) { LOG.info(" Keeping everything. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); // Move files from data to archive // delete it all! String destination = rootDir + "/" + dcNumber + "/" + service + "/" + logdir + "/" + matchDate + "/" + matchHour + "/" + matchComponent + "/archive/"; PathInfo pathInfo = new PathInfo(); pathInfo.setDcNumber(dcNumber); pathInfo.setService(service); pathInfo.setLogdir(logdir); pathInfo.setDate(matchDate); pathInfo.setHour(matchHour); pathInfo.setComponent(matchComponent); try { lockUtil.acquireWriteLock(lockUtil.getLockPath(pathInfo)); fs.mkdirs(new Path(destination)); for (FileStatus f : fs.listStatus(p)) { fs.rename(f.getPath(), new Path(destination)); } } finally { lockUtil.releaseWriteLock(lockUtil.getLockPath(pathInfo)); } } else if (filters.size() == 1 && filters.get(0) instanceof BoomFilterMapper.DropAllFilter) { LOG.info(" Dropping everything. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); PathInfo pathInfo = new PathInfo(); pathInfo.setDcNumber(dcNumber); pathInfo.setService(service); pathInfo.setLogdir(logdir); pathInfo.setDate(matchDate); pathInfo.setHour(matchHour); pathInfo.setComponent(matchComponent); try { lockUtil.acquireWriteLock(lockUtil.getLockPath(pathInfo)); fs.delete(p, true); } finally { lockUtil.releaseWriteLock(lockUtil.getLockPath(pathInfo)); } } else { LOG.info(" Run Filter/Archive job {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); jobRunner.submit(jobProps); } } else { LOG.warn("Skipping filter job, since no filter file exists"); } addChildren = false; } } matcher = archivePathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { String matchDate = matcher.group(1); String matchHour = matcher.group(2); String timestamp = matchDate + matchHour; if (doDelete && timestamp.compareTo(deleteCutoffDate) < 0) { LOG.info("Deleting old directory: {}", p); fs.delete(p, true); addChildren = false; } } matcher = workingPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.info(" Matches working pattern ({})", p); if (resetOrphanedJobs) { String matchDate = matcher.group(1); String matchHour = matcher.group(2); String matchComponent = matcher.group(3); // Move everything from working/xxx/incoming/ to incoming/ PathInfo lockPathInfo = new PathInfo(logdir, rootDir + "/" + dcNumber + "/" + service + "/" + logdir + "/" + matchDate + "/" + matchHour + "/" + matchComponent); lockUtil.acquireWriteLock(lockUtil.getLockPath(lockPathInfo)); FileStatus[] fileStatuses = fs.listStatus(new Path(p.toUri().getPath() + "/incoming/")); if (fileStatuses != null) { for (FileStatus fileStatus : fileStatuses) { Path toPath = new Path( fileStatus.getPath().getParent().getParent().getParent().getParent(), "incoming/" + fileStatus.getPath().getName()); LOG.info(" Moving data from {} to {}", fileStatus.getPath(), toPath); LOG.info(" mkdir {}", toPath); fs.mkdirs(toPath); Path fromDir = new Path(p.toUri().getPath(), "incoming/" + fileStatus.getPath().getName()); LOG.info(" moving from {}", fromDir); FileStatus[] files = fs.listStatus(fromDir); if (files == null || files.length == 0) { LOG.info(" Nothing to move from {}", fromDir); } else { for (FileStatus f : files) { LOG.info(" rename {} {}", f.getPath(), new Path(toPath, f.getPath().getName())); fs.rename(f.getPath(), new Path(toPath, f.getPath().getName())); } } LOG.info(" rm {}", fileStatus.getPath()); fs.delete(fileStatus.getPath(), true); } lockUtil.releaseWriteLock(lockUtil.getLockPath(lockPathInfo)); fs.delete(new Path(p.toUri().getPath()), true); } } addChildren = false; } } // Add any children which are directories to the stack. if (addChildren) { for (int i = children.length - 1; i >= 0; i--) { FileStatus child = children[i]; if (child.isDirectory()) { paths.push(child.getPath()); } } } } // Since we may have deleted a bunch of directories, delete any unused // locks // from ZooKeeper. { LOG.info("Checking for unused locks in ZooKeeper"); String scanPath = rootDir + "/" + dcNumber + "/" + service + "/" + logdir; if (date != null) { scanPath += "/" + date; if (hour != null) { scanPath += "/" + hour; } } List<LockInfo> lockInfo = lockUtil.scan(scanPath); for (LockInfo li : lockInfo) { // Check if the lock path still exists in HDFS. If it doesn't, then // delete it from ZooKeeper. String path = li.getPath(); String hdfsPath = path.substring(LockUtil.ROOT.length()); if (!fs.exists(new Path(hdfsPath))) { ZooKeeper zk = lockUtil.getZkClient(); while (!path.equals(LockUtil.ROOT)) { try { zk.delete(path, -1); } catch (KeeperException.NotEmptyException e) { // That's fine. just stop trying then. break; } catch (Exception e) { LOG.error("Caught exception trying to delete from ZooKeeper.", e); break; } LOG.info("Deleted from ZooKeeper: {}", path); path = path.substring(0, path.lastIndexOf('/')); } } } } // Now that we're done, wait for the Oozie Runner to stop, and print the // results. LOG.info("Waiting for Oozie jobs to complete."); jobRunner.shutdown(); jobRunnerThread.join(); LOG.info("Job Stats : Started={} Succeeded={} failed={} errors={}", new Object[] { jobRunner.getStarted(), jobRunner.getSucceeded(), jobRunner.getFailed(), jobRunner.getErrors() }); lockUtil.close(); } catch (Exception e) { LOG.error("Unexpected exception caught.", e); return 1; } return 0; }
From source file:org.apache.hadoop.hive.metastore.cache.CachedStore.java
@VisibleForTesting /**/*from w w w. j ava 2s. com*/ * This starts a background thread, which initially populates the SharedCache and later * periodically gets updates from the metastore db * * @param conf * @param runOnlyOnce * @param shouldRunPrewarm */ static synchronized void startCacheUpdateService(Configuration conf, boolean runOnlyOnce, boolean shouldRunPrewarm) { if (cacheUpdateMaster == null) { initBlackListWhiteList(conf); if (!MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST)) { cacheRefreshPeriodMS = MetastoreConf.getTimeVar(conf, ConfVars.CACHED_RAW_STORE_CACHE_UPDATE_FREQUENCY, TimeUnit.MILLISECONDS); } LOG.info("CachedStore: starting cache update service (run every {} ms", cacheRefreshPeriodMS); cacheUpdateMaster = Executors.newScheduledThreadPool(1, new ThreadFactory() { @Override public Thread newThread(Runnable r) { Thread t = Executors.defaultThreadFactory().newThread(r); t.setName("CachedStore-CacheUpdateService: Thread-" + t.getId()); t.setDaemon(true); return t; } }); if (!runOnlyOnce) { cacheUpdateMaster.scheduleAtFixedRate(new CacheUpdateMasterWork(conf, shouldRunPrewarm), 0, cacheRefreshPeriodMS, TimeUnit.MILLISECONDS); } } if (runOnlyOnce) { // Some tests control the execution of the background update thread cacheUpdateMaster.schedule(new CacheUpdateMasterWork(conf, shouldRunPrewarm), 0, TimeUnit.MILLISECONDS); } }