List of usage examples for java.lang Thread isAlive
public final native boolean isAlive();
From source file:com.petpet.c3po.controller.Controller.java
public void resolveConflicts(Map<String, String> options) { if (processingQueue != null && !processingQueue.isEmpty()) { LOG.info("Tried to resolve conflicts while gathering, exiting"); System.out.println("Tried to resolve conflicts while gathering, exiting"); return;/*from w w w . j a v a 2 s . c o m*/ } System.out.println("Conflict resolution process started"); int consThreads = this.configurator.getIntProperty(Constants.CNF_CONSOLIDATORS_COUNT, 2); this.consolidatorPool = Executors.newFixedThreadPool(consThreads); List<Consolidator> consolidators = new ArrayList<Consolidator>(); final String pathToRules = options.get(Constants.CNF_DROOLS_PATH); final String collectionName = options.get(Constants.OPT_COLLECTION_NAME); final DroolsConflictResolutionProcessingRule resolver = new DroolsConflictResolutionProcessingRule( pathToRules); Thread thread = new Thread(new Runnable() { public void run() { //DroolsConflictResolutionProcessingRule resolver = new DroolsConflictResolutionProcessingRule(pathToRules); Iterator<Element> elementIterator = Configurator.getDefaultConfigurator().getPersistence() .find(Element.class, new Filter(new FilterCondition("collection", collectionName))); long i = 0; while (elementIterator.hasNext()) { try { Element elementProcessed = resolver.process(elementIterator.next()); processingQueue.put(elementProcessed); i++; if (i % 10000 == 0) { System.out.println("Processed " + i + " objects"); } } catch (Exception e) { e.printStackTrace(); } } } }); thread.start(); LOG.debug("Initializing consolidators..."); Consolidator cons = new Consolidator(this.persistence, this.processingQueue); Thread threadCons = new Thread(cons); threadCons.start(); while (true) { try { System.out.println("Waiting for new objects..."); if (!thread.isAlive()) { break; } Thread.sleep(1000); } catch (InterruptedException e) { } } System.out.println("Done resolving conflicts"); cons.setRunning(false); resolver.onCommandFinished(); }
From source file:org.apache.hadoop.hbase.PerformanceEvaluationDoubleTable.java
@SuppressWarnings("unused") private void doMultipleClients(final String cmd) throws IOException { final List<Thread> threads = new ArrayList<Thread>(this.N); final int perClientRows = R / N; for (int i = 0; i < this.N; i++) { Thread t = new Thread(Integer.toString(i)) { public void run() { super.run(); PerformanceEvaluationDoubleTable pe = new PerformanceEvaluationDoubleTable(conf); int index = Integer.parseInt(getName()); try { long elapsedTime = pe.runOneClient(cmd, index * perClientRows, perClientRows, perClientRows, new Status() { public void setStatus(final String msg) throws IOException { LOG.info("client-" + getName() + " " + msg); }/*w ww . ja v a 2 s .c o m*/ }); LOG.info("Finished " + getName() + " in " + elapsedTime + "ms writing " + perClientRows + " rows"); } catch (IOException e) { throw new RuntimeException(e); } } }; threads.add(t); } for (Thread t : threads) { t.start(); } for (Thread t : threads) { while (t.isAlive()) { try { t.join(); } catch (InterruptedException e) { LOG.debug("Interrupted, continuing" + e.toString()); } } } }
From source file:org.kurento.test.browser.Browser.java
public void createRemoteDriver(final DesiredCapabilities capabilities) throws MalformedURLException { assertPublicIpNotNull();// w ww .j a v a 2 s .co m String remoteHubUrl = getProperty(SELENIUM_REMOTE_HUB_URL_PROPERTY); GridNode gridNode = null; if (remoteHubUrl == null) { log.debug("Creating remote webdriver for {}", id); if (!GridHandler.getInstance().containsSimilarBrowserKey(id)) { if (login != null) { System.setProperty(TEST_NODE_LOGIN_PROPERTY, login); } if (passwd != null) { System.setProperty(TEST_NODE_PASSWD_PROPERTY, passwd); } if (pem != null) { System.setProperty(TEST_NODE_PEM_PROPERTY, pem); } // Filtering valid nodes (just the first time will be effective) GridHandler.getInstance().filterValidNodes(); if (!node.equals(host) && login != null && !login.isEmpty() && (passwd != null && !passwd.isEmpty() || pem != null && !pem.isEmpty())) { gridNode = new GridNode(node, browserType, browserPerInstance, login, passwd, pem); GridHandler.getInstance().addNode(id, gridNode); } else { gridNode = GridHandler.getInstance().getRandomNodeFromList(id, browserType, browserPerInstance); } // Start Hub (just the first time will be effective) GridHandler.getInstance().startHub(); // Start node GridHandler.getInstance().startNode(gridNode); // Copy video (if necessary) if (video != null && browserType == BrowserType.CHROME) { GridHandler.getInstance().copyRemoteVideo(gridNode, video); } } else { // Wait for node boolean started = false; do { gridNode = GridHandler.getInstance().getNode(id); if (gridNode != null) { started = gridNode.isStarted(); } if (!started) { log.debug("Node {} is not started ... waiting 1 second", id); waitSeconds(1); } } while (!started); } // At this moment we are able to use the argument for remote video if (video != null && browserType == BrowserType.CHROME) { ChromeOptions options = (ChromeOptions) capabilities.getCapability(ChromeOptions.CAPABILITY); options.addArguments("--use-file-for-fake-video-capture=" + GridHandler.getInstance().getFirstNode(id).getRemoteVideo(video)); capabilities.setCapability(ChromeOptions.CAPABILITY, options); } final int hubPort = GridHandler.getInstance().getHubPort(); final String hubHost = GridHandler.getInstance().getHubHost(); log.debug("Creating remote webdriver of {} ({})", id, gridNode.getHost()); remoteHubUrl = "http://" + hubHost + ":" + hubPort + "/wd/hub"; } final String remoteHub = remoteHubUrl; Thread t = new Thread() { @Override public void run() { boolean exception = false; do { try { driver = new RemoteWebDriver(new URL(remoteHub), capabilities); exception = false; } catch (MalformedURLException | WebDriverException e) { log.error("Exception {} creating RemoteWebDriver ... retrying in 1 second", e.getClass()); waitSeconds(1); exception = true; } } while (exception); } }; t.start(); int timeout = getProperty(SELENIUM_REMOTEWEBDRIVER_TIME_PROPERTY, SELENIUM_REMOTEWEBDRIVER_TIME_DEFAULT); String nodeMsg = gridNode != null ? " (" + gridNode.getHost() + ")" : ""; for (int i = 0; i < timeout; i++) { if (t.isAlive()) { log.debug("Waiting for RemoteWebDriver {}{}", id, nodeMsg); } else { log.debug("Remote webdriver of {}{} created", id, nodeMsg); return; } waitSeconds(1); } String exceptionMessage = "Remote webdriver of " + id + nodeMsg + " not created in " + timeout + "seconds"; log.error(">>>>>>>>>> " + exceptionMessage); throw new RuntimeException(exceptionMessage); }
From source file:br.org.acessobrasil.nucleuSilva.util.PegarPaginaWEB.java
/** * Mtodo que extra o contedo de uma pgina web. * /*from w ww . j a va 2s.c o m*/ * @param url * Pgina que vai ser pesquisada. * @return Contedo da pgina HTML. * @throws IOException * Erro ao tentar extrair o contedo da pgina html. */ public void getContent(final RelatorioDaUrl relatorio) { /* * Melhorar este cdigo!!! */ log.addLog("getContent(" + relatorio.getUrl() + ")\n"); int status = 9999; setAtivo(false); metodo = null; final int mb = 1024; String type = new String(); InputStream ist = null; StringBuilder sbd = null; setPaginaObtida(true); TimeOut tc = new TimeOut(this); setAtivo(true); Thread thrTc = new Thread(tc); thrTc.start(); try { metodo = new GetMethod(relatorio.getUrl()); metodo.setRequestHeader("user-agent", "Mozilla/5.0"); metodo.setFollowRedirects(true); } catch (Exception e) { log.addLog("Erro no GetMetodo: " + e.getMessage() + "\n"); //Ini - nati code colocaNaTabelaErros("timeOut", relatorio); setPaginaObtida(false); setAtivo(false); //Fim - nati code metodo = null; thrTc = null; tc = null; return; } if (!tc.timeOut) { // httpClient.setConnectionTimeout(arg0) try { status = httpClient.executeMethod(metodo); type = getContentType(metodo); String tam = getContentLength(metodo); String location = getLocation(metodo); if (location != "") { //System.out.print(relatorio.getUrl()+" to "+location+"\n"); } log.addLog("type=" + type + " tam=" + tam + "\n"); if ((status == HttpStatus.SC_OK) && (type.toUpperCase().indexOf("TEXT/HTML") > -1)) { if (!tc.timeOut) { sbd = new StringBuilder(); //ist = metodo.getResponseBodyAsStream(); } else { colocaNaTabelaErros("timeOut", relatorio); setPaginaObtida(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); setAtivo(false); } if (!tc.timeOut) { /* byte[] dados = new byte[mb]; int bytesLidos = 0; while ((bytesLidos = ist.read(dados)) > 0) { sbd.append(new String(dados, 0, bytesLidos)); } ist.close(); */ sbd.append(metodo.getResponseBodyAsString()); } else { colocaNaTabelaErros("timeOut", relatorio); setPaginaObtida(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); setAtivo(false); } //verifica se existe contedo if (sbd.toString().equals("") || sbd == null || sbd.toString().trim().length() <= 1) { colocaNaTabelaErros("Sem contedo", relatorio); setPaginaObtida(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); setAtivo(false); } } else { //verifica se o tipo est errado if (type.toUpperCase().indexOf("TEXT/HTML") == -1) { colocaNaTabelaErros("No HTML", relatorio); } else if (status == HttpStatus.SC_NOT_FOUND) { colocaNaTabelaErros("No Encontrado", relatorio); } else { colocaNaTabelaErros("Status error " + status, relatorio); } setPaginaObtida(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); setAtivo(false); } if (!tc.timeOut) { metodo.abort(); metodo.releaseConnection(); } else { colocaNaTabelaErros("timeOut", relatorio); setAtivo(false); setPaginaObtida(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); } if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); } catch (Exception e) { log.addLog("Erro: " + e.getMessage() + "\n"); colocaNaTabelaErros("Erro: " + e.getMessage(), relatorio); setAtivo(false); setPaginaObtida(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); } } else { colocaNaTabelaErros("timeOut", relatorio); setPaginaObtida(false); setAtivo(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); } try { metodo.abort(); metodo.releaseConnection(); } catch (Exception e) { } if (sbd != null && (type.toUpperCase().indexOf("TEXT") > -1) && !tc.timeOut && isAtivo()) { setAtivo(false); //System.out.println("PPW:\n"+sbd.toString()); relatorio.setConteudo(sbd); } if (!tc.isTimeOut()) { setPaginaObtida(true); } tc.timeOut = false; setAtivo(false); if (!thrTc.interrupted()) if (thrTc.isAlive()) thrTc.interrupt(); log.addLog("Ok \n"); }
From source file:org.apache.flink.test.runtime.leaderelection.ZooKeeperLeaderElectionITCase.java
/** * Tests that a job can be executed after a new leader has been elected. For all except for the * last leader, the job is blocking. The JobManager will be terminated while executing the * blocking job. Once only one JobManager is left, it is checked that a non-blocking can be * successfully executed.//w w w .j av a 2 s . c om */ @Test public void testJobExecutionOnClusterWithLeaderReelection() throws Exception { int numJMs = 10; int numTMs = 2; int numSlotsPerTM = 3; int parallelism = numTMs * numSlotsPerTM; Configuration configuration = new Configuration(); configuration.setString(ConfigConstants.HA_MODE, "zookeeper"); configuration.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, numJMs); configuration.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTMs); configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlotsPerTM); configuration.setString(ConfigConstants.STATE_BACKEND, "filesystem"); configuration.setString(ConfigConstants.HA_ZOOKEEPER_STORAGE_PATH, tempDirectory.getAbsoluteFile().toURI().toString()); // we "effectively" disable the automatic RecoverAllJobs message and sent it manually to make // sure that all TMs have registered to the JM prior to issueing the RecoverAllJobs message configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, AkkaUtils.INF_TIMEOUT().toString()); Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(true); JobVertex sender = new JobVertex("sender"); JobVertex receiver = new JobVertex("receiver"); sender.setInvokableClass(Tasks.Sender.class); receiver.setInvokableClass(Tasks.BlockingOnceReceiver.class); sender.setParallelism(parallelism); receiver.setParallelism(parallelism); receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE); SlotSharingGroup slotSharingGroup = new SlotSharingGroup(); sender.setSlotSharingGroup(slotSharingGroup); receiver.setSlotSharingGroup(slotSharingGroup); final JobGraph graph = new JobGraph("Blocking test job", sender, receiver); final ForkableFlinkMiniCluster cluster = new ForkableFlinkMiniCluster(configuration); ActorSystem clientActorSystem = null; Thread thread = null; JobSubmitterRunnable jobSubmission = null; try { cluster.start(); clientActorSystem = cluster.startJobClientActorSystem(graph.getJobID()); final ActorSystem clientAS = clientActorSystem; jobSubmission = new JobSubmitterRunnable(clientAS, cluster, graph); thread = new Thread(jobSubmission); thread.start(); Deadline deadline = timeout.$times(3).fromNow(); // Kill all JobManager except for two for (int i = 0; i < numJMs; i++) { ActorGateway jm = cluster.getLeaderGateway(deadline.timeLeft()); cluster.waitForTaskManagersToBeRegisteredAtJobManager(jm.actor()); // recover all jobs, sent manually log.info("Sent recover all jobs manually to job manager {}.", jm.path()); jm.tell(JobManagerMessages.getRecoverAllJobs()); if (i < numJMs - 1) { Future<Object> future = jm.ask(new WaitForAllVerticesToBeRunningOrFinished(graph.getJobID()), deadline.timeLeft()); Await.ready(future, deadline.timeLeft()); cluster.clearLeader(); if (i == numJMs - 2) { Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(false); } log.info("Kill job manager {}.", jm.path()); jm.tell(TestingJobManagerMessages.getDisablePostStop()); jm.tell(Kill.getInstance()); } } log.info("Waiting for submitter thread to terminate."); thread.join(deadline.timeLeft().toMillis()); log.info("Submitter thread has terminated."); if (thread.isAlive()) { fail("The job submission thread did not stop (meaning it did not succeeded in" + "executing the test job."); } Await.result(jobSubmission.resultPromise.future(), deadline.timeLeft()); } finally { if (clientActorSystem != null) { cluster.shutdownJobClientActorSystem(clientActorSystem); } if (thread != null && thread.isAlive()) { jobSubmission.finished = true; } cluster.stop(); } }
From source file:org.apache.flink.test.recovery.AbstractProcessFailureRecoveryTest.java
@Test public void testTaskManagerProcessFailure() { final StringWriter processOutput1 = new StringWriter(); final StringWriter processOutput2 = new StringWriter(); final StringWriter processOutput3 = new StringWriter(); ActorSystem jmActorSystem = null;/*from w w w . j ava 2 s.co m*/ Process taskManagerProcess1 = null; Process taskManagerProcess2 = null; Process taskManagerProcess3 = null; File coordinateTempDir = null; try { // check that we run this test only if the java command // is available on this machine String javaCommand = getJavaCommandPath(); if (javaCommand == null) { System.out.println("---- Skipping Process Failure test : Could not find java executable ----"); return; } // create a logging file for the process File tempLogFile = File.createTempFile(getClass().getSimpleName() + "-", "-log4j.properties"); tempLogFile.deleteOnExit(); CommonTestUtils.printLog4jDebugConfig(tempLogFile); // coordination between the processes goes through a directory coordinateTempDir = createTempDirectory(); // find a free port to start the JobManager final int jobManagerPort = NetUtils.getAvailablePort(); // start a JobManager Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", jobManagerPort); Configuration jmConfig = new Configuration(); jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "1000 ms"); jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "6 s"); jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9); jmConfig.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "10 s"); jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress)); ActorRef jmActor = JobManager.startJobManagerActors(jmConfig, jmActorSystem, StreamingMode.STREAMING) ._1(); // the TaskManager java command String[] command = new String[] { javaCommand, "-Dlog.level=DEBUG", "-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath(), "-Xms80m", "-Xmx80m", "-classpath", getCurrentClasspath(), TaskManagerProcessEntryPoint.class.getName(), String.valueOf(jobManagerPort) }; // start the first two TaskManager processes taskManagerProcess1 = new ProcessBuilder(command).start(); new PipeForwarder(taskManagerProcess1.getErrorStream(), processOutput1); taskManagerProcess2 = new ProcessBuilder(command).start(); new PipeForwarder(taskManagerProcess2.getErrorStream(), processOutput2); // we wait for the JobManager to have the two TaskManagers available // since some of the CI environments are very hostile, we need to give this a lot of time (2 minutes) waitUntilNumTaskManagersAreRegistered(jmActor, 2, 120000); // the program will set a marker file in each of its parallel tasks once they are ready, so that // this coordinating code is aware of this. // the program will very slowly consume elements until the marker file (later created by the // test driver code) is present final File coordinateDirClosure = coordinateTempDir; final Throwable[] errorRef = new Throwable[1]; // we trigger program execution in a separate thread Thread programTrigger = new Thread("Program Trigger") { @Override public void run() { try { testProgram(jobManagerPort, coordinateDirClosure); } catch (Throwable t) { t.printStackTrace(); errorRef[0] = t; } } }; //start the test program programTrigger.start(); // wait until all marker files are in place, indicating that all tasks have started // max 20 seconds waitForMarkerFiles(coordinateTempDir, PARALLELISM, 20000); // start the third TaskManager taskManagerProcess3 = new ProcessBuilder(command).start(); new PipeForwarder(taskManagerProcess3.getErrorStream(), processOutput3); // we wait for the third TaskManager to register // since some of the CI environments are very hostile, we need to give this a lot of time (2 minutes) waitUntilNumTaskManagersAreRegistered(jmActor, 3, 120000); // kill one of the previous TaskManagers, triggering a failure and recovery taskManagerProcess1.destroy(); taskManagerProcess1 = null; // we create the marker file which signals the program functions tasks that they can complete touchFile(new File(coordinateTempDir, PROCEED_MARKER_FILE)); // wait for at most 5 minutes for the program to complete programTrigger.join(300000); // check that the program really finished assertFalse("The program did not finish in time", programTrigger.isAlive()); // check whether the program encountered an error if (errorRef[0] != null) { Throwable error = errorRef[0]; error.printStackTrace(); fail("The program encountered a " + error.getClass().getSimpleName() + " : " + error.getMessage()); } // all seems well :-) } catch (Exception e) { e.printStackTrace(); printProcessLog("TaskManager 1", processOutput1.toString()); printProcessLog("TaskManager 2", processOutput2.toString()); printProcessLog("TaskManager 3", processOutput3.toString()); fail(e.getMessage()); } catch (Error e) { e.printStackTrace(); printProcessLog("TaskManager 1", processOutput1.toString()); printProcessLog("TaskManager 2", processOutput2.toString()); printProcessLog("TaskManager 3", processOutput3.toString()); throw e; } finally { if (taskManagerProcess1 != null) { taskManagerProcess1.destroy(); } if (taskManagerProcess2 != null) { taskManagerProcess2.destroy(); } if (taskManagerProcess3 != null) { taskManagerProcess3.destroy(); } if (jmActorSystem != null) { jmActorSystem.shutdown(); } if (coordinateTempDir != null) { try { FileUtils.deleteDirectory(coordinateTempDir); } catch (Throwable t) { // we can ignore this } } } }
From source file:org.apache.flink.test.recovery.AbstractJobManagerProcessFailureRecoveryITCase.java
@Test public void testJobManagerProcessFailure() throws Exception { // Config/*from www .j a v a 2s. c o m*/ final int numberOfJobManagers = 2; final int numberOfTaskManagers = 2; final int numberOfSlotsPerTaskManager = 2; assertEquals(PARALLELISM, numberOfTaskManagers * numberOfSlotsPerTaskManager); // Setup // Test actor system ActorSystem testActorSystem; // Job managers final JobManagerProcess[] jmProcess = new JobManagerProcess[numberOfJobManagers]; // Task managers final ActorSystem[] tmActorSystem = new ActorSystem[numberOfTaskManagers]; // Leader election service LeaderRetrievalService leaderRetrievalService = null; // Coordination between the processes goes through a directory File coordinateTempDir = null; try { final Deadline deadline = TestTimeOut.fromNow(); // Coordination directory coordinateTempDir = createTempDirectory(); // Job Managers Configuration config = ZooKeeperTestUtils.createZooKeeperRecoveryModeConfig( ZooKeeper.getConnectString(), FileStateBackendBasePath.getPath()); // Start first process jmProcess[0] = new JobManagerProcess(0, config); jmProcess[0].createAndStart(); // Task manager configuration config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4); config.setInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, 100); config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2); // Start the task manager process for (int i = 0; i < numberOfTaskManagers; i++) { tmActorSystem[i] = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig()); TaskManager.startTaskManagerComponentsAndActor(config, tmActorSystem[i], "localhost", Option.<String>empty(), Option.<LeaderRetrievalService>empty(), false, TaskManager.class); } // Test actor system testActorSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig()); jmProcess[0].getActorRef(testActorSystem, deadline.timeLeft()); // Leader listener TestingListener leaderListener = new TestingListener(); leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config); leaderRetrievalService.start(leaderListener); // Initial submission leaderListener.waitForNewLeader(deadline.timeLeft().toMillis()); String leaderAddress = leaderListener.getAddress(); UUID leaderId = leaderListener.getLeaderSessionID(); // Get the leader ref ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testActorSystem, deadline.timeLeft()); ActorGateway leaderGateway = new AkkaActorGateway(leaderRef, leaderId); // Wait for all task managers to connect to the leading job manager JobManagerActorTestUtils.waitForTaskManagers(numberOfTaskManagers, leaderGateway, deadline.timeLeft()); final File coordinateDirClosure = coordinateTempDir; final Throwable[] errorRef = new Throwable[1]; // we trigger program execution in a separate thread Thread programTrigger = new Thread("Program Trigger") { @Override public void run() { try { testJobManagerFailure(ZooKeeper.getConnectString(), coordinateDirClosure); } catch (Throwable t) { t.printStackTrace(); errorRef[0] = t; } } }; //start the test program programTrigger.start(); // wait until all marker files are in place, indicating that all tasks have started AbstractTaskManagerProcessFailureRecoveryTest.waitForMarkerFiles(coordinateTempDir, READY_MARKER_FILE_PREFIX, PARALLELISM, deadline.timeLeft().toMillis()); // Kill one of the job managers and trigger recovery jmProcess[0].destroy(); jmProcess[1] = new JobManagerProcess(1, config); jmProcess[1].createAndStart(); jmProcess[1].getActorRef(testActorSystem, deadline.timeLeft()); // we create the marker file which signals the program functions tasks that they can complete AbstractTaskManagerProcessFailureRecoveryTest .touchFile(new File(coordinateTempDir, PROCEED_MARKER_FILE)); programTrigger.join(deadline.timeLeft().toMillis()); // We wait for the finish marker file. We don't wait for the program trigger, because // we submit in detached mode. AbstractTaskManagerProcessFailureRecoveryTest.waitForMarkerFiles(coordinateTempDir, FINISH_MARKER_FILE_PREFIX, 1, deadline.timeLeft().toMillis()); // check that the program really finished assertFalse("The program did not finish in time", programTrigger.isAlive()); // check whether the program encountered an error if (errorRef[0] != null) { Throwable error = errorRef[0]; error.printStackTrace(); fail("The program encountered a " + error.getClass().getSimpleName() + " : " + error.getMessage()); } } catch (Exception e) { e.printStackTrace(); for (JobManagerProcess p : jmProcess) { if (p != null) { p.printProcessLog(); } } fail(e.getMessage()); } finally { for (int i = 0; i < numberOfTaskManagers; i++) { if (tmActorSystem[i] != null) { tmActorSystem[i].shutdown(); } } if (leaderRetrievalService != null) { leaderRetrievalService.stop(); } for (JobManagerProcess jmProces : jmProcess) { if (jmProces != null) { jmProces.destroy(); } } // Delete coordination directory if (coordinateTempDir != null) { try { FileUtils.deleteDirectory(coordinateTempDir); } catch (Throwable ignored) { } } } }
From source file:com.meltmedia.cadmium.servlets.ClassLoaderLeakPreventor.java
/** * Deregister shutdown hook and execute it immediately *//* w w w . j a v a2 s. c o m*/ @SuppressWarnings("deprecation") protected void removeShutdownHook(Thread shutdownHook) { final String displayString = "'" + shutdownHook + "' of type " + shutdownHook.getClass().getName(); error("Removing shutdown hook: " + displayString); Runtime.getRuntime().removeShutdownHook(shutdownHook); if (executeShutdownHooks) { // Shutdown hooks should be executed info("Executing shutdown hook now: " + displayString); // Make sure it's from this web app instance shutdownHook.start(); // Run cleanup immediately if (shutdownHookWaitMs > 0) { // Wait for shutdown hook to finish try { shutdownHook.join(shutdownHookWaitMs); // Wait for thread to run } catch (InterruptedException e) { // Do nothing } if (shutdownHook.isAlive()) { warn(shutdownHook + "still running after " + shutdownHookWaitMs + " ms - Stopping!"); shutdownHook.stop(); } } } }
From source file:org.apache.flink.test.recovery.ProcessFailureBatchRecoveryITCase.java
@Test public void testTaskManagerProcessFailure() { final StringWriter processOutput1 = new StringWriter(); final StringWriter processOutput2 = new StringWriter(); final StringWriter processOutput3 = new StringWriter(); ActorSystem jmActorSystem = null;//w w w. j a v a 2 s. c om Process taskManagerProcess1 = null; Process taskManagerProcess2 = null; Process taskManagerProcess3 = null; File coordinateTempDir = null; try { // check that we run this test only if the java command // is available on this machine String javaCommand = getJavaCommandPath(); if (javaCommand == null) { System.out.println( "---- Skipping ProcessFailureBatchRecoveryITCase : Could not find java executable"); return; } // create a logging file for the process File tempLogFile = File.createTempFile(getClass().getSimpleName() + "-", "-log4j.properties"); tempLogFile.deleteOnExit(); CommonTestUtils.printLog4jDebugConfig(tempLogFile); // coordination between the processes goes through a directory coordinateTempDir = createTempDirectory(); // find a free port to start the JobManager final int jobManagerPort = NetUtils.getAvailablePort(); // start a JobManager Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", jobManagerPort); Configuration jmConfig = new Configuration(); jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "500 ms"); jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2 s"); jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 2); jmConfig.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "4 s"); jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress)); ActorRef jmActor = JobManager.startJobManagerActors(jmConfig, jmActorSystem)._1(); // the TaskManager java command String[] command = new String[] { javaCommand, "-Dlog.level=DEBUG", "-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath(), "-Xms80m", "-Xmx80m", "-classpath", getCurrentClasspath(), TaskManagerProcessEntryPoint.class.getName(), String.valueOf(jobManagerPort) }; // start the first two TaskManager processes taskManagerProcess1 = new ProcessBuilder(command).start(); new PipeForwarder(taskManagerProcess1.getErrorStream(), processOutput1); taskManagerProcess2 = new ProcessBuilder(command).start(); new PipeForwarder(taskManagerProcess2.getErrorStream(), processOutput2); // we wait for the JobManager to have the two TaskManagers available // wait for at most 20 seconds waitUntilNumTaskManagersAreRegistered(jmActor, 2, 20000); // the program will set a marker file in each of its parallel tasks once they are ready, so that // this coordinating code is aware of this. // the program will very slowly consume elements until the marker file (later created by the // test driver code) is present final File coordinateDirClosure = coordinateTempDir; final Throwable[] errorRef = new Throwable[1]; ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", jobManagerPort); env.setDegreeOfParallelism(PARALLELISM); env.setNumberOfExecutionRetries(1); final long NUM_ELEMENTS = 1000000L; final DataSet<Long> result = env.generateSequence(1, NUM_ELEMENTS) // make sure every mapper is involved (no one is skipped because of lazy split assignment) .rebalance() // the majority of the behavior is in the MapFunction .map(new RichMapFunction<Long, Long>() { private final File proceedFile = new File(coordinateDirClosure, PROCEED_MARKER_FILE); private boolean markerCreated = false; private boolean checkForProceedFile = true; @Override public Long map(Long value) throws Exception { if (!markerCreated) { int taskIndex = getRuntimeContext().getIndexOfThisSubtask(); touchFile(new File(coordinateDirClosure, READY_MARKER_FILE_PREFIX + taskIndex)); markerCreated = true; } // check if the proceed file exists if (checkForProceedFile) { if (proceedFile.exists()) { checkForProceedFile = false; } else { // otherwise wait so that we make slow progress Thread.sleep(10); } } return value; } }).reduce(new ReduceFunction<Long>() { @Override public Long reduce(Long value1, Long value2) { return value1 + value2; } }); // we trigger a program now (in a separate thread) Thread programTrigger = new Thread("ProcessFailureBatchRecoveryITCase Program Trigger") { @Override public void run() { try { long sum = result.collect().get(0); assertEquals(NUM_ELEMENTS * (NUM_ELEMENTS + 1L) / 2L, sum); } catch (Throwable t) { t.printStackTrace(); errorRef[0] = t; } } }; programTrigger.start(); // wait until all marker files are in place, indicating that all tasks have started // max 20 seconds waitForMarkerFiles(coordinateTempDir, PARALLELISM, 20000); // start the third TaskManager taskManagerProcess3 = new ProcessBuilder(command).start(); new PipeForwarder(taskManagerProcess3.getErrorStream(), processOutput3); // we wait for the third TaskManager to register (20 seconds max) waitUntilNumTaskManagersAreRegistered(jmActor, 3, 20000); // kill one of the previous TaskManagers, triggering a failure and recovery taskManagerProcess1.destroy(); taskManagerProcess1 = null; // we create the marker file which signals the program functions tasks that they can complete touchFile(new File(coordinateTempDir, PROCEED_MARKER_FILE)); // wait for at most 30 seconds for the program to complete programTrigger.join(30000); // check that the program really finished assertFalse("The program did not finish in time", programTrigger.isAlive()); // check whether the program encountered an error if (errorRef[0] != null) { Throwable error = errorRef[0]; error.printStackTrace(); fail("The program encountered a " + error.getClass().getSimpleName() + " : " + error.getMessage()); } // all seems well :-) } catch (Exception e) { e.printStackTrace(); printProcessLog("TaskManager 1", processOutput1.toString()); printProcessLog("TaskManager 2", processOutput2.toString()); printProcessLog("TaskManager 3", processOutput3.toString()); fail(e.getMessage()); } catch (Error e) { e.printStackTrace(); printProcessLog("TaskManager 1", processOutput1.toString()); printProcessLog("TaskManager 2", processOutput2.toString()); printProcessLog("TaskManager 3", processOutput3.toString()); throw e; } finally { if (taskManagerProcess1 != null) { taskManagerProcess1.destroy(); } if (taskManagerProcess2 != null) { taskManagerProcess2.destroy(); } if (taskManagerProcess3 != null) { taskManagerProcess3.destroy(); } if (jmActorSystem != null) { jmActorSystem.shutdown(); } if (coordinateTempDir != null) { try { FileUtils.deleteDirectory(coordinateTempDir); } catch (Throwable t) { // we can ignore this } } } }
From source file:org.apache.sysml.debug.DMLDebugger.java
/** * Controls the communication between debugger CLI and DML runtime. *//*from w ww .j av a 2 s. c o m*/ @SuppressWarnings("deprecation") public synchronized void runSystemMLDebugger() { debuggerUI.setOptions(); debuggerUI.getDebuggerCLI(); Thread runtime = new Thread(DMLRuntime); boolean isRuntimeInstruction = false; while (!quit) { try { //get debugger function from CLI getCommand(); if (cmd != null) { isRuntimeInstruction = false; //check for help if (cmd.hasOption("h")) { debuggerUI.getDebuggerCLI(); } //check for exit else if (cmd.hasOption("q")) { synchronized (DMLDebugger.class) { quit = true; } runtime.stop(); } else if (cmd.hasOption("r")) { if (currEC != null) { System.out.println( "Runtime has already started. Try \"s\" to go to next line, or \"c\" to continue running your DML script."); } else { currEC = preEC; runtime.start(); isRuntimeInstruction = true; } } else if (cmd.hasOption("c")) { if (currEC == null) System.out.println( "Runtime has not been started. Try \"r\" to start DML runtime execution."); else if (!runtime.isAlive()) { System.err.println("Invalid debug state."); //System.out.println("Runtime terminated. Try \"-c\" to recompile followed by \"r\" to restart DML runtime execution."); } else { System.out.println("Resuming DML script execution ..."); preEC.getDebugState().setCommand(null); runtime.resume(); isRuntimeInstruction = true; } } else if (cmd.hasOption("si")) { if (!runtime.isAlive()) { currEC = preEC; runtime.start(); isRuntimeInstruction = true; } preEC.getDebugState().setCommand("step_instruction"); runtime.resume(); isRuntimeInstruction = true; } else if (cmd.hasOption("s")) { if (!runtime.isAlive()) { currEC = preEC; runtime.start(); isRuntimeInstruction = true; } preEC.getDebugState().setCommand("step_line"); runtime.resume(); isRuntimeInstruction = true; } else if (cmd.hasOption("b")) { int lineNumber = dbFunctions.getValue(cmd.getOptionValues("b"), lines.length); if (lineNumber > 0) { if (DMLBreakpointManager.getBreakpoint(lineNumber) == null) System.out.println("Sorry, a breakpoint cannot be inserted at line " + lineNumber + ". Please try a different line number."); else { if (DMLBreakpointManager.getBreakpoint(lineNumber) .getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE) { System.out.format("Breakpoint at line %d already exists.\n", lineNumber); } else { dbprog.accessBreakpoint(lineNumber, 0, BPINSTRUCTION_STATUS.ENABLED); } } } } else if (cmd.hasOption("d")) { int lineNumber = dbFunctions.getValue(cmd.getOptionValues("d"), lines.length); if (lineNumber > 0 && DMLBreakpointManager.getBreakpoint(lineNumber) != null && DMLBreakpointManager.getBreakpoint(lineNumber) .getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE) { dbprog.accessBreakpoint(lineNumber, 1, BPINSTRUCTION_STATUS.INVISIBLE); } else { System.out.println("Sorry, a breakpoint cannot be deleted at line " + lineNumber + ". Please try a different line number."); } } else if (cmd.hasOption("i")) { String[] infoOptions = cmd.getOptionValues("i"); if (infoOptions == null || infoOptions.length == 0) { System.err.println( "The command \"info\" requires option. Try \"info break\" or \"info frame\"."); } else if (infoOptions[0].trim().equals("break")) { dbFunctions.listBreakpoints(DMLBreakpointManager.getBreakpoints()); } else if (infoOptions[0].trim().equals("frame")) { if (!runtime.isAlive()) System.err.println( "Runtime has not been started. Try \"r\" or \"s\" to start DML runtime execution."); else dbFunctions.printCallStack(currEC.getDebugState().getCurrentFrame(), currEC.getDebugState().getCallStack()); } else { System.err.println( "Invalid option for command \"info\". Try \"info break\" or \"info frame\"."); } } else if (cmd.hasOption("p")) { String[] pOptions = cmd.getOptionValues("p"); if (pOptions == null || pOptions.length != 1) { System.err.println("Incorrect options for command \"print\""); } else { String varName = pOptions[0].trim(); if (runtime.isAlive()) { if (varName.contains("[")) { // matrix with index: can be cell or column or row try { String variableNameWithoutIndices = varName.split("\\[")[0].trim(); String indexString = (varName.split("\\[")[1].trim()).split("\\]")[0] .trim(); String rowIndexStr = ""; String colIndexStr = ""; if (indexString.startsWith(",")) { colIndexStr = indexString.split(",")[1].trim(); } else if (indexString.endsWith(",")) { rowIndexStr = indexString.split(",")[0].trim(); } else { rowIndexStr = indexString.split(",")[0].trim(); colIndexStr = indexString.split(",")[1].trim(); } int rowIndex = -1; int colIndex = -1; if (!rowIndexStr.isEmpty()) { rowIndex = Integer.parseInt(rowIndexStr); } if (!colIndexStr.isEmpty()) { colIndex = Integer.parseInt(colIndexStr); } dbFunctions.print(currEC.getDebugState().getVariables(), variableNameWithoutIndices, "value", rowIndex, colIndex); } catch (Exception indicesException) { System.err.println( "Incorrect format for \"p\". If you are trying to print matrix variable M, you can use M[1,] or M[,1] or M[1,1] (without spaces)."); } } else { // Print entire matrix dbFunctions.print(currEC.getDebugState().getVariables(), varName, "value", -1, -1); } } else System.err.println( "Runtime has not been started. Try \"r\" or \"s\" to start DML runtime execution."); } } else if (cmd.hasOption("whatis")) { String[] pOptions = cmd.getOptionValues("whatis"); if (pOptions == null || pOptions.length != 1) { System.err.println("Incorrect options for command \"whatis\""); } else { String varName = pOptions[0].trim(); dbFunctions.print(currEC.getDebugState().getVariables(), varName, "metadata", -1, -1); } } else if (cmd.hasOption("set")) { String[] pOptions = cmd.getOptionValues("set"); if (pOptions == null || pOptions.length != 2) { System.err.println("Incorrect options for command \"set\""); } else { try { if (pOptions[0].contains("[")) { String[] paramsToSetMatrix = new String[4]; paramsToSetMatrix[0] = pOptions[0].split("\\[")[0].trim(); String indexString = (pOptions[0].split("\\[")[1].trim()).split("\\]")[0] .trim(); paramsToSetMatrix[1] = indexString.split(",")[0].trim(); paramsToSetMatrix[2] = indexString.split(",")[1].trim(); paramsToSetMatrix[3] = pOptions[1].trim(); dbFunctions.setMatrixCell(currEC.getDebugState().getVariables(), paramsToSetMatrix); } else { dbFunctions.setScalarValue(currEC.getDebugState().getVariables(), pOptions); } } catch (Exception exception1) { System.out.println( "Only scalar variable or a matrix cell available in current frame can be set in current version."); } } } else if (cmd.hasOption("l")) { String[] pOptions = cmd.getOptionValues("l"); String[] argsForRange = new String[2]; int currentPC = 1; if (runtime.isAlive()) { currentPC = currEC.getDebugState().getPC().getLineNumber(); } IntRange range = null; if (pOptions == null) { // Print first 10 lines range = new IntRange(currentPC, Math.min(lines.length, currentPC + 10)); } else if (pOptions.length == 1 && pOptions[0].trim().toLowerCase().equals("all")) { // Print entire program range = new IntRange(1, lines.length); } else if (pOptions.length == 2 && pOptions[0].trim().toLowerCase().equals("next")) { int numLines = 10; try { numLines = Integer.parseInt(pOptions[1]); } catch (Exception e1) { } argsForRange[0] = "" + currentPC; argsForRange[1] = "" + Math.min(lines.length, numLines + currentPC); range = dbFunctions.getRange(argsForRange, lines.length); } else if (pOptions.length == 2 && pOptions[0].trim().toLowerCase().equals("prev")) { int numLines = 10; try { numLines = Integer.parseInt(pOptions[1]); } catch (Exception e1) { } argsForRange[0] = "" + Math.max(1, currentPC - numLines); argsForRange[1] = "" + currentPC; range = dbFunctions.getRange(argsForRange, lines.length); } if (range == null) { System.err.println( "Incorrect usage of command \"l\". Try \"l\" or \"l all\" or \"l next 5\" or \"l prev 5\"."); } else { if (range.getMinimumInteger() > 0) { dbFunctions.printLines(lines, range); } else { System.err.println( "Sorry no lines that can be printed. Try \"l\" or \"l all\" or \"l next 5\" or \"l prev 5\"."); } } } else if (cmd.hasOption("li")) { String[] pOptions = cmd.getOptionValues("li"); String[] argsForRange = new String[2]; int currentPC = 1; if (runtime.isAlive()) { currentPC = currEC.getDebugState().getPC().getLineNumber(); } IntRange range = null; if (pOptions == null) { // Print first 10 lines range = new IntRange(currentPC, Math.min(lines.length, currentPC + 10)); } else if (pOptions.length == 1 && pOptions[0].trim().toLowerCase().equals("all")) { // Print entire program range = new IntRange(1, lines.length); } else if (pOptions.length == 2 && pOptions[0].trim().toLowerCase().equals("next")) { int numLines = 10; try { numLines = Integer.parseInt(pOptions[1]); } catch (Exception e1) { } argsForRange[0] = "" + currentPC; argsForRange[1] = "" + Math.min(lines.length, numLines + currentPC); range = dbFunctions.getRange(argsForRange, lines.length); } else if (pOptions.length == 2 && pOptions[0].trim().toLowerCase().equals("prev")) { int numLines = 10; try { numLines = Integer.parseInt(pOptions[1]); } catch (Exception e1) { } argsForRange[0] = "" + Math.max(1, currentPC - numLines); argsForRange[1] = "" + currentPC; range = dbFunctions.getRange(argsForRange, lines.length); } if (range == null) { System.err.println( "Incorrect usage of command \"li\". Try \"li\" or \"li all\" or \"li next 5\" or \"li prev 5\"."); } else { if (range.getMinimumInteger() > 0) { dbFunctions.printInstructions(lines, dbprog.getDMLInstMap(), range, false); } else { System.err.println( "Sorry no lines that can be printed. Try \"li\" or \"li all\" or \"li next 5\" or \"li prev 5\"."); } } } else if (cmd.hasOption("set_scalar")) { if (!runtime.isAlive()) System.err.println( "Runtime has not been started. Try \"r\" to start DML runtime execution."); else dbFunctions.setScalarValue(currEC.getDebugState().getVariables(), cmd.getOptionValues("set_scalar")); } else if (cmd.hasOption("m")) { String varname = dbFunctions.getValue(cmd.getOptionValues("m")); if (runtime.isAlive()) dbFunctions.printMatrixVariable(currEC.getDebugState().getVariables(), varname); else System.err.println( "Runtime has not been started. Try \"r\" to start DML runtime execution."); } else if (cmd.hasOption("x")) { if (!runtime.isAlive()) System.err.println( "Runtime has not been started. Try \"r\" to start DML runtime execution."); else { dbFunctions.printMatrixCell(currEC.getDebugState().getVariables(), cmd.getOptionValues("x")); } } else if (cmd.hasOption("set_cell")) { if (!runtime.isAlive()) System.err.println( "Runtime has not been started. Try \"r\" to start DML runtime execution."); else { dbFunctions.setMatrixCell(currEC.getDebugState().getVariables(), cmd.getOptionValues("set_cell")); } } else { System.err.println("Undefined command. Try \"help\"."); } //block until runtime suspends execution or terminates //while(runtime.isAlive() && !currEC.getProgram().isStopped()) { wait(300); // To avoid race condition between submitting job and //System.out.println(">> Before while"); while (isRuntimeInstruction && !currEC.getDebugState().canAcceptNextCommand()) { if (quit) { break; } else { wait(300); //wait } } } wait(300); } catch (Exception e) { System.err.println("Error processing debugger command. Try \"help\"."); } } }