List of usage examples for java.util.concurrent Semaphore availablePermits
public int availablePermits()
From source file:com.thoughtworks.go.agent.bootstrapper.AgentBootstrapperTest.java
@Test public void shouldNotDieWhenCreationOfLauncherRaisesException() throws InterruptedException { final Semaphore waitForLauncherCreation = new Semaphore(1); waitForLauncherCreation.acquire();/* www . j av a 2s.com*/ final boolean[] reLaunchWaitIsCalled = new boolean[1]; final AgentBootstrapper bootstrapper = new AgentBootstrapper() { @Override void waitForRelaunchTime() { assertThat(waitTimeBeforeRelaunch, is(0)); reLaunchWaitIsCalled[0] = true; super.waitForRelaunchTime(); } @Override AgentLauncherCreator getLauncherCreator() { return new AgentLauncherCreator() { public AgentLauncher createLauncher() { try { throw new RuntimeException("i bombed"); } finally { if (waitForLauncherCreation.availablePermits() == 0) { waitForLauncherCreation.release(); } } } @Override public void close() { } }; } }; final AgentBootstrapper spyBootstrapper = stubJVMExit(bootstrapper); Thread stopLoopThd = new Thread(new Runnable() { public void run() { try { waitForLauncherCreation.acquire(); } catch (InterruptedException e) { throw new RuntimeException(e); } ReflectionUtil.setField(spyBootstrapper, "loop", false); } }); stopLoopThd.start(); try { spyBootstrapper.go(true, new AgentBootstrapperArgs(new URL("http://" + "ghost-name" + ":" + 3518 + "/go"), null, AgentBootstrapperArgs.SslMode.NONE)); stopLoopThd.join(); } catch (Exception e) { fail("should not have propagated exception thrown while creating launcher"); } assertThat(reLaunchWaitIsCalled[0], is(true)); }
From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessMutexBase.java
@Test public void testReentrantSingleLock() throws Exception { final int THREAD_QTY = 10; CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1)); client.start();//from w w w.j av a 2 s. c om try { final AtomicBoolean hasLock = new AtomicBoolean(false); final AtomicBoolean isFirst = new AtomicBoolean(true); final Semaphore semaphore = new Semaphore(1); final InterProcessLock mutex = makeLock(client); List<Future<Object>> threads = Lists.newArrayList(); ExecutorService service = Executors.newCachedThreadPool(); for (int i = 0; i < THREAD_QTY; ++i) { Future<Object> t = service.submit(new Callable<Object>() { @Override public Object call() throws Exception { semaphore.acquire(); mutex.acquire(); Assert.assertTrue(hasLock.compareAndSet(false, true)); try { if (isFirst.compareAndSet(true, false)) { semaphore.release(THREAD_QTY - 1); while (semaphore.availablePermits() > 0) { Thread.sleep(100); } } else { Thread.sleep(100); } } finally { mutex.release(); hasLock.set(false); } return null; } }); threads.add(t); } for (Future<Object> t : threads) { t.get(); } } finally { client.close(); } }
From source file:de.codecentric.batch.jsr352.CustomJsrJobOperator.java
@Override public long start(String jobName, Properties params) throws JobStartException, JobSecurityException { final JsrXmlApplicationContext batchContext = new JsrXmlApplicationContext(params); batchContext.setValidating(false);/*from w w w . j a va 2 s .co m*/ Resource batchXml = new ClassPathResource("/META-INF/batch.xml"); String jobConfigurationLocation = "/META-INF/batch-jobs/" + jobName + ".xml"; Resource jobXml = new ClassPathResource(jobConfigurationLocation); if (batchXml.exists()) { batchContext.load(batchXml); } if (jobXml.exists()) { batchContext.load(jobXml); } AbstractBeanDefinition beanDefinition = BeanDefinitionBuilder .genericBeanDefinition("org.springframework.batch.core.jsr.JsrJobContextFactoryBean") .getBeanDefinition(); beanDefinition.setScope(BeanDefinition.SCOPE_SINGLETON); batchContext.registerBeanDefinition(JSR_JOB_CONTEXT_BEAN_NAME, beanDefinition); batchContext.setParent(parentContext); try { batchContext.refresh(); } catch (BeanCreationException e) { throw new JobStartException(e); } Assert.notNull(jobName, "The job name must not be null."); final org.springframework.batch.core.JobExecution jobExecution; try { JobParameters jobParameters = jobParametersConverter.getJobParameters(params); String[] jobNames = batchContext.getBeanNamesForType(Job.class); if (jobNames == null || jobNames.length <= 0) { throw new BatchRuntimeException("No Job defined in current context"); } org.springframework.batch.core.JobInstance jobInstance = jobRepository.createJobInstance(jobNames[0], jobParameters); jobExecution = jobRepository.createJobExecution(jobInstance, jobParameters, jobConfigurationLocation); } catch (Exception e) { throw new JobStartException(e); } try { final Semaphore semaphore = new Semaphore(1); final List<Exception> exceptionHolder = Collections.synchronizedList(new ArrayList<Exception>()); semaphore.acquire(); taskExecutor.execute(new Runnable() { @Override public void run() { JsrJobContextFactoryBean factoryBean = null; try { factoryBean = (JsrJobContextFactoryBean) batchContext .getBean("&" + JSR_JOB_CONTEXT_BEAN_NAME); factoryBean.setJobExecution(jobExecution); final AbstractJob job = batchContext.getBean(AbstractJob.class); addListenerToJobService.addListenerToJob(job); semaphore.release(); // Initialization of the JobExecution for job level dependencies jobRegistry.register(job, jobExecution); job.execute(jobExecution); jobRegistry.remove(jobExecution); } catch (Exception e) { exceptionHolder.add(e); } finally { if (factoryBean != null) { factoryBean.close(); } batchContext.close(); if (semaphore.availablePermits() == 0) { semaphore.release(); } } } }); semaphore.acquire(); if (exceptionHolder.size() > 0) { semaphore.release(); throw new JobStartException(exceptionHolder.get(0)); } } catch (Exception e) { if (jobRegistry.exists(jobExecution.getId())) { jobRegistry.remove(jobExecution); } jobExecution.upgradeStatus(BatchStatus.FAILED); if (jobExecution.getExitStatus().equals(ExitStatus.UNKNOWN)) { jobExecution.setExitStatus(ExitStatus.FAILED.addExitDescription(e)); } jobRepository.update(jobExecution); if (batchContext.isActive()) { batchContext.close(); } throw new JobStartException(e); } return jobExecution.getId(); }
From source file:com.amazonaws.services.sqs.buffered.SendQueueBuffer.java
/** * Submits an outbound request for delivery to the queue associated with * this buffer.// w w w. java 2 s . c o m * <p> * * @param operationLock * the lock synchronizing calls for the call type ( * {@code sendMessage}, {@code deleteMessage}, * {@code changeMessageVisibility} ) * @param openOutboundBatchTask * the open batch task for this call type * @param request * the request to submit * @param inflightOperationBatches * the permits controlling the batches for this type of request * @return never null * @throws AmazonClientException * (see the various outbound calls for details) */ @SuppressWarnings("unchecked") <OBT extends OutboundBatchTask<R, Result>, R extends AmazonWebServiceRequest, Result> QueueBufferFuture<R, Result> submitOutboundRequest( Object operationLock, OBT[] openOutboundBatchTask, R request, final Semaphore inflightOperationBatches, QueueBufferCallback<R, Result> callback) { /* * Callers add requests to a single batch task (openOutboundBatchTask) * until it is full or maxBatchOpenMs elapses. The total number of batch * task in flight is controlled by the inflightOperationBatch semaphore * capped at maxInflightOutboundBatches. */ QueueBufferFuture<R, Result> theFuture = null; try { synchronized (operationLock) { if (openOutboundBatchTask[0] == null || ((theFuture = openOutboundBatchTask[0].addRequest(request, callback))) == null) { OBT obt = (OBT) newOutboundBatchTask(request); inflightOperationBatches.acquire(); openOutboundBatchTask[0] = obt; // Register a listener for the event signaling that the // batch task has completed (successfully or not). openOutboundBatchTask[0].onCompleted = new Listener<OutboundBatchTask<R, Result>>() { public void invoke(OutboundBatchTask<R, Result> task) { inflightOperationBatches.release(); } }; if (log.isTraceEnabled()) { log.trace("Queue " + qUrl + " created new batch for " + request.getClass().toString() + " " + inflightOperationBatches.availablePermits() + " free slots remain"); } theFuture = openOutboundBatchTask[0].addRequest(request, callback); executor.execute(openOutboundBatchTask[0]); if (null == theFuture) { //this can happen only if the request itself is flawed, //so that it can't be added to any batch, even a brand //new one throw new AmazonClientException("Failed to schedule request " + request + " for execution"); } } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); AmazonClientException toThrow = new AmazonClientException("Interrupted while waiting for lock."); toThrow.initCause(e); throw toThrow; } return theFuture; }
From source file:com.alibaba.napoli.client.benchmark.NapoliNormalQueueTest.java
@Test public void sendMessageWithSenderStoreEnableTest() throws Exception { log.info("start to execute sendMessageWithSenderStoreEnableTest"); long beginQueueSize = JmxUtil.getQueueSize(sendConnector.getAddress(), queueName); qSender = new DefaultAsyncSender(); qSender.setConnector(sendConnector); qSender.setName(queueName);/* w w w . ja v a 2 s . co m*/ qSender.setStoreEnable(true); qSender.setReprocessInterval(10000 * 1000 * 1000); qSender.init(); int tc = 10; log.info("yanny requestcount = " + System.getProperty("requestCount") + ", begin queue size is " + beginQueueSize); final int tp = Integer.parseInt(System.getProperty("requestCount", "20")); final Semaphore semaphore = new Semaphore(tc); final AtomicInteger sumCount = new AtomicInteger(); final AtomicInteger requestCount = new AtomicInteger(); long startTime = System.currentTimeMillis(); log.info("Yanny start send request " + startTime); for (int i = 0; i < tc; i++) { Thread t = new Thread("thread--" + i) { public void run() { try { //?tringap??Serializable semaphore.acquire(); Person person = new Person(); person.setLoginName("superman"); person.setEmail("sm@1.com"); person.setPenName("pname"); person.setStatus(PersonStatus.ENABLED); for (int j = 0; j < tp; j++) { // log.info("hello"); int id = requestCount.incrementAndGet(); person.setPersonId("" + id); //?? ??true???alse boolean result = qSender.send(person); if (!result) { log.info("----------------send to queue " + "result is false. personid=" + j); } else { sumCount.incrementAndGet(); } } } catch (Throwable t) { t.printStackTrace(); } finally { semaphore.release(); } } }; t.start(); } while (semaphore.availablePermits() != tc) { Thread.sleep(100); } int totalRequest = tc * tp; long endTime = System.currentTimeMillis(); log.info("yanny: send " + totalRequest + " message, take " + (endTime - startTime) + " milseconds"); JmxUtil.waitTillQueueSizeAsTarget(sendConnector.getAddress(), queueName, beginQueueSize); endTime = System.currentTimeMillis(); String errorMessage = ""; long qBdbCount = NapoliTestUtil.getStoreSize(sendConnector.getSenderKVStore(qSender.getName())); log.info("yanny totalRequest " + totalRequest + " send queue success " + sumCount + " local store count:" + qBdbCount + " queue received " + qWorker.getAccessNum() + " take " + (endTime - startTime) + " milseconds"); log.info(initConsumeMessage); log.info("NapoliNormalQueueTest's success=" + qWorker.getAccessNum() + " bdb's size=" + qBdbCount); //with store enabled, all send should succeed. if (qSender.getStoreEnable()) { if (sumCount.get() != totalRequest) { errorMessage += ";with store enabled, all send should return success, but not equal now. send succeed " + sumCount.get() + "; total request:" + totalRequest; } } else { if (sumCount.get() < totalRequest * 0.95) { errorMessage += ";with store disabled, expected more than 95% message send succeed, total request:" + totalRequest + "; send succeed " + sumCount.get(); } } if (sumCount.get() < qWorker.getAccessNum()) { errorMessage += ";queue should not have success messages more than send succeed" + sumCount.get() + " (success " + qWorker.getAccessNum() + ")"; } if ((sumCount.get() - qBdbCount) > qWorker.getAccessNum()) { errorMessage += ";queue received message (" + qWorker.getAccessNum() + ") less than send succeed - local stored message, message lost " + (sumCount.get() - qBdbCount); } int allowedDiff = (int) Math.round(sumCount.get() * 0.001); if (((qWorker.getAccessNum() + qBdbCount) - sumCount.get()) > allowedDiff) { errorMessage += "queue received message should not have more than send succeed + " + allowedDiff + " than allowed (0.1%), gap " + ((qWorker.getAccessNum() + qBdbCount) - sumCount.get()); } assertTrue(errorMessage, errorMessage.equals("")); verify(napoliSenderStat, atMost(qWorker.getAccessNum())).sendSuccess(anyLong(), anyLong()); verify(napoliSenderStat, atLeast((int) (sumCount.get() - qBdbCount))).sendSuccess(anyLong(), anyLong()); verify(napoliSenderStat, times((int) qBdbCount)).sendFailure(anyLong(), anyLong()); verify(napoliReceiverStat, times((int) qWorker.getAccessNum())).receiveSuccess(anyLong(), anyLong()); }
From source file:com.alibaba.napoli.client.benchmark.NapoliNormalQueueTest.java
@Test public void sendMessageWithSenderStoreDisabledTest() throws Exception { log.info("start to execute sendMessageWithSenderStoreDisabledTest"); long beginQueueSize = JmxUtil.getQueueSize(sendConnector.getAddress(), queueName); qSender = new DefaultAsyncSender(); qSender.setConnector(sendConnector); qSender.setName(queueName);/*from w ww. ja v a 2 s . c om*/ qSender.setStoreEnable(false); qSender.setReprocessInterval(10000 * 1000 * 1000); qSender.init(); int tc = 10; log.info("yanny requestcount = " + System.getProperty("requestCount")); final int tp = Integer.parseInt(System.getProperty("requestCount", "20")); final Semaphore semaphore = new Semaphore(tc); final AtomicInteger sumCount = new AtomicInteger(); final AtomicInteger requestCount = new AtomicInteger(); long startTime = System.currentTimeMillis(); log.info("Yanny start send request " + startTime); for (int i = 0; i < tc; i++) { Thread t = new Thread("thread--" + i) { public void run() { try { //?tringap??Serializable semaphore.acquire(); Person person = new Person(); person.setLoginName("superman"); person.setEmail("sm@1.com"); person.setPenName("pname"); person.setStatus(PersonStatus.ENABLED); for (int j = 0; j < tp; j++) { // log.info("hello"); int id = requestCount.incrementAndGet(); person.setPersonId("" + id); //?? ??true???alse boolean result = qSender.send(person); if (!result) { log.info("----------------send to queue " + "result is false. personid=" + j); } else { sumCount.incrementAndGet(); } } } catch (Throwable t) { t.printStackTrace(); } finally { semaphore.release(); } } }; t.start(); } while (semaphore.availablePermits() != tc) { Thread.sleep(100); } int totalRequest = tc * tp; long endTime = System.currentTimeMillis(); log.info("yanny: send " + totalRequest + " message, take " + (endTime - startTime) + " milseconds"); JmxUtil.waitTillQueueSizeAsTarget(sendConnector.getAddress(), queueName, beginQueueSize); endTime = System.currentTimeMillis(); String errorMessage = ""; long qBdbCount = NapoliTestUtil.getStoreSize(sendConnector.getSenderKVStore(qSender.getName())); log.info("yanny totalRequest " + totalRequest + " send queue success " + sumCount + " local store count:" + qBdbCount + " queue received " + qWorker.getAccessNum() + " take " + (endTime - startTime) + " milseconds"); log.info(initConsumeMessage); log.info("NapoliNormalQueueTest's success=" + qWorker.getAccessNum() + " bdb's size=" + qBdbCount); if (qBdbCount > 0) { errorMessage += ";with store disabled, local store count should be empty, but is " + qBdbCount; } //with store enabled, all send should succeed. if (qSender.getStoreEnable()) { if (sumCount.get() != totalRequest) { errorMessage += ";with store enabled, all send should return success, but not equal now. send succeed " + sumCount.get() + "; total request:" + totalRequest; } } else { if (sumCount.get() < totalRequest * 0.95) { errorMessage += ";with store disabled, expected more than 95% message send succeed, total request:" + totalRequest + "; send succeed " + sumCount.get(); } } //?????otalRequest,??>=sum if (totalRequest < qWorker.getAccessNum()) { errorMessage += ";queue should not have success messages more than send succeed" + sumCount.get() + " (success " + qWorker.getAccessNum() + ")"; } //?qBdbCount? if ((sumCount.get() - qBdbCount) > qWorker.getAccessNum()) { errorMessage += ";queue received message (" + qWorker.getAccessNum() + ") less than send succeed - local stored message, message lost " + (sumCount.get() - qBdbCount); } int allowedDiff = (int) Math.round(sumCount.get() * 0.001); if (((qWorker.getAccessNum() + qBdbCount) - sumCount.get()) > allowedDiff) { errorMessage += "queue received message should not have more than send succeed + " + allowedDiff + " than allowed (0.1%), gap " + ((qWorker.getAccessNum() + qBdbCount) - sumCount.get()); } assertTrue(errorMessage, errorMessage.equals("")); verify(napoliSenderStat, atMost(qWorker.getAccessNum())).sendSuccess(anyLong(), anyLong()); verify(napoliSenderStat, atLeast((int) (sumCount.get() - qBdbCount))).sendSuccess(anyLong(), anyLong()); verify(napoliSenderStat, times(totalRequest - sumCount.get())).sendFalse(anyLong(), anyLong()); verify(napoliSenderStat, times((int) qBdbCount)).sendFailure(anyLong(), anyLong()); verify(napoliReceiverStat, times((int) qWorker.getAccessNum())).receiveSuccess(anyLong(), anyLong()); }
From source file:com.netflix.curator.framework.recipes.leader.TestLeaderSelectorCluster.java
@Test public void testLostRestart() throws Exception { final Timing timing = new Timing(); CuratorFramework client = null;//from w w w . j a v a 2 s. c o m TestingCluster cluster = new TestingCluster(3); cluster.start(); try { client = CuratorFrameworkFactory.newClient(cluster.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); client.start(); client.sync("/", null); final AtomicReference<Exception> error = new AtomicReference<Exception>(null); final AtomicReference<String> lockNode = new AtomicReference<String>(null); final Semaphore semaphore = new Semaphore(0); final CountDownLatch lostLatch = new CountDownLatch(1); final CountDownLatch internalLostLatch = new CountDownLatch(1); LeaderSelectorListener listener = new LeaderSelectorListener() { @Override public void takeLeadership(CuratorFramework client) throws Exception { try { List<String> names = client.getChildren().forPath("/leader"); if (names.size() != 1) { semaphore.release(); Exception exception = new Exception("Names size isn't 1: " + names.size()); error.set(exception); return; } lockNode.set(names.get(0)); semaphore.release(); if (!timing.multiple(4).awaitLatch(internalLostLatch)) { error.set(new Exception("internalLostLatch await failed")); } } finally { lostLatch.countDown(); } } @Override public void stateChanged(CuratorFramework client, ConnectionState newState) { if (newState == ConnectionState.LOST) { internalLostLatch.countDown(); } } }; LeaderSelector selector = new LeaderSelector(client, "/leader", listener); selector.start(); Assert.assertTrue(timing.multiple(4).acquireSemaphore(semaphore)); if (error.get() != null) { throw new AssertionError(error.get()); } Collection<InstanceSpec> instances = cluster.getInstances(); cluster.stop(); Assert.assertTrue(timing.multiple(4).awaitLatch(lostLatch)); timing.sleepABit(); Assert.assertFalse(selector.hasLeadership()); Assert.assertNotNull(lockNode.get()); cluster = new TestingCluster(instances.toArray(new InstanceSpec[instances.size()])); cluster.start(); try { client.delete().forPath(ZKPaths.makePath("/leader", lockNode.get())); // simulate the lock deleting due to session expiration } catch (Exception ignore) { // ignore } Assert.assertTrue(semaphore.availablePermits() == 0); Assert.assertFalse(selector.hasLeadership()); selector.requeue(); Assert.assertTrue(timing.multiple(4).acquireSemaphore(semaphore)); } finally { IOUtils.closeQuietly(client); IOUtils.closeQuietly(cluster); } }
From source file:org.apache.kylin.storage.hbase.util.HbaseStreamingInput.java
public static void addData(String tableName) throws IOException { createTable(tableName);/*from ww w . java 2 s.c o m*/ final Semaphore semaphore = new Semaphore(0); new Thread(new Runnable() { @Override public void run() { scheduleJob(semaphore, 300000);//5 minutes a batch } }).start(); while (true) { try { semaphore.acquire(); int waiting = semaphore.availablePermits(); if (waiting > 0) { logger.warn("There are another " + waiting + " batches waiting to be added"); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); e.printStackTrace(); } Connection conn = getConnection(); Table table = conn.getTable(TableName.valueOf(tableName)); byte[] key = new byte[8 + 4];//time + id logger.info("============================================"); long startTime = System.currentTimeMillis(); logger.info("data load start time in millis: " + startTime); logger.info("data load start at " + formatTime(startTime)); List<Put> buffer = Lists.newArrayList(); for (int i = 0; i < (1 << 10); ++i) { long time = System.currentTimeMillis(); Bytes.putLong(key, 0, time); Bytes.putInt(key, 8, i); Put put = new Put(key); byte[] cell = randomBytes(CELL_SIZE); put.addColumn(CF, QN, cell); buffer.add(put); } table.put(buffer); table.close(); conn.close(); long endTime = System.currentTimeMillis(); logger.info("data load end at " + formatTime(endTime)); logger.info("data load time consumed: " + (endTime - startTime)); logger.info("============================================"); } }
From source file:org.apache.solr.request.SimpleFacets.java
License:asdf
/** * Returns a list of value constraints and the associated facet counts * for each facet field specified in the params. * * @see FacetParams#FACET_FIELD//from w ww . j av a 2 s . c om * @see #getFieldMissingCount * @see #getFacetTermEnumCounts */ @SuppressWarnings("unchecked") public NamedList<Object> getFacetFieldCounts() throws IOException, SyntaxError { NamedList<Object> res = new SimpleOrderedMap<>(); String[] facetFs = global.getParams(FacetParams.FACET_FIELD); if (null == facetFs) { return res; } // Passing a negative number for FACET_THREADS implies an unlimited number of threads is acceptable. // Also, a subtlety of directExecutor is that no matter how many times you "submit" a job, it's really // just a method call in that it's run by the calling thread. int maxThreads = req.getParams().getInt(FacetParams.FACET_THREADS, 0); Executor executor = maxThreads == 0 ? directExecutor : facetExecutor; final Semaphore semaphore = new Semaphore((maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads); List<Future<NamedList>> futures = new ArrayList<>(facetFs.length); if (fdebugParent != null) { fdebugParent.putInfoItem("maxThreads", maxThreads); } try { //Loop over fields; submit to executor, keeping the future for (String f : facetFs) { if (fdebugParent != null) { fdebug = new FacetDebugInfo(); fdebugParent.addChild(fdebug); } final ParsedParams parsed = parseParams(FacetParams.FACET_FIELD, f); final SolrParams localParams = parsed.localParams; final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS); final String key = parsed.key; final String facetValue = parsed.facetValue; Callable<NamedList> callable = () -> { try { NamedList<Object> result = new SimpleOrderedMap<>(); if (termList != null) { List<String> terms = StrUtils.splitSmart(termList, ",", true); result.add(key, getListedTermCounts(facetValue, parsed, terms)); } else { result.add(key, getTermCounts(facetValue, parsed)); } return result; } catch (SolrException se) { throw se; } catch (Exception e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Exception during facet.field: " + facetValue, e); } finally { semaphore.release(); } }; RunnableFuture<NamedList> runnableFuture = new FutureTask<>(callable); semaphore.acquire();//may block and/or interrupt executor.execute(runnableFuture);//releases semaphore when done futures.add(runnableFuture); } //facetFs loop //Loop over futures to get the values. The order is the same as facetFs but shouldn't matter. for (Future<NamedList> future : futures) { res.addAll(future.get()); } assert semaphore.availablePermits() >= maxThreads; } catch (InterruptedException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while processing facet fields: InterruptedException", e); } catch (ExecutionException ee) { Throwable e = ee.getCause();//unwrap if (e instanceof RuntimeException) { throw (RuntimeException) e; } throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while processing facet fields: " + e.toString(), e); } return res; }
From source file:org.jumpmind.symmetric.service.impl.DataExtractorService.java
protected OutgoingBatch extractOutgoingBatch(ProcessInfo processInfo, Node targetNode, IDataWriter dataWriter, OutgoingBatch currentBatch, boolean useStagingDataWriter, boolean updateBatchStatistics, ExtractMode mode) {//from ww w . ja va 2 s. c o m if (currentBatch.getStatus() != Status.OK || ExtractMode.EXTRACT_ONLY == mode) { Node sourceNode = nodeService.findIdentity(); TransformWriter transformExtractWriter = null; if (useStagingDataWriter) { long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD); transformExtractWriter = createTransformDataWriter(sourceNode, targetNode, new ProcessInfoDataWriter( new StagingDataWriter(memoryThresholdInBytes, nodeService.findIdentityNodeId(), Constants.STAGING_CATEGORY_OUTGOING, stagingManager), processInfo)); } else { transformExtractWriter = createTransformDataWriter(sourceNode, targetNode, new ProcessInfoDataWriter(dataWriter, processInfo)); } long ts = System.currentTimeMillis(); long extractTimeInMs = 0l; long byteCount = 0l; long transformTimeInMs = 0l; if (currentBatch.getStatus() == Status.IG) { Batch batch = new Batch(BatchType.EXTRACT, currentBatch.getBatchId(), currentBatch.getChannelId(), symmetricDialect.getBinaryEncoding(), sourceNode.getNodeId(), currentBatch.getNodeId(), currentBatch.isCommonFlag()); batch.setIgnored(true); try { IStagedResource resource = getStagedResource(currentBatch); if (resource != null) { resource.delete(); } DataContext ctx = new DataContext(batch); ctx.put("targetNode", targetNode); ctx.put("sourceNode", sourceNode); transformExtractWriter.open(ctx); transformExtractWriter.start(batch); transformExtractWriter.end(batch, false); } finally { transformExtractWriter.close(); } } else if (!isPreviouslyExtracted(currentBatch)) { int maxPermits = parameterService.getInt(ParameterConstants.CONCURRENT_WORKERS); String semaphoreKey = useStagingDataWriter ? Long.toString(currentBatch.getBatchId()) : currentBatch.getNodeBatchId(); Semaphore lock = null; try { synchronized (locks) { lock = locks.get(semaphoreKey); if (lock == null) { lock = new Semaphore(maxPermits); locks.put(semaphoreKey, lock); } try { lock.acquire(); } catch (InterruptedException e) { throw new org.jumpmind.exception.InterruptedException(e); } } synchronized (lock) { if (!isPreviouslyExtracted(currentBatch)) { currentBatch.setExtractCount(currentBatch.getExtractCount() + 1); if (updateBatchStatistics) { changeBatchStatus(Status.QY, currentBatch, mode); } currentBatch.resetStats(); IDataReader dataReader = new ExtractDataReader(symmetricDialect.getPlatform(), new SelectFromSymDataSource(currentBatch, sourceNode, targetNode, processInfo)); DataContext ctx = new DataContext(); ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode); ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_ID, targetNode.getNodeId()); ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_EXTERNAL_ID, targetNode.getExternalId()); ctx.put(Constants.DATA_CONTEXT_TARGET_NODE_GROUP_ID, targetNode.getNodeGroupId()); ctx.put(Constants.DATA_CONTEXT_TARGET_NODE, targetNode); ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE, sourceNode); ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_ID, sourceNode.getNodeId()); ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_EXTERNAL_ID, sourceNode.getExternalId()); ctx.put(Constants.DATA_CONTEXT_SOURCE_NODE_GROUP_ID, sourceNode.getNodeGroupId()); new DataProcessor(dataReader, transformExtractWriter, "extract").process(ctx); extractTimeInMs = System.currentTimeMillis() - ts; Statistics stats = transformExtractWriter.getNestedWriter().getStatistics().values() .iterator().next(); transformTimeInMs = stats.get(DataWriterStatisticConstants.TRANSFORMMILLIS); extractTimeInMs = extractTimeInMs - transformTimeInMs; byteCount = stats.get(DataWriterStatisticConstants.BYTECOUNT); } } } catch (RuntimeException ex) { IStagedResource resource = getStagedResource(currentBatch); if (resource != null) { resource.close(); resource.delete(); } throw ex; } finally { lock.release(); synchronized (locks) { if (lock.availablePermits() == maxPermits) { locks.remove(semaphoreKey); } } } } if (updateBatchStatistics) { long dataEventCount = currentBatch.getDataEventCount(); long insertEventCount = currentBatch.getInsertEventCount(); currentBatch = requeryIfEnoughTimeHasPassed(ts, currentBatch); // preserve in the case of a reload event if (dataEventCount > currentBatch.getDataEventCount()) { currentBatch.setDataEventCount(dataEventCount); } // preserve in the case of a reload event if (insertEventCount > currentBatch.getInsertEventCount()) { currentBatch.setInsertEventCount(insertEventCount); } // only update the current batch after we have possibly // "re-queried" if (extractTimeInMs > 0) { currentBatch.setExtractMillis(extractTimeInMs); } if (byteCount > 0) { currentBatch.setByteCount(byteCount); statisticManager.incrementDataBytesExtracted(currentBatch.getChannelId(), byteCount); statisticManager.incrementDataExtracted(currentBatch.getChannelId(), currentBatch.getExtractCount()); } } } return currentBatch; }