Example usage for java.util.concurrent Semaphore Semaphore

List of usage examples for java.util.concurrent Semaphore Semaphore

Introduction

In this page you can find the example usage for java.util.concurrent Semaphore Semaphore.

Prototype

public Semaphore(int permits) 

Source Link

Document

Creates a Semaphore with the given number of permits and nonfair fairness setting.

Usage

From source file:org.jahia.services.render.filter.cache.LegacyAclCacheKeyPartGenerator.java

@SuppressWarnings("unchecked")
private Map<String, Set<String>> getPrincipalAcl(final String aclKey, final String siteKey)
        throws RepositoryException {

    final String cacheKey = siteKey != null ? aclKey + ":" + siteKey : aclKey;
    Element element = cache.get(cacheKey);
    if (element == null) {
        Semaphore semaphore = processings.get(cacheKey);
        if (semaphore == null) {
            semaphore = new Semaphore(1);
            processings.putIfAbsent(cacheKey, semaphore);
        }/*ww w.ja v  a 2  s .  c  o  m*/
        try {
            semaphore.tryAcquire(500, TimeUnit.MILLISECONDS);
            element = cache.get(cacheKey);
            if (element != null) {
                return (Map<String, Set<String>>) element.getObjectValue();
            }

            logger.debug("Getting ACL for {}", cacheKey);
            long l = System.currentTimeMillis();

            Map<String, Set<String>> map = template.doExecuteWithSystemSessionAsUser(null,
                    Constants.LIVE_WORKSPACE, null, new JCRCallback<Map<String, Set<String>>>() {

                        @Override
                        public Map<String, Set<String>> doInJCR(JCRSessionWrapper session)
                                throws RepositoryException {
                            Query query = session.getWorkspace().getQueryManager()
                                    .createQuery("select * from [jnt:ace] as ace where ace.[j:principal] = '"
                                            + JCRContentUtils.sqlEncode(aclKey) + "'", Query.JCR_SQL2);
                            QueryResult queryResult = query.execute();
                            NodeIterator rowIterator = queryResult.getNodes();

                            Map<String, Set<String>> mapGranted = new ConcurrentHashMap<String, Set<String>>();
                            Map<String, Set<String>> mapDenied = new LinkedHashMap<String, Set<String>>();

                            while (rowIterator.hasNext()) {
                                JCRNodeWrapper node = (JCRNodeWrapper) rowIterator.next();
                                if (siteKey != null && !node.getResolveSite().getName().equals(siteKey)) {
                                    continue;
                                }
                                String path = node.getParent().getParent().getPath();
                                Set<String> foundRoles = new HashSet<String>();
                                boolean granted = node.getProperty("j:aceType").getString().equals("GRANT");
                                Value[] roles = node.getProperty(Constants.J_ROLES).getValues();
                                for (Value r : roles) {
                                    String role = r.getString();
                                    if (!foundRoles.contains(role)) {
                                        foundRoles.add(role);
                                    }
                                }
                                if (path.equals("/")) {
                                    path = "";
                                }
                                if (granted) {
                                    mapGranted.put(path, foundRoles);
                                } else {
                                    mapDenied.put(path, foundRoles);
                                }
                            }
                            for (String deniedPath : mapDenied.keySet()) {
                                String grantedPath = deniedPath;
                                while (grantedPath.length() > 0) {
                                    grantedPath = StringUtils.substringBeforeLast(grantedPath, "/");
                                    if (mapGranted.containsKey(grantedPath)) {
                                        Collection<String> intersection = CollectionUtils.intersection(
                                                mapGranted.get(grantedPath), mapDenied.get(deniedPath));
                                        for (String s : intersection) {
                                            mapGranted.get(grantedPath).add(s + " -> " + deniedPath);
                                        }
                                    }
                                }
                            }
                            return mapGranted;
                        }
                    });
            element = new Element(cacheKey, map);
            element.setEternal(true);
            cache.put(element);
            logger.debug("Getting ACL for {} took {} ms", cacheKey, System.currentTimeMillis() - l);
        } catch (InterruptedException e) {
            logger.debug(e.getMessage(), e);
        } finally {
            semaphore.release();
        }
    }
    return (Map<String, Set<String>>) element.getObjectValue();
}

From source file:com.mtgi.analytics.BehaviorTrackingManagerTest.java

@Test
public void testThreadSafety() throws InterruptedException, SQLException {

    EventGenerator[] threads = new EventGenerator[20];
    Semaphore in = new Semaphore(0), out = new Semaphore(0);
    for (int i = 0; i < threads.length; ++i) {
        threads[i] = new EventGenerator("thread[" + i + "]", in, out);
        threads[i].start();/*from ww  w . j av a 2  s  .  c  o  m*/
    }

    //release the threads to do their work
    in.release(threads.length);
    //wait for all to finish
    out.acquire(threads.length);

    //let them all quiesce.
    for (int i = 0; i < threads.length; ++i) {
        threads[i].join(10000);
        assertFalse("thread[" + i + "] has exited", threads[i].isAlive());
    }

    //make sure lingering autoflushes are done, and perform a manual flush to pick up stragglers.
    flushTaskExecutions();
    manager.flush();
    assertEquals("no uncommitted events remain", 0, manager.getEventsPendingFlush());

    //do an initial count to see how we look.
    ResultSet rs = stmt.executeQuery(
            "select count(event_id) from BEHAVIOR_TRACKING_EVENT where event_type != 'behavior-tracking'");
    assertTrue(rs.next());
    int ret = rs.getInt(1);
    rs.close();
    assertEquals("all threads' events are committed", 39 * threads.length, ret);

    //let each thread verify that all of its data was committed
    for (EventGenerator g : threads)
        g.verifyEvents();
}

From source file:com.liveramp.hank.partition_server.UpdateManager.java

@Override
public void update() throws IOException {
    HankTimer timer = new HankTimer();
    try {/*  ww w. j  av  a  2  s.  com*/

        // Delete unknown files
        deleteUnknownFiles();
        // Perform update
        Semaphore concurrentUpdatesSemaphore = new Semaphore(configurator.getNumConcurrentUpdates());
        List<Throwable> encounteredThrowables = new ArrayList<Throwable>();
        PartitionUpdateTaskStatisticsAggregator partitionUpdateTaskStatisticsAggregator = new PartitionUpdateTaskStatisticsAggregator();
        Map<String, Queue<PartitionUpdateTask>> dataDirectoryToUpdateTasks = new HashMap<String, Queue<PartitionUpdateTask>>();
        List<PartitionUpdateTask> allUpdateTasks = buildPartitionUpdateTasks(
                partitionUpdateTaskStatisticsAggregator, encounteredThrowables);
        // Build and organize update tasks per data directory
        for (PartitionUpdateTask updateTask : allUpdateTasks) {
            String dataDirectory = updateTask.getDataDirectory();
            Queue<PartitionUpdateTask> updateTasks = dataDirectoryToUpdateTasks.get(dataDirectory);
            if (updateTasks == null) {
                updateTasks = new LinkedList<PartitionUpdateTask>();
                dataDirectoryToUpdateTasks.put(dataDirectory, updateTasks);
            }
            updateTasks.add(updateTask);
        }

        // Logging
        LOG.info("Number of update tasks: " + allUpdateTasks.size());
        for (Map.Entry<String, Queue<PartitionUpdateTask>> entry : dataDirectoryToUpdateTasks.entrySet()) {
            LOG.info("Number of update tasks scheduled in " + entry.getKey() + ": " + entry.getValue().size());
        }

        // Build executor services
        Map<String, ExecutorService> dataDirectoryToExecutorService = new HashMap<String, ExecutorService>();
        for (String dataDirectory : dataDirectoryToUpdateTasks.keySet()) {
            dataDirectoryToExecutorService.put(dataDirectory,
                    new UpdateThreadPoolExecutor(configurator.getMaxConcurrentUpdatesPerDataDirectory(),
                            new UpdaterThreadFactory(dataDirectory), concurrentUpdatesSemaphore));
        }

        LOG.info("Submitting update tasks for " + dataDirectoryToUpdateTasks.size() + " directories.");

        // Execute tasks. We execute one task for each data directory and loop around so that the tasks
        // attempt to acquire the semaphore in a reasonable order.
        boolean remaining = true;
        while (remaining) {
            remaining = false;
            for (Map.Entry<String, Queue<PartitionUpdateTask>> entry : dataDirectoryToUpdateTasks.entrySet()) {
                // Pop next task
                Queue<PartitionUpdateTask> partitionUpdateTasks = entry.getValue();
                if (!partitionUpdateTasks.isEmpty()) {
                    PartitionUpdateTask partitionUpdateTask = partitionUpdateTasks.remove();
                    // Execute task
                    dataDirectoryToExecutorService.get(entry.getKey()).execute(partitionUpdateTask);
                }
                if (!partitionUpdateTasks.isEmpty()) {
                    remaining = true;
                }
            }
        }

        LOG.info("All update tasks submitted, shutting down executor services");

        // Shutdown executors
        for (ExecutorService executorService : dataDirectoryToExecutorService.values()) {
            executorService.shutdown();
        }

        LOG.info("Waiting for executors to finish.");

        // Wait for executors to finish
        for (Map.Entry<String, ExecutorService> entry : dataDirectoryToExecutorService.entrySet()) {
            String directory = entry.getKey();
            ExecutorService executorService = entry.getValue();

            boolean keepWaiting = true;
            while (keepWaiting) {
                try {
                    LOG.info("Waiting for updates to complete on data directory: " + directory);
                    boolean terminated = executorService.awaitTermination(
                            UPDATE_EXECUTOR_TERMINATION_CHECK_TIMEOUT_VALUE,
                            UPDATE_EXECUTOR_TERMINATION_CHECK_TIMEOUT_UNIT);
                    if (terminated) {
                        // We finished executing all tasks
                        // Otherwise, timeout elapsed and current thread was not interrupted. Keep waiting.
                        LOG.info("Finished updates for directory: " + directory);
                        keepWaiting = false;
                    }
                    // Record update ETA
                    Hosts.setUpdateETA(host, partitionUpdateTaskStatisticsAggregator.computeETA());
                } catch (InterruptedException e) {
                    // Received interruption (stop request).
                    // Swallow the interrupted state and ask the executor to shutdown immediately. Also, keep waiting.
                    LOG.info(
                            "The update manager was interrupted. Stopping the update process (stop executing new partition update tasks"
                                    + " and wait for those that were running to finish).");
                    // Shutdown all executors
                    for (ExecutorService otherExecutorService : dataDirectoryToExecutorService.values()) {
                        otherExecutorService.shutdownNow();
                    }
                    // Record failed update exception (we need to keep waiting)
                    encounteredThrowables.add(
                            new IOException("Failed to complete update: update interruption was requested."));
                }
            }
        }

        LOG.info("All executors have finished updates");

        // Shutdown all executors
        for (ExecutorService executorService : dataDirectoryToExecutorService.values()) {
            executorService.shutdownNow();
        }

        LOG.info("Finished with " + encounteredThrowables.size() + " errors.");

        // Detect failures
        if (!encounteredThrowables.isEmpty()) {
            LOG.error(String.format("%d exceptions encountered while running partition update tasks:",
                    encounteredThrowables.size()));
            int i = 0;
            for (Throwable t : encounteredThrowables) {
                LOG.error(String.format("Exception %d/%d:", ++i, encounteredThrowables.size()), t);
            }
            throw new IOException(String.format(
                    "Failed to complete update: %d exceptions encountered while running partition update tasks.",
                    encounteredThrowables.size()));
        }

        // Garbage collect useless host domains
        garbageCollectHostDomains(host);

        // Log statistics
        partitionUpdateTaskStatisticsAggregator.logStats();

    } catch (IOException e) {
        LOG.info("Update failed and took " + FormatUtils.formatSecondsDuration(timer.getDurationMs() / 1000));
        throw e;
    }
    LOG.info("Update succeeded and took " + FormatUtils.formatSecondsDuration(timer.getDurationMs() / 1000));
}

From source file:com.netflix.curator.framework.recipes.cache.TestPathChildrenCache.java

@Test
public void testIssue27Alt() throws Exception {
    CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    client.start();/*from   w ww  .java  2  s  . co m*/
    try {
        client.create().forPath("/base");
        client.create().forPath("/base/a");
        client.create().forPath("/base/b");
        client.create().forPath("/base/c");

        client.getChildren().forPath("/base");

        final List<PathChildrenCacheEvent.Type> events = Lists.newArrayList();
        final Semaphore semaphore = new Semaphore(0);
        PathChildrenCache cache = new PathChildrenCache(client, "/base", true);
        cache.getListenable().addListener(new PathChildrenCacheListener() {
            @Override
            public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                events.add(event.getType());
                semaphore.release();
            }
        });
        cache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);

        client.delete().forPath("/base/a");
        Assert.assertTrue(semaphore.tryAcquire(1, 10, TimeUnit.SECONDS));

        client.create().forPath("/base/a");
        Assert.assertTrue(semaphore.tryAcquire(1, 10, TimeUnit.SECONDS));

        List<PathChildrenCacheEvent.Type> expected = Lists.newArrayList(
                PathChildrenCacheEvent.Type.CHILD_REMOVED, PathChildrenCacheEvent.Type.CHILD_ADDED);
        Assert.assertEquals(expected, events);
    } finally {
        client.close();
    }
}

From source file:com.bt.aloha.batchtest.PerformanceBatchTest.java

@Override
public void run() {
    normalizeWeightings();// ww w.jav  a2  s .c  o  m
    try {
        finishSemaphore = new Semaphore(1 - getNumberOfConcurrentStarts());
        String[] beans = new String[getNumberOfConcurrentStarts()];
        BatchTestScenario[] concurrentScenarios = new BatchTestScenario[getNumberOfConcurrentStarts()];
        for (int j = 0; j < getNumberOfConcurrentStarts(); j++) {
            beans[j] = pickScenarioName();

            BatchTestScenario s = (BatchTestScenario) applicationContext.getBean(beans[j]);

            if (s.getBatchTestScenarioResultListener() == null)
                s.setBatchTestScenarioResultListener(this);
            concurrentScenarios[j] = s;
        }

        startTestTime = System.currentTimeMillis();
        for (int j = 0; j < getNumberOfConcurrentStarts(); j++) {
            if (concurrentScenarios[j] == null)
                break;
            RunScenario rs = new RunScenario(concurrentScenarios[j], beans[j], getNumberOfRuns());
            executorService.execute(rs);
        }

        finishSemaphore.acquire();
        endTestTime = System.currentTimeMillis();
        analyze();
    } catch (Throwable t) {
        log.warn("The main thread was interrupted for some reason!");
        log.error(t.getMessage());
    }
}

From source file:com.netflix.curator.framework.recipes.queue.DistributedQueue.java

private void processChildren(List<String> children, long currentVersion) throws Exception {
    final Semaphore processedLatch = new Semaphore(0);
    final boolean isUsingLockSafety = (lockPath != null);
    int min = minItemsBeforeRefresh;
    for (final String itemNode : children) {
        if (Thread.currentThread().isInterrupted()) {
            processedLatch.release(children.size());
            break;
        }/* w  ww. j  ava2 s  .  c om*/

        if (!itemNode.startsWith(QUEUE_ITEM_NAME)) {
            log.warn("Foreign node in queue path: " + itemNode);
            processedLatch.release();
            continue;
        }

        if (min-- <= 0) {
            if (refreshOnWatch && (currentVersion != childrenCache.getData().version)) {
                processedLatch.release(children.size());
                break;
            }
        }

        if (getDelay(itemNode) > 0) {
            processedLatch.release();
            continue;
        }

        executor.execute(new Runnable() {
            @Override
            public void run() {
                try {
                    if (isUsingLockSafety) {
                        processWithLockSafety(itemNode, ProcessType.NORMAL);
                    } else {
                        processNormally(itemNode, ProcessType.NORMAL);
                    }
                } catch (Exception e) {
                    log.error("Error processing message at " + itemNode, e);
                } finally {
                    processedLatch.release();
                }
            }
        });
    }

    processedLatch.acquire(children.size());
}

From source file:org.apache.lucene.server.handlers.BulkCSVAddDocumentHandlerNonBinary.java

@Override
public String handleStreamed(Reader reader, Map<String, List<String>> params) throws Exception {
    char delimChar;
    List<String> delims = params.get("delimChar");
    if (delims != null) {
        if (delims.size() != 1) {
            throw new IllegalArgumentException("delim parameter should only be specified once; got: " + delims);
        }// w  w  w  . j  a  v a2 s  . c  o  m
        if (delims.get(0).length() != 1) {
            throw new IllegalArgumentException("delim must be comma or tab character; got: " + delims.get(0));
        }
        delimChar = delims.get(0).charAt(0);
        if (delimChar != ',' && delimChar != '\t') {
            throw new IllegalArgumentException("delim must be comma or tab character; got: " + delims.get(0));
        }
    } else {
        delimChar = ',';
    }

    if (params.get("indexName") == null) {
        throw new IllegalArgumentException("required parameter \"indexName\" is missing");
    }

    if (params.get("indexName").size() != 1) {
        throw new IllegalArgumentException("only one \"indexName\" value is allowed");
    }

    String indexName = params.get("indexName").get(0);

    // Make sure the index does in fact exist
    IndexState indexState = globalState.get(indexName);

    // Make sure the index is started:
    if (indexState.isStarted() == false) {
        throw new IllegalArgumentException("index \"" + indexName + "\" isn't started: cannot index documents");
    }

    ShardState shardState = indexState.getShard(0);

    // parse fields header and lookup fields:
    List<FieldDef> fieldsList = new ArrayList<>();
    StringBuilder curField = new StringBuilder();
    long globalOffset = 0;

    char[] buffer = new char[1024];
    int bufferUpto = 0;
    int bufferLimit = 0;
    while (true) {
        if (bufferUpto == bufferLimit) {
            globalOffset += bufferLimit;
            bufferLimit = reader.read(buffer, 0, buffer.length);
            if (bufferLimit == -1) {
                throw new IllegalArgumentException("hit end while parsing header");
            }
            bufferUpto = 0;
        }
        char c = buffer[bufferUpto++];
        if (c == delimChar) {
            fieldsList.add(indexState.getField(curField.toString()));
            curField.setLength(0);
        } else if (c == CSVParserChars.NEWLINE) {
            fieldsList.add(indexState.getField(curField.toString()));
            break;
        } else {
            curField.append(c);
        }
    }

    FieldDef[] fields = fieldsList.toArray(new FieldDef[fieldsList.size()]);

    ShardState.IndexingContext ctx = new ShardState.IndexingContext();
    char[] bufferOld = buffer;

    // nocommit tune this .. core count?

    // Use this to limit how many in-flight 256 KB chunks we allow into the JVM at once:

    // nocommit this should be in GlobalState so it's across all incoming indexing:
    Semaphore semaphore = new Semaphore(64);

    boolean done = false;

    // create first chunk buffer, and carry over any leftovers from the header processing:
    buffer = new char[CHUNK_SIZE_KB * 1024 / 2];
    System.arraycopy(bufferOld, bufferUpto, buffer, 0, bufferLimit - bufferUpto);
    bufferUpto = bufferLimit - bufferUpto;

    globalOffset += bufferUpto;

    ParseAndIndexOneChunk prev = null;
    int phase = ctx.inFlightChunks.getPhase();

    while (done == false && ctx.getError() == null) {
        int count = reader.read(buffer, bufferUpto, buffer.length - bufferUpto);
        if (count == -1 || bufferUpto + count == buffer.length) {
            if (count != -1) {
                bufferUpto += count;
            }
            if (bufferUpto < buffer.length) {
                char[] realloc = new char[bufferUpto];
                System.arraycopy(buffer, 0, realloc, 0, bufferUpto);
                buffer = realloc;
            }
            // NOTE: This ctor will stall when it tries to acquire the semaphore if we already have too many in-flight indexing chunks:
            prev = new ParseAndIndexOneChunk(delimChar, globalOffset, ctx, prev, shardState, fields, buffer,
                    semaphore);
            globalState.submitIndexingTask(prev);
            if (count == -1) {
                // the end
                prev.setNextStartFragment(new char[0], 0, 0);
                done = true;
                break;
            } else {
                globalOffset += buffer.length;
                // not done yet, make the next buffer:
                buffer = new char[CHUNK_SIZE_KB * 1024 / 2];
                bufferUpto = 0;
            }
        } else {
            bufferUpto += count;
        }
    }

    if (done == false) {
        // we exited loop due to error; force last indexing chunk to finish up:
        prev.setNextStartFragment(new char[0], 0, 0);
    }

    // Wait for all chunks to finish indexing:
    ctx.inFlightChunks.awaitAdvance(phase);

    Throwable t = ctx.getError();
    if (t != null) {
        IOUtils.reThrow(t);
        return null;
    } else {
        JSONObject o = new JSONObject();
        o.put("indexGen", shardState.writer.getMaxCompletedSequenceNumber());
        o.put("indexedDocumentCount", ctx.addCount.get());
        return o.toString();
    }
}

From source file:org.apache.lucene.server.handlers.BulkAddDocumentHandler.java

@Override
public String handleStreamed(Reader reader, Map<String, List<String>> params) throws Exception {

    if (params.get("indexName") == null) {
        throw new IllegalArgumentException("required parameter \"indexName\" is missing");
    }//from  ww  w .  j  a  va 2s . co  m

    if (params.get("indexName").size() != 1) {
        throw new IllegalArgumentException("only one \"indexName\" value is allowed");
    }

    String indexName = params.get("indexName").get(0);

    // Make sure the index does in fact exist
    IndexState indexState = globalState.get(indexName);
    ShardState shardState = indexState.getShard(0);

    // Make sure the index is started:
    if (indexState.isStarted() == false) {
        throw new IllegalArgumentException("index \"" + indexName + "\" isn't started: cannot index documents");
    }

    ShardState.IndexingContext ctx = new ShardState.IndexingContext();

    // nocommit tune this .. core count?

    // Use this to limit how many in-flight 256 KB chunks we allow into the JVM at once:

    // nocommit this should be in GlobalState so it's across all incoming indexing:
    Semaphore semaphore = new Semaphore(64);

    boolean done = false;

    // create first chunk buffer, and carry over any leftovers from the header processing:
    char[] buffer = new char[CHUNK_SIZE_KB * 1024 / 2];
    int bufferUpto = 0;

    long globalOffset = 0;

    ParseAndIndexOneChunk prev = null;
    int phase = ctx.inFlightChunks.getPhase();

    AddDocumentHandler addDocHandler = (AddDocumentHandler) globalState.getHandler("addDocument");

    while (done == false && ctx.getError() == null) {
        int count = reader.read(buffer, bufferUpto, buffer.length - bufferUpto);
        if (count == -1 || bufferUpto + count == buffer.length) {
            if (count != -1) {
                bufferUpto += count;
            } else if (bufferUpto < buffer.length) {
                char[] realloc = new char[bufferUpto];
                System.arraycopy(buffer, 0, realloc, 0, bufferUpto);
                buffer = realloc;
            }
            // NOTE: This ctor will stall when it tries to acquire the semaphore if we already have too many in-flight indexing chunks:
            prev = new ParseAndIndexOneChunk(globalOffset, ctx, prev, indexState, buffer, semaphore,
                    addDocHandler);
            globalState.submitIndexingTask(prev);
            if (count == -1) {
                // the end
                prev.setNextStartFragment(new char[0], 0, 0);
                done = true;
                break;
            } else {
                globalOffset += buffer.length;
                // not done yet, make the next buffer:
                buffer = new char[CHUNK_SIZE_KB * 1024 / 2];
                bufferUpto = 0;
            }
        } else {
            bufferUpto += count;
        }
    }

    if (done == false) {
        // we exited loop due to error; force last indexing chunk to finish up:
        prev.setNextStartFragment(new char[0], 0, 0);
    }

    // Wait for all chunks to finish indexing:
    ctx.inFlightChunks.awaitAdvance(phase);

    Throwable t = ctx.getError();
    if (t != null) {
        IOUtils.reThrow(t);
        return null;
    } else {
        JSONObject o = new JSONObject();
        o.put("indexGen", shardState.writer.getMaxCompletedSequenceNumber());
        o.put("indexedDocumentCount", ctx.addCount.get());
        return o.toString();
    }
}

From source file:org.commoncrawl.service.queryserver.query.DomainListQuery.java

@Override
protected long executeRemote(final FileSystem fileSystem, final Configuration conf, EventLoop eventLoop,
        SlaveDatabaseIndex instanceIndex, File tempFirDir,
        QueryProgressCallback<DomainListQueryInfo, Text, SubDomainMetadata> progressCallback)
        throws IOException {

    int shardsProcessed = 0;

    // ok create a semaphore for the number of shard we are going to query ...
    final Semaphore semaphore = new Semaphore(-(getCommonQueryInfo().getRelevantShardIds().size() - 1));
    // and create a record count array 
    final long recordCounts[] = new long[getCommonQueryInfo().getRelevantShardIds().size()];
    final IOException exceptions[] = new IOException[getCommonQueryInfo().getRelevantShardIds().size()];

    int threadIdx = 0;
    // ok dispatch queries for each shard we are responsible for ... 
    for (int shardId : getCommonQueryInfo().getRelevantShardIds()) {

        final int currentShardId = shardId;
        final int currentThreadIdx = threadIdx++;

        Thread subQueryThread = new Thread(new Runnable() {

            @Override//from  w  ww  .j  ava  2  s .c  om
            public void run() {
                Path shardOutputPath = getHDFSQueryResultsFilePathForShard(currentShardId);

                LOG.info("Execute Remote for Query:" + getQueryId() + " for shardId:" + currentShardId
                        + "  Creating spill file @:" + shardOutputPath);

                try {
                    // create SequenceFile Spill Writer ... 
                    SequenceFileSpillWriter<Text, SubDomainMetadata> spillWriter = new SequenceFileSpillWriter<Text, SubDomainMetadata>(
                            fileSystem, conf, shardOutputPath, Text.class, SubDomainMetadata.class, null, true);
                    try {
                        LOG.info("Execute Remote for Query:" + getQueryId()
                                + " calling executeDomainListQuery on index");
                        // scan index for matching patterns ... spill into writer ...
                        recordCounts[currentThreadIdx] += _slaveDatabaseIndex.queryDomainsGivenPattern(
                                getQueryData().getSearchPattern(), currentShardId, spillWriter);
                        LOG.info("Execute Remote for Query:" + getQueryId()
                                + " executeDomainListQuery returned:" + recordCounts[currentThreadIdx]);
                    } finally {
                        spillWriter.close();
                        // increment semaphore count 
                        semaphore.release();
                    }
                } catch (IOException e) {
                    LOG.error("Execute Remote for Query:" + getQueryId()
                            + " executeDomainListQuery failed with error:"
                            + CCStringUtils.stringifyException(e));
                    exceptions[currentThreadIdx] = e;
                }
            }
        });
        subQueryThread.start();
    }

    // ok block until all queries are complete
    LOG.info("Query:" + getQueryId() + " Waiting on Worker Threads");
    semaphore.acquireUninterruptibly();
    LOG.info("Query:" + getQueryId() + " All Threads Compelted");

    for (IOException e : exceptions) {
        if (e != null) {
            LOG.error(
                    "Query:" + getQueryId() + " Failed with Exception:" + CCStringUtils.stringifyException(e));
            throw e;
        }
    }
    long cumilativeRecordCount = 0L;
    for (long recordCount : recordCounts)
        cumilativeRecordCount += recordCount;
    return cumilativeRecordCount;
}

From source file:com.thoughtworks.go.server.service.BackupServiceIntegrationTest.java

@Test
public void shouldReturnBackupStartedBy() throws InterruptedException {
    assertThat(backupService.backupStartedBy(), is(Optional.empty()));

    final Semaphore waitForBackupToStart = new Semaphore(1);
    final Semaphore waitForAssertionToCompleteWhileBackupIsOn = new Semaphore(1);
    final BackupUpdateListener backupUpdateListener = new BackupUpdateListener() {
        private boolean backupStarted = false;

        @Override//www  . j  av a2 s.c  om
        public void updateStep(BackupProgressStatus step) {
            if (!backupStarted) {
                backupStarted = true;
                waitForBackupToStart.release();
                try {
                    waitForAssertionToCompleteWhileBackupIsOn.acquire();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
        }

        @Override
        public void error(String message) {
        }

        @Override
        public void completed() {
        }
    };

    waitForAssertionToCompleteWhileBackupIsOn.acquire();
    waitForBackupToStart.acquire();
    Thread backupThd = new Thread(() -> backupService.startBackup(admin, backupUpdateListener));

    backupThd.start();
    waitForBackupToStart.acquire();
    String backupStartedBy = backupService.backupStartedBy().get();
    ServerBackup runningBackup = (ServerBackup) ReflectionUtil.getField(backupService, "runningBackup");

    assertThat(runningBackup.getUsername(), is(backupStartedBy));
    waitForAssertionToCompleteWhileBackupIsOn.release();
    backupThd.join();
}