Example usage for java.util.concurrent ExecutorCompletionService submit

List of usage examples for java.util.concurrent ExecutorCompletionService submit

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. ExecutorCompletionService
  6. submit

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorCompletionService submit.

Prototype

public Future<V> submit(Callable<V> task)

Source Link

Usage

From source file:com.alibaba.otter.manager.biz.monitor.impl.GlobalMonitor.java

private void concurrentProcess(Map<Long, List<AlarmRule>> rules) {
    ExecutorCompletionService completionExecutor = new ExecutorCompletionService(executor);
    List<Future> futures = new ArrayList<Future>();
    for (Entry<Long, List<AlarmRule>> entry : rules.entrySet()) {
        final List<AlarmRule> alarmRules = entry.getValue();
        futures.add(completionExecutor.submit(new Callable<Object>() {

            @Override/*  w  w w.ja  va2s  . c o  m*/
            public Object call() throws Exception {
                pipelineMonitor.explore(alarmRules);
                return null;
            }
        }));
    }

    List<Throwable> exceptions = new ArrayList<Throwable>();
    int index = 0;
    int size = futures.size();
    while (index < size) {
        try {
            Future<?> future = completionExecutor.take();
            future.get();
        } catch (InterruptedException e) {
            exceptions.add(e);
        } catch (ExecutionException e) {
            exceptions.add(e);
        }
        index++;
    }

    if (!exceptions.isEmpty()) {
        StringBuilder sb = new StringBuilder(exceptions.size() + " exception happens in global monitor\n");
        sb.append("exception stack start :\n");
        for (Throwable t : exceptions) {
            sb.append(ExceptionUtils.getStackTrace(t));
        }
        sb.append("exception stack end \n");
        throw new IllegalStateException(sb.toString());
    }
}

From source file:com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl.java

public Object call(final String[] addrs, final Event event) {
    Assert.notNull(this.factory, "No factory specified");
    if (addrs == null || addrs.length == 0) {
        throw new IllegalArgumentException("addrs example: 127.0.0.1:1099");
    }//from   w w  w  . j  av  a2s. co  m

    ExecutorCompletionService completionService = new ExecutorCompletionService(executor);
    List<Future<Object>> futures = new ArrayList<Future<Object>>(addrs.length);
    List result = new ArrayList(10);
    for (final String addr : addrs) {
        futures.add(completionService.submit((new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                return DefaultCommunicationClientImpl.this.call(addr, event);
            }
        })));
    }

    Exception ex = null;
    int errorIndex = 0;
    while (errorIndex < futures.size()) {
        try {
            Future future = completionService.take();// ?
            future.get();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            ex = e;
            break;
        } catch (ExecutionException e) {
            ex = e;
            break;
        }

        errorIndex++;
    }

    if (errorIndex < futures.size()) {
        for (int index = 0; index < futures.size(); index++) {
            Future<Object> future = futures.get(index);
            if (future.isDone() == false) {
                future.cancel(true);
            }
        }
    } else {
        for (int index = 0; index < futures.size(); index++) {
            Future<Object> future = futures.get(index);
            try {
                result.add(future.get());
            } catch (InterruptedException e) {
                // ignore
                Thread.currentThread().interrupt();
            } catch (ExecutionException e) {
                // ignore
            }
        }
    }

    if (ex != null) {
        throw new CommunicationException(
                String.format("call addr[%s] error by %s", addrs[errorIndex], ex.getMessage()), ex);
    } else {
        return result;
    }
}

From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessMutexBase.java

@Test
public void testWaitingProcessKilledServer() throws Exception {
    final Timing timing = new Timing();
    final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(),
            new RetryOneTime(1));
    try {/*from ww w.ja  va  2  s  . c  o m*/
        client.start();

        final CountDownLatch latch = new CountDownLatch(1);
        ConnectionStateListener listener = new ConnectionStateListener() {
            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {
                if (newState == ConnectionState.LOST) {
                    latch.countDown();
                }
            }
        };
        client.getConnectionStateListenable().addListener(listener);

        final AtomicBoolean isFirst = new AtomicBoolean(true);
        ExecutorCompletionService<Object> service = new ExecutorCompletionService<Object>(
                Executors.newFixedThreadPool(2));
        for (int i = 0; i < 2; ++i) {
            service.submit(new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                    InterProcessLock lock = makeLock(client);
                    lock.acquire();
                    try {
                        if (isFirst.compareAndSet(true, false)) {
                            timing.sleepABit();

                            server.stop();
                            Assert.assertTrue(timing.awaitLatch(latch));
                            server = new TestingServer(server.getPort(), server.getTempDirectory());
                        }
                    } finally {
                        try {
                            lock.release();
                        } catch (Exception e) {
                            // ignore
                        }
                    }
                    return null;
                }
            });
        }

        for (int i = 0; i < 2; ++i) {
            service.take().get(timing.forWaiting().milliseconds(), TimeUnit.MILLISECONDS);
        }
    } finally {
        IOUtils.closeQuietly(client);
    }
}

From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessSemaphore.java

@Test
public void testReleaseInChunks() throws Exception {
    final int MAX_LEASES = 11;
    final int THREADS = 100;

    final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(),
            new RetryOneTime(1));
    client.start();//from   w w  w  .  j a  v  a2s .  c o  m
    try {
        final Stepper latch = new Stepper();
        final Random random = new Random();
        final Counter counter = new Counter();
        ExecutorService service = Executors.newCachedThreadPool();
        ExecutorCompletionService<Object> completionService = new ExecutorCompletionService<Object>(service);
        for (int i = 0; i < THREADS; ++i) {
            completionService.submit(new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                    InterProcessSemaphoreV2 semaphore = new InterProcessSemaphoreV2(client, "/test",
                            MAX_LEASES);
                    Lease lease = semaphore.acquire(10, TimeUnit.SECONDS);
                    if (lease == null) {
                        throw new Exception("timed out");
                    }
                    try {
                        synchronized (counter) {
                            ++counter.currentCount;
                            if (counter.currentCount > counter.maxCount) {
                                counter.maxCount = counter.currentCount;
                            }
                            counter.notifyAll();
                        }

                        latch.await();
                    } finally {
                        synchronized (counter) {
                            --counter.currentCount;
                        }
                        semaphore.returnLease(lease);
                    }
                    return null;
                }
            });
        }

        int remaining = THREADS;
        while (remaining > 0) {
            int times = Math.min(random.nextInt(5) + 1, remaining);
            latch.countDown(times);
            remaining -= times;
            Thread.sleep(random.nextInt(100) + 1);
        }

        for (int i = 0; i < THREADS; ++i) {
            completionService.take();
        }

        synchronized (counter) {
            Assert.assertTrue(counter.currentCount == 0);
            Assert.assertTrue(counter.maxCount > 0);
            Assert.assertTrue(counter.maxCount <= MAX_LEASES);
            System.out.println(counter.maxCount);
        }
    } finally {
        client.close();
    }
}

From source file:com.alibaba.otter.node.etl.load.loader.db.FileLoadAction.java

/**
 * ? fast-fail // w  ww  .j  a  v a2s .c  om
 */
private void moveFiles(FileLoadContext context, List<FileData> fileDatas, File rootDir) {
    Exception exception = null;
    adjustPoolSize(context);
    ExecutorCompletionService<Exception> executorComplition = new ExecutorCompletionService<Exception>(
            executor);

    List<Future<Exception>> results = new ArrayList<Future<Exception>>();
    for (FileData fileData : fileDatas) {
        Future<Exception> future = executorComplition.submit(new FileLoadWorker(context, rootDir, fileData));
        results.add(future);

        // fast fail
        if (future.isDone()) { // ( CallerRunsPolicy)
            try {
                exception = future.get();
            } catch (Exception e) {
                exception = e;
            }
            if (exception != null) {
                for (Future<Exception> result : results) {
                    if (!result.isDone() && !result.isCancelled()) {
                        result.cancel(true);
                    }
                }
                throw exception instanceof LoadException ? (LoadException) exception
                        : new LoadException(exception);
            }
        }

    }

    int resultSize = results.size();
    int cursor = 0;
    while (cursor < resultSize) {
        try {
            Future<Exception> result = executorComplition.take();
            exception = result.get();
        } catch (Exception e) {
            exception = e;
            break;
        }
        cursor++;
    }

    if (cursor != resultSize) { // ??
        for (Future<Exception> future : results) {
            if (!future.isDone() && !future.isCancelled()) {
                future.cancel(true);
            }
        }

    }

    if (exception != null) {
        throw exception instanceof LoadException ? (LoadException) exception : new LoadException(exception);
    }
}

From source file:com.netflix.curator.framework.recipes.leader.TestLeaderLatch.java

@Test
public void testWaiting() throws Exception {
    final int PARTICIPANT_QTY = 10;

    ExecutorService executorService = Executors.newFixedThreadPool(PARTICIPANT_QTY);
    ExecutorCompletionService<Void> service = new ExecutorCompletionService<Void>(executorService);

    final Timing timing = new Timing();
    final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(),
            timing.session(), timing.connection(), new RetryOneTime(1));
    try {// w w  w. jav  a  2 s.  co m
        client.start();

        final AtomicBoolean thereIsALeader = new AtomicBoolean(false);
        for (int i = 0; i < PARTICIPANT_QTY; ++i) {
            service.submit(new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    LeaderLatch latch = new LeaderLatch(client, PATH_NAME);
                    try {
                        latch.start();
                        Assert.assertTrue(latch.await(timing.forWaiting().seconds(), TimeUnit.SECONDS));
                        Assert.assertTrue(thereIsALeader.compareAndSet(false, true));
                        Thread.sleep((int) (10 * Math.random()));
                    } finally {
                        thereIsALeader.set(false);
                        latch.close();
                    }
                    return null;
                }
            });
        }

        for (int i = 0; i < PARTICIPANT_QTY; ++i) {
            service.take().get();
        }
    } finally {
        executorService.shutdown();
        IOUtils.closeQuietly(client);
    }
}

From source file:com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl.CallbackSearchServiceImpl.java

/**
 * @param context ?/*from   ww  w . j a v  a  2  s . c o m*/
 * @param query ?
 * @return 
 * @throws IndexAndSearchException exception occurred when 
 */
public SearchIndexResultSet query(QueryContext context, QueryRequest query) throws IndexAndSearchException {
    LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_BEGIN, "callbackquery",
            "[callbackquery:" + query + "]"));
    if (query == null || context == null || StringUtils.isEmpty(query.getCubeId())) {
        LOGGER.error(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_EXCEPTION, "callbackquery",
                "[callbackquery:" + query + "]"));
        throw new IndexAndSearchException(
                TesseractExceptionUtils.getExceptionMessage(IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                        IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION),
                IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION);
    }
    // TODO ???
    if (query.getGroupBy() == null || query.getSelect() == null) {
        return null;
    }
    Map<String, String> requestParams = ((QueryContextAdapter) context).getQuestionModel().getRequestParams();
    // Build query target map
    Map<String, List<MiniCubeMeasure>> callbackMeasures = context.getQueryMeasures().stream()
            .filter(m -> m.getType().equals(MeasureType.CALLBACK)).map(m -> {
                CallbackMeasure tmp = (CallbackMeasure) m;
                for (Map.Entry<String, String> entry : tmp.getCallbackParams().entrySet()) {
                    if (requestParams.containsKey(entry.getKey())) {
                        tmp.getCallbackParams().put(entry.getKey(), requestParams.get(entry.getKey()));
                    }
                }
                return m;
            }).collect(Collectors.groupingBy(c -> ((CallbackMeasure) c).getCallbackUrl(), Collectors.toList()));
    if (callbackMeasures == null || callbackMeasures.isEmpty()) {
        LOGGER.error(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_EXCEPTION, "Empty callback measure",
                "[callbackquery:" + query + "]"));
        throw new IndexAndSearchException(
                TesseractExceptionUtils.getExceptionMessage(IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                        IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION),
                IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION);
    }
    LOGGER.info("Find callback targets " + callbackMeasures);

    // Keep group-by sequence.
    List<String> groupby = new ArrayList<String>(query.getGroupBy().getGroups());
    LinkedHashMap<String, List<String>> groupbyParams = new LinkedHashMap<String, List<String>>(groupby.size());
    for (String g : groupby) {
        groupbyParams.put(g, new ArrayList<String>());
    }

    LinkedHashMap<String, List<String>> whereParams = new LinkedHashMap<String, List<String>>();
    for (Expression e : query.getWhere().getAndList()) {
        List<String> l = e.getQueryValues().stream().filter(v -> !StringUtils.isEmpty(v.getValue()))
                .map(v -> v.getValue()).collect(Collectors.toList());
        if (groupbyParams.containsKey(e.getProperties())) {
            // if not contains SUMMARY_KEY, add it into group by list
            if (!l.contains(TesseractConstant.SUMMARY_KEY)) {
                l.add(TesseractConstant.SUMMARY_KEY);
            }
            // Put it into group by field
            groupbyParams.get(e.getProperties()).addAll(l);
        } else {
            // Put it into filter field
            if (CollectionUtils.isEmpty(l)) {
                List<Set<String>> tmp = e.getQueryValues().stream().map(v -> v.getLeafValues())
                        .collect(Collectors.toList());
                List<String> values = Lists.newArrayList();
                tmp.forEach(t -> values.addAll(t));
                whereParams.put(e.getProperties(), values);
            } else {
                whereParams.put(e.getProperties(), new ArrayList<String>(l));
            }
        }
    }

    // Prepare query tools
    //        CountDownLatch latch = new CountDownLatch(response.size());
    //        List<Future<CallbackResponse>> results = Lists.newArrayList();
    Map<CallbackExecutor, Future<CallbackResponse>> results = Maps.newHashMap();
    ExecutorCompletionService<CallbackResponse> service = new ExecutorCompletionService<CallbackResponse>(
            taskExecutor);
    StringBuilder callbackMeasureNames = new StringBuilder();
    for (Entry<String, List<MiniCubeMeasure>> e : callbackMeasures.entrySet()) {
        CallbackExecutor ce = new CallbackExecutor(e, groupbyParams, whereParams);
        results.put(ce, service.submit(ce));
        e.getValue().forEach(m -> {
            callbackMeasureNames.append(" " + m.getCaption() + " ");
        });
    }
    //        }
    Map<CallbackExecutor, CallbackResponse> response = new ConcurrentHashMap<CallbackExecutor, CallbackResponse>(
            callbackMeasures.size());
    StringBuffer sb = new StringBuffer();
    results.forEach((k, v) -> {
        try {
            response.put(k, v.get());
        } catch (Exception e1) {
            LOGGER.error(e1.getMessage(), e1);
            sb.append(": " + callbackMeasureNames.toString()
                    + " ??, ?");
        }
    });
    if (!StringUtils.isEmpty(sb.toString())) {
        if (ThreadLocalPlaceholder.getProperty(ThreadLocalPlaceholder.ERROR_MSG_KEY) != null) {
            ThreadLocalPlaceholder.unbindProperty(ThreadLocalPlaceholder.ERROR_MSG_KEY);
        }
        ThreadLocalPlaceholder.bindProperty(ThreadLocalPlaceholder.ERROR_MSG_KEY, sb.toString());
    }
    // Package result
    SqlQuery sqlQuery = QueryRequestUtil.transQueryRequest2SqlQuery(query);
    SearchIndexResultSet result = null;
    if (!response.isEmpty()) {
        result = packageResultRecords(query, sqlQuery, response);
    } else {
        result = new SearchIndexResultSet(new Meta(query.getGroupBy().getGroups().toArray(new String[0])), 0);
    }

    LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_END, "query", "[query:" + query + "]"));
    return result;
}

From source file:com.splout.db.hadoop.GeneratorCMD.java

public int run(String[] args) throws Exception {
    JCommander jComm = new JCommander(this);
    jComm.setProgramName(//from   w  w w.  jav  a2  s  .c o  m
            "Splout Tablespaces Generator. Generates tablespaces, ready to be deployed to a Splout Cluster.");
    try {
        jComm.parse(args);
    } catch (Throwable t) {
        t.printStackTrace();
        jComm.usage();
        return -1;
    }

    if (parallelism < 1) {
        System.err.println("Parallelism must be greater than 0.");
        System.exit(1);
    }

    log.info("Parsing input parameters...");

    // All the tablespaces that will be generated and deployed atomically, hashed by their name
    // We generate this first so we can detect errors in the configuration before even using Hadoop
    Map<String, TablespaceSpec> tablespacesToGenerate = new HashMap<String, TablespaceSpec>();

    // Partition maps to reuse at indexation. Used when sampling is skipped.
    final Map<String, PartitionMap> partitionMapsToReuse = new HashMap<String, PartitionMap>();

    for (String tablespaceFile : tablespaceFiles) {
        Path file = new Path(tablespaceFile);
        FileSystem fS = FileSystem.get(file.toUri(), getConf());

        if (!fS.exists(file)) {
            throw new IllegalArgumentException("Config input file: " + file + " doesn't exist!");
        }

        String strContents = HadoopUtils.fileToString(fS, file);
        JSONTablespaceDefinition def = JSONSerDe.deSer(strContents, JSONTablespaceDefinition.class);
        TablespaceSpec spec = def.build(conf);
        String name = def.getName();

        tablespacesToGenerate.put(name, spec);

        // Reusing partition maps?
        if (qnodeURL != null) {
            partitionMapsToReuse.put(name, retrievePartitionMapfromQNode(name));
        }
    }

    if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) {
        File nativeLibs = new File("native");
        if (nativeLibs.exists()) {
            SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf);
        }
    }

    Path out = new Path(output);
    FileSystem outFs = out.getFileSystem(getConf());
    HadoopUtils.deleteIfExists(outFs, out);

    ExecutorService executor = Executors.newFixedThreadPool(parallelism);
    ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(executor);
    ArrayList<Future<Boolean>> generatorFutures = new ArrayList<Future<Boolean>>();

    // Generate each tablespace
    for (final Map.Entry<String, TablespaceSpec> tablespace : tablespacesToGenerate.entrySet()) {
        Path tablespaceOut = new Path(out, tablespace.getKey());
        TablespaceSpec spec = tablespace.getValue();

        log.info("Generating view with Hadoop (" + tablespace.getKey() + ")");
        final TablespaceGenerator viewGenerator = new TablespaceGenerator(spec, tablespaceOut, this.getClass());

        generatorFutures.add(ecs.submit(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                if (qnodeURL == null) {
                    viewGenerator.generateView(conf, samplingType, new TupleSampler.RandomSamplingOptions());
                    return true;
                } else {
                    viewGenerator.generateView(conf, partitionMapsToReuse.get(tablespace.getKey()));
                    return true;
                }
            }
        }));
    }

    // Waiting all tasks to finish.
    for (int i = 0; i < tablespacesToGenerate.size(); i++) {
        // Get will throw an exception if the callable returned it.
        try {
            ecs.take().get();
        } catch (ExecutionException e) {
            // One job was wrong. Stopping the rest.
            for (Future<Boolean> task : generatorFutures) {
                task.cancel(true);
            }
            executor.shutdown();
            throw e;
        }
    }

    executor.shutdown();

    log.info("Done!");
    return 0;
}

From source file:com.netflix.curator.framework.recipes.locks.TestReaper.java

private void testSimulationWithLocks(String namespace) throws Exception {
    final int LOCK_CLIENTS = 10;
    final int ITERATIONS = 250;
    final int MAX_WAIT_MS = 10;

    ExecutorService service = Executors.newFixedThreadPool(LOCK_CLIENTS);
    ExecutorCompletionService<Object> completionService = new ExecutorCompletionService<Object>(service);

    Timing timing = new Timing();
    Reaper reaper = null;/*from   w  ww  .j av  a  2  s.c om*/
    final CuratorFramework client = makeClient(timing, namespace);
    try {
        client.start();

        reaper = new Reaper(client, MAX_WAIT_MS / 2);
        reaper.start();
        reaper.addPath("/a/b");

        for (int i = 0; i < LOCK_CLIENTS; ++i) {
            completionService.submit(new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                    final InterProcessMutex lock = new InterProcessMutex(client, "/a/b");
                    for (int i = 0; i < ITERATIONS; ++i) {
                        lock.acquire();
                        try {
                            Thread.sleep((int) (Math.random() * MAX_WAIT_MS));
                        } finally {
                            lock.release();
                        }
                    }
                    return null;
                }
            });
        }

        for (int i = 0; i < LOCK_CLIENTS; ++i) {
            completionService.take().get();
        }

        Thread.sleep(timing.session());
        timing.sleepABit();

        Stat stat = client.checkExists().forPath("/a/b");
        Assert.assertNull("Child qty: " + ((stat != null) ? stat.getNumChildren() : 0), stat);
    } finally {
        service.shutdownNow();
        IOUtils.closeQuietly(reaper);
        IOUtils.closeQuietly(client);
    }
}

From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessSemaphoreCluster.java

@Test
public void testCluster() throws Exception {
    final int QTY = 20;
    final int OPERATION_TIME_MS = 1000;
    final String PATH = "/foo/bar/lock";

    ExecutorService executorService = Executors.newFixedThreadPool(QTY);
    ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<Void>(executorService);
    final Timing timing = new Timing();
    TestingCluster cluster = new TestingCluster(3);
    List<SemaphoreClient> semaphoreClients = Lists.newArrayList();
    try {/* w w w  . java2  s  .c o  m*/
        cluster.start();

        final AtomicInteger opCount = new AtomicInteger(0);
        for (int i = 0; i < QTY; ++i) {
            SemaphoreClient semaphoreClient = new SemaphoreClient(cluster.getConnectString(), PATH,
                    new Callable<Void>() {
                        @Override
                        public Void call() throws Exception {
                            opCount.incrementAndGet();
                            Thread.sleep(OPERATION_TIME_MS);
                            return null;
                        }
                    });
            completionService.submit(semaphoreClient);
            semaphoreClients.add(semaphoreClient);
        }

        timing.forWaiting().sleepABit();

        Assert.assertNotNull(SemaphoreClient.getActiveClient());

        final CountDownLatch latch = new CountDownLatch(1);
        CuratorFramework client = CuratorFrameworkFactory.newClient(cluster.getConnectString(),
                timing.session(), timing.connection(), new ExponentialBackoffRetry(100, 3));
        ConnectionStateListener listener = new ConnectionStateListener() {
            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {
                if (newState == ConnectionState.LOST) {
                    latch.countDown();
                }
            }
        };
        client.getConnectionStateListenable().addListener(listener);
        client.start();
        try {
            client.getZookeeperClient().blockUntilConnectedOrTimedOut();

            cluster.stop();

            latch.await();
        } finally {
            IOUtils.closeQuietly(client);
        }

        long startTicks = System.currentTimeMillis();
        for (;;) {
            int thisOpCount = opCount.get();
            Thread.sleep(2 * OPERATION_TIME_MS);
            if (thisOpCount == opCount.get()) {
                break; // checking that the op count isn't increasing
            }
            Assert.assertTrue((System.currentTimeMillis() - startTicks) < timing.forWaiting().milliseconds());
        }

        int thisOpCount = opCount.get();

        Iterator<InstanceSpec> iterator = cluster.getInstances().iterator();
        cluster = new TestingCluster(iterator.next(), iterator.next());
        cluster.start();
        timing.forWaiting().sleepABit();

        startTicks = System.currentTimeMillis();
        for (;;) {
            Thread.sleep(2 * OPERATION_TIME_MS);
            if (opCount.get() > thisOpCount) {
                break; // checking that semaphore has started working again
            }
            Assert.assertTrue((System.currentTimeMillis() - startTicks) < timing.forWaiting().milliseconds());
        }
    } finally {
        for (SemaphoreClient semaphoreClient : semaphoreClients) {
            IOUtils.closeQuietly(semaphoreClient);
        }
        IOUtils.closeQuietly(cluster);
        executorService.shutdownNow();
    }
}