List of usage examples for java.util.concurrent LinkedBlockingQueue size
public int size()
From source file:org.apache.bigtop.bigpetstore.qstream.HttpLoadGen.java
/** * Appends via REST calls.//from w w w . j av a 2 s . c o m */ public LinkedBlockingQueue<Transaction> startWriteQueue(final int milliseconds) { /** * Write queue. Every 5 seconds, write */ final LinkedBlockingQueue<Transaction> transactionQueue = new LinkedBlockingQueue<Transaction>( getQueueSize()); new Thread() { @Override public void run() { int fileNumber = 0; while (true) { waitFor(milliseconds, transactionQueue); System.out.println("CLEARING " + transactionQueue.size() + " elements from queue."); Stack<Transaction> transactionsToWrite = new Stack<Transaction>(); transactionQueue.drainTo(transactionsToWrite); /** * pop transactions from the queue, and sent them over http as json. */ while (!transactionsToWrite.isEmpty()) { try { String trAsJson = URLEncoder.encode(Utils.toJson(transactionsToWrite.pop())); /** * i.e. wget http://localhost:3000/rpush/guestbook/{"name":"cos boudnick", "state":"...",...} */ HttpResponse resp = Utils.get(path + "/" + trAsJson); if (total % 20 == 0) System.out.println("wrote customer " + trAsJson); total++; } catch (Throwable t) { System.err.println("transaction failed.... !"); t.printStackTrace(); } System.out.println("TRANSACTIONS SO FAR " + total++ + " RATE " + total / ((System.currentTimeMillis() - startTime) / 1000)); } } } }.start(); return transactionQueue; }
From source file:se.vgregion.pubsub.push.impl.DefaultPushSubscriberVerifyTest.java
@Test @Transactional//w ww .j a va 2 s . co m @Rollback public void verify() throws Exception { final LinkedBlockingQueue<HttpRequest> requests = new LinkedBlockingQueue<HttpRequest>(); server.register("/*", new HttpRequestHandler() { @Override public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { requests.add(request); response.setEntity( new StringEntity(getQueryParamValue(request.getRequestLine().getUri(), "hub.challenge"))); } }); subscriber.verify(SubscriptionMode.SUBSCRIBE); Assert.assertEquals(1, requests.size()); HttpRequest actualRequest = requests.poll(); String requestUri = actualRequest.getRequestLine().getUri(); Assert.assertEquals("subscribe", getQueryParamValue(requestUri, "hub.mode")); Assert.assertEquals(subscriber.getTopic().toString(), URLDecoder.decode(getQueryParamValue(requestUri, "hub.topic"), "UTF-8")); Assert.assertNotNull(getQueryParamValue(requestUri, "hub.challenge")); Assert.assertEquals("123", getQueryParamValue(requestUri, "hub.lease_seconds")); Assert.assertEquals(subscriber.getVerifyToken(), getQueryParamValue(requestUri, "hub.verify_token")); }
From source file:org.apache.distributedlog.auditor.DLAuditor.java
private void collectLedgersFromDL(final URI uri, final Namespace namespace, final Set<Long> ledgers) throws IOException { logger.info("Enumerating {} to collect streams.", uri); Iterator<String> streams = namespace.getLogs(); final LinkedBlockingQueue<String> streamQueue = new LinkedBlockingQueue<String>(); while (streams.hasNext()) { streamQueue.add(streams.next()); }/* w ww.j a v a 2 s .c o m*/ logger.info("Collected {} streams from uri {} : {}", new Object[] { streamQueue.size(), uri, streams }); executeAction(streamQueue, 10, new Action<String>() { @Override public void execute(String stream) throws IOException { collectLedgersFromStream(namespace, stream, ledgers); } }); }
From source file:org.wso2.carbon.databridge.agent.thrift.internal.publisher.client.EventPublisher.java
private void notifyConnectionFailure(ArrayList<Event> tempEventBundleHolder) { if (null != receiverStateObserver) { ReceiverConfiguration conf = dataPublisherConfiguration.getReceiverConfiguration(); StringBuilder url = new StringBuilder(conf.getDataReceiverProtocol().toString()); url.append("://"); url.append(conf.getDataReceiverIp()); url.append(":"); url.append(conf.getDataReceiverPort()); receiverStateObserver.notifyConnectionFailure(url.toString(), conf.getUserName(), conf.getPassword()); if (null != eventQueue) { LinkedBlockingQueue<Event> events = eventQueue.getAndResetQueue(); if (null != tempEventBundleHolder) { for (Event event : tempEventBundleHolder) { try { events.put(event); } catch (InterruptedException ignore) { log.error("Error while populating resend events list", ignore); }//from ww w . j a v a 2 s . c o m } } if (log.isDebugEnabled()) { log.debug("resending events size: " + events.size() + "in thread id :" + Thread.currentThread().getId()); } receiverStateObserver.resendEvents(events); } } }
From source file:disko.flow.analyzers.FullRelexAnalyzer.java
public void process(AnalysisContext<TextDocument> ctx, Ports ports) throws InterruptedException { if (pool == null) init();//from w ww .ja va2 s . c o m final InputPort<EntityMaintainer> inputPort = ports.getInput(EntityAnalyzer.ENTITY_CHANNEL); final OutputPort<RelexTaskResult> outputPort = ports.getOutput(PARSE_CHANNEL); final LinkedBlockingQueue<Future<RelexTaskResult>> futureResults = new LinkedBlockingQueue<Future<RelexTaskResult>>( outputPort.getChannel().getCapacity()); log.debug("Starting LinkGrammarAnalyzer..."); exec.submit(new Callable<Integer>() { public Integer call() throws Exception { try { log.debug("LinkGrammarAnalyzer from channel + " + inputPort.getChannel()); for (EntityMaintainer em = inputPort.take(); !inputPort.isEOS(em); em = inputPort.take()) submitTask(em, futureResults); } catch (Throwable t) { log.error("Unable to submit parsing task.", t); } finally { futureResults.put(new FutureRelexTaskResultEOS()); } return (futureResults.size() - 1); } }); try { while (true) { try { Future<RelexTaskResult> futureResult = futureResults.take(); RelexTaskResult relexTaskResult; relexTaskResult = futureResult.get(); if (relexTaskResult == null) break; log.debug("LinkGrammarAnalyzer received " + relexTaskResult.index + ": " + relexTaskResult.result.getParses().size() + " parses of sentences " + relexTaskResult.sentence); relexTaskResult.result.setSentence(relexTaskResult.entityMaintainer.getOriginalSentence()); outputPort.put(relexTaskResult); } catch (InterruptedException e) { for (Future<RelexTaskResult> future : futureResults) { try { future.cancel(true); } catch (Throwable t) { log.error(t); } } break; } } for (Future<RelexTaskResult> future : futureResults) { future.cancel(true); } } catch (ExecutionException e) { throw new RuntimeException(e); } finally { outputPort.close(); /* * exec.shutdown(); for (RelexContext context: pool){ * context.getLinkParserClient().close(); } */ destroy(); } }
From source file:org.apache.bookkeeper.metadata.etcd.EtcdRegistrationTest.java
private void testWatchBookies(boolean readonly) throws Exception { LinkedBlockingQueue<Versioned<Set<BookieSocketAddress>>> writableChanges = new LinkedBlockingQueue<>(); LinkedBlockingQueue<Versioned<Set<BookieSocketAddress>>> readonlyChanges = new LinkedBlockingQueue<>(); result(regClient.watchReadOnlyBookies(newRegistrationListener(readonlyChanges))); result(regClient.watchWritableBookies(newRegistrationListener(writableChanges))); Versioned<Set<BookieSocketAddress>> versionedBookies = writableChanges.take(); assertTrue(versionedBookies.getValue().isEmpty()); versionedBookies = readonlyChanges.take(); assertTrue(versionedBookies.getValue().isEmpty()); final int numBookies = 3; final List<EtcdRegistrationManager> bookies = createNumBookies(readonly, numBookies, scope, 1); LinkedBlockingQueue<Versioned<Set<BookieSocketAddress>>> changes; if (readonly) { changes = readonlyChanges;//ww w . j ava2 s .c o m } else { changes = writableChanges; } Version preVersion = new LongVersion(-1); Set<BookieSocketAddress> expectedBookies = new HashSet<>(); for (int i = 0; i < numBookies; i++) { BookieSocketAddress address = new BookieSocketAddress(newBookie(i)); expectedBookies.add(address); versionedBookies = changes.take(); Version curVersion = versionedBookies.getVersion(); assertEquals(Occurred.AFTER, curVersion.compare(preVersion)); assertEquals(expectedBookies, versionedBookies.getValue()); preVersion = curVersion; } bookies.forEach(EtcdRegistrationManager::close); for (int i = 0; i < numBookies; i++) { versionedBookies = changes.take(); Version curVersion = versionedBookies.getVersion(); assertEquals(Occurred.AFTER, curVersion.compare(preVersion)); assertEquals(numBookies - i - 1, versionedBookies.getValue().size()); preVersion = curVersion; } if (readonly) { assertEquals(0, writableChanges.size()); } else { assertEquals(0, readonlyChanges.size()); } }
From source file:org.ala.spatial.analysis.index.LayerDistanceIndex.java
/** * @param threadcount number of threads to run analysis. * @param onlyThesePairs array of distances to run as fieldId1 + " " + * fieldId2 where fieldId1.compareTo(fieldId2) < 0 or null for all missing * distances.//from w ww. ja v a 2s. co m * @throws InterruptedException */ public void occurrencesUpdate(int threadcount, String[] onlyThesePairs) throws InterruptedException { //create distances file if it does not exist. File layerDistancesFile = new File(AlaspatialProperties.getAnalysisWorkingDir() + LAYER_DISTANCE_FILE); if (!layerDistancesFile.exists()) { try { FileWriter fw = new FileWriter(layerDistancesFile); fw.close(); } catch (Exception e) { e.printStackTrace(); } } Map<String, Double> map = loadDistances(); LinkedBlockingQueue<String> todo = new LinkedBlockingQueue(); if (onlyThesePairs != null && onlyThesePairs.length > 0) { for (String s : onlyThesePairs) { todo.add(s); } } else { //find all environmental layer analysis files File root = new File(AlaspatialProperties.getAnalysisLayersDir()); File[] dirs = root.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname != null && pathname.isDirectory(); } }); HashMap<String, String> domains = new HashMap<String, String>(); for (File dir : dirs) { //iterate through files so we get everything File[] files = new File(dir.getPath()).listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname.getName().endsWith(".grd") && pathname.getName().startsWith("el"); } }); for (int i = 0; i < files.length; i++) { for (int j = i + 1; j < files.length; j++) { String file1 = files[i].getName().replace(".grd", ""); String file2 = files[j].getName().replace(".grd", ""); //only operate on file names that are valid fields if (Client.getFieldDao().getFieldById(file1) != null && Client.getFieldDao().getFieldById(file2) != null) { String domain1 = domains.get(file1); if (domain1 == null) { String pid1 = Client.getFieldDao().getFieldById(file1).getSpid(); domain1 = Client.getLayerDao().getLayerById(Integer.parseInt(pid1)).getdomain(); domains.put(file1, domain1); } String domain2 = domains.get(file2); if (domain2 == null) { String pid2 = Client.getFieldDao().getFieldById(file2).getSpid(); domain2 = Client.getLayerDao().getLayerById(Integer.parseInt(pid2)).getdomain(); domains.put(file2, domain2); } String key = (file1.compareTo(file2) < 0) ? file1 + " " + file2 : file2 + " " + file1; //domain test if (isSameDomain(parseDomain(domain1), parseDomain(domain2))) { if (!map.containsKey(key) && !todo.contains(key)) { todo.put(key); } } } } } } } LinkedBlockingQueue<String> toDisk = new LinkedBlockingQueue<String>(); CountDownLatch cdl = new CountDownLatch(todo.size()); CalcThread[] threads = new CalcThread[threadcount]; for (int i = 0; i < threadcount; i++) { threads[i] = new CalcThread(cdl, todo, toDisk); threads[i].start(); } ToDiskThread toDiskThread = new ToDiskThread( AlaspatialProperties.getAnalysisWorkingDir() + LAYER_DISTANCE_FILE, toDisk); toDiskThread.start(); cdl.await(); for (int i = 0; i < threadcount; i++) { threads[i].interrupt(); } toDiskThread.interrupt(); }
From source file:au.org.ala.spatial.analysis.layers.LayerDistanceIndex.java
/** * @param threadcount number of threads to run analysis. * @param onlyThesePairs array of distances to run as fieldId1 + " " + * fieldId2 where fieldId1.compareTo(fieldId2) < 0 or null for all missing * distances./*from w w w. ja va 2 s .c om*/ * @throws InterruptedException */ public void occurrencesUpdate(int threadcount, String[] onlyThesePairs) throws InterruptedException { //create distances file if it does not exist. File layerDistancesFile = new File(IntersectConfig.getAlaspatialOutputPath() + LAYER_DISTANCE_FILE); if (!layerDistancesFile.exists()) { FileWriter fw = null; try { fw = new FileWriter(layerDistancesFile); fw.flush(); } catch (Exception e) { logger.error(e.getMessage(), e); } finally { if (fw != null) { try { fw.close(); } catch (Exception e) { logger.error(e.getMessage(), e); } } } } Map<String, Double> map = loadDistances(); LinkedBlockingQueue<String> todo = new LinkedBlockingQueue(); if (onlyThesePairs != null && onlyThesePairs.length > 0) { for (String s : onlyThesePairs) { todo.add(s); } } else { //find all environmental layer analysis files File root = new File(IntersectConfig.getAlaspatialOutputPath()); File[] dirs = root.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname != null && pathname.isDirectory(); } }); HashMap<String, String> domains = new HashMap<String, String>(); for (File dir : dirs) { //iterate through files so we get everything File[] files = new File(dir.getPath()).listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname.getName().endsWith(".grd") && pathname.getName().startsWith("el"); } }); for (int i = 0; i < files.length; i++) { for (int j = i + 1; j < files.length; j++) { String file1 = files[i].getName().replace(".grd", ""); String file2 = files[j].getName().replace(".grd", ""); //only operate on file names that are valid fields if (Client.getFieldDao().getFieldById(file1) != null && Client.getFieldDao().getFieldById(file2) != null) { String domain1 = domains.get(file1); if (domain1 == null) { String pid1 = Client.getFieldDao().getFieldById(file1).getSpid(); domain1 = Client.getLayerDao().getLayerById(Integer.parseInt(pid1)).getdomain(); domains.put(file1, domain1); } String domain2 = domains.get(file2); if (domain2 == null) { String pid2 = Client.getFieldDao().getFieldById(file2).getSpid(); domain2 = Client.getLayerDao().getLayerById(Integer.parseInt(pid2)).getdomain(); domains.put(file2, domain2); } String key = (file1.compareTo(file2) < 0) ? file1 + " " + file2 : file2 + " " + file1; //domain test if (isSameDomain(parseDomain(domain1), parseDomain(domain2))) { if (!map.containsKey(key) && !todo.contains(key)) { todo.put(key); } } } } } } } LinkedBlockingQueue<String> toDisk = new LinkedBlockingQueue<String>(); CountDownLatch cdl = new CountDownLatch(todo.size()); CalcThread[] threads = new CalcThread[threadcount]; for (int i = 0; i < threadcount; i++) { threads[i] = new CalcThread(cdl, todo, toDisk); threads[i].start(); } ToDiskThread toDiskThread = new ToDiskThread( IntersectConfig.getAlaspatialOutputPath() + LAYER_DISTANCE_FILE, toDisk); toDiskThread.start(); cdl.await(); for (int i = 0; i < threadcount; i++) { threads[i].interrupt(); } toDiskThread.interrupt(); }
From source file:com.SecUpwN.AIMSICD.service.AimsicdService.java
/** * Updates Neighbouring Cell details/*from w ww .j a v a 2 s . co m*/ */ public List<Cell> updateNeighbouringCells() { List<Cell> neighboringCells = new ArrayList<>(); List<NeighboringCellInfo> neighboringCellInfo; neighboringCellInfo = tm.getNeighboringCellInfo(); if (neighboringCellInfo.size() == 0) { // try to poll the neighboring cells for a few seconds final LinkedBlockingQueue<NeighboringCellInfo> neighboringCellBlockingQueue = new LinkedBlockingQueue<>( 100); final PhoneStateListener listener = new PhoneStateListener() { private void handle() { List<NeighboringCellInfo> neighboringCellInfo; neighboringCellInfo = tm.getNeighboringCellInfo(); if (neighboringCellInfo.size() == 0) { return; } Log.i(TAG, "neighbouringCellInfo empty - event based polling succeeded!"); tm.listen(this, PhoneStateListener.LISTEN_NONE); neighboringCellBlockingQueue.addAll(neighboringCellInfo); } @Override public void onServiceStateChanged(ServiceState serviceState) { handle(); } @Override public void onDataConnectionStateChanged(int state) { handle(); } @Override public void onDataConnectionStateChanged(int state, int networkType) { handle(); } @Override public void onSignalStrengthsChanged(SignalStrength signalStrength) { handle(); } @Override public void onCellInfoChanged(List<CellInfo> cellInfo) { handle(); } }; Log.i(TAG, "neighbouringCellInfo empty - start polling"); //LISTEN_CELL_INFO added in API 17 if (Build.VERSION.SDK_INT > 16) { tm.listen(listener, PhoneStateListener.LISTEN_CELL_INFO | PhoneStateListener.LISTEN_CELL_LOCATION | PhoneStateListener.LISTEN_DATA_CONNECTION_STATE | PhoneStateListener.LISTEN_SERVICE_STATE | PhoneStateListener.LISTEN_SIGNAL_STRENGTHS); } else { tm.listen(listener, PhoneStateListener.LISTEN_CELL_LOCATION | PhoneStateListener.LISTEN_DATA_CONNECTION_STATE | PhoneStateListener.LISTEN_SERVICE_STATE | PhoneStateListener.LISTEN_SIGNAL_STRENGTHS); } for (int i = 0; i < 10 && neighboringCellInfo.size() == 0; i++) { try { Log.i(TAG, "neighbouringCellInfo empty - try " + i); NeighboringCellInfo info = neighboringCellBlockingQueue.poll(1, TimeUnit.SECONDS); if (info == null) { neighboringCellInfo = tm.getNeighboringCellInfo(); if (neighboringCellInfo.size() > 0) { Log.i(TAG, "neighbouringCellInfo empty - try " + i + " succeeded time based"); break; } else { continue; } } ArrayList<NeighboringCellInfo> cellInfoList = new ArrayList<NeighboringCellInfo>( neighboringCellBlockingQueue.size() + 1); while (info != null) { cellInfoList.add(info); info = neighboringCellBlockingQueue.poll(1, TimeUnit.SECONDS); } neighboringCellInfo = cellInfoList; } catch (InterruptedException e) { // normal } } } Log.i(TAG, "neighbouringCellInfo Size - " + neighboringCellInfo.size()); for (NeighboringCellInfo neighbourCell : neighboringCellInfo) { Log.i(TAG, "neighbouringCellInfo - CID:" + neighbourCell.getCid() + " LAC:" + neighbourCell.getLac() + " RSSI:" + neighbourCell.getRssi() + " PSC:" + neighbourCell.getPsc()); final Cell cell = new Cell(neighbourCell.getCid(), neighbourCell.getLac(), neighbourCell.getRssi(), neighbourCell.getPsc(), neighbourCell.getNetworkType(), false); neighboringCells.add(cell); } return neighboringCells; }
From source file:org.commoncrawl.service.pagerank.slave.PageRankUtils.java
public static void distributeRank(final PRValueMap valueMap, final Path outlinksFile, final boolean outlinksIsRemote, File localOutputDir, String remoteOutputDir, int thisNodeIdx, int nodeCount, int iterationNumber, final ProgressAndCancelCheckCallback progressCallback) throws IOException { final Configuration conf = CrawlEnvironment.getHadoopConfig(); Vector<PRValueOutputStream> outputStreamVector = new Vector<PRValueOutputStream>(); // allocate a queue ... final LinkedBlockingQueue<OutlinkItem> queue = new LinkedBlockingQueue<OutlinkItem>(20000); try {//from w ww .j a va 2 s . c om // start the loader thread ... Thread loaderThread = new Thread(new Runnable() { final BytesWritable key = new BytesWritable(); final BytesWritable value = new BytesWritable(); final DataInputBuffer keyStream = new DataInputBuffer(); final DataInputBuffer valueStream = new DataInputBuffer(); @Override public void run() { LOG.info("Opening Outlinks File at:" + outlinksFile); SequenceFile.Reader reader = null; try { FileSystem fsForOutlinksFile = null; if (outlinksIsRemote) { fsForOutlinksFile = CrawlEnvironment.getDefaultFileSystem(); } else { fsForOutlinksFile = FileSystem.getLocal(conf); } FileStatus outlinksFileStatus = fsForOutlinksFile.getFileStatus(outlinksFile); long bytesToReadTotal = (outlinksFileStatus != null) ? outlinksFileStatus.getLen() : 0; reader = new SequenceFile.Reader(fsForOutlinksFile, outlinksFile, conf); OutlinkItem item = new OutlinkItem(); int itemCount = 0; boolean isCancelled = false; while (!isCancelled && reader.next(key, value)) { keyStream.reset(key.getBytes(), 0, key.getLength()); valueStream.reset(value.getBytes(), 0, value.getLength()); //populate item from data readURLFPFromStream(keyStream, item.targetFingerprint); item.urlCount = readURLFPAndCountFromStream(valueStream, item.sourceFingerprint); try { long blockTimeStart = System.currentTimeMillis(); queue.put(item); long blockTimeEnd = System.currentTimeMillis(); } catch (InterruptedException e) { } item = new OutlinkItem(); if (itemCount++ % 10000 == 0 && progressCallback != null) { float percentComplete = (float) reader.getPosition() / (float) bytesToReadTotal; if (progressCallback.updateProgress(percentComplete)) { LOG.info("Cancel check callback returned true.Cancelling outlink item load"); isCancelled = true; } } } item.sourceFingerprint = null; item.targetFingerprint = null; // add empty item try { if (!isCancelled) { queue.put(item); } else { queue.put(new OutlinkItem(new IOException("Operation Cancelled"))); } } catch (InterruptedException e) { } } catch (IOException e) { // add error item to queue. try { queue.put(new OutlinkItem(e)); } catch (InterruptedException e1) { } } finally { if (reader != null) try { reader.close(); } catch (IOException e) { } } } }); loaderThread.start(); // first things first ... initialize output stream vector FileSystem fileSystem = buildDistributionOutputStreamVector(true, getOutlinksBaseName(thisNodeIdx, iterationNumber), localOutputDir, remoteOutputDir, thisNodeIdx, nodeCount, outputStreamVector); try { // open outlinks file . LOG.info("Iterating Items in Outlinks File and Writing Test Value"); int itemCount = 0; int totalOutlinkCount = 0; int iterationOutlinkCount = 0; long iterationStart = System.currentTimeMillis(); long timeStart = iterationStart; boolean done = false; ArrayList<OutlinkItem> items = new ArrayList<OutlinkItem>(); // start iterating outlinks while (!done) { //OutlinkItem item = null; //try { long waitTimeStart = System.currentTimeMillis(); queue.drainTo(items); long waitTimeEnd = System.currentTimeMillis(); //} catch (InterruptedException e) { //} for (OutlinkItem item : items) { if (item.error != null) { LOG.info( "Loader Thread Returned Error:" + CCStringUtils.stringifyException(item.error)); throw item.error; } else if (item.sourceFingerprint == null) { LOG.info("Loader Thread Indicated EOF via emtpy item"); done = true; } else { ++itemCount; /* LOG.info("SourceFP-DomainHash:" + item.sourceFingerprint.getDomainHash() + " URLHash:" + item.sourceFingerprint.getUrlHash() + " PartitionIdx:" + ((item.sourceFingerprint.hashCode() & Integer.MAX_VALUE) % CrawlEnvironment.PR_NUMSLAVES) ); */ // now get pr value for fingerprint (random seek in memory here!!!) float prValue = valueMap.getPRValue(item.sourceFingerprint) / (float) Math.max(item.urlCount, 1); // write value out int nodeIndex = (item.targetFingerprint.hashCode() & Integer.MAX_VALUE) % nodeCount; outputStreamVector.get(nodeIndex).writePRValue(item.targetFingerprint, item.sourceFingerprint, prValue); if (itemCount % 10000 == 0) { long timeEnd = System.currentTimeMillis(); int milliseconds = (int) (timeEnd - iterationStart); LOG.info("Distribute PR for 10000 Items with:" + iterationOutlinkCount + " Outlinks Took:" + milliseconds + " Milliseconds" + " QueueCount:" + queue.size()); iterationStart = System.currentTimeMillis(); totalOutlinkCount += iterationOutlinkCount; iterationOutlinkCount = 0; } } } items.clear(); } totalOutlinkCount += iterationOutlinkCount; LOG.info("Distribute Finished for a total of:" + itemCount + " Items with:" + totalOutlinkCount + " Outlinks Took:" + (System.currentTimeMillis() - timeStart) + " Milliseconds"); LOG.info("Waiting for Loader Thread to Die"); try { loaderThread.join(); } catch (InterruptedException e) { } LOG.info("Loader Thread Died - Moving on..."); } finally { for (PRValueOutputStream info : outputStreamVector) { if (info != null) { info.close(false); } } if (fileSystem != null) { fileSystem.close(); } } } catch (IOException e) { LOG.error("Exception caught while distributing outlinks:" + CCStringUtils.stringifyException(e)); throw e; } }