List of usage examples for java.util.concurrent ArrayBlockingQueue put
public void put(E e) throws InterruptedException
From source file:com.esri.geoevent.clusterSimulator.ui.CertificateCheckerDialog.java
@Override public boolean allowConnection(final X509Certificate[] chain) { if (trustedCerts.contains(chain[0])) { return true; }/*from ww w. j a v a 2s.co m*/ final ArrayBlockingQueue<Boolean> responseQueue = new ArrayBlockingQueue<Boolean>(1); Runnable runnable = new Runnable() { @Override public void run() { try { final Stage dialog = new Stage(); dialog.initModality(Modality.APPLICATION_MODAL); dialog.initOwner(MainApplication.primaryStage); dialog.setTitle("Certificate Check"); FXMLLoader loader = new FXMLLoader(getClass().getResource("CertificateCheckerDialog.fxml")); Parent parent = (Parent) loader.load(); CertCheckController controller = (CertCheckController) loader.getController(); controller.certText.setText(chain[0].toString()); Scene scene = new Scene(parent); dialog.setScene(scene); dialog.showAndWait(); responseQueue.put(Boolean.valueOf(controller.allowConnection)); } catch (Exception e) { e.printStackTrace(); } } }; if (Platform.isFxApplicationThread()) { runnable.run(); } else { Platform.runLater(runnable); } try { boolean retVal = responseQueue.take(); if (retVal) { trustedCerts.add(chain[0]); } return retVal; } catch (Exception e) { e.printStackTrace(); } return false; }
From source file:org.kurento.rabbitmq.RabbitTemplate.java
protected Message doSendAndReceiveWithTemporary(final String exchange, final String routingKey, final Message message) { return this.execute(new ChannelCallback<Message>() { @Override/*from w ww . j a v a 2s.c o m*/ public Message doInRabbit(Channel channel) throws Exception { final ArrayBlockingQueue<Message> replyHandoff = new ArrayBlockingQueue<Message>(1); Assert.isNull(message.getMessageProperties().getReplyTo(), "Send-and-receive methods can only be used if the Message does not already have a replyTo property."); DeclareOk queueDeclaration = channel.queueDeclare(); String replyTo = queueDeclaration.getQueue(); message.getMessageProperties().setReplyTo(replyTo); String consumerTag = UUID.randomUUID().toString(); DefaultConsumer consumer = new DefaultConsumer(channel) { @Override public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException { MessageProperties messageProperties = messagePropertiesConverter .toMessageProperties(properties, envelope, encoding); Message reply = new Message(body, messageProperties); if (logger.isTraceEnabled()) { logger.trace("Message received " + reply); } try { replyHandoff.put(reply); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } }; channel.basicConsume(replyTo, true, consumerTag, true, true, null, consumer); doSend(channel, exchange, routingKey, message, null); Message reply = (replyTimeout < 0) ? replyHandoff.take() : replyHandoff.poll(replyTimeout, TimeUnit.MILLISECONDS); channel.basicCancel(consumerTag); return reply; } }); }
From source file:com.eucalyptus.blockstorage.S3SnapshotTransfer.java
/** * Compresses the snapshot and uploads it to a bucket in objectstorage gateway as a single or multipart upload based on the configuration in * {@link StorageInfo}. Bucket name should be configured before invoking this method. It can be looked up and initialized by {@link #prepareForUpload()} or * explicitly set using {@link #setBucketName(String)} * //from w w w . j av a 2 s . c o m * @param sourceFileName * absolute path to the snapshot on the file system */ @Override public void upload(String sourceFileName) throws SnapshotTransferException { validateInput(); // Validate input loadTransferConfig(); // Load the transfer configuration parameters from database SnapshotProgressCallback progressCallback = new SnapshotProgressCallback(snapshotId); // Setup the progress callback Boolean error = Boolean.FALSE; ArrayBlockingQueue<SnapshotPart> partQueue = null; SnapshotPart part = null; SnapshotUploadInfo snapUploadInfo = null; Future<List<PartETag>> uploadPartsFuture = null; Future<String> completeUploadFuture = null; byte[] buffer = new byte[READ_BUFFER_SIZE]; Long readOffset = 0L; Long bytesRead = 0L; Long bytesWritten = 0L; int len; int partNumber = 1; try { // Get the uncompressed file size for uploading as metadata Long uncompressedSize = getFileSize(sourceFileName); // Setup the snapshot and part entities. snapUploadInfo = SnapshotUploadInfo.create(snapshotId, bucketName, keyName); Path zipFilePath = Files.createTempFile(keyName + '-', '-' + String.valueOf(partNumber)); part = SnapshotPart.createPart(snapUploadInfo, zipFilePath.toString(), partNumber, readOffset); FileInputStream inputStream = new FileInputStream(sourceFileName); ByteArrayOutputStream baos = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream(baos); FileOutputStream outputStream = new FileOutputStream(zipFilePath.toString()); try { LOG.debug("Reading snapshot " + snapshotId + " and compressing it to disk in chunks of size " + partSize + " bytes or greater"); while ((len = inputStream.read(buffer)) > 0) { bytesRead += len; gzipStream.write(buffer, 0, len); if ((bytesWritten + baos.size()) < partSize) { baos.writeTo(outputStream); bytesWritten += baos.size(); baos.reset(); } else { gzipStream.close(); baos.writeTo(outputStream); // Order is important. Closing the gzip stream flushes stuff bytesWritten += baos.size(); baos.reset(); outputStream.close(); if (partNumber > 1) {// Update the part status part = part.updateStateCreated(bytesWritten, bytesRead, Boolean.FALSE); } else {// Initialize multipart upload only once after the first part is created LOG.info("Uploading snapshot " + snapshotId + " to objectstorage using multipart upload"); progressCallback.setUploadSize(uncompressedSize); uploadId = initiateMulitpartUpload(uncompressedSize); snapUploadInfo = snapUploadInfo.updateUploadId(uploadId); part = part.updateStateCreated(uploadId, bytesWritten, bytesRead, Boolean.FALSE); partQueue = new ArrayBlockingQueue<SnapshotPart>(queueSize); uploadPartsFuture = Threads.enqueue(serviceConfig, UploadPartTask.class, poolSize, new UploadPartTask(partQueue, progressCallback)); } // Check for the future task before adding part to the queue. if (uploadPartsFuture != null && uploadPartsFuture.isDone()) { // This task shouldn't be done until the last part is added. If it is done at this point, then something might have gone wrong throw new SnapshotUploadPartException( "Error uploading parts, aborting part creation process. Check previous log messages for the exact error"); } // Add part to the queue partQueue.put(part); // Prep the metadata for the next part readOffset += bytesRead; bytesRead = 0L; bytesWritten = 0L; // Setup the part entity for next part zipFilePath = Files.createTempFile(keyName + '-', '-' + String.valueOf((++partNumber))); part = SnapshotPart.createPart(snapUploadInfo, zipFilePath.toString(), partNumber, readOffset); gzipStream = new GZIPOutputStream(baos); outputStream = new FileOutputStream(zipFilePath.toString()); } } gzipStream.close(); baos.writeTo(outputStream); bytesWritten += baos.size(); baos.reset(); outputStream.close(); inputStream.close(); // Update the part status part = part.updateStateCreated(bytesWritten, bytesRead, Boolean.TRUE); // Update the snapshot upload info status snapUploadInfo = snapUploadInfo.updateStateCreatedParts(partNumber); } catch (Exception e) { LOG.error("Failed to upload " + snapshotId + " due to: ", e); error = Boolean.TRUE; throw new SnapshotTransferException("Failed to upload " + snapshotId + " due to: ", e); } finally { if (inputStream != null) { inputStream.close(); } if (gzipStream != null) { gzipStream.close(); } if (outputStream != null) { outputStream.close(); } baos.reset(); } if (partNumber > 1) { // Check for the future task before adding the last part to the queue. if (uploadPartsFuture != null && uploadPartsFuture.isDone()) { // This task shouldn't be done until the last part is added. If it is done at this point, then something might have gone wrong throw new SnapshotUploadPartException( "Error uploading parts, aborting part upload process. Check previous log messages for the exact error"); } // Add the last part to the queue partQueue.put(part); // Kick off the completion task completeUploadFuture = Threads.enqueue(serviceConfig, CompleteMpuTask.class, poolSize, new CompleteMpuTask(uploadPartsFuture, snapUploadInfo, partNumber)); } else { try { LOG.info("Uploading snapshot " + snapshotId + " to objectstorage as a single object. Compressed size of snapshot (" + bytesWritten + " bytes) is less than minimum part size (" + partSize + " bytes) for multipart upload"); PutObjectResult putResult = uploadSnapshotAsSingleObject(zipFilePath.toString(), bytesWritten, uncompressedSize, progressCallback); markSnapshotAvailable(); try { part = part.updateStateUploaded(putResult.getETag()); snapUploadInfo = snapUploadInfo.updateStateUploaded(putResult.getETag()); } catch (Exception e) { LOG.debug("Failed to update status in DB for " + snapUploadInfo); } LOG.info("Uploaded snapshot " + snapshotId + " to objectstorage"); } catch (Exception e) { error = Boolean.TRUE; LOG.error("Failed to upload snapshot " + snapshotId + " due to: ", e); throw new SnapshotTransferException("Failed to upload snapshot " + snapshotId + " due to: ", e); } finally { deleteFile(zipFilePath); } } } catch (SnapshotTransferException e) { error = Boolean.TRUE; throw e; } catch (Exception e) { error = Boolean.TRUE; LOG.error("Failed to upload snapshot " + snapshotId + " due to: ", e); throw new SnapshotTransferException("Failed to upload snapshot " + snapshotId + " due to: ", e); } finally { if (error) { abortUpload(snapUploadInfo); if (uploadPartsFuture != null && !uploadPartsFuture.isDone()) { uploadPartsFuture.cancel(true); } if (completeUploadFuture != null && !completeUploadFuture.isDone()) { completeUploadFuture.cancel(true); } } } }
From source file:it.unimi.di.big.mg4j.document.WikipediaDocumentSequence.java
@Override public DocumentIterator iterator() throws IOException { final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final MutableString nameSpaceAccumulator = new MutableString(); final ObjectOpenHashSet<MutableString> nameSpacesAccumulator = new ObjectOpenHashSet<MutableString>(); final ArrayBlockingQueue<DocumentFactory> freeFactories = new ArrayBlockingQueue<DocumentFactory>(16); for (int i = freeFactories.remainingCapacity(); i-- != 0;) freeFactories.add(this.factory.copy()); final ArrayBlockingQueue<DocumentAndFactory> readyDocumentsAndFactories = new ArrayBlockingQueue<DocumentAndFactory>( freeFactories.size());// w ww. j a v a 2 s . co m final SAXParser parser; try { parser = saxParserFactory.newSAXParser(); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } final DefaultHandler handler = new DefaultHandler() { private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); private boolean inText; private boolean inTitle; private boolean inId; private boolean inTimestamp; private boolean inNamespaceDef; private boolean redirect; private MutableString text = new MutableString(); private MutableString title = new MutableString(); private MutableString id = new MutableString(); private MutableString timestamp = new MutableString(); private final Reference2ObjectMap<Enum<?>, Object> metadata = new Reference2ObjectOpenHashMap<Enum<?>, Object>(); { metadata.put(PropertyBasedDocumentFactory.MetadataKeys.ENCODING, "UTF-8"); metadata.put(MetadataKeys.REDIRECT, redirectAnchors); } @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { redirect = inText = inTitle = inId = inTimestamp = false; text.length(0); title.length(0); id.length(0); timestamp.length(0); } else if ("text".equals(localName)) inText = true; else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first id/title elements. else if ("id".equals(localName) && id.length() == 0) inId = true; else if ("timestamp".equals(localName) && timestamp.length() == 0) inTimestamp = true; else if ("redirect".equals(localName)) { redirect = true; if (attributes.getValue("title") != null) // Accumulate the title of the page as virtual text of the redirect page. synchronized (redirectAnchors) { final String link = Encoder.encodeTitleToUrl(attributes.getValue("title"), true); redirectAnchors.add( new AnchorExtractor.Anchor(new MutableString(baseURL.length() + link.length()) .append(baseURL).append(link), title.copy())); } } else if ("namespace".equals(localName)) { // Found a new namespace inNamespaceDef = true; nameSpaceAccumulator.length(0); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("namespace".equals(localName)) { // Collecting a namespace if (nameSpaceAccumulator.length() != 0) nameSpacesAccumulator.add(nameSpaceAccumulator.copy().toLowerCase()); return; } if ("namespaces".equals(localName)) { // All namespaces collected nameSpaces = ImmutableSet.copyOf(nameSpacesAccumulator); return; } if (!redirect) { if ("title".equals(localName)) { // Set basic metadata for the page metadata.put(PropertyBasedDocumentFactory.MetadataKeys.TITLE, title.copy()); String link = Encoder.encodeTitleToUrl(title.toString(), true); metadata.put(PropertyBasedDocumentFactory.MetadataKeys.URI, new MutableString(baseURL.length() + link.length()).append(baseURL).append(link)); inTitle = false; } else if ("id".equals(localName)) { metadata.put(MetadataKeys.ID, Long.valueOf(id.toString())); inId = false; } else if ("timestamp".equals(localName)) { try { metadata.put(MetadataKeys.LASTEDIT, dateFormat.parse(timestamp.toString())); } catch (ParseException e) { throw new RuntimeException(e.getMessage(), e); } inTimestamp = false; } else if ("text".equals(localName)) { inText = false; if (!keepNamespaced) { // Namespaces are case-insensitive and language-dependent final int pos = title.indexOf(':'); if (pos != -1 && nameSpaces.contains(title.substring(0, pos).toLowerCase())) return; } try { final MutableString html = new MutableString(); DocumentFactory freeFactory; try { freeFactory = freeFactories.take(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } if (parseText) { if (DISAMBIGUATION.search(text) != -1) { // It's a disambiguation page. /* Roi's hack: duplicate links using the page title, so the generic name will end up as anchor text. */ final MutableString newLinks = new MutableString(); for (int start = 0, end; (start = BRACKETS_OPEN.search(text, start)) != -1; start = end) { end = start; final int endOfLink = text.indexOfAnyOf(END_OF_DISAMBIGUATION_LINK, start); // Note that we don't escape title because we are working at the Wikipedia raw text level. if (endOfLink != -1) { newLinks.append(text.array(), start, endOfLink - start).append('|') .append(title).append("]]\n"); end = endOfLink; } end++; } text.append(newLinks); } // We separate categories by OXOXO, so we don't get overflowing phrases. final MutableString category = new MutableString(); for (int start = 0, end; (start = CATEGORY_START.search(text, start)) != -1; start = end) { end = BRACKETS_CLOSED.search(text, start += CATEGORY_START.length()); if (end != -1) category.append(text.subSequence(start, end)).append(" OXOXO "); else break; } metadata.put(MetadataKeys.CATEGORY, category); // Heuristics to get the first paragraph metadata.put(MetadataKeys.FIRSTPAR, new MutableString()); String plainText = wikiModel.render(new PlainTextConverter(true), text.toString()); for (int start = 0; start < plainText.length(); start++) { //System.err.println("Examining " + plainText.charAt( start ) ); if (Character.isWhitespace(plainText.charAt(start))) continue; if (plainText.charAt(start) == '{') { //System.err.print( "Braces " + start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\" -> " ); start = BRACES_CLOSED.search(plainText, start); //System.err.println( start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\"" ); if (start == -1) break; start++; } else if (plainText.charAt(start) == '[') { start = BRACKETS_CLOSED.search(plainText, start); if (start == -1) break; start++; } else { final int end = plainText.indexOf('\n', start); if (end != -1) metadata.put(MetadataKeys.FIRSTPAR, new MutableString(plainText.substring(start, end))); break; } } try { wikiModel.render(new HTMLConverter(), text.toString(), html, false, true); final Map<String, String> categories = wikiModel.getCategories(); // Put back category links in the page (they have been parsed by bliki and to not appear anymore in the HTML rendering) for (Entry<String, String> entry : categories.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue().trim(); if (value.length() != 0) // There are empty such things html.append("\n<a href=\"").append(baseURL).append("Category:") .append(Encoder.encodeTitleToUrl(key, true)).append("\">") .append(HtmlEscapers.htmlEscaper().escape(key)) .append("</a>\n"); } } catch (Exception e) { LOGGER.error("Unexpected exception while parsing " + title, e); } } readyDocumentsAndFactories.put(new DocumentAndFactory( freeFactory.getDocument(IOUtils.toInputStream(html, Charsets.UTF_8), new Reference2ObjectOpenHashMap<Enum<?>, Object>(metadata)), freeFactory)); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } } } } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); if (inId) id.append(ch, start, length); if (inTimestamp) timestamp.append(ch, start, length); if (inNamespaceDef) { nameSpaceAccumulator.append(ch, start, length); inNamespaceDef = false; // Dirty, but it works } } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); } }; final Thread parsingThread = new Thread() { public void run() { try { InputStream in = new FileInputStream(wikipediaXmlDump); if (bzipped) in = new BZip2CompressorInputStream(in); parser.parse( new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); readyDocumentsAndFactories.put(END); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } }; parsingThread.start(); return new AbstractDocumentIterator() { private DocumentFactory lastFactory; @Override public Document nextDocument() throws IOException { try { final DocumentAndFactory documentAndFactory = readyDocumentsAndFactories.take(); if (lastFactory != null) freeFactories.put(lastFactory); if (documentAndFactory == END) return null; lastFactory = documentAndFactory.factory; return documentAndFactory.document; } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } }; }
From source file:efen.parsewiki.WikipediaDocumentSequence.java
@Override public DocumentIterator iterator() throws IOException { final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); saxParserFactory.setNamespaceAware(true); final MutableString nameSpaceAccumulator = new MutableString(); final ObjectOpenHashSet<MutableString> nameSpacesAccumulator = new ObjectOpenHashSet<MutableString>(); final ArrayBlockingQueue<DocumentFactory> freeFactories = new ArrayBlockingQueue<DocumentFactory>(16); for (int i = freeFactories.remainingCapacity(); i-- != 0;) freeFactories.add(this.factory.copy()); final ArrayBlockingQueue<DocumentAndFactory> readyDocumentsAndFactories = new ArrayBlockingQueue<DocumentAndFactory>( freeFactories.size());//from www. ja v a 2s . c o m final SAXParser parser; try { parser = saxParserFactory.newSAXParser(); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } final DefaultHandler handler = new DefaultHandler() { private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); private boolean inText; private boolean inTitle; private boolean inId; private boolean inTimestamp; private boolean inNamespaceDef; private boolean redirect; private MutableString text = new MutableString(); private MutableString title = new MutableString(); private MutableString id = new MutableString(); private MutableString timestamp = new MutableString(); private final Reference2ObjectMap<Enum<?>, Object> metadata = new Reference2ObjectOpenHashMap<Enum<?>, Object>(); { metadata.put(PropertyBasedDocumentFactory.MetadataKeys.ENCODING, "UTF-8"); metadata.put(MetadataKeys.REDIRECT, redirectAnchors); } @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if ("page".equals(localName)) { redirect = inText = inTitle = inId = inTimestamp = false; text.length(0); title.length(0); id.length(0); timestamp.length(0); } else if ("text".equals(localName)) inText = true; else if ("title".equals(localName) && title.length() == 0) inTitle = true; // We catch only the first id/title elements. else if ("id".equals(localName) && id.length() == 0) inId = true; else if ("timestamp".equals(localName) && timestamp.length() == 0) inTimestamp = true; else if ("redirect".equals(localName)) { redirect = true; if (attributes.getValue("title") != null) // Accumulate the title of the page as virtual text of the redirect page. synchronized (redirectAnchors) { final String link = Encoder.encodeTitleToUrl(attributes.getValue("title"), true); redirectAnchors.add( new AnchorExtractor.Anchor(new MutableString(baseURL.length() + link.length()) .append(baseURL).append(link), title.copy())); } } else if ("namespace".equals(localName)) { // Found a new namespace inNamespaceDef = true; nameSpaceAccumulator.length(0); } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if ("namespace".equals(localName)) { // Collecting a namespace if (nameSpaceAccumulator.length() != 0) nameSpacesAccumulator.add(nameSpaceAccumulator.copy().toLowerCase()); return; } if ("namespaces".equals(localName)) { // All namespaces collected nameSpaces = ImmutableSet.copyOf(nameSpacesAccumulator); return; } if (!redirect) { if ("title".equals(localName)) { // Set basic metadata for the page metadata.put(PropertyBasedDocumentFactory.MetadataKeys.TITLE, title.copy()); String link = Encoder.encodeTitleToUrl(title.toString(), true); metadata.put(PropertyBasedDocumentFactory.MetadataKeys.URI, new MutableString(baseURL.length() + link.length()).append(baseURL).append(link)); inTitle = false; } else if ("id".equals(localName)) { metadata.put(MetadataKeys.ID, Long.valueOf(id.toString())); inId = false; } else if ("timestamp".equals(localName)) { try { metadata.put(MetadataKeys.LASTEDIT, dateFormat.parse(timestamp.toString())); } catch (ParseException e) { throw new RuntimeException(e.getMessage(), e); } inTimestamp = false; } else if ("text".equals(localName)) { inText = false; if (!keepNamespaced) { // Namespaces are case-insensitive and language-dependent final int pos = title.indexOf(':'); if (pos != -1 && isATrueNamespace(title.substring(0, pos))) return; } try { final MutableString html = new MutableString(); DocumentFactory freeFactory; try { freeFactory = freeFactories.take(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } if (parseText) { if (DISAMBIGUATION.search(text) != -1) { // It's a disambiguation page. /* Roi's hack: duplicate links using the page title, so the generic name will end up as anchor text. */ final MutableString newLinks = new MutableString(); for (int start = 0, end; (start = BRACKETS_OPEN.search(text, start)) != -1; start = end) { end = start; final int endOfLink = text.indexOfAnyOf(END_OF_DISAMBIGUATION_LINK, start); // Note that we don't escape title because we are working at the Wikipedia raw text level. if (endOfLink != -1) { newLinks.append(text.array(), start, endOfLink - start).append('|') .append(title).append("]]\n"); end = endOfLink; } end++; } text.append(newLinks); } // We separate categories by OXOXO, so we don't get overflowing phrases. final MutableString category = new MutableString(); for (int start = 0, end; (start = CATEGORY_START.search(text, start)) != -1; start = end) { end = BRACKETS_CLOSED.search(text, start += CATEGORY_START.length()); if (end != -1) category.append(text.subSequence(start, end)).append(" OXOXO "); else break; } metadata.put(MetadataKeys.CATEGORY, category); // Heuristics to get the first paragraph metadata.put(MetadataKeys.FIRSTPAR, new MutableString()); String plainText = new WikiModel(imageBaseURL, linkBaseURL) .render(new PlainTextConverter(true), text.toString()); for (int start = 0; start < plainText.length(); start++) { //System.err.println("Examining " + plainText.charAt( start ) ); if (Character.isWhitespace(plainText.charAt(start))) continue; if (plainText.charAt(start) == '{') { //System.err.print( "Braces " + start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\" -> " ); start = BRACES_CLOSED.search(plainText, start); //System.err.println( start + " text: \"" + plainText.subSequence( start, start + 10 ) + "\"" ); if (start == -1) break; start++; } else if (plainText.charAt(start) == '[') { start = BRACKETS_CLOSED.search(plainText, start); if (start == -1) break; start++; } else { final int end = plainText.indexOf('\n', start); if (end != -1) metadata.put(MetadataKeys.FIRSTPAR, new MutableString(plainText.substring(start, end)));//new MutableString( new WikiModel( imageBaseURL, linkBaseURL ).render( new PlainTextConverter( true ), text.substring( start, end ).toString() ) ) ); break; } } try { WikiModel wikiModel = new WikiModel(imageBaseURL, linkBaseURL); wikiModel.render(new HTMLConverter(), text.toString(), html, false, false); final Map<String, String> categories = wikiModel.getCategories(); // Put back category links in the page (they have been parsed by bliki and to not appear anymore in the HTML rendering) for (Entry<String, String> entry : categories.entrySet()) { final String key = entry.getKey(); final String value = entry.getValue().trim(); if (value.length() != 0) // There are empty such things html.append("\n<a href=\"").append(baseURL).append("Category:") .append(Encoder.encodeTitleToUrl(key, true)).append("\">") .append(HtmlEscapers.htmlEscaper().escape(key)) .append("</a>\n"); } } catch (Exception e) { LOGGER.error("Unexpected exception while parsing " + title, e); } } readyDocumentsAndFactories.put(new DocumentAndFactory( freeFactory.getDocument(IOUtils.toInputStream(html, Charsets.UTF_8), new Reference2ObjectOpenHashMap<Enum<?>, Object>(metadata)), freeFactory)); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } catch (IOException e) { throw new RuntimeException(e.getMessage(), e); } } } } @Override public void characters(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); if (inId) id.append(ch, start, length); if (inTimestamp) timestamp.append(ch, start, length); if (inNamespaceDef) { nameSpaceAccumulator.append(ch, start, length); inNamespaceDef = false; // Dirty, but it works } } @Override public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { if (inText && parseText) text.append(ch, start, length); if (inTitle) title.append(ch, start, length); } }; final Thread parsingThread = new Thread() { public void run() { try { InputStream in = new FileInputStream(wikipediaXmlDump); if (bzipped) in = new BZip2CompressorInputStream(in); parser.parse( new InputSource(new InputStreamReader(new FastBufferedInputStream(in), Charsets.UTF_8)), handler); readyDocumentsAndFactories.put(END); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } }; parsingThread.start(); return new AbstractDocumentIterator() { private DocumentFactory lastFactory; @Override public Document nextDocument() throws IOException { try { final DocumentAndFactory documentAndFactory = readyDocumentsAndFactories.take(); if (lastFactory != null) freeFactories.put(lastFactory); if (documentAndFactory == END) return null; lastFactory = documentAndFactory.factory; return documentAndFactory.document; } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } } }; }
From source file:org.apache.kylin.cube.inmemcubing.InMemCubeBuilderTest.java
static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<List<String>> queue, int count, long randSeed) throws IOException, InterruptedException { CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), null); int nColumns = flatTableDesc.getColumnList().size(); @SuppressWarnings("unchecked") Set<String>[] distinctSets = new Set[nColumns]; for (int i = 0; i < nColumns; i++) distinctSets[i] = new TreeSet<String>(); // get distinct values on each column List<String> lines = FileUtils.readLines(new File(flatTable), "UTF-8"); for (String line : lines) { String[] row = line.trim().split(","); assert row.length == nColumns; for (int i = 0; i < nColumns; i++) distinctSets[i].add(row[i]); }//from w w w. j a v a 2 s .c om List<String[]> distincts = new ArrayList<String[]>(); for (int i = 0; i < nColumns; i++) { distincts.add((String[]) distinctSets[i].toArray(new String[distinctSets[i].size()])); } Random rand = new Random(); if (randSeed != 0) rand.setSeed(randSeed); // output with random data for (; count > 0; count--) { ArrayList<String> row = new ArrayList<String>(nColumns); for (int i = 0; i < nColumns; i++) { String[] candidates = distincts.get(i); row.add(candidates[rand.nextInt(candidates.length)]); } queue.put(row); } queue.put(new ArrayList<String>(0)); }
From source file:org.apache.kylin.cube.inmemcubing.ITInMemCubeBuilderTest.java
static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<List<String>> queue, int count, long randSeed) throws IOException, InterruptedException { IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor()); int nColumns = flatDesc.getAllColumns().size(); @SuppressWarnings("unchecked") Set<String>[] distinctSets = new Set[nColumns]; for (int i = 0; i < nColumns; i++) distinctSets[i] = new TreeSet<String>(); // get distinct values on each column List<String> lines = FileUtils.readLines(new File(flatTable), "UTF-8"); for (String line : lines) { String[] row = line.trim().split(","); assert row.length == nColumns; for (int i = 0; i < nColumns; i++) distinctSets[i].add(row[i]); }/*www .j a va 2 s. c o m*/ List<String[]> distincts = new ArrayList<String[]>(); for (int i = 0; i < nColumns; i++) { distincts.add((String[]) distinctSets[i].toArray(new String[distinctSets[i].size()])); } Random rand = new Random(); if (randSeed != 0) rand.setSeed(randSeed); // output with random data for (; count > 0; count--) { ArrayList<String> row = new ArrayList<String>(nColumns); for (int i = 0; i < nColumns; i++) { String[] candidates = distincts.get(i); row.add(candidates[rand.nextInt(candidates.length)]); } queue.put(row); } queue.put(new ArrayList<String>(0)); }
From source file:org.codice.ddf.commands.catalog.IngestCommand.java
private void buildQueue(Stream<Path> ingestStream, ArrayBlockingQueue<Metacard> metacardQueue, long start) { ingestStream.filter(a -> !a.toFile().isDirectory()).forEach(a -> { File file = a.toFile();//from w w w . ja va2 s . c o m if (file.isHidden()) { ignoreCount.incrementAndGet(); } else { String extension = file.getName(); if (extension.contains(".")) { int x = extension.indexOf('.'); extension = extension.substring(x); } if (ignoreList != null && (ignoreList.contains(extension) || ignoreList.contains(file.getName()))) { ignoreCount.incrementAndGet(); printProgressAndFlush(start, fileCount.get(), ingestCount.get() + ignoreCount.get()); } else { Metacard result; try { result = readMetacard(file); } catch (IngestException e) { result = null; logIngestException(e, file); if (failedIngestDirectory != null) { moveToFailedIngestDirectory(file); } printErrorMessage("Failed to ingest file [" + file.getAbsolutePath() + "]."); if (INGEST_LOGGER.isWarnEnabled()) { INGEST_LOGGER.warn("Failed to ingest file [{}].", file.getAbsolutePath()); } } if (result != null) { try { metacardQueue.put(result); } catch (InterruptedException e) { INGEST_LOGGER.error("Thread interrupted while waiting to 'put' metacard: {}", e); } } } } }); doneBuildingQueue.set(true); }
From source file:org.dragoneronca.nlp.wol.disambiguation.Disambiguation.java
@Override public void run() { if (executed) { return;/* ww w.jav a2 s .c om*/ } else { executed = true; } LOG.info("Start"); long startTime = System.currentTimeMillis(); LOG.info("Graph loading..."); LightWolGraph lightWolGraph = new LightWolGraph(domainContext.edgesIterator()); int iteration = 0; int totConvergedTerms; int totProcessedTerms; int totImpossibleToDisambiguateTerms; do { LOG.info("Iteration n." + iteration); int processedSenses = 0; HashMap<SenseSolver, Thread> threadsMap = new HashMap<>(); for (int i = 0; i < PARALLELISM_DEGREE; i++) { SenseSolver solver = new SenseSolver(cycleScorer, quasiCycleScorer); addConsumer(solver); Thread thread = new Thread(solver); thread.start(); threadsMap.put(solver, thread); } totConvergedTerms = 0; totProcessedTerms = 0; totImpossibleToDisambiguateTerms = 0; try { Iterator<Integer> senseIdIterator = lightWolGraph.getSenses().iterator(); while (senseIdIterator.hasNext()) { for (ArrayBlockingQueue<LightSense> queue : consumersMap.values()) { if (senseIdIterator.hasNext()) { queue.put(new LightSense(senseIdIterator.next(), lightWolGraph)); processedSenses++; } else { break; } } long endTime = System.currentTimeMillis(); if (endTime - startTime > DELTA_TIME) { LOG.info("processed " + processedSenses + " senses"); startTime = endTime; } } for (ArrayBlockingQueue<LightSense> queue : consumersMap.values()) { queue.put(new LightSense()); // eof } for (Map.Entry<SenseSolver, Thread> entry : threadsMap.entrySet()) { try { entry.getValue().join(); totConvergedTerms += entry.getKey().getConvergedTerms(); totProcessedTerms += entry.getKey().getProcessedTerms(); totImpossibleToDisambiguateTerms += entry.getKey().getImpossibleToDisambiguate(); } catch (InterruptedException e) { LOG.warn("Exception while joining", e); } finally { removeConsumer(entry.getKey()); } } LOG.info("Converged terms: " + totConvergedTerms + " / " + totProcessedTerms); LOG.info("Terms impossible to disambiguate: " + totImpossibleToDisambiguateTerms); } catch (InterruptedException e) { LOG.warn("Exception while producing an element", e); } iteration++; } while ((double) totConvergedTerms / totProcessedTerms < CONVERGENCE_RATIO); LOG.info("The disambiguation algorithm has converged"); LOG.info("Updating scores in the database..."); updateDatabaseScores(lightWolGraph); }
From source file:org.openiot.gsn.vsensor.RVirtualSensor.java
public void dataAvailable(String inputStreamName, StreamElement streamElement) { ArrayBlockingQueue<StreamElement> circularBuffer = circularBuffers.get(inputStreamName); // Get the circular buffer that matches the input stream. Create a new one // if none exists if (circularBuffer == null) { circularBuffer = new ArrayBlockingQueue<StreamElement>(windowSize); circularBuffers.put(inputStreamName, circularBuffer); }//from ww w. j a v a 2 s .c o m try { circularBuffer.put(streamElement); logger.debug( "Window for " + inputStreamName + " contains: " + circularBuffer.size() + " of " + windowSize); if (circularBuffer.size() == windowSize) { logger.info("Window for " + inputStreamName + " contains: " + circularBuffer.size() + " of " + windowSize); // Connect to Rserve and assign global variables try { rc = new RConnection(params.get(SERVER), Integer.parseInt(params.get(PORT))); logger.info("Connected to R server " + params.get(SERVER) + ":" + params.get(PORT)); String[] fieldname = streamElement.getFieldNames(); logger.info("Sending " + fieldname.length + " data attributes to R server."); // Assign R vector variables prior the script for (int n = 0; n < fieldname.length; n++) { // Build the window double[] values = new double[windowSize]; StreamElement elt = null; // convert the circular buffer to an array Object[] elts = circularBuffer.toArray(); for (int i = 0; i < elts.length; i++) { elt = (StreamElement) elts[i]; values[i] = ((Number) elt.getData()[n]).doubleValue(); // } // assign vectors as R variables rc.assign("gsn_" + fieldname[n].toLowerCase(), values); } logger.info("Done."); // read the R script // open the script file every time we do some processing (this can be // improved). File file = new File(params.get(SCRIPT).toString()); script = FileUtils.readFileToString(file, "UTF-8"); logger.info("Sending R script."); // evaluate the R script rc.voidEval(script); logger.info("Done."); // get the output timestamp logger.info("Performing computation in R server (please wait)."); // collect vector values after computation DataField[] outStructure = null; outStructure = getVirtualSensorConfiguration().getOutputStructure(); String[] plotFieldName = new String[outStructure.length]; Byte[] plotFieldType = new Byte[outStructure.length]; for (int w = 0; w < outStructure.length; w++) { plotFieldName[w] = outStructure[w].getName(); plotFieldType[w] = outStructure[w].getDataTypeID(); } Serializable[] outputData = null; StreamElement se = null; Byte[] fieldType = streamElement.getFieldTypes(); // check if we have defined more attributes in the output structure if (outStructure.length > fieldname.length) { outputData = new Serializable[outStructure.length]; } else { outputData = new Serializable[fieldname.length]; } for (int n = 0; n < fieldname.length; n++) { // evaluate/get attribute data from R server xp = rc.parseAndEval(fieldname[n].toLowerCase()); if (fieldType[n] == DataTypes.DOUBLE) { double[] b1 = xp.asDoubles(); outputData[n] = b1[b1.length - 1]; } if (fieldType[n] == DataTypes.INTEGER) { int[] b1 = xp.asIntegers(); outputData[n] = b1[b1.length - 1]; } if (fieldType[n] == DataTypes.BIGINT) { int[] b1 = xp.asIntegers(); outputData[n] = (long) b1[b1.length - 1]; } } int len1 = outStructure.length; int len2 = fieldname.length; // check if we have defined more attributes in the output structure if (len1 > len2) { if (stype.equals("plot")) { xp = rc.parseAndEval("gsn_plot"); outputData[len2] = xp.asBytes(); se = new StreamElement(plotFieldName, plotFieldType, outputData); } } else { se = new StreamElement(fieldname, fieldType, outputData); } logger.info("Computation finished."); dataProduced(se); logger.debug("Stream published: " + se.toString().toLowerCase()); // Close connection to R server rc.close(); logger.info("Connection to R server closed."); } catch (Exception e) { logger.warn(e); // Close connection to R server logger.info("Connection to R server closed."); rc.close(); } // Remove step size elements from the beginning of the buffer for (int i = 0; i < stepSize; i++) { try { circularBuffer.take(); } catch (InterruptedException e) { logger.warn(e.getMessage(), e); } } } // end if if for window } catch (InterruptedException e) { logger.warn(e.getMessage(), e); } }