List of usage examples for java.io PipedInputStream PipedInputStream
public PipedInputStream(PipedOutputStream src, int pipeSize) throws IOException
PipedInputStream so that it is connected to the piped output stream src and uses the specified pipe size for the pipe's buffer. From source file:org.wikidata.wdtk.client.DumpProcessingOutputAction.java
/** * Creates a separate thread for writing into the given output stream and * returns a pipe output stream that can be used to pass data to this * thread./* ww w.j a v a 2 s .com*/ * <p> * This code is inspired by * http://stackoverflow.com/questions/12532073/gzipoutputstream * -that-does-its-compression-in-a-separate-thread * * @param outputStream * the stream to write to in the thread * @return a new stream that data should be written to * @throws IOException * if the pipes could not be created for some reason */ protected OutputStream getAsynchronousOutputStream(final OutputStream outputStream) throws IOException { final int SIZE = 1024 * 1024 * 10; final PipedOutputStream pos = new PipedOutputStream(); final PipedInputStream pis = new PipedInputStream(pos, SIZE); final FinishableRunnable run = new FinishableRunnable() { volatile boolean finish = false; volatile boolean hasFinished = false; @Override public void finish() { this.finish = true; while (!this.hasFinished) { // loop until thread is really finished } } @Override public void run() { try { byte[] bytes = new byte[SIZE]; // Note that we finish really gently here, writing all data // that is still in the input first (in theory, new data // could arrive asynchronously, so that the thread never // finishes, but this is not the intended mode of // operation). for (int len; (!this.finish || pis.available() > 0) && (len = pis.read(bytes)) > 0;) { outputStream.write(bytes, 0, len); } } catch (IOException e) { e.printStackTrace(); } finally { close(pis); close(outputStream); this.hasFinished = true; } } }; new Thread(run, "async-output-stream").start(); this.outputStreams.add(new Closeable() { @Override public void close() throws IOException { run.finish(); } }); return pos; }
From source file:de.upb.wdqa.wdvd.FeatureExtractor.java
private static InputStream getPipedDumpFileStream(final InputStream inputStream) throws IOException { final PipedOutputStream pipedOutputStream = new PipedOutputStream(); final PipedInputStream pipedInputStream = new PipedInputStream(pipedOutputStream, BUFFER_SIZE); new Thread("Dump File Reader") { @Override//from ww w . ja v a2 s .c o m public void run() { try { IOUtils.copy(inputStream, pipedOutputStream); inputStream.close(); pipedOutputStream.close(); } catch (Throwable t) { logger.error("", t); } } }.start(); return pipedInputStream; }
From source file:de.upb.wdqa.wdvd.FeatureExtractor.java
private static InputStream getUncompressedStream(final InputStream inputStream) throws IOException { // the decompression is a major bottleneck, make sure that it does not // have to wait for the buffer to empty final PipedOutputStream pipedOutputStream = new PipedOutputStream(); final PipedInputStream pipedInputStream = new PipedInputStream(pipedOutputStream, BUFFER_SIZE); new Thread("Dump File Decompressor") { @Override/*from ww w . j ava 2 s . c om*/ public void run() { try { InputStream compressorInputStream = new BZip2CompressorInputStream(inputStream); IOUtils.copy(compressorInputStream, pipedOutputStream); compressorInputStream.close(); pipedOutputStream.close(); } catch (IOException e) { logger.error("", e); } } }.start(); return pipedInputStream; }
From source file:org.apache.nifi.remote.util.SiteToSiteRestApiClient.java
public void openConnectionForSend(final String transactionUrl, final Peer peer) throws IOException { final CommunicationsSession commSession = peer.getCommunicationsSession(); final String flowFilesPath = transactionUrl + "/flow-files"; final HttpPost post = createPost(flowFilesPath); // Set uri so that it'll be used as transit uri. ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(post.getURI().toString()); post.setHeader("Content-Type", "application/octet-stream"); post.setHeader("Accept", "text/plain"); post.setHeader(HttpHeaders.PROTOCOL_VERSION, String.valueOf(transportProtocolVersionNegotiator.getVersion())); setHandshakeProperties(post);//from w w w . ja v a2 s . c o m final CountDownLatch initConnectionLatch = new CountDownLatch(1); final URI requestUri = post.getURI(); final PipedOutputStream outputStream = new PipedOutputStream(); final PipedInputStream inputStream = new PipedInputStream(outputStream, DATA_PACKET_CHANNEL_READ_BUFFER_SIZE); final ReadableByteChannel dataPacketChannel = Channels.newChannel(inputStream); final HttpAsyncRequestProducer asyncRequestProducer = new HttpAsyncRequestProducer() { private final ByteBuffer buffer = ByteBuffer.allocate(DATA_PACKET_CHANNEL_READ_BUFFER_SIZE); private int totalRead = 0; private int totalProduced = 0; private boolean requestHasBeenReset = false; @Override public HttpHost getTarget() { return URIUtils.extractHost(requestUri); } @Override public HttpRequest generateRequest() throws IOException, HttpException { // Pass the output stream so that Site-to-Site client thread can send // data packet through this connection. logger.debug("sending data to {} has started...", flowFilesPath); ((HttpOutput) commSession.getOutput()).setOutputStream(outputStream); initConnectionLatch.countDown(); final BasicHttpEntity entity = new BasicHttpEntity(); entity.setChunked(true); entity.setContentType("application/octet-stream"); post.setEntity(entity); return post; } private final AtomicBoolean bufferHasRemainingData = new AtomicBoolean(false); /** * If the proxy server requires authentication, the same POST request has to be sent again. * The first request will result 407, then the next one will be sent with auth headers and actual data. * This method produces a content only when it's need to be sent, to avoid producing the flow-file contents twice. * Whether we need to wait auth is determined heuristically by the previous POST request which creates transaction. * See {@link SiteToSiteRestApiClient#initiateTransactionForSend(HttpPost)} for further detail. */ @Override public void produceContent(final ContentEncoder encoder, final IOControl ioControl) throws IOException { if (shouldCheckProxyAuth() && proxyAuthRequiresResend.get() && !requestHasBeenReset) { logger.debug("Need authentication with proxy server. Postpone producing content."); encoder.complete(); return; } if (bufferHasRemainingData.get()) { // If there's remaining buffer last time, send it first. writeBuffer(encoder); if (bufferHasRemainingData.get()) { return; } } int read; // This read() blocks until data becomes available, // or corresponding outputStream is closed. if ((read = dataPacketChannel.read(buffer)) > -1) { logger.trace("Read {} bytes from dataPacketChannel. {}", read, flowFilesPath); totalRead += read; buffer.flip(); writeBuffer(encoder); } else { final long totalWritten = commSession.getOutput().getBytesWritten(); logger.debug( "sending data to {} has reached to its end. produced {} bytes by reading {} bytes from channel. {} bytes written in this transaction.", flowFilesPath, totalProduced, totalRead, totalWritten); if (totalRead != totalWritten || totalProduced != totalWritten) { final String msg = "Sending data to %s has reached to its end, but produced : read : wrote byte sizes (%d : %d : %d) were not equal. Something went wrong."; throw new RuntimeException( String.format(msg, flowFilesPath, totalProduced, totalRead, totalWritten)); } transferDataLatch.countDown(); encoder.complete(); dataPacketChannel.close(); } } private void writeBuffer(ContentEncoder encoder) throws IOException { while (buffer.hasRemaining()) { final int written = encoder.write(buffer); logger.trace("written {} bytes to encoder.", written); if (written == 0) { logger.trace("Buffer still has remaining. {}", buffer); bufferHasRemainingData.set(true); return; } totalProduced += written; } bufferHasRemainingData.set(false); buffer.clear(); } @Override public void requestCompleted(final HttpContext context) { logger.debug("Sending data to {} completed.", flowFilesPath); debugProxyAuthState(context); } @Override public void failed(final Exception ex) { final String msg = String.format("Failed to send data to %s due to %s", flowFilesPath, ex.toString()); logger.error(msg, ex); eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, msg); } @Override public boolean isRepeatable() { // In order to pass authentication, request has to be repeatable. return true; } @Override public void resetRequest() throws IOException { logger.debug("Sending data request to {} has been reset...", flowFilesPath); requestHasBeenReset = true; } @Override public void close() throws IOException { logger.debug("Closing sending data request to {}", flowFilesPath); closeSilently(outputStream); closeSilently(dataPacketChannel); stopExtendingTtl(); } }; postResult = getHttpAsyncClient().execute(asyncRequestProducer, new BasicAsyncResponseConsumer(), null); try { // Need to wait the post request actually started so that we can write to its output stream. if (!initConnectionLatch.await(connectTimeoutMillis, TimeUnit.MILLISECONDS)) { throw new IOException("Awaiting initConnectionLatch has been timeout."); } // Started. transferDataLatch = new CountDownLatch(1); startExtendingTtl(transactionUrl, dataPacketChannel, null); } catch (final InterruptedException e) { throw new IOException("Awaiting initConnectionLatch has been interrupted.", e); } }