List of usage examples for java.nio.channels WritableByteChannel close
public void close() throws IOException;
From source file:org.alfresco.repo.transfer.HttpClientTransmitterImpl.java
/** * *///from ww w . jav a 2s .c om public void getTransferReport(Transfer transfer, OutputStream result) { TransferTarget target = transfer.getTransferTarget(); PostMethod getReportRequest = getPostMethod(); try { HostConfiguration hostConfig = getHostConfig(target); HttpState httpState = getHttpState(target); try { getReportRequest.setPath(target.getEndpointPath() + "/report"); //Put the transferId on the query string getReportRequest.setQueryString( new NameValuePair[] { new NameValuePair("transferId", transfer.getTransferId()) }); int responseStatus = httpClient.executeMethod(hostConfig, getReportRequest, httpState); checkResponseStatus("getReport", responseStatus, getReportRequest); InputStream is = getReportRequest.getResponseBodyAsStream(); // Now copy the response input stream to result. final ReadableByteChannel inputChannel = Channels.newChannel(is); final WritableByteChannel outputChannel = Channels.newChannel(result); try { // copy the channels channelCopy(inputChannel, outputChannel); } finally { // closing the channels inputChannel.close(); outputChannel.close(); } return; } catch (RuntimeException e) { throw e; } catch (Exception e) { String error = "Failed to execute HTTP request to target"; log.debug(error, e); throw new TransferException(MSG_HTTP_REQUEST_FAILED, new Object[] { "getTransferReport", target.toString(), e.toString() }, e); } } finally { getReportRequest.releaseConnection(); } }
From source file:com.google.cloud.hadoop.gcsio.GoogleCloudStorageIntegrationHelper.java
/** * Writes a file with the given buffer repeated numWrites times. * * @param bucketName name of the bucket to create object in * @param objectName name of the object to create * @param buffer Data to write//from w ww . j a va 2 s .c o m * @param numWrites number of times to repeat the data * @return number of bytes written */ protected int writeFile(String bucketName, String objectName, ByteBuffer buffer, int numWrites) throws IOException { int numBytesWritten = -1; int totalBytesWritten = 0; WritableByteChannel writeChannel = null; try { writeChannel = create(bucketName, objectName, new CreateFileOptions(false /* overwrite existing */)); for (int i = 0; i < numWrites; i++) { buffer.clear(); numBytesWritten = writeChannel.write(buffer); Assert.assertEquals("could not write the entire buffer", buffer.capacity(), numBytesWritten); totalBytesWritten += numBytesWritten; } } finally { if (writeChannel != null) { writeChannel.close(); } } return totalBytesWritten; }
From source file:gobblin.data.management.copy.writer.TarArchiveInputStreamDataWriter.java
/** * Untars the passed in {@link FileAwareInputStream} to the task's staging directory. Uses the name of the root * {@link TarArchiveEntry} in the stream as the directory name for the untarred file. The method also commits the data * by moving the file from staging to output directory. * * @see gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter#write(gobblin.data.management.copy.FileAwareInputStream) *//*from w ww .ja va 2 s . c om*/ @Override public void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile) throws IOException { this.closer.register(inputStream); TarArchiveInputStream tarIn = new TarArchiveInputStream(inputStream); final ReadableByteChannel inputChannel = Channels.newChannel(tarIn); TarArchiveEntry tarEntry; // flush the first entry in the tar, which is just the root directory tarEntry = tarIn.getNextTarEntry(); String tarEntryRootName = StringUtils.remove(tarEntry.getName(), Path.SEPARATOR); log.info("Unarchiving at " + writeAt); try { while ((tarEntry = tarIn.getNextTarEntry()) != null) { // the API tarEntry.getName() is misleading, it is actually the path of the tarEntry in the tar file String newTarEntryPath = tarEntry.getName().replace(tarEntryRootName, writeAt.getName()); Path tarEntryStagingPath = new Path(writeAt.getParent(), newTarEntryPath); if (tarEntry.isDirectory() && !this.fs.exists(tarEntryStagingPath)) { this.fs.mkdirs(tarEntryStagingPath); } else if (!tarEntry.isDirectory()) { FSDataOutputStream out = this.fs.create(tarEntryStagingPath, true); final WritableByteChannel outputChannel = Channels.newChannel(out); try { StreamCopier copier = new StreamCopier(inputChannel, outputChannel); if (isInstrumentationEnabled()) { copier.withCopySpeedMeter(this.copySpeedMeter); } this.bytesWritten.addAndGet(copier.copy()); if (isInstrumentationEnabled()) { log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(), this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate()); } else { log.info("File {} copied.", copyableFile.getOrigin().getPath()); } } finally { out.close(); outputChannel.close(); } } } } finally { tarIn.close(); inputChannel.close(); inputStream.close(); } }
From source file:org.wymiwyg.wrhapi.test.BaseTests.java
@Test public void testEmptyBodyAnHeader() throws Exception { final String body = ""; WebServer webServer = createServer().startNewWebServer(new Handler() { public void handle(Request request, final Response response) throws HandlerException { log.info("handling testEmptyBody"); response.setResponseStatus(ResponseStatus.CREATED); response.setBody(new MessageBody2Write() { public void writeTo(WritableByteChannel out) throws IOException { try { response.setHeader(HeaderName.CONTENT_TYPE, "text/plain"); } catch (HandlerException ex) { throw new RuntimeException(ex); }/* ww w . ja va 2s. c o m*/ out.write(ByteBuffer.wrap(body.getBytes())); out.close(); } }); } }, serverBinding); try { URL serverURL = new URL("http://" + serverBinding.getInetAddress().getHostAddress() + ":" + serverBinding.getPort() + "/"); HttpURLConnection connection = (HttpURLConnection) serverURL.openConnection(); assertEquals("text/plain", connection.getHeaderField("Content-Type")); assertEquals(ResponseStatus.CREATED.getCode(), connection.getResponseCode()); Reader reader = new InputStreamReader(connection.getInputStream()); StringWriter stringWriter = new StringWriter(); for (int ch = reader.read(); ch != -1; ch = reader.read()) { stringWriter.write(ch); } assertEquals(body, stringWriter.toString()); } catch (MalformedURLException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } finally { webServer.stop(); } }
From source file:org.apache.gobblin.data.management.copy.writer.TarArchiveInputStreamDataWriter.java
/** * Untars the passed in {@link FileAwareInputStream} to the task's staging directory. Uses the name of the root * {@link TarArchiveEntry} in the stream as the directory name for the untarred file. The method also commits the data * by moving the file from staging to output directory. * * @see org.apache.gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter#write(org.apache.gobblin.data.management.copy.FileAwareInputStream) *///from w w w . j av a 2 s .c om @Override public void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile) throws IOException { this.closer.register(inputStream); TarArchiveInputStream tarIn = new TarArchiveInputStream(inputStream); final ReadableByteChannel inputChannel = Channels.newChannel(tarIn); TarArchiveEntry tarEntry; // flush the first entry in the tar, which is just the root directory tarEntry = tarIn.getNextTarEntry(); String tarEntryRootName = StringUtils.remove(tarEntry.getName(), Path.SEPARATOR); log.info("Unarchiving at " + writeAt); try { while ((tarEntry = tarIn.getNextTarEntry()) != null) { // the API tarEntry.getName() is misleading, it is actually the path of the tarEntry in the tar file String newTarEntryPath = tarEntry.getName().replace(tarEntryRootName, writeAt.getName()); Path tarEntryStagingPath = new Path(writeAt.getParent(), newTarEntryPath); if (!FileUtils.isSubPath(writeAt.getParent(), tarEntryStagingPath)) { throw new IOException( String.format("Extracted file: %s is trying to write outside of output directory: %s", tarEntryStagingPath, writeAt.getParent())); } if (tarEntry.isDirectory() && !this.fs.exists(tarEntryStagingPath)) { this.fs.mkdirs(tarEntryStagingPath); } else if (!tarEntry.isDirectory()) { FSDataOutputStream out = this.fs.create(tarEntryStagingPath, true); final WritableByteChannel outputChannel = Channels.newChannel(out); try { StreamCopier copier = new StreamCopier(inputChannel, outputChannel); if (isInstrumentationEnabled()) { copier.withCopySpeedMeter(this.copySpeedMeter); } this.bytesWritten.addAndGet(copier.copy()); if (isInstrumentationEnabled()) { log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(), this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate()); } else { log.info("File {} copied.", copyableFile.getOrigin().getPath()); } } finally { out.close(); outputChannel.close(); } } } } finally { tarIn.close(); inputChannel.close(); inputStream.close(); } }
From source file:org.bytesoft.openjtcc.supports.logger.DbTransactionLoggerImpl.java
private byte[] streamToByteArray(InputStream input) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ReadableByteChannel in = null; WritableByteChannel out = null; ByteBuffer buffer = ByteBuffer.allocate(1024); try {//from w ww . j av a 2 s . co m in = Channels.newChannel(input); out = Channels.newChannel(baos); while (in.read(buffer) != -1) { buffer.flip(); out.write(buffer); buffer.clear(); } } catch (IOException ex) { // ignore } finally { if (out != null) { try { out.close(); } catch (IOException e) { // ignore } } if (baos != null) { try { baos.close(); } catch (IOException e) { // ignore } } } return baos.toByteArray(); }
From source file:com.linkedin.databus.core.TestDbusEventBufferMult.java
@Test public void testMultiPPartionStreamStats() throws Exception { createBufMult();//from ww w . ja v a 2 s . c o m PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() }; //generate a bunch of windows for 3 partitions int windowsNum = 10; for (int i = 1; i <= windowsNum; ++i) { DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 1, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 2, null); } String[] pnames = new String[p.length]; int count = 0; for (PhysicalPartition ip : p) { pnames[count++] = ip.toSimpleString(); } StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames); PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) }; CheckpointMult cpMult = new CheckpointMult(); for (int i = 0; i < 3; ++i) { Checkpoint cp = new Checkpoint(); cp.setFlexible(); cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cpMult.addCheckpoint(p[i], cp); } DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl); ByteArrayOutputStream baos = new ByteArrayOutputStream(); WritableByteChannel writeChannel = Channels.newChannel(baos); reader.streamEvents(false, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter()); writeChannel.close(); baos.close(); //make sure we got the physical partition names right List<String> ppartNames = statsColl.getStatsCollectorKeys(); assertEquals(ppartNames.size(), 3); HashSet<String> expectedPPartNames = new HashSet<String>( Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString())); for (String ppartName : ppartNames) { assertTrue(expectedPPartNames.contains(ppartName)); } //verify event counts per partition DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() }; assertEquals(ppartStats[0].getNumDataEvents(), windowsNum); assertEquals(ppartStats[1].getNumDataEvents(), windowsNum * 2); assertEquals(ppartStats[2].getNumDataEvents(), windowsNum * 3); assertEquals(ppartStats[0].getNumSysEvents(), windowsNum); assertEquals(ppartStats[1].getNumSysEvents(), windowsNum); assertEquals(ppartStats[2].getNumSysEvents(), windowsNum); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), windowsNum * (1 + 2 + 3)); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), windowsNum * 3); assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math .max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math .min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); }
From source file:com.linkedin.databus.core.TestDbusEventBufferMult.java
@Test public void testSinglePPartionStreamFromLatest() throws Exception { createBufMult();//from w ww .j a v a 2 s . c om PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition() }; //generate a bunch of windows for 3 partitions int windowsNum = 10; for (int i = 1; i <= windowsNum; ++i) { DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i, null); } String[] pnames = new String[p.length]; int count = 0; for (PhysicalPartition ip : p) { pnames[count++] = ip.toSimpleString(); } StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames); PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]) }; CheckpointMult cpMult = new CheckpointMult(); Checkpoint cp = new Checkpoint(); cp.setFlexible(); cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cpMult.addCheckpoint(p[0], cp); DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl); ByteArrayOutputStream baos = new ByteArrayOutputStream(); WritableByteChannel writeChannel = Channels.newChannel(baos); // Set streamFromLatestScn == true reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter()); writeChannel.close(); baos.close(); //make sure we got the physical partition names right List<String> ppartNames = statsColl.getStatsCollectorKeys(); assertEquals(ppartNames.size(), 1); HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString())); for (String ppartName : ppartNames) { assertTrue(expectedPPartNames.contains(ppartName)); } //verify event counts per partition DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats() }; // Only the last window is returned in each of the partitions assertEquals(ppartStats[0].getNumDataEvents(), 1); assertEquals(ppartStats[0].getNumSysEvents(), 1); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1)); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1)); assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), ppartStats[0].getTimeLag()); assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), ppartStats[0].getTimeLag()); }
From source file:com.linkedin.databus.core.TestDbusEventBufferMult.java
@Test public void testMultiPPartionStreamFromLatest() throws Exception { createBufMult();/* w w w .j a v a 2s .c o m*/ PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() }; //generate a bunch of windows for 3 partitions int windowsNum = 10; for (int i = 1; i <= windowsNum; ++i) { DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]); buf.startEvents(); byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset()); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 1, null); buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]); buf.startEvents(); assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null)); buf.endEvents(100 * i + 2, null); } String[] pnames = new String[p.length]; int count = 0; for (PhysicalPartition ip : p) { pnames[count++] = ip.toSimpleString(); } StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames); PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) }; CheckpointMult cpMult = new CheckpointMult(); for (int i = 0; i < 3; ++i) { Checkpoint cp = new Checkpoint(); cp.setFlexible(); cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION); cpMult.addCheckpoint(p[i], cp); } DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl); ByteArrayOutputStream baos = new ByteArrayOutputStream(); WritableByteChannel writeChannel = Channels.newChannel(baos); // Set streamFromLatestScn == true reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter()); writeChannel.close(); baos.close(); //make sure we got the physical partition names right List<String> ppartNames = statsColl.getStatsCollectorKeys(); assertEquals(ppartNames.size(), 3); HashSet<String> expectedPPartNames = new HashSet<String>( Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString())); for (String ppartName : ppartNames) { assertTrue(expectedPPartNames.contains(ppartName)); } //verify event counts per partition DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() }; // Only the last window is returned in each of the partitions assertEquals(ppartStats[0].getNumDataEvents(), 1); assertEquals(ppartStats[1].getNumDataEvents(), 2); assertEquals(ppartStats[2].getNumDataEvents(), 3); assertEquals(ppartStats[0].getNumSysEvents(), 1); assertEquals(ppartStats[1].getNumSysEvents(), 1); assertEquals(ppartStats[2].getNumSysEvents(), 1); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1 + 2 + 3)); assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1 + 1 + 1)); assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math .max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math .min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag()))); }