List of usage examples for java.nio.channels SeekableByteChannel size
long size() throws IOException;
From source file:Main.java
public static void printDetails(SeekableByteChannel seekableChannel, String msg) { try {// w w w . ja v a 2 s . c o m System.out.println( msg + ": Size = " + seekableChannel.size() + ", Position = " + seekableChannel.position()); } catch (IOException e) { e.printStackTrace(); } }
From source file:com.spectralogic.ds3client.helpers.channels.WindowedSeekableByteChannel_Test.java
@Test(timeout = 1000) public void readChannelSections() throws IOException { try (final SeekableByteChannel channel = stringToChannel("aabbbcccc")) { final Object lock = new Object(); final SeekableByteChannel channelOfAs = new WindowedSeekableByteChannel(channel, lock, 0L, 2L); final SeekableByteChannel channelOfBs = new WindowedSeekableByteChannel(channel, lock, 2L, 3L); final SeekableByteChannel channelOfCs = new WindowedSeekableByteChannel(channel, lock, 5L, 4L); assertThat(channelOfAs.size(), is(2L)); assertThat(channelOfBs.size(), is(3L)); assertThat(channelOfCs.size(), is(4L)); assertThat(channelToString(channelOfAs), is("aa")); assertThat(channelToString(channelOfBs), is("bbb")); assertThat(channelToString(channelOfCs), is("cccc")); }/*from w w w. j a v a 2s .co m*/ }
From source file:com.spectralogic.ds3client.helpers.channels.WindowedSeekableByteChannel_Test.java
@Test(timeout = 1000) public void writeChannelSections() throws IOException { try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { final Object lock = new Object(); final SeekableByteChannel channelOfAs = new WindowedSeekableByteChannel(channel, lock, 0L, 2L); final SeekableByteChannel channelOfBs = new WindowedSeekableByteChannel(channel, lock, 2L, 3L); final SeekableByteChannel channelOfCs = new WindowedSeekableByteChannel(channel, lock, 5L, 4L); assertThat(channelOfAs.size(), is(2L)); assertThat(channelOfBs.size(), is(3L)); assertThat(channelOfCs.size(), is(4L)); writeToChannel("cccc", channelOfCs); writeToChannel("bbb", channelOfBs); writeToChannel("aa", channelOfAs); channel.position(0);//from w ww . j av a 2 s . com assertThat(channel.toString(), is("aabbbcccc")); } }
From source file:com.arpnetworking.metrics.common.tailer.StatefulTailer.java
private boolean readLines(final SeekableByteChannel reader) throws IOException { // Compute the hash if not already set if (!_hash.isPresent() && reader.size() >= REQUIRED_BYTES_FOR_HASH) { _hash = computeHash(reader, REQUIRED_BYTES_FOR_HASH); }//from w ww . j av a 2 s .c o m // Track current position in file and next read position // NOTE: The next read position is always the beginning of a line long position = reader.position(); long nextReadPosition = position; // Reset buffers _buffer.clear(); _lineBuffer.reset(); // Process available data int bufferSize = reader.read(_buffer); boolean hasData = false; boolean hasCR = false; while (isRunning() && bufferSize != -1) { hasData = true; for (int i = 0; i < bufferSize; i++) { final byte ch = _buffer.get(i); switch (ch) { case '\n': hasCR = false; handleLine(); nextReadPosition = position + i + 1; updateCheckpoint(nextReadPosition); break; case '\r': if (hasCR) { _lineBuffer.write('\r'); } hasCR = true; break; default: if (hasCR) { hasCR = false; handleLine(); nextReadPosition = position + i + 1; updateCheckpoint(nextReadPosition); } _lineBuffer.write(ch); } } position = reader.position(); _buffer.clear(); bufferSize = reader.read(_buffer); } reader.position(nextReadPosition); return hasData; }
From source file:com.arpnetworking.tsdcore.tailer.StatefulTailer.java
private Optional<String> computeHash(final SeekableByteChannel reader, final int hashSize) throws IOException { // Don't hash empty data sets if (hashSize <= 0) { return Optional.absent(); }// w w w . j a v a 2 s .c o m // Validate sufficient data to compute the hash final long oldPosition = reader.position(); reader.position(0); if (reader.size() < hashSize) { reader.position(oldPosition); LOGGER.trace(String.format("Reader size insufficient to compute hash; hashSize=%s, hashSize=%d", Integer.valueOf(hashSize), Long.valueOf(reader.size()))); return Optional.absent(); } // Read the data to hash final ByteBuffer buffer = ByteBuffer.allocate(hashSize); int totalBytesRead = 0; while (totalBytesRead < hashSize) { final int bytesRead = reader.read(buffer); if (bytesRead < 0) { LOGGER.warn(String.format("Unexpected end of file reached; totalBytesRead=%d", Long.valueOf(totalBytesRead))); return Optional.absent(); } totalBytesRead += bytesRead; } // Compute the hash _md5.reset(); final byte[] digest = _md5.digest(buffer.array()); final String hash = Hex.encodeHexString(digest); LOGGER.trace(String.format("Computed hash; hash=%s, bytes=%s", hash, Hex.encodeHexString(buffer.array()))); // Return the reader to its original state reader.position(oldPosition); return Optional.of(hash); }
From source file:com.arpnetworking.metrics.common.tailer.StatefulTailer.java
private Optional<String> computeHash(final SeekableByteChannel reader, final int hashSize) throws IOException { // Don't hash empty data sets if (hashSize <= 0) { return Optional.empty(); }// w w w . j a v a 2s.com // Validate sufficient data to compute the hash final long oldPosition = reader.position(); reader.position(0); if (reader.size() < hashSize) { reader.position(oldPosition); LOGGER.trace().setMessage("Reader size insufficient to compute hash").addData("hashSize", hashSize) .addData("readerSize", reader.size()).log(); return Optional.empty(); } // Read the data to hash final ByteBuffer buffer = ByteBuffer.allocate(hashSize); int totalBytesRead = 0; while (totalBytesRead < hashSize) { final int bytesRead = reader.read(buffer); if (bytesRead < 0) { LOGGER.warn().setMessage("Unexpected end of file reached").addData("totalBytesRead", totalBytesRead) .log(); return Optional.empty(); } totalBytesRead += bytesRead; } // Compute the hash _md5.reset(); final byte[] digest = _md5.digest(buffer.array()); final String hash = Hex.encodeHexString(digest); LOGGER.trace().setMessage("Computed hash").addData("hash", hash).log(); // Return the reader to its original state reader.position(oldPosition); return Optional.of(hash); }
From source file:com.arpnetworking.metrics.common.tailer.StatefulTailer.java
private void resume(final SeekableByteChannel reader, final InitialPosition initialPosition) throws IOException { // Attempt to resume from checkpoint long position = initialPosition.get(reader); // Override position with last known position from store _hash = computeHash(reader, REQUIRED_BYTES_FOR_HASH); if (_hash.isPresent()) { final Optional<Long> storedPosition = _positionStore.getPosition(_hash.get()); if (storedPosition.isPresent()) { // Optionally limit the size of the backlog to process final long fileSize = reader.size(); if (_maximumOffsetOnResume.isPresent() && fileSize - storedPosition.get() > _maximumOffsetOnResume.get()) { position = fileSize - _maximumOffsetOnResume.get(); // TODO(vkoskela): Discard the current potentially partial line [AINT-584] } else { position = storedPosition.get(); }/*w ww. j ava 2s . c o m*/ } } LOGGER.info().setMessage("Starting tailer").addData("file", _file).addData("position", position).log(); reader.position(position); }
From source file:com.arpnetworking.metrics.common.tailer.StatefulTailer.java
private void readLoop(final SeekableByteChannel reader) throws IOException, InterruptedException { Optional<Long> lastChecked = Optional.empty(); Optional<String> currentReaderPrefixHash = Optional.empty(); int currentReaderPrefixHashLength = 0; while (isRunning()) { // Obtain properties of file we expect we are reading final Attributes attributes; try {//ww w . j a v a 2 s . c o m attributes = getAttributes(_file, lastChecked); } catch (final NoSuchFileException t) { rotate(Optional.of(reader), String.format("File rotation detected based attributes access failure; file=%s", _file)); // Return to the file loop return; } if (attributes.getLength() < reader.position()) { // File was rotated; either: // 1) Position is past the length of the file // 2) The expected file is smaller than the current file rotate(Optional.of(reader), String.format( "File rotation detected based on length, position and size; file=%s, length=%d, position=%d, size=%d", _file, attributes.getLength(), reader.position(), reader.size())); // Return to the file loop return; } else { // File was _likely_ not rotated if (reader.size() > reader.position()) { // There is more data in the file if (!readLines(reader)) { // There actually isn't any more data in the file; this // means the file was rotated and the new file has more // data than the old file (e.g. rotation from empty). // TODO(vkoskela): Account for missing final newline. [MAI-322] // There is a degenerate case where the last line in a // file does not have a newline. Then readLines will // always find new data, but the file has been rotated // away. We should buffer the contents of partial lines // thereby detecting when the length grows whether we // actually got more data in the current file. rotate(Optional.<SeekableByteChannel>empty(), String.format( "File rotation detected based on length and no new data; file=%s, length=%d, position=%d", _file, attributes.getLength(), reader.position())); // Return to the file loop return; } lastChecked = Optional.of(attributes.getLastModifiedTime()); // This control path, specifically, successfully reading // data from the file does not trigger a wait. This permits // continuous reading without pausing. } else if (attributes.isNewer()) { // The file does not contain any additional data, but its // last modified date is after the last read date. The file // must have rotated and contains the same length of // content. This can happen on periodic systems which log // the same data at the beginning of each period. rotate(Optional.<SeekableByteChannel>empty(), String.format( "File rotation detected based equal length and position but newer" + "; file=%s, length=%d, position=%d, lastChecked=%s, attributes=%s", _file, attributes.getLength(), reader.position(), lastChecked.get(), attributes)); // Return to the file loop return; } else { // The files are the same size and the timestamps are the // same. This is more common than it sounds since file // modification timestamps are not very precise on many // file systems. // // Since we're not doing anything at this point let's hash // the first N bytes of the current file and the expected // file to see if we're still working on the same file. final Optional<Boolean> hashesSame = compareByHash(currentReaderPrefixHash, currentReaderPrefixHashLength); if (hashesSame.isPresent() && !hashesSame.get()) { // The file rotated with the same length! rotate(Optional.<SeekableByteChannel>empty(), String.format("File rotation detected based on hash; file=%s", _file)); // Return to the file loop return; } // else: the files are empty or the hashes are the same. In // either case we don't have enough data to determine if // the files are different; we'll need to wait and see when // more data is written if the size and age diverge. // TODO(vkoskela): Configurable maximum rotation hash size. [MAI-323] // TODO(vkoskela): Configurable minimum rotation hash size. [MAI-324] // TODO(vkoskela): Configurable identity hash size. [MAI-325] // TODO(vkoskela): We should add a rehash interval. [MAI-326] // This interval would be separate from the read interval, // and generally longer, preventing us from rehashing the // file every interval; but short enough that we don't wait // too long before realizing a slowly growing file was // rotated. // Read interval _trigger.waitOnTrigger(); } } // Compute the prefix hash unless we have an identity final int newPrefixHashLength = (int) Math.min(reader.size(), REQUIRED_BYTES_FOR_HASH); if (!_hash.isPresent() && (currentReaderPrefixHashLength != newPrefixHashLength || !currentReaderPrefixHash.isPresent())) { currentReaderPrefixHashLength = newPrefixHashLength; currentReaderPrefixHash = computeHash(reader, currentReaderPrefixHashLength); } // Update the reader position updateCheckpoint(reader.position()); } }
From source file:com.arpnetworking.tsdcore.tailer.StatefulTailer.java
private void readLoop(final SeekableByteChannel reader) throws IOException, InterruptedException { Optional<Long> lastChecked = Optional.absent(); Optional<String> currentReaderPrefixHash = Optional.absent(); int currentReaderPrefixHashLength = 0; while (isRunning()) { // Obtain properties of file we expect we are reading final Attributes attributes; try {/*from w ww. j av a2 s .c o m*/ attributes = getAttributes(_file, lastChecked); } catch (final NoSuchFileException t) { rotate(Optional.of(reader), String.format("File rotation detected based attributes access failure; file=%s", _file)); // Return to the file loop return; } if (attributes.getLength() < reader.position()) { // File was rotated; either: // 1) Position is past the length of the file // 2) The expected file is smaller than the current file rotate(Optional.of(reader), String.format( "File rotation detected based on length, position and size; file=%s, length=%d, position=%d, size=%d", _file, Long.valueOf(attributes.getLength()), Long.valueOf(reader.position()), Long.valueOf(reader.size()))); // Return to the file loop return; } else { // File was _likely_ not rotated if (reader.size() > reader.position()) { // There is more data in the file if (!readLines(reader)) { // There actually isn't any more data in the file; this // means the file was rotated and the new file has more // data than the old file (e.g. rotation from empty). // TODO(vkoskela): Account for missing final newline. [MAI-322] // There is a degenerate case where the last line in a // file does not have a newline. Then readLines will // always find new data, but the file has been rotated // away. We should buffer the contents of partial lines // thereby detecting when the length grows whether we // actually got more data in the current file. rotate(Optional.<SeekableByteChannel>absent(), String.format( "File rotation detected based on length and no new data; file=%s, length=%d, position=%d", _file, Long.valueOf(attributes.getLength()), Long.valueOf(reader.position()))); // Return to the file loop return; } lastChecked = Optional.of(Long.valueOf(_file.lastModified())); } else if (attributes.isNewer()) { // The file does not contain any additional data, but its // last modified date is after the last read date. The file // must have rotated and contains the same length of // content. This can happen on periodic systems which log // the same data at the beginning of each period. rotate(Optional.<SeekableByteChannel>absent(), String.format( "File rotation detected based equal length and position but newer" + "; file=%s, length=%d, position=%d, lastChecked=%s, attributes=%s", _file, Long.valueOf(attributes.getLength()), Long.valueOf(reader.position()), lastChecked.get(), attributes)); // Return to the file loop return; } else { // The files are the same size and the timestamps are the // same. This is more common than it sounds since file // modification timestamps are not very precise on many // file systems. // // Since we're not doing anything at this point let's hash // the first N bytes of the current file and the expected // file to see if we're still working on the same file. final Optional<Boolean> hashesSame = compareByHash(currentReaderPrefixHash, currentReaderPrefixHashLength); if (hashesSame.isPresent() && !hashesSame.get().booleanValue()) { // The file rotated with the same length! rotate(Optional.<SeekableByteChannel>absent(), String.format("File rotation detected based on hash; file=%s", _file)); // Return to the file loop return; } // else: the files are empty or the hashes are the same. In // either case we don't have enough data to determine if // the files are different; we'll need to wait and see when // more data is written if the size and length diverge. // TODO(vkoskela): Configurable maximum rotation hash size. [MAI-323] // TODO(vkoskela): Configurable minimum rotation hash size. [MAI-324] // TODO(vkoskela): Configurable identity hash size. [MAI-325] // TODO(vkoskela): We should add a rehash interval. [MAI-326] // This interval would be separate from the read interval, // and generally longer, preventing us from rehashing the // file every interval; but short enough that we don't wait // too long before realizing a slowly growing file was // rotated. } } // Compute the prefix hash unless we have an identity if (!_hash.isPresent()) { currentReaderPrefixHashLength = (int) Math.min(reader.size(), REQUIRED_BYTES_FOR_HASH); currentReaderPrefixHash = computeHash(reader, currentReaderPrefixHashLength); } // Read interval _trigger.waitOnTrigger(); // Update the reader position updateCheckpoint(reader.position()); } }