List of usage examples for java.nio.channels FileChannel open
public static FileChannel open(Path path, OpenOption... options) throws IOException
From source file:org.apache.kylin.cube.inmemcubing.ConcurrentDiskStore.java
private void openReadChannel() throws IOException { if (readChannel == null) { readChannel = FileChannel.open(diskFile.toPath(), StandardOpenOption.READ); }//from w ww. j a va 2 s . co m }
From source file:org.apache.nifi.processors.standard.TailFile.java
/** * Updates member variables to reflect the "expected recovery checksum" and * seek to the appropriate location in the tailed file, updating our * checksum, so that we are ready to proceed with the * {@link #onTrigger(ProcessContext, ProcessSession)} call. * * @param context the ProcessContext/* w ww . j av a2 s. c om*/ * @param stateValues the values that were recovered from state that was * previously stored. This Map should be populated with the keys defined in * {@link TailFileState.StateKeys}. * @param filePath the file of the file for which state must be recovered * @throws IOException if unable to seek to the appropriate location in the * tailed file. */ private void recoverState(final ProcessContext context, final Map<String, String> stateValues, final String filePath) throws IOException { final String prefix = MAP_PREFIX + states.get(filePath).getFilenameIndex() + '.'; if (!stateValues.containsKey(prefix + TailFileState.StateKeys.FILENAME)) { resetState(filePath); return; } if (!stateValues.containsKey(prefix + TailFileState.StateKeys.POSITION)) { resetState(filePath); return; } if (!stateValues.containsKey(prefix + TailFileState.StateKeys.TIMESTAMP)) { resetState(filePath); return; } if (!stateValues.containsKey(prefix + TailFileState.StateKeys.LENGTH)) { resetState(filePath); return; } final String checksumValue = stateValues.get(prefix + TailFileState.StateKeys.CHECKSUM); final boolean checksumPresent = (checksumValue != null); final String storedStateFilename = stateValues.get(prefix + TailFileState.StateKeys.FILENAME); final long position = Long.parseLong(stateValues.get(prefix + TailFileState.StateKeys.POSITION)); final long timestamp = Long.parseLong(stateValues.get(prefix + TailFileState.StateKeys.TIMESTAMP)); final long length = Long.parseLong(stateValues.get(prefix + TailFileState.StateKeys.LENGTH)); FileChannel reader = null; File tailFile = null; if (checksumPresent && filePath.equals(storedStateFilename)) { states.get(filePath).setExpectedRecoveryChecksum(Long.parseLong(checksumValue)); // We have an expected checksum and the currently configured filename is the same as the state file. // We need to check if the existing file is the same as the one referred to in the state file based on // the checksum. final Checksum checksum = new CRC32(); final File existingTailFile = new File(storedStateFilename); if (existingTailFile.length() >= position) { try (final InputStream tailFileIs = new FileInputStream(existingTailFile); final CheckedInputStream in = new CheckedInputStream(tailFileIs, checksum)) { StreamUtils.copy(in, new NullOutputStream(), states.get(filePath).getState().getPosition()); final long checksumResult = in.getChecksum().getValue(); if (checksumResult == states.get(filePath).getExpectedRecoveryChecksum()) { // Checksums match. This means that we want to resume reading from where we left off. // So we will populate the reader object so that it will be used in onTrigger. If the // checksums do not match, then we will leave the reader object null, so that the next // call to onTrigger will result in a new Reader being created and starting at the // beginning of the file. getLogger().debug( "When recovering state, checksum of tailed file matches the stored checksum. Will resume where left off."); tailFile = existingTailFile; reader = FileChannel.open(tailFile.toPath(), StandardOpenOption.READ); getLogger().debug("Created FileChannel {} for {} in recoverState", new Object[] { reader, tailFile }); reader.position(position); } else { // we don't seek the reader to the position, so our reader will start at beginning of file. getLogger().debug( "When recovering state, checksum of tailed file does not match the stored checksum. Will begin tailing current file from beginning."); } } } else { // fewer bytes than our position, so we know we weren't already reading from this file. Keep reader at a position of 0. getLogger().debug( "When recovering state, existing file to tail is only {} bytes but position flag is {}; " + "this indicates that the file has rotated. Will begin tailing current file from beginning.", new Object[] { existingTailFile.length(), position }); } states.get(filePath).setState(new TailFileState(filePath, tailFile, reader, position, timestamp, length, checksum, ByteBuffer.allocate(65536))); } else { resetState(filePath); } getLogger().debug("Recovered state {}", new Object[] { states.get(filePath).getState() }); }
From source file:org.apache.nifi.processors.standard.TailFile.java
private void processTailFile(final ProcessContext context, final ProcessSession session, final String tailFile) { // If user changes the file that is being tailed, we need to consume the already-rolled-over data according // to the Initial Start Position property boolean rolloverOccurred; TailFileObject tfo = states.get(tailFile); if (tfo.isTailFileChanged()) { rolloverOccurred = false;// w w w . j av a 2 s . co m final String recoverPosition = context.getProperty(START_POSITION).getValue(); if (START_BEGINNING_OF_TIME.getValue().equals(recoverPosition)) { recoverRolledFiles(context, session, tailFile, tfo.getExpectedRecoveryChecksum(), tfo.getState().getTimestamp(), tfo.getState().getPosition()); } else if (START_CURRENT_FILE.getValue().equals(recoverPosition)) { cleanup(); tfo.setState(new TailFileState(tailFile, null, null, 0L, 0L, 0L, null, tfo.getState().getBuffer())); } else { final String filename = tailFile; final File file = new File(filename); try { final FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ); getLogger().debug("Created FileChannel {} for {}", new Object[] { fileChannel, file }); final Checksum checksum = new CRC32(); final long position = file.length(); final long timestamp = file.lastModified(); try (final InputStream fis = new FileInputStream(file); final CheckedInputStream in = new CheckedInputStream(fis, checksum)) { StreamUtils.copy(in, new NullOutputStream(), position); } fileChannel.position(position); cleanup(); tfo.setState(new TailFileState(filename, file, fileChannel, position, timestamp, file.length(), checksum, tfo.getState().getBuffer())); } catch (final IOException ioe) { getLogger().error( "Attempted to position Reader at current position in file {} but failed to do so due to {}", new Object[] { file, ioe.toString() }, ioe); context.yield(); return; } } tfo.setTailFileChanged(false); } else { // Recover any data that may have rolled over since the last time that this processor ran. // If expectedRecoveryChecksum != null, that indicates that this is the first iteration since processor was started, so use whatever checksum value // was present when the state was last persisted. In this case, we must then null out the value so that the next iteration won't keep using the "recovered" // value. If the value is null, then we know that either the processor has already recovered that data, or there was no state persisted. In either case, // use whatever checksum value is currently in the state. Long expectedChecksumValue = tfo.getExpectedRecoveryChecksum(); if (expectedChecksumValue == null) { expectedChecksumValue = tfo.getState().getChecksum() == null ? null : tfo.getState().getChecksum().getValue(); } rolloverOccurred = recoverRolledFiles(context, session, tailFile, expectedChecksumValue, tfo.getState().getTimestamp(), tfo.getState().getPosition()); tfo.setExpectedRecoveryChecksum(null); } // initialize local variables from state object; this is done so that we can easily change the values throughout // the onTrigger method and then create a new state object after we finish processing the files. TailFileState state = tfo.getState(); File file = state.getFile(); FileChannel reader = state.getReader(); Checksum checksum = state.getChecksum(); if (checksum == null) { checksum = new CRC32(); } long position = state.getPosition(); long timestamp = state.getTimestamp(); long length = state.getLength(); // Create a reader if necessary. if (file == null || reader == null) { file = new File(tailFile); reader = createReader(file, position); if (reader == null) { context.yield(); return; } } final long startNanos = System.nanoTime(); // Check if file has rotated if (rolloverOccurred || (timestamp <= file.lastModified() && length > file.length()) || (timestamp < file.lastModified() && length >= file.length())) { // Since file has rotated, we close the reader, create a new one, and then reset our state. try { reader.close(); getLogger().debug("Closed FileChannel {}", new Object[] { reader, reader }); } catch (final IOException ioe) { getLogger().warn("Failed to close reader for {} due to {}", new Object[] { file, ioe }); } reader = createReader(file, 0L); position = 0L; checksum.reset(); } if (file.length() == position || !file.exists()) { // no data to consume so rather than continually running, yield to allow other processors to use the thread. getLogger().debug("No data to consume; created no FlowFiles"); tfo.setState(new TailFileState(tailFile, file, reader, position, timestamp, length, checksum, state.getBuffer())); persistState(tfo, context); context.yield(); return; } // If there is data to consume, read as much as we can. final TailFileState currentState = state; final Checksum chksum = checksum; // data has been written to file. Stream it to a new FlowFile. FlowFile flowFile = session.create(); final FileChannel fileReader = reader; final AtomicLong positionHolder = new AtomicLong(position); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream rawOut) throws IOException { try (final OutputStream out = new BufferedOutputStream(rawOut)) { positionHolder.set(readLines(fileReader, currentState.getBuffer(), out, chksum)); } } }); // If there ended up being no data, just remove the FlowFile if (flowFile.getSize() == 0) { session.remove(flowFile); getLogger().debug("No data to consume; removed created FlowFile"); } else { // determine filename for FlowFile by using <base filename of log file>.<initial offset>-<final offset>.<extension> final String tailFilename = file.getName(); final String baseName = StringUtils.substringBeforeLast(tailFilename, "."); final String flowFileName; if (baseName.length() < tailFilename.length()) { flowFileName = baseName + "." + position + "-" + positionHolder.get() + "." + StringUtils.substringAfterLast(tailFilename, "."); } else { flowFileName = baseName + "." + position + "-" + positionHolder.get(); } final Map<String, String> attributes = new HashMap<>(3); attributes.put(CoreAttributes.FILENAME.key(), flowFileName); attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain"); attributes.put("tailfile.original.path", tailFile); flowFile = session.putAllAttributes(flowFile, attributes); session.getProvenanceReporter().receive(flowFile, file.toURI().toString(), "FlowFile contains bytes " + position + " through " + positionHolder.get() + " of source file", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos)); session.transfer(flowFile, REL_SUCCESS); position = positionHolder.get(); // Set timestamp to the latest of when the file was modified and the current timestamp stored in the state. // We do this because when we read a file that has been rolled over, we set the state to 1 millisecond later than the last mod date // in order to avoid ingesting that file again. If we then read from this file during the same second (or millisecond, depending on the // operating system file last mod precision), then we could set the timestamp to a smaller value, which could result in reading in the // rotated file a second time. timestamp = Math.max(state.getTimestamp(), file.lastModified()); length = file.length(); getLogger().debug("Created {} and routed to success", new Object[] { flowFile }); } // Create a new state object to represent our current position, timestamp, etc. tfo.setState(new TailFileState(tailFile, file, reader, position, timestamp, length, checksum, state.getBuffer())); // We must commit session before persisting state in order to avoid data loss on restart session.commit(); persistState(tfo, context); }
From source file:org.apache.nifi.processors.standard.TailFile.java
private FileChannel createReader(final File file, final long position) { final FileChannel reader; try {//from w w w . j a va2s . c om reader = FileChannel.open(file.toPath(), StandardOpenOption.READ); } catch (final IOException ioe) { getLogger().warn( "Unable to open file {}; will attempt to access file again after the configured Yield Duration has elapsed: {}", new Object[] { file, ioe }); return null; } getLogger().debug("Created FileChannel {} for {}", new Object[] { reader, file }); try { reader.position(position); } catch (final IOException ioe) { getLogger().error("Failed to read from {} due to {}", new Object[] { file, ioe }); try { reader.close(); getLogger().debug("Closed FileChannel {}", new Object[] { reader }); } catch (final IOException ioe2) { } return null; } return reader; }
From source file:org.cryptomator.cryptofs.CryptoFileSystemProviderIntegrationTest.java
@Test public void testOpenAndCloseFileChannel() throws IOException { FileSystem fs = CryptoFileSystemProvider.newFileSystem(tmpPath, cryptoFileSystemProperties().withPassphrase("asd").build()); try (FileChannel ch = FileChannel.open(fs.getPath("/foo"), EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW))) { Assert.assertTrue(ch instanceof CryptoFileChannel); }// w ww. ja v a 2s . com }
From source file:org.darkware.wpman.security.ChecksumDatabase.java
/** * Calculate a checksum on the given file. * * @param file The file to calculate a checksum for. * @return The checksum as a {@code String}. * @see #doChecksum(ReadableByteChannel) *//*from ww w .j a va 2 s. co m*/ protected String calculateChecksum(final Path file) { try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { return this.doChecksum(channel); } catch (IOException e) { ChecksumDatabase.log.error("Failed to calculate checksum on {}: {}", file, e.getLocalizedMessage()); return ""; } }
From source file:org.eclipse.packagedrone.utils.rpm.build.PayloadRecorder.java
@Override public FileChannel openChannel() throws IOException { checkFinished(true);//from ww w . j a v a 2 s.c om return FileChannel.open(this.tempFile, StandardOpenOption.READ); }
From source file:org.lockss.repository.TestRepositoryNodeImpl.java
void findMaxDirPathNio(File root) { int maxName = findMaxDirname(root) - 10; String one = mkstr("onedir", maxName) + "/"; for (int rpt = 1; rpt < 1000; rpt++) { String path = StringUtils.repeat(one, rpt); File dir = new File(root, path); String dirstr = dir.getPath(); boolean res = dir.mkdirs(); if (!res) { log.info("mkdirs failed at " + dirstr.length() + " chars"); break; }/* ww w.j a v a2 s.c o m*/ log.info("mkdirs ok: " + dirstr.length()); File f = new File(dir, "foobbb"); try { Path npath = Paths.get(f.getPath()); Files.createFile(npath); FileChannel ochan = FileChannel.open(npath, StandardOpenOption.WRITE); OutputStream os = Channels.newOutputStream(ochan); os.write((byte) 44); os.close(); FileChannel ichan = FileChannel.open(npath, StandardOpenOption.READ); InputStream is = Channels.newInputStream(ichan); int bb = is.read(); is.close(); assertEquals(44, bb); log.info("file ok at " + npath.toString().length() + " chars"); } catch (FileNotFoundException fnfe) { log.error("FNF: " + f.getPath().length(), fnfe); } catch (IOException ioe) { log.error("IOE: " + f.getPath().length() + ", " + ioe.getMessage()); } } }
From source file:org.neo4j.io.fs.FileUtils.java
public static FileChannel open(Path path, String mode) throws IOException { return FileChannel.open(path, convertOpenMode(mode)); }
From source file:org.opencb.opencga.server.rest.utils.FileRanges.java
@GET @Path("/ranges/{file}") @ApiOperation(value = "Fetchs alignment files using HTTP Ranges protocol") @Produces("text/plain") public Response getRanges(@Context HttpHeaders headers, @ApiParam(value = "File id, name or path") @PathParam("file") String fileIdStr, @ApiParam(value = "Study [[user@]project:]study where study and project can be either the id or alias") @QueryParam("study") String studyStr) { DataInputStream stream = null; try {/*ww w .j a v a 2 s . com*/ AbstractManager.MyResourceId resource = catalogManager.getFileManager().getId(fileIdStr, studyStr, sessionId); catalogManager.getAuthorizationManager().checkFilePermission(resource.getResourceId(), resource.getUser(), FileAclEntry.FilePermissions.DOWNLOAD); QueryResult<File> queryResult = catalogManager.getFile(resource.getResourceId(), this.queryOptions, sessionId); File file = queryResult.getResult().get(0); List<String> rangeList = headers.getRequestHeader("range"); if (rangeList != null) { long from; long to; String[] acceptedRanges = rangeList.get(0).split("=")[1].split("-"); from = Long.parseLong(acceptedRanges[0]); to = Long.parseLong(acceptedRanges[1]); int length = (int) (to - from) + 1; ByteBuffer buf = ByteBuffer.allocate(length); logger.debug("from: {} , to: {}, length:{}", from, to, length); StopWatch t = StopWatch.createStarted(); java.nio.file.Path filePath = Paths.get(file.getUri()); try (FileChannel fc = (FileChannel.open(filePath, StandardOpenOption.READ))) { fc.position(from); fc.read(buf); } t.stop(); logger.debug("Skip {}B and read {}B in {}s", from, length, t.getTime(TimeUnit.MILLISECONDS) / 1000.0); return Response.ok(buf.array(), MediaType.APPLICATION_OCTET_STREAM_TYPE) .header("Accept-Ranges", "bytes").header("Access-Control-Allow-Origin", "*") .header("Access-Control-Allow-Headers", "x-requested-with, content-type, range") .header("Access-Control-Allow-Credentials", "true") .header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") .header("Content-Range", "bytes " + from + "-" + to + "/" + file.getSize()) .header("Content-length", to - from + 1).status(Response.Status.PARTIAL_CONTENT).build(); } else { stream = catalogManager.downloadFile(resource.getResourceId(), sessionId); return createOkResponse(stream, MediaType.APPLICATION_OCTET_STREAM_TYPE, file.getName()); } } catch (Exception e) { if (stream != null) { try { stream.close(); } catch (IOException ignore) { } } return createErrorResponse(e); } }