List of usage examples for org.apache.commons.lang3 StringUtils substringAfterLast
public static String substringAfterLast(final String str, final String separator)
Gets the substring after the last occurrence of a separator.
From source file:org.apache.nifi.processors.standard.FetchFile.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { FlowFile flowFile = session.get();/*from w w w . j a v a 2 s .com*/ if (flowFile == null) { return; } final StopWatch stopWatch = new StopWatch(true); final String filename = context.getProperty(FILENAME).evaluateAttributeExpressions(flowFile).getValue(); final LogLevel levelFileNotFound = LogLevel .valueOf(context.getProperty(FILE_NOT_FOUND_LOG_LEVEL).getValue()); final LogLevel levelPermDenied = LogLevel.valueOf(context.getProperty(PERM_DENIED_LOG_LEVEL).getValue()); final File file = new File(filename); // Verify that file system is reachable and file exists Path filePath = file.toPath(); if (!Files.exists(filePath) && !Files.notExists(filePath)) { // see https://docs.oracle.com/javase/tutorial/essential/io/check.html for more details getLogger().log(levelFileNotFound, "Could not fetch file {} from file system for {} because the existence of the file cannot be verified; routing to failure", new Object[] { file, flowFile }); session.transfer(session.penalize(flowFile), REL_FAILURE); return; } else if (!Files.exists(filePath)) { getLogger().log(levelFileNotFound, "Could not fetch file {} from file system for {} because the file does not exist; routing to not.found", new Object[] { file, flowFile }); session.getProvenanceReporter().route(flowFile, REL_NOT_FOUND); session.transfer(session.penalize(flowFile), REL_NOT_FOUND); return; } // Verify read permission on file final String user = System.getProperty("user.name"); if (!isReadable(file)) { getLogger().log(levelPermDenied, "Could not fetch file {} from file system for {} due to user {} not having sufficient permissions to read the file; routing to permission.denied", new Object[] { file, flowFile, user }); session.getProvenanceReporter().route(flowFile, REL_PERMISSION_DENIED); session.transfer(session.penalize(flowFile), REL_PERMISSION_DENIED); return; } // If configured to move the file and fail if unable to do so, check that the existing file does not exist and that we have write permissions // for the parent file. final String completionStrategy = context.getProperty(COMPLETION_STRATEGY).getValue(); final String targetDirectoryName = context.getProperty(MOVE_DESTINATION_DIR) .evaluateAttributeExpressions(flowFile).getValue(); if (targetDirectoryName != null) { final File targetDir = new File(targetDirectoryName); if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) { if (targetDir.exists() && (!isWritable(targetDir) || !isDirectory(targetDir))) { getLogger().error( "Could not fetch file {} from file system for {} because Completion Strategy is configured to move the original file to {}, " + "but that is not a directory or user {} does not have permissions to write to that directory", new Object[] { file, flowFile, targetDir, user }); session.transfer(flowFile, REL_FAILURE); return; } final String conflictStrategy = context.getProperty(CONFLICT_STRATEGY).getValue(); if (CONFLICT_FAIL.getValue().equalsIgnoreCase(conflictStrategy)) { final File targetFile = new File(targetDir, file.getName()); if (targetFile.exists()) { getLogger().error( "Could not fetch file {} from file system for {} because Completion Strategy is configured to move the original file to {}, " + "but a file with name {} already exists in that directory and the Move Conflict Strategy is configured for failure", new Object[] { file, flowFile, targetDir, file.getName() }); session.transfer(flowFile, REL_FAILURE); return; } } } } // import content from file system try (final FileInputStream fis = new FileInputStream(file)) { flowFile = session.importFrom(fis, flowFile); } catch (final IOException ioe) { getLogger().error("Could not fetch file {} from file system for {} due to {}; routing to failure", new Object[] { file, flowFile, ioe.toString() }, ioe); session.transfer(session.penalize(flowFile), REL_FAILURE); return; } session.getProvenanceReporter().modifyContent(flowFile, "Replaced content of FlowFile with contents of " + file.toURI(), stopWatch.getElapsed(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); // It is critical that we commit the session before we perform the Completion Strategy. Otherwise, we could have a case where we // ingest the file, delete/move the file, and then NiFi is restarted before the session is committed. That would result in data loss. // As long as we commit the session right here, before we perform the Completion Strategy, we are safe. session.commit(); // Attempt to perform the Completion Strategy action Exception completionFailureException = null; if (COMPLETION_DELETE.getValue().equalsIgnoreCase(completionStrategy)) { // convert to path and use Files.delete instead of file.delete so that if we fail, we know why try { delete(file); } catch (final IOException ioe) { completionFailureException = ioe; } } else if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) { final File targetDirectory = new File(targetDirectoryName); final File targetFile = new File(targetDirectory, file.getName()); try { if (targetFile.exists()) { final String conflictStrategy = context.getProperty(CONFLICT_STRATEGY).getValue(); if (CONFLICT_KEEP_INTACT.getValue().equalsIgnoreCase(conflictStrategy)) { // don't move, just delete the original Files.delete(file.toPath()); } else if (CONFLICT_RENAME.getValue().equalsIgnoreCase(conflictStrategy)) { // rename to add a random UUID but keep the file extension if it has one. final String simpleFilename = targetFile.getName(); final String newName; if (simpleFilename.contains(".")) { newName = StringUtils.substringBeforeLast(simpleFilename, ".") + "-" + UUID.randomUUID().toString() + "." + StringUtils.substringAfterLast(simpleFilename, "."); } else { newName = simpleFilename + "-" + UUID.randomUUID().toString(); } move(file, new File(targetDirectory, newName), false); } else if (CONFLICT_REPLACE.getValue().equalsIgnoreCase(conflictStrategy)) { move(file, targetFile, true); } } else { move(file, targetFile, false); } } catch (final IOException ioe) { completionFailureException = ioe; } } // Handle completion failures if (completionFailureException != null) { getLogger().warn( "Successfully fetched the content from {} for {} but failed to perform Completion Action due to {}; routing to success", new Object[] { file, flowFile, completionFailureException }, completionFailureException); } }
From source file:org.apache.nifi.processors.standard.FetchFileTransfer.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { FlowFile flowFile = session.get();//from w w w. ja v a 2 s. co m if (flowFile == null) { return; } final StopWatch stopWatch = new StopWatch(true); final String host = context.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue(); final int port = context.getProperty(UNDEFAULTED_PORT).evaluateAttributeExpressions(flowFile).asInteger(); final String filename = context.getProperty(REMOTE_FILENAME).evaluateAttributeExpressions(flowFile) .getValue(); // Try to get a FileTransfer object from our cache. BlockingQueue<FileTransferIdleWrapper> transferQueue; synchronized (fileTransferMap) { final Tuple<String, Integer> tuple = new Tuple<>(host, port); transferQueue = fileTransferMap.get(tuple); if (transferQueue == null) { transferQueue = new LinkedBlockingQueue<>(); fileTransferMap.put(tuple, transferQueue); } // periodically close idle connections if (System.currentTimeMillis() - lastClearTime > IDLE_CONNECTION_MILLIS) { closeConnections(false); lastClearTime = System.currentTimeMillis(); } } // we have a queue of FileTransfer Objects. Get one from the queue or create a new one. FileTransfer transfer; FileTransferIdleWrapper transferWrapper = transferQueue.poll(); if (transferWrapper == null) { transfer = createFileTransfer(context); } else { transfer = transferWrapper.getFileTransfer(); } // Pull data from remote system. final InputStream in; try { in = transfer.getInputStream(filename, flowFile); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { StreamUtils.copy(in, out); transfer.flush(); } }); transferQueue.offer(new FileTransferIdleWrapper(transfer, System.nanoTime())); } catch (final FileNotFoundException e) { getLogger().error( "Failed to fetch content for {} from filename {} on remote host {} because the file could not be found on the remote system; routing to {}", new Object[] { flowFile, filename, host, REL_NOT_FOUND.getName() }); session.transfer(session.penalize(flowFile), REL_NOT_FOUND); session.getProvenanceReporter().route(flowFile, REL_NOT_FOUND); return; } catch (final PermissionDeniedException e) { getLogger().error( "Failed to fetch content for {} from filename {} on remote host {} due to insufficient permissions; routing to {}", new Object[] { flowFile, filename, host, REL_PERMISSION_DENIED.getName() }); session.transfer(session.penalize(flowFile), REL_PERMISSION_DENIED); session.getProvenanceReporter().route(flowFile, REL_PERMISSION_DENIED); return; } catch (final ProcessException | IOException e) { try { transfer.close(); } catch (final IOException e1) { getLogger().warn("Failed to close connection to {}:{} due to {}", new Object[] { host, port, e.toString() }, e); } getLogger().error( "Failed to fetch content for {} from filename {} on remote host {}:{} due to {}; routing to comms.failure", new Object[] { flowFile, filename, host, port, e.toString() }, e); session.transfer(session.penalize(flowFile), REL_COMMS_FAILURE); return; } // Add FlowFile attributes final String protocolName = transfer.getProtocolName(); final Map<String, String> attributes = new HashMap<>(); attributes.put(protocolName + ".remote.host", host); attributes.put(protocolName + ".remote.port", String.valueOf(port)); attributes.put(protocolName + ".remote.filename", filename); if (filename.contains("/")) { final String path = StringUtils.substringBeforeLast(filename, "/"); final String filenameOnly = StringUtils.substringAfterLast(filename, "/"); attributes.put(CoreAttributes.PATH.key(), path); attributes.put(CoreAttributes.FILENAME.key(), filenameOnly); } else { attributes.put(CoreAttributes.FILENAME.key(), filename); } flowFile = session.putAllAttributes(flowFile, attributes); // emit provenance event and transfer FlowFile session.getProvenanceReporter().fetch(flowFile, protocolName + "://" + host + ":" + port + "/" + filename, stopWatch.getElapsed(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); // it is critical that we commit the session before moving/deleting the remote file. Otherwise, we could have a situation where // we ingest the data, delete/move the remote file, and then NiFi dies/is shut down before the session is committed. This would // result in data loss! If we commit the session first, we are safe. session.commit(); final String completionStrategy = context.getProperty(COMPLETION_STRATEGY).getValue(); if (COMPLETION_DELETE.getValue().equalsIgnoreCase(completionStrategy)) { try { transfer.deleteFile(null, filename); } catch (final FileNotFoundException e) { // file doesn't exist -- effectively the same as removing it. Move on. } catch (final IOException ioe) { getLogger().warn( "Successfully fetched the content for {} from {}:{}{} but failed to remove the remote file due to {}", new Object[] { flowFile, host, port, filename, ioe }, ioe); } } else if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) { String targetDir = context.getProperty(MOVE_DESTINATION_DIR).evaluateAttributeExpressions(flowFile) .getValue(); if (!targetDir.endsWith("/")) { targetDir = targetDir + "/"; } final String simpleFilename = StringUtils.substringAfterLast(filename, "/"); final String target = targetDir + simpleFilename; try { transfer.rename(filename, target); } catch (final IOException ioe) { getLogger().warn( "Successfully fetched the content for {} from {}:{}{} but failed to rename the remote file due to {}", new Object[] { flowFile, host, port, filename, ioe }, ioe); } } }
From source file:org.apache.nifi.processors.standard.TailFile.java
private void processTailFile(final ProcessContext context, final ProcessSession session, final String tailFile) { // If user changes the file that is being tailed, we need to consume the already-rolled-over data according // to the Initial Start Position property boolean rolloverOccurred; TailFileObject tfo = states.get(tailFile); if (tfo.isTailFileChanged()) { rolloverOccurred = false;// www. j a va2s. c o m final String recoverPosition = context.getProperty(START_POSITION).getValue(); if (START_BEGINNING_OF_TIME.getValue().equals(recoverPosition)) { recoverRolledFiles(context, session, tailFile, tfo.getExpectedRecoveryChecksum(), tfo.getState().getTimestamp(), tfo.getState().getPosition()); } else if (START_CURRENT_FILE.getValue().equals(recoverPosition)) { cleanup(); tfo.setState(new TailFileState(tailFile, null, null, 0L, 0L, 0L, null, tfo.getState().getBuffer())); } else { final String filename = tailFile; final File file = new File(filename); try { final FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ); getLogger().debug("Created FileChannel {} for {}", new Object[] { fileChannel, file }); final Checksum checksum = new CRC32(); final long position = file.length(); final long timestamp = file.lastModified(); try (final InputStream fis = new FileInputStream(file); final CheckedInputStream in = new CheckedInputStream(fis, checksum)) { StreamUtils.copy(in, new NullOutputStream(), position); } fileChannel.position(position); cleanup(); tfo.setState(new TailFileState(filename, file, fileChannel, position, timestamp, file.length(), checksum, tfo.getState().getBuffer())); } catch (final IOException ioe) { getLogger().error( "Attempted to position Reader at current position in file {} but failed to do so due to {}", new Object[] { file, ioe.toString() }, ioe); context.yield(); return; } } tfo.setTailFileChanged(false); } else { // Recover any data that may have rolled over since the last time that this processor ran. // If expectedRecoveryChecksum != null, that indicates that this is the first iteration since processor was started, so use whatever checksum value // was present when the state was last persisted. In this case, we must then null out the value so that the next iteration won't keep using the "recovered" // value. If the value is null, then we know that either the processor has already recovered that data, or there was no state persisted. In either case, // use whatever checksum value is currently in the state. Long expectedChecksumValue = tfo.getExpectedRecoveryChecksum(); if (expectedChecksumValue == null) { expectedChecksumValue = tfo.getState().getChecksum() == null ? null : tfo.getState().getChecksum().getValue(); } rolloverOccurred = recoverRolledFiles(context, session, tailFile, expectedChecksumValue, tfo.getState().getTimestamp(), tfo.getState().getPosition()); tfo.setExpectedRecoveryChecksum(null); } // initialize local variables from state object; this is done so that we can easily change the values throughout // the onTrigger method and then create a new state object after we finish processing the files. TailFileState state = tfo.getState(); File file = state.getFile(); FileChannel reader = state.getReader(); Checksum checksum = state.getChecksum(); if (checksum == null) { checksum = new CRC32(); } long position = state.getPosition(); long timestamp = state.getTimestamp(); long length = state.getLength(); // Create a reader if necessary. if (file == null || reader == null) { file = new File(tailFile); reader = createReader(file, position); if (reader == null) { context.yield(); return; } } final long startNanos = System.nanoTime(); // Check if file has rotated if (rolloverOccurred || (timestamp <= file.lastModified() && length > file.length()) || (timestamp < file.lastModified() && length >= file.length())) { // Since file has rotated, we close the reader, create a new one, and then reset our state. try { reader.close(); getLogger().debug("Closed FileChannel {}", new Object[] { reader, reader }); } catch (final IOException ioe) { getLogger().warn("Failed to close reader for {} due to {}", new Object[] { file, ioe }); } reader = createReader(file, 0L); position = 0L; checksum.reset(); } if (file.length() == position || !file.exists()) { // no data to consume so rather than continually running, yield to allow other processors to use the thread. getLogger().debug("No data to consume; created no FlowFiles"); tfo.setState(new TailFileState(tailFile, file, reader, position, timestamp, length, checksum, state.getBuffer())); persistState(tfo, context); context.yield(); return; } // If there is data to consume, read as much as we can. final TailFileState currentState = state; final Checksum chksum = checksum; // data has been written to file. Stream it to a new FlowFile. FlowFile flowFile = session.create(); final FileChannel fileReader = reader; final AtomicLong positionHolder = new AtomicLong(position); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream rawOut) throws IOException { try (final OutputStream out = new BufferedOutputStream(rawOut)) { positionHolder.set(readLines(fileReader, currentState.getBuffer(), out, chksum)); } } }); // If there ended up being no data, just remove the FlowFile if (flowFile.getSize() == 0) { session.remove(flowFile); getLogger().debug("No data to consume; removed created FlowFile"); } else { // determine filename for FlowFile by using <base filename of log file>.<initial offset>-<final offset>.<extension> final String tailFilename = file.getName(); final String baseName = StringUtils.substringBeforeLast(tailFilename, "."); final String flowFileName; if (baseName.length() < tailFilename.length()) { flowFileName = baseName + "." + position + "-" + positionHolder.get() + "." + StringUtils.substringAfterLast(tailFilename, "."); } else { flowFileName = baseName + "." + position + "-" + positionHolder.get(); } final Map<String, String> attributes = new HashMap<>(3); attributes.put(CoreAttributes.FILENAME.key(), flowFileName); attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain"); attributes.put("tailfile.original.path", tailFile); flowFile = session.putAllAttributes(flowFile, attributes); session.getProvenanceReporter().receive(flowFile, file.toURI().toString(), "FlowFile contains bytes " + position + " through " + positionHolder.get() + " of source file", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos)); session.transfer(flowFile, REL_SUCCESS); position = positionHolder.get(); // Set timestamp to the latest of when the file was modified and the current timestamp stored in the state. // We do this because when we read a file that has been rolled over, we set the state to 1 millisecond later than the last mod date // in order to avoid ingesting that file again. If we then read from this file during the same second (or millisecond, depending on the // operating system file last mod precision), then we could set the timestamp to a smaller value, which could result in reading in the // rotated file a second time. timestamp = Math.max(state.getTimestamp(), file.lastModified()); length = file.length(); getLogger().debug("Created {} and routed to success", new Object[] { flowFile }); } // Create a new state object to represent our current position, timestamp, etc. tfo.setState(new TailFileState(tailFile, file, reader, position, timestamp, length, checksum, state.getBuffer())); // We must commit session before persisting state in order to avoid data loss on restart session.commit(); persistState(tfo, context); }
From source file:org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository.java
/** * <p>/*from ww w . j a v a 2s . c o m*/ * Merges all of the given Journal Files into a single, merged Provenance Event Log File. As these records are merged, they will be compressed, if the repository is configured to compress records * </p> * <p> * <p> * If the repository is configured to compress the data, the file written to may not be the same as the <code>suggestedMergeFile</code>, as a filename extension of '.gz' may be appended. If the * journals are successfully merged, the file that they were merged into will be returned. If unable to merge the records (for instance, because the repository has been closed or because the list * of journal files was empty), this method will return <code>null</code>. * </p> * * @param journalFiles the journal files to merge * @param suggestedMergeFile the file to write the merged records to * @param eventReporter the event reporter to report any warnings or errors to; may be null. * @return the file that the given journals were merged into, or <code>null</code> if no records were merged. * @throws IOException if a problem occurs writing to the mergedFile, reading from a journal */ File mergeJournals(final List<File> journalFiles, final File suggestedMergeFile, final EventReporter eventReporter) throws IOException { if (this.closed.get()) { logger.info("Provenance Repository has been closed; will not merge journal files to {}", suggestedMergeFile); return null; } if (journalFiles.isEmpty()) { logger.debug("Couldn't merge journals: Journal Files is empty; won't merge journals"); return null; } Collections.sort(journalFiles, new Comparator<File>() { @Override public int compare(final File o1, final File o2) { final String suffix1 = StringUtils.substringAfterLast(o1.getName(), "."); final String suffix2 = StringUtils.substringAfterLast(o2.getName(), "."); try { final int journalIndex1 = Integer.parseInt(suffix1); final int journalIndex2 = Integer.parseInt(suffix2); return Integer.compare(journalIndex1, journalIndex2); } catch (final NumberFormatException nfe) { return o1.getName().compareTo(o2.getName()); } } }); final String firstJournalFile = journalFiles.get(0).getName(); final String firstFileSuffix = StringUtils.substringAfterLast(firstJournalFile, "."); final boolean allPartialFiles = firstFileSuffix.equals("0"); // check if we have all of the "partial" files for the journal. if (allPartialFiles) { if (suggestedMergeFile.exists()) { // we have all "partial" files and there is already a merged file. Delete the data from the index // because the merge file may not be fully merged. We will re-merge. logger.warn("Merged Journal File {} already exists; however, all partial journal files also exist " + "so assuming that the merge did not finish. Repeating procedure in order to ensure consistency."); // Since we only store the file's basename, block offset, and event ID, and because the newly created file could end up on // a different Storage Directory than the original, we need to ensure that we delete both the partially merged // file and the TOC file. Otherwise, we could get the wrong copy and have issues retrieving events. if (!suggestedMergeFile.delete()) { logger.error( "Failed to delete partially written Provenance Journal File {}. This may result in events from this journal " + "file not being able to be displayed. This file should be deleted manually.", suggestedMergeFile); } final File tocFile = TocUtil.getTocFile(suggestedMergeFile); if (tocFile.exists() && !tocFile.delete()) { logger.error( "Failed to delete .toc file {}; this may result in not being able to read the Provenance Events from the {} Journal File. " + "This can be corrected by manually deleting the {} file", tocFile, suggestedMergeFile, tocFile); } } } else { logger.warn("Cannot merge journal files {} because expected first file to end with extension '.0' " + "but it did not; assuming that the files were already merged but only some finished deletion " + "before restart. Deleting remaining partial journal files.", journalFiles); for (final File file : journalFiles) { if (!file.delete() && file.exists()) { logger.warn( "Failed to delete unneeded journal file {}; this file should be cleaned up manually", file); } } return null; } final long startNanos = System.nanoTime(); // Map each journal to a RecordReader final List<RecordReader> readers = new ArrayList<>(); int records = 0; final boolean isCompress = configuration.isCompressOnRollover(); final File writerFile = isCompress ? new File(suggestedMergeFile.getParentFile(), suggestedMergeFile.getName() + ".gz") : suggestedMergeFile; try { for (final File journalFile : journalFiles) { try { // Use MAX_VALUE for number of chars because we don't want to truncate the value as we write it // out. This allows us to later decide that we want more characters and still be able to retrieve // the entire event. readers.add(RecordReaders.newRecordReader(journalFile, null, Integer.MAX_VALUE)); } catch (final EOFException eof) { // there's nothing here. Skip over it. } catch (final IOException ioe) { logger.warn("Unable to merge {} with other Journal Files due to {}", journalFile, ioe.toString()); if (logger.isDebugEnabled()) { logger.warn("", ioe); } if (eventReporter != null) { eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "re " + ioe.toString()); } } } // Create a Map so that the key is the next record available from a reader and the value is the Reader from which // the record came. This sorted map is then used so that we are able to always get the first entry, which is the next // lowest record id final SortedMap<StandardProvenanceEventRecord, RecordReader> recordToReaderMap = new TreeMap<>( new Comparator<StandardProvenanceEventRecord>() { @Override public int compare(final StandardProvenanceEventRecord o1, final StandardProvenanceEventRecord o2) { return Long.compare(o1.getEventId(), o2.getEventId()); } }); long minEventId = 0L; long earliestTimestamp = System.currentTimeMillis(); for (final RecordReader reader : readers) { StandardProvenanceEventRecord record = null; try { record = reader.nextRecord(); } catch (final EOFException eof) { } catch (final Exception e) { logger.warn("Failed to generate Provenance Event Record from Journal due to " + e + "; it's possible that the record wasn't " + "completely written to the file. This record will be skipped."); if (logger.isDebugEnabled()) { logger.warn("", e); } if (eventReporter != null) { eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to read Provenance Event Record from Journal due to " + e + "; it's possible that hte record wasn't completely written to the file. This record will be skipped."); } } if (record == null) { continue; } if (record.getEventTime() < earliestTimestamp) { earliestTimestamp = record.getEventTime(); } if (record.getEventId() < minEventId) { minEventId = record.getEventId(); } recordToReaderMap.put(record, reader); } // loop over each entry in the map, persisting the records to the merged file in order, and populating the map // with the next entry from the journal file from which the previous record was written. try (final RecordWriter writer = RecordWriters.newSchemaRecordWriter(writerFile, configuration.isCompressOnRollover(), true)) { writer.writeHeader(minEventId); while (!recordToReaderMap.isEmpty()) { final Map.Entry<StandardProvenanceEventRecord, RecordReader> entry = recordToReaderMap .entrySet().iterator().next(); final StandardProvenanceEventRecord record = entry.getKey(); final RecordReader reader = entry.getValue(); writer.writeRecord(record, record.getEventId()); final int blockIndex = writer.getTocWriter().getCurrentBlockIndex(); records++; // Remove this entry from the map recordToReaderMap.remove(record); // Get the next entry from this reader and add it to the map StandardProvenanceEventRecord nextRecord = null; try { nextRecord = reader.nextRecord(); } catch (final EOFException eof) { } if (nextRecord != null) { recordToReaderMap.put(nextRecord, reader); } } } } finally { for (final RecordReader reader : readers) { try { reader.close(); } catch (final IOException ioe) { } } } // Success. Remove all of the journal files, as they're no longer needed, now that they've been merged. for (final File journalFile : journalFiles) { if (!journalFile.delete() && journalFile.exists()) { logger.warn("Failed to remove temporary journal file {}; this file should be cleaned up manually", journalFile.getAbsolutePath()); if (eventReporter != null) { eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to remove temporary journal file " + journalFile.getAbsolutePath() + "; this file should be cleaned up manually"); } } final File tocFile = getTocFile(journalFile); if (!tocFile.delete() && tocFile.exists()) { logger.warn( "Failed to remove temporary journal TOC file {}; this file should be cleaned up manually", tocFile.getAbsolutePath()); if (eventReporter != null) { eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to remove temporary journal TOC file " + tocFile.getAbsolutePath() + "; this file should be cleaned up manually"); } } } if (records == 0) { writerFile.delete(); logger.debug("Couldn't merge journals: No Records to merge"); return null; } else { final long nanos = System.nanoTime() - startNanos; final long millis = TimeUnit.MILLISECONDS.convert(nanos, TimeUnit.NANOSECONDS); logger.info( "Successfully merged {} journal files ({} records) into single Provenance Log File {} in {} milliseconds", journalFiles.size(), records, suggestedMergeFile, millis); } return writerFile; }
From source file:org.apache.nifi.web.api.AccessResource.java
/** * Gets the status the client's access.//from ww w. j ava 2s . c om * * @param httpServletRequest the servlet request * @return A accessStatusEntity */ @GET @Consumes(MediaType.WILDCARD) @Produces(MediaType.APPLICATION_JSON) @Path("") @ApiOperation(value = "Gets the status the client's access", notes = NON_GUARANTEED_ENDPOINT, response = AccessStatusEntity.class) @ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Unable to determine access status because the client could not be authenticated."), @ApiResponse(code = 403, message = "Unable to determine access status because the client is not authorized to make this request."), @ApiResponse(code = 409, message = "Unable to determine access status because NiFi is not in the appropriate state."), @ApiResponse(code = 500, message = "Unable to determine access status because an unexpected error occurred.") }) public Response getAccessStatus(@Context HttpServletRequest httpServletRequest) { // only consider user specific access over https if (!httpServletRequest.isSecure()) { throw new IllegalStateException( "User authentication/authorization is only supported when running over HTTPS."); } final AccessStatusDTO accessStatus = new AccessStatusDTO(); try { final X509Certificate[] certificates = certificateExtractor .extractClientCertificate(httpServletRequest); // if there is not certificate, consider a token if (certificates == null) { // look for an authorization token final String authorization = httpServletRequest.getHeader(JwtAuthenticationFilter.AUTHORIZATION); // if there is no authorization header, we don't know the user if (authorization == null) { accessStatus.setStatus(AccessStatusDTO.Status.UNKNOWN.name()); accessStatus.setMessage("No credentials supplied, unknown user."); } else { try { // Extract the Base64 encoded token from the Authorization header final String token = StringUtils.substringAfterLast(authorization, " "); final JwtAuthenticationRequestToken jwtRequest = new JwtAuthenticationRequestToken(token, httpServletRequest.getRemoteAddr()); final NiFiAuthenticationToken authenticationResponse = (NiFiAuthenticationToken) jwtAuthenticationProvider .authenticate(jwtRequest); final NiFiUser nifiUser = ((NiFiUserDetails) authenticationResponse.getDetails()) .getNiFiUser(); // set the user identity accessStatus.setIdentity(nifiUser.getIdentity()); // attempt authorize to /flow accessStatus.setStatus(AccessStatusDTO.Status.ACTIVE.name()); accessStatus.setMessage("You are already logged in."); } catch (JwtException e) { throw new InvalidAuthenticationException(e.getMessage(), e); } } } else { try { final X509AuthenticationRequestToken x509Request = new X509AuthenticationRequestToken( httpServletRequest.getHeader(ProxiedEntitiesUtils.PROXY_ENTITIES_CHAIN), principalExtractor, certificates, httpServletRequest.getRemoteAddr()); final NiFiAuthenticationToken authenticationResponse = (NiFiAuthenticationToken) x509AuthenticationProvider .authenticate(x509Request); final NiFiUser nifiUser = ((NiFiUserDetails) authenticationResponse.getDetails()).getNiFiUser(); // set the user identity accessStatus.setIdentity(nifiUser.getIdentity()); // attempt authorize to /flow accessStatus.setStatus(AccessStatusDTO.Status.ACTIVE.name()); accessStatus.setMessage("You are already logged in."); } catch (final IllegalArgumentException iae) { throw new InvalidAuthenticationException(iae.getMessage(), iae); } } } catch (final UntrustedProxyException upe) { throw new AccessDeniedException(upe.getMessage(), upe); } catch (final AuthenticationServiceException ase) { throw new AdministrationException(ase.getMessage(), ase); } // create the entity final AccessStatusEntity entity = new AccessStatusEntity(); entity.setAccessStatus(accessStatus); return generateOkResponse(entity).build(); }
From source file:org.apache.nifi.web.ContentViewerController.java
/** * @param request request//from w w w. j a v a 2s .c o m * @return Get the content request context based on the specified request */ private ContentRequestContext getContentRequest(final HttpServletRequest request) { final String ref = request.getParameter("ref"); final String clientId = request.getParameter("clientId"); final URI refUri = URI.create(ref); final String query = refUri.getQuery(); String rawClusterNodeId = null; if (query != null) { final String[] queryParameters = query.split("&"); for (int i = 0; i < queryParameters.length; i++) { if (queryParameters[0].startsWith("clusterNodeId=")) { rawClusterNodeId = StringUtils.substringAfterLast(queryParameters[0], "clusterNodeId="); } } } final String clusterNodeId = rawClusterNodeId; return new ContentRequestContext() { @Override public String getDataUri() { return ref; } @Override public String getClusterNodeId() { return clusterNodeId; } @Override public String getClientId() { return clientId; } @Override public String getProxiedEntitiesChain() { return null; } }; }
From source file:org.apache.nifi.web.dao.impl.StandardSnippetDAO.java
@Override public FlowSnippetDTO copySnippet(final String groupId, final String snippetId, final Double originX, final Double originY, final String idGenerationSeed) { try {// w ww .j av a 2 s . c o m // ensure the parent group exist final ProcessGroup processGroup = flowController.getGroup(groupId); if (processGroup == null) { throw new IllegalArgumentException("The specified parent process group could not be found"); } // get the existing snippet Snippet existingSnippet = getSnippet(snippetId); // get the process group ProcessGroup existingSnippetProcessGroup = flowController.getGroup(existingSnippet.getParentGroupId()); // ensure the group could be found if (existingSnippetProcessGroup == null) { throw new IllegalStateException( "The parent process group for the existing snippet could not be found."); } // generate the snippet contents FlowSnippetDTO snippetContents = snippetUtils.populateFlowSnippet(existingSnippet, true, false, false); // resolve sensitive properties lookupSensitiveProperties(snippetContents); // copy snippet snippetContents = snippetUtils.copy(snippetContents, processGroup, idGenerationSeed, true); // move the snippet if necessary if (originX != null && originY != null) { org.apache.nifi.util.SnippetUtils.moveSnippet(snippetContents, originX, originY); } try { // instantiate the snippet and return the contents flowController.instantiateSnippet(processGroup, snippetContents); return snippetContents; } catch (IllegalStateException ise) { // illegal state will be thrown from instantiateSnippet when there is an issue with the snippet _before_ any of the // components are actually created. if we've received this exception we want to attempt to roll back any of the // policies that we've already cloned for this request snippetUtils.rollbackClonedPolicies(snippetContents); // rethrow the same exception throw ise; } } catch (ProcessorInstantiationException pie) { throw new NiFiCoreException( String.format("Unable to copy snippet because processor type '%s' is unknown to this NiFi.", StringUtils.substringAfterLast(pie.getMessage(), "."))); } }
From source file:org.apache.nifi.web.dao.impl.StandardTemplateDAO.java
@Override public FlowSnippetDTO instantiateTemplate(String groupId, Double originX, Double originY, String templateId, String idGenerationSeed) { ProcessGroup group = locateProcessGroup(flowController, groupId); // get the template id and find the template Template template = getTemplate(templateId); // ensure the template could be found if (template == null) { throw new ResourceNotFoundException( String.format("Unable to locate template with id '%s'.", templateId)); }//w ww. j a v a 2s . c o m try { // copy the template which pre-processes all ids TemplateDTO templateDetails = template.getDetails(); FlowSnippetDTO snippet = snippetUtils.copy(templateDetails.getSnippet(), group, idGenerationSeed, false); // calculate scaling factors based on the template encoding version // attempt to parse the encoding version final FlowEncodingVersion templateEncodingVersion = FlowEncodingVersion .parse(templateDetails.getEncodingVersion()); // get the major version, or 0 if no version could be parsed int templateEncodingMajorVersion = templateEncodingVersion != null ? templateEncodingVersion.getMajorVersion() : 0; // based on the major version < 1, use the default scaling factors. Otherwise, don't scale (use factor of 1.0) double factorX = templateEncodingMajorVersion < 1 ? FlowController.DEFAULT_POSITION_SCALE_FACTOR_X : 1.0; double factorY = templateEncodingMajorVersion < 1 ? FlowController.DEFAULT_POSITION_SCALE_FACTOR_Y : 1.0; // reposition and scale the template contents org.apache.nifi.util.SnippetUtils.moveAndScaleSnippet(snippet, originX, originY, factorX, factorY); // find all the child process groups in each process group in the top level of this snippet final List<ProcessGroupDTO> childProcessGroups = org.apache.nifi.util.SnippetUtils .findAllProcessGroups(snippet); // scale (but don't reposition) child process groups childProcessGroups.stream().forEach(processGroup -> org.apache.nifi.util.SnippetUtils .scaleSnippet(processGroup.getContents(), factorX, factorY)); // instantiate the template into this group flowController.instantiateSnippet(group, snippet); return snippet; } catch (ProcessorInstantiationException pie) { throw new NiFiCoreException(String.format( "Unable to instantiate template because processor type '%s' is unknown to this NiFi.", StringUtils.substringAfterLast(pie.getMessage(), "."))); } }
From source file:org.apache.nifi.web.docs.DocumentationController.java
/** * * @param request servlet request//from w ww.ja v a 2s .c om * @param response servlet response * @throws ServletException if a servlet-specific error occurs * @throws IOException if an I/O error occurs */ @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { final ExtensionMapping extensionMappings = (ExtensionMapping) servletContext .getAttribute("nifi-extension-mapping"); final Collator collator = Collator.getInstance(Locale.US); // create the processors lookup final Map<String, String> processors = new TreeMap<>(collator); for (final String processorClass : extensionMappings.getProcessorNames()) { processors.put(StringUtils.substringAfterLast(processorClass, "."), processorClass); } // create the controller service lookup final Map<String, String> controllerServices = new TreeMap<>(collator); for (final String controllerServiceClass : extensionMappings.getControllerServiceNames()) { controllerServices.put(StringUtils.substringAfterLast(controllerServiceClass, "."), controllerServiceClass); } // create the reporting task lookup final Map<String, String> reportingTasks = new TreeMap<>(collator); for (final String reportingTaskClass : extensionMappings.getReportingTaskNames()) { reportingTasks.put(StringUtils.substringAfterLast(reportingTaskClass, "."), reportingTaskClass); } // make the available components available to the documentation jsp request.setAttribute("processors", processors); request.setAttribute("controllerServices", controllerServices); request.setAttribute("reportingTasks", reportingTasks); request.setAttribute("totalComponents", GENERAL_LINK_COUNT + processors.size() + controllerServices.size() + reportingTasks.size() + DEVELOPER_LINK_COUNT); // forward appropriately request.getRequestDispatcher("/WEB-INF/jsp/documentation.jsp").forward(request, response); }
From source file:org.apache.nifi.web.security.jwt.JwtAuthenticationFilter.java
@Override public Authentication attemptAuthentication(final HttpServletRequest request) { // only support jwt login when running securely if (!request.isSecure()) { return null; }/* w ww.j ava 2 s . c o m*/ // TODO: Refactor request header extraction logic to shared utility as it is duplicated in AccessResource // get the principal out of the user token final String authorization = request.getHeader(AUTHORIZATION); // if there is no authorization header, we don't know the user if (authorization == null || !StringUtils.startsWith(authorization, BEARER)) { return null; } else { // Extract the Base64 encoded token from the Authorization header final String token = StringUtils.substringAfterLast(authorization, " "); return new JwtAuthenticationRequestToken(token, request.getRemoteAddr()); } }