List of usage examples for org.apache.commons.lang3 StringUtils substringBefore
public static String substringBefore(final String str, final String separator)
Gets the substring before the first occurrence of a separator.
From source file:net.thirdy.blackmarket.util.BlackmarketConfig.java
private static Map<String, String> parseBlackmarketRawFile(File blackarketFile) { Map<String, String> map = new HashMap<>(); String raw = readFileToString(blackarketFile); if (StringUtils.isNotBlank(raw)) { String[] lines = StringUtils.split(raw, BlackmarketUtil.lineSep()); for (String line : lines) { if (isLineNotComment(line) && StringUtils.isNotBlank(line)) { // substringBefore/substringAfter is first occurance String key = StringUtils.substringBefore(line, "="); String value = StringUtils.substringAfter(line, "="); key = StringUtils.trim(key); value = StringUtils.trim(value); map.put(key, value);/* w w w . j a va 2s . co m*/ } } } return map; }
From source file:net.tirasa.olingooauth2.AzureADOAuth2HttpClientFactory.java
@Override protected void init() throws OAuth2Exception { final DefaultHttpClient httpClient = wrapped.create(null, null); // 1. access the OAuth2 grant service (with authentication) String code = null;/*w ww . ja va2 s .co m*/ try { final URIBuilder builder = new URIBuilder(oauth2GrantServiceURI).addParameter("response_type", "code") .addParameter("client_id", clientId).addParameter("redirect_uri", redirectURI); HttpResponse response = httpClient.execute(new HttpGet(builder.build())); final String loginPage = EntityUtils.toString(response.getEntity()); String postURL = StringUtils.substringBefore( StringUtils.substringAfter(loginPage, "<form id=\"credentials\" method=\"post\" action=\""), "\">"); final String ppsx = StringUtils.substringBefore(StringUtils.substringAfter(loginPage, "<input type=\"hidden\" id=\"PPSX\" name=\"PPSX\" value=\""), "\"/>"); final String ppft = StringUtils.substringBefore(StringUtils.substringAfter(loginPage, "<input type=\"hidden\" name=\"PPFT\" id=\"i0327\" value=\""), "\"/>"); List<BasicNameValuePair> data = new ArrayList<BasicNameValuePair>(); data.add(new BasicNameValuePair("login", creds.getUserName())); data.add(new BasicNameValuePair("passwd", creds.getPassword())); data.add(new BasicNameValuePair("PPSX", ppsx)); data.add(new BasicNameValuePair("PPFT", ppft)); HttpPost post = new HttpPost(postURL); post.setEntity(new UrlEncodedFormEntity(data, "UTF-8")); response = httpClient.execute(post); final String samlPage = EntityUtils.toString(response.getEntity()); postURL = StringUtils.substringBefore( StringUtils.substringAfter(samlPage, "<form name=\"fmHF\" id=\"fmHF\" action=\""), "\" method=\"post\" target=\"_top\">"); final String wctx = StringUtils.substringBefore(StringUtils.substringAfter(samlPage, "<input type=\"hidden\" name=\"wctx\" id=\"wctx\" value=\""), "\">"); final String wresult = StringUtils.substringBefore(StringUtils.substringAfter(samlPage, "<input type=\"hidden\" name=\"wresult\" id=\"wresult\" value=\""), "\">"); final String wa = StringUtils.substringBefore( StringUtils.substringAfter(samlPage, "<input type=\"hidden\" name=\"wa\" id=\"wa\" value=\""), "\">"); data = new ArrayList<BasicNameValuePair>(); data.add(new BasicNameValuePair("wctx", wctx)); data.add(new BasicNameValuePair("wresult", wresult.replace(""", "\""))); data.add(new BasicNameValuePair("wa", wa)); post = new HttpPost(postURL); post.setEntity(new UrlEncodedFormEntity(data, "UTF-8")); response = httpClient.execute(post); final Header locationHeader = response.getFirstHeader("Location"); if (response.getStatusLine().getStatusCode() != 302 || locationHeader == null) { throw new OAuth2Exception("Unexpected response from server"); } final String[] oauth2Info = StringUtils .split(StringUtils.substringAfter(locationHeader.getValue(), "?"), '&'); code = StringUtils.substringAfter(oauth2Info[0], "="); EntityUtils.consume(response.getEntity()); } catch (Exception e) { throw new OAuth2Exception(e); } if (code == null) { throw new OAuth2Exception("No OAuth2 grant"); } // 2. ask the OAuth2 token service final List<BasicNameValuePair> data = new ArrayList<BasicNameValuePair>(); data.add(new BasicNameValuePair("grant_type", "authorization_code")); data.add(new BasicNameValuePair("code", code)); data.add(new BasicNameValuePair("client_id", clientId)); data.add(new BasicNameValuePair("redirect_uri", redirectURI)); data.add(new BasicNameValuePair("resource", resourceURI)); fetchAccessToken(httpClient, data); if (token == null) { throw new OAuth2Exception("No OAuth2 access token"); } }
From source file:nl.armatiek.xslweb.configuration.Context.java
public WebApp getWebApp(String path) { String name = StringUtils.substringBefore(path.substring(1), "/"); WebApp webApp = webApps.get(name);//from w ww .jav a2 s . c o m if (webApp == null) { webApp = webApps.get("ROOT"); } return webApp; }
From source file:nlp.corpora.SwitchboardDialogueDB.java
/************************************************************************************ * This method takes the converted data and * stores in a text file format which is required for the ChatBot. *//* w w w . ja v a 2s. com*/ private void writeFile(List<String> rawData, String fileName, String folderName) { String addToFileName; String addToFolderName; String addToFilePath; Path addToFolderPath; try { addToFileName = StringUtils.substringBefore(FilenameUtils.getBaseName(fileName), "."); addToFolderName = folderName.substring(folderName.lastIndexOf("/") + 1); addToFolderPath = Paths.get(addToDirectory + "/" + addToFolderName + "_parsed/"); Files.createDirectories(addToFolderPath); addToFilePath = addToFolderPath + "/" + addToFileName + "_parsed.txt"; Path file = Paths.get(addToFilePath); Files.write(file, rawData, Charset.forName("UTF-8")); } catch (Exception ex) { System.out.println(ex.toString()); System.out.println("writeFile() Method: Could not find the file"); } }
From source file:org.apache.cassandra.index.SecondaryIndexManager.java
/** * Returns the parent name of the specified {@link ColumnFamilyStore}. * * @param cfName the <code>ColumnFamilyStore</code> name * @return the parent name of the specified <code>ColumnFamilyStore</code> *//* w w w . ja v a2 s.co m*/ public static String getParentCfsName(String cfName) { assert isIndexColumnFamily(cfName); return StringUtils.substringBefore(cfName, Directories.SECONDARY_INDEX_NAME_SEPARATOR); }
From source file:org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository.java
private void recover() throws IOException { long maxId = -1L; final List<File> filesToRecover = new ArrayList<>(); for (final File file : configuration.getStorageDirectories()) { final File[] matchingFiles = file.listFiles(new FileFilter() { @Override/*from w ww. j a v a 2 s. c om*/ public boolean accept(final File pathname) { final String filename = pathname.getName(); if (!filename.contains(FILE_EXTENSION) || filename.endsWith(TEMP_FILE_SUFFIX)) { return false; } final String baseFilename = filename.substring(0, filename.indexOf(".")); return NUMBER_PATTERN.matcher(baseFilename).matches(); } }); for (final File matchingFile : matchingFiles) { filesToRecover.add(matchingFile); } } final SortedMap<Long, Path> sortedPathMap = new TreeMap<>(new Comparator<Long>() { @Override public int compare(final Long o1, final Long o2) { return Long.compare(o1, o2); } }); File maxIdFile = null; for (final File file : filesToRecover) { final String filename = file.getName(); final String baseName = filename.substring(0, filename.indexOf(".")); final long fileFirstId = Long.parseLong(baseName); sortedPathMap.put(fileFirstId, file.toPath()); if (fileFirstId > maxId) { maxId = fileFirstId; maxIdFile = file; } } if (maxIdFile != null) { // Determine the max ID in the last file. try (final RecordReader reader = RecordReaders.newRecordReader(maxIdFile, getAllLogFiles(), maxAttributeChars)) { final long eventId = reader.getMaxEventId(); if (eventId > maxId) { maxId = eventId; checkAndSetMaxEventId(maxId); } } catch (final IOException ioe) { logger.error("Failed to read Provenance Event File {} due to {}", maxIdFile, ioe); logger.error("", ioe); } } // Establish current max event ID and increment generator to pick up from this point checkAndSetMaxEventId(maxId); idGenerator.set(maxId + 1); try { final Set<File> recoveredJournals = recoverJournalFiles(); filesToRecover.addAll(recoveredJournals); // Find the file that has the greatest ID File greatestMinIdFile = null; long greatestMinId = 0L; for (final File recoveredJournal : recoveredJournals) { // if the file was removed because the journals were empty, don't count it if (!recoveredJournal.exists()) { continue; } final String basename = StringUtils.substringBefore(recoveredJournal.getName(), "."); try { final long minId = Long.parseLong(basename); sortedPathMap.put(minId, recoveredJournal.toPath()); if (greatestMinIdFile == null || minId > greatestMinId) { greatestMinId = minId; greatestMinIdFile = recoveredJournal; } } catch (final NumberFormatException nfe) { // not a file we care about... } } // Read the records in the last file to find its max id if (greatestMinIdFile != null) { try (final RecordReader recordReader = RecordReaders.newRecordReader(greatestMinIdFile, Collections.<Path>emptyList(), maxAttributeChars)) { maxId = recordReader.getMaxEventId(); } } // set the ID Generator 1 greater than the max id idGenerator.set(maxId + 1); } catch (final IOException ioe) { logger.error("Failed to recover Journal Files due to {}", ioe.toString()); logger.error("", ioe); } idToPathMap.set(Collections.unmodifiableSortedMap(sortedPathMap)); logger.trace("In recovery, path map: {}", sortedPathMap); logger.info("Recovered records"); recoveryFinished.set(true); }
From source file:org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository.java
/** * Purges old events from the repository * * @throws IOException if unable to purge old events due to an I/O problem *///from w w w . jav a2 s . c o m synchronized void purgeOldEvents() throws IOException { while (!recoveryFinished.get()) { try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } final List<File> toPurge = new ArrayList<>(); final long timeCutoff = System.currentTimeMillis() - configuration.getMaxRecordLife(TimeUnit.MILLISECONDS); final List<File> sortedByBasename = getLogFiles(); long bytesUsed = getSize(sortedByBasename, timeCutoff); for (final Path path : idToPathMap.get().values()) { final File file = path.toFile(); final long lastModified = file.lastModified(); if (lastModified > 0L && lastModified < timeCutoff) { toPurge.add(file); } } // This comparator sorts the data based on the "basename" of the files. I.e., the numeric portion. // We do this because the numeric portion represents the ID of the first event in the log file. // As a result, we are sorting based on time, since the ID is monotonically increasing. By doing this, // are able to avoid hitting disk continually to check timestamps final Comparator<File> sortByBasenameComparator = new Comparator<File>() { @Override public int compare(final File o1, final File o2) { final String baseName1 = StringUtils.substringBefore(o1.getName(), "."); final String baseName2 = StringUtils.substringBefore(o2.getName(), "."); Long id1 = null; Long id2 = null; try { id1 = Long.parseLong(baseName1); } catch (final NumberFormatException nfe) { id1 = null; } try { id2 = Long.parseLong(baseName2); } catch (final NumberFormatException nfe) { id2 = null; } if (id1 == null && id2 == null) { return 0; } if (id1 == null) { return 1; } if (id2 == null) { return -1; } return Long.compare(id1, id2); } }; // If we have too much data (at least 90% of our max capacity), start aging it off if (bytesUsed > configuration.getMaxStorageCapacity() * 0.9) { Collections.sort(sortedByBasename, sortByBasenameComparator); for (final File file : sortedByBasename) { toPurge.add(file); bytesUsed -= file.length(); if (bytesUsed < configuration.getMaxStorageCapacity()) { // we've shrunk the repo size down enough to stop break; } } } // Sort all of the files that we want to purge such that the oldest events are aged off first Collections.sort(toPurge, sortByBasenameComparator); logger.debug("Purging old event files: {}", toPurge); // Remove any duplicates that we may have. final Set<File> uniqueFilesToPurge = new LinkedHashSet<>(toPurge); // Age off the data. final Set<String> removed = new LinkedHashSet<>(); for (File file : uniqueFilesToPurge) { final String baseName = StringUtils.substringBefore(file.getName(), "."); ExpirationAction currentAction = null; try { for (final ExpirationAction action : expirationActions) { currentAction = action; if (!action.hasBeenPerformed(file)) { final File fileBeforeAction = file; final StopWatch stopWatch = new StopWatch(true); file = action.execute(file); stopWatch.stop(); logger.info("Successfully performed Expiration Action {} on Provenance Event file {} in {}", action, fileBeforeAction, stopWatch.getDuration()); } } removed.add(baseName); } catch (final FileNotFoundException fnf) { logger.warn( "Failed to perform Expiration Action {} on Provenance Event file {} because the file no longer exists; will not " + "perform additional Expiration Actions on this file", currentAction, file); removed.add(baseName); } catch (final Throwable t) { logger.warn( "Failed to perform Expiration Action {} on Provenance Event file {} due to {}; will not perform additional " + "Expiration Actions on this file at this time", currentAction, file, t.toString()); logger.warn("", t); eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to perform Expiration Action " + currentAction + " on Provenance Event file " + file + " due to " + t.toString() + "; will not perform additional Expiration Actions " + "on this file at this time"); } } // Update the Map ID to Path map to not include the removed file // We cannot obtain the write lock here because there may be a need for the lock in the rollover method, // if we have 'backpressure applied'. This would result in a deadlock because the rollover method would be // waiting for purgeOldEvents, and purgeOldEvents would be waiting for the write lock held by rollover. boolean updated = false; while (!updated) { final SortedMap<Long, Path> existingPathMap = idToPathMap.get(); final SortedMap<Long, Path> newPathMap = new TreeMap<>(new PathMapComparator()); newPathMap.putAll(existingPathMap); final Iterator<Map.Entry<Long, Path>> itr = newPathMap.entrySet().iterator(); while (itr.hasNext()) { final Map.Entry<Long, Path> entry = itr.next(); final String filename = entry.getValue().toFile().getName(); final String baseName = StringUtils.substringBefore(filename, "."); if (removed.contains(baseName)) { itr.remove(); } } updated = idToPathMap.compareAndSet(existingPathMap, newPathMap); logger.debug("After expiration, path map: {}", newPathMap); } }
From source file:org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository.java
/** * <p>/* w ww . ja va 2 s.c om*/ * MUST be called with the write lock held. * </p> * <p> * Rolls over the data in the journal files, merging them into a single Provenance Event Log File, and * compressing as needed. * * @param force if true, will force a rollover regardless of whether or not data has been written * @throws IOException if unable to complete rollover */ private void rollover(final boolean force) throws IOException { if (!configuration.isAllowRollover()) { return; } // If this is the first time we're creating the out stream, or if we // have written something to the stream, then roll over if (force || recordsWrittenSinceRollover.get() > 0L || dirtyWriterCount.get() > 0) { final List<File> journalsToMerge = new ArrayList<>(); for (final RecordWriter writer : writers) { if (!writer.isClosed()) { final File writerFile = writer.getFile(); journalsToMerge.add(writerFile); try { writer.close(); } catch (final IOException ioe) { logger.warn("Failed to close {} due to {}", writer, ioe.toString()); if (logger.isDebugEnabled()) { logger.warn("", ioe); } } } } if (logger.isDebugEnabled()) { if (journalsToMerge.isEmpty()) { logger.debug("No journals to merge; all RecordWriters were already closed"); } else { logger.debug("Going to merge {} files for journals starting with ID {}", journalsToMerge.size(), StringUtils.substringBefore(journalsToMerge.get(0).getName(), ".")); } } // Choose a storage directory to store the merged file in. final long storageDirIdx = storageDirectoryIndex.getAndIncrement(); final List<File> storageDirs = configuration.getStorageDirectories(); final File storageDir = storageDirs.get((int) (storageDirIdx % storageDirs.size())); Future<?> future = null; if (!journalsToMerge.isEmpty()) { // Run the rollover logic in a background thread. final AtomicReference<Future<?>> futureReference = new AtomicReference<>(); final int recordsWritten = recordsWrittenSinceRollover.getAndSet(0); final Runnable rolloverRunnable = new Runnable() { @Override public void run() { try { final File fileRolledOver; try { fileRolledOver = mergeJournals(journalsToMerge, getMergeFile(journalsToMerge, storageDir), eventReporter); } catch (final IOException ioe) { logger.error( "Failed to merge Journal Files {} into a Provenance Log File due to {}", journalsToMerge, ioe.toString()); logger.error("", ioe); return; } if (fileRolledOver == null) { logger.debug( "Couldn't merge journals. Will try again in 10 seconds. journalsToMerge: {}, storageDir: {}", journalsToMerge, storageDir); return; } final File file = fileRolledOver; // update our map of id to Path // We need to make sure that another thread doesn't also update the map at the same time. We cannot // use the write lock when purging old events, and we want to use the same approach here. boolean updated = false; final Long fileFirstEventId = Long .valueOf(StringUtils.substringBefore(fileRolledOver.getName(), ".")); while (!updated) { final SortedMap<Long, Path> existingPathMap = idToPathMap.get(); final SortedMap<Long, Path> newIdToPathMap = new TreeMap<>(new PathMapComparator()); newIdToPathMap.putAll(existingPathMap); newIdToPathMap.put(fileFirstEventId, file.toPath()); updated = idToPathMap.compareAndSet(existingPathMap, newIdToPathMap); } logger.info("Successfully Rolled over Provenance Event file containing {} records", recordsWritten); rolloverCompletions.getAndIncrement(); // We have finished successfully. Cancel the future so that we don't run anymore Future<?> future; while ((future = futureReference.get()) == null) { try { Thread.sleep(10L); } catch (final InterruptedException ie) { } } future.cancel(false); } catch (final Throwable t) { logger.error("Failed to rollover Provenance repository due to {}", t.toString()); logger.error("", t); } } }; // We are going to schedule the future to run immediately and then repeat every 10 seconds. This allows us to keep retrying if we // fail for some reason. When we succeed, the Runnable will cancel itself. future = rolloverExecutor.scheduleWithFixedDelay(rolloverRunnable, 0, 10, TimeUnit.SECONDS); futureReference.set(future); } streamStartTime.set(System.currentTimeMillis()); bytesWrittenSinceRollover.set(0); // We don't want to create new 'writers' until the number of unmerged journals falls below our threshold. So we wait // here before we repopulate the 'writers' member variable and release the lock. int journalFileCount = getJournalCount(); long repoSize = getSize(getLogFiles(), 0L); final int journalCountThreshold = configuration.getJournalCount() * 5; final long sizeThreshold = (long) (configuration.getMaxStorageCapacity() * 1.1D); // do not go over 10% of max capacity // check if we need to apply backpressure. // If we have too many journal files, or if the repo becomes too large, backpressure is necessary. Without it, // if the rate at which provenance events are registered exceeds the rate at which we can compress/merge them, // then eventually we will end up with all of the data stored in the 'journals' directory. This // would mean that the data would never even be accessible. In order to prevent this, if we exceeds 110% of the configured // max capacity for the repo, or if we have 5 sets of journal files waiting to be merged, we will block here until // that is no longer the case. if (journalFileCount > journalCountThreshold || repoSize > sizeThreshold) { logger.warn("The rate of the dataflow is exceeding the provenance recording rate. " + "Slowing down flow to accommodate. Currently, there are {} journal files ({} bytes) and " + "threshold for blocking is {} ({} bytes)", journalFileCount, repoSize, journalCountThreshold, sizeThreshold); eventReporter.reportEvent(Severity.WARNING, "Provenance Repository", "The rate of the dataflow is " + "exceeding the provenance recording rate. Slowing down flow to accommodate"); while (journalFileCount > journalCountThreshold || repoSize > sizeThreshold) { // if a shutdown happens while we are in this loop, kill the rollover thread and break if (this.closed.get()) { if (future != null) { future.cancel(true); } break; } if (repoSize > sizeThreshold) { logger.debug( "Provenance Repository has exceeded its size threshold; will trigger purging of oldest events"); purgeOldEvents(); journalFileCount = getJournalCount(); repoSize = getSize(getLogFiles(), 0L); continue; } else { // if we are constrained by the number of journal files rather than the size of the repo, // then we will just sleep a bit because another thread is already actively merging the journals, // due to the runnable that we scheduled above try { Thread.sleep(100L); } catch (final InterruptedException ie) { } } logger.debug( "Provenance Repository is still behind. Keeping flow slowed down " + "to accommodate. Currently, there are {} journal files ({} bytes) and " + "threshold for blocking is {} ({} bytes)", journalFileCount, repoSize, journalCountThreshold, sizeThreshold); journalFileCount = getJournalCount(); repoSize = getSize(getLogFiles(), 0L); } logger.info( "Provenance Repository has now caught up with rolling over journal files. Current number of " + "journal files to be rolled over is {}", journalFileCount); } // we've finished rolling over successfully. Create new writers and reset state. writers = createWriters(configuration, idGenerator.get()); dirtyWriterCount.set(0); streamStartTime.set(System.currentTimeMillis()); recordsWrittenSinceRollover.getAndSet(0); } }
From source file:org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository.java
protected Set<File> recoverJournalFiles() throws IOException { if (!configuration.isAllowRollover()) { return Collections.emptySet(); }/*w w w. jav a2 s . c o m*/ final Map<String, List<File>> journalMap = new HashMap<>(); // Map journals' basenames to the files with that basename. final List<File> storageDirs = configuration.getStorageDirectories(); for (final File storageDir : configuration.getStorageDirectories()) { final File journalDir = new File(storageDir, "journals"); if (!journalDir.exists()) { continue; } final File[] journalFiles = journalDir.listFiles(); if (journalFiles == null) { continue; } for (final File journalFile : journalFiles) { if (journalFile.isDirectory()) { continue; } final String basename = StringUtils.substringBefore(journalFile.getName(), "."); List<File> files = journalMap.get(basename); if (files == null) { files = new ArrayList<>(); journalMap.put(basename, files); } files.add(journalFile); } } final Set<File> mergedFiles = new HashSet<>(); for (final List<File> journalFileSet : journalMap.values()) { final long storageDirIdx = storageDirectoryIndex.getAndIncrement(); final File storageDir = storageDirs.get((int) (storageDirIdx % storageDirs.size())); final File mergedFile = mergeJournals(journalFileSet, getMergeFile(journalFileSet, storageDir), eventReporter); if (mergedFile != null) { mergedFiles.add(mergedFile); } } return mergedFiles; }
From source file:org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository.java
static File getMergeFile(final List<File> journalFiles, final File storageDir) { // verify that all Journal files have the same basename String canonicalBaseName = null; for (final File journal : journalFiles) { final String basename = StringUtils.substringBefore(journal.getName(), "."); if (canonicalBaseName == null) { canonicalBaseName = basename; }// ww w. j av a2 s. co m if (!canonicalBaseName.equals(basename)) { throw new IllegalArgumentException( "Cannot merge journal files because they do not contain the same basename, which means that they are not correlated properly"); } } final File mergedFile = new File(storageDir, canonicalBaseName + ".prov"); return mergedFile; }