List of usage examples for org.apache.commons.io IOUtils readLines
public static List readLines(Reader input) throws IOException
Reader as a list of Strings, one entry per line. From source file:bixo.examples.webmining.AnalyzeHtml.java
private Set<String> loadAnalyzedPhrases(String fileName, PhraseShingleAnalyzer analyzer) { InputStream is = AnalyzeHtml.class.getResourceAsStream(fileName); Set<String> result = new HashSet<String>(); try {//from w w w. j ava 2s. c om List<String> lines = IOUtils.readLines(is); for (String line : lines) { if (line.trim().startsWith("#")) { continue; } String analyzedPhrase = _analyzer.getAnalyzedPhrase(line); result.add(analyzedPhrase); } } catch (Exception e) { throw new RuntimeException("Error loading file:" + fileName, e); } return result; }
From source file:com.splunk.shuttl.archiver.endtoend.ArchiverEndToEndTest.java
private List<String> getLinesFromResponse(HttpResponse response) { try {//from ww w .ja va 2 s.co m return IOUtils.readLines(response.getEntity().getContent()); } catch (IOException e) { TUtilsTestNG.failForException("Could not read lines for http response.", e); return null; } }
From source file:eu.annocultor.context.EnvironmentImpl.java
@Override public String getBuildSignature() { try {// w w w. ja v a 2 s . co m InputStream is = getClass().getResourceAsStream("/build.txt"); if (is == null) { return "(build signature could not be read)"; } return IOUtils.readLines(is).get(0); } catch (Exception e) { return null; } }
From source file:gov.nih.nci.nbia.StandaloneDMV2.java
private static List<String> connectAndReadFromURL(URL url, String fileName, String userId, String passWd) { List<String> data = null; DefaultHttpClient httpClient = null; TrustStrategy easyStrategy = new TrustStrategy() { @Override//from ww w.ja va 2 s. com public boolean isTrusted(X509Certificate[] certificate, String authType) throws CertificateException { return true; } }; try { // SSLContext sslContext = SSLContext.getInstance("SSL"); // set up a TrustManager that trusts everything // sslContext.init(null, new TrustManager[] { new // EasyX509TrustManager(null)}, null); SSLSocketFactory sslsf = new SSLSocketFactory(easyStrategy, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); Scheme httpsScheme = new Scheme("https", 443, sslsf); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(httpsScheme); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); ClientConnectionManager ccm = new ThreadSafeClientConnManager(schemeRegistry); HttpParams httpParams = new BasicHttpParams(); HttpConnectionParams.setConnectionTimeout(httpParams, 50000); HttpConnectionParams.setSoTimeout(httpParams, new Integer(12000)); httpClient = new DefaultHttpClient(ccm, httpParams); httpClient.setRoutePlanner(new ProxySelectorRoutePlanner(schemeRegistry, ProxySelector.getDefault())); // // Additions by lrt for tcia - // // attempt to reduce errors going through a Coyote Point // Equalizer load balance switch httpClient.getParams().setParameter("http.socket.timeout", new Integer(12000)); httpClient.getParams().setParameter("http.socket.receivebuffer", new Integer(16384)); httpClient.getParams().setParameter("http.tcp.nodelay", true); httpClient.getParams().setParameter("http.connection.stalecheck", false); // // end lrt additions HttpPost httpPostMethod = new HttpPost(url.toString()); List<BasicNameValuePair> postParams = new ArrayList<BasicNameValuePair>(); postParams.add(new BasicNameValuePair("serverManifestLoc", fileName)); if (userId != null && passWd != null) { postParams.add(new BasicNameValuePair("userId", userId)); httpPostMethod.addHeader("password", passWd); } UrlEncodedFormEntity query = new UrlEncodedFormEntity(postParams); httpPostMethod.setEntity(query); HttpResponse response = httpClient.execute(httpPostMethod); int responseCode = response.getStatusLine().getStatusCode(); // System.out.println("!!!!!Response code for requesting datda file: // " + responseCode); if (responseCode != HttpURLConnection.HTTP_OK) { return null; } else { InputStream inputStream = response.getEntity().getContent(); data = IOUtils.readLines(inputStream); } } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (KeyManagementException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (NoSuchAlgorithmException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (KeyStoreException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (UnrecoverableKeyException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (httpClient != null) { httpClient.getConnectionManager().shutdown(); } } return data; }
From source file:com.izforge.izpack.compiler.packager.impl.Packager.java
/** * Write manifest in the install jar.//from w w w .j a v a 2s . c om */ @Override public void writeManifest() throws IOException { IXMLElement data = resourceFinder.getXMLTree(); IXMLElement guiPrefsElement = data.getFirstChildNamed("guiprefs"); // Add splash screen configuration List<String> lines = IOUtils.readLines(getClass().getResourceAsStream("MANIFEST.MF")); IXMLElement splashNode = guiPrefsElement.getFirstChildNamed("splash"); if (splashNode != null) { // Add splash image to installer jar File splashImage = FileUtils .toFile(resourceFinder.findProjectResource(splashNode.getContent(), "Resource", splashNode)); String destination = String.format("META-INF/%s", splashImage.getName()); mergeManager.addResourceToMerge(splashImage.getAbsolutePath(), destination); lines.add(String.format("SplashScreen-Image: %s", destination)); } lines.add(""); File tempManifest = com.izforge.izpack.util.file.FileUtils.createTempFile("MANIFEST", ".MF"); FileUtils.writeLines(tempManifest, lines); mergeManager.addResourceToMerge(tempManifest.getAbsolutePath(), "META-INF/MANIFEST.MF"); }
From source file:br.gov.lexml.parser.documentoarticulado.LexMLParserFromText.java
private List<String> getLines(String text) { try {//from w w w . j a v a 2s . co m return IOUtils.readLines(new StringReader(text)); } catch (Exception e) { throw new IllegalArgumentException(e); } }
From source file:com.nridge.connector.common.con_com.crawl.CrawlStart.java
/** * Parses a file identified by the path/file name parameter * and loads it into an internally managed crawler start URI * list.// w w w . j a va 2 s . c o m * * @param aPathFileName Absolute file name (e.g. 'crawl_start.txt'). * * @throws IOException I/O related exception. */ public void load(String aPathFileName) throws IOException { List<String> lineList; Logger appLogger = mAppMgr.getLogger(this, "load"); appLogger.trace(mAppMgr.LOGMSG_TRACE_ENTER); try (FileReader fileReader = new FileReader(aPathFileName)) { lineList = IOUtils.readLines(fileReader); } for (String followString : lineList) { if (!StringUtils.startsWith(followString, "#")) mStartList.add(followString); } appLogger.trace(mAppMgr.LOGMSG_TRACE_DEPART); }
From source file:com.linkedin.pinot.perf.QueryRunner.java
/** * Use multiple threads to run query at an increasing target QPS. * * Use a concurrent linked queue to buffer the queries to be sent. Use the main thread to insert queries into the * queue at the target QPS, and start {numThreads} worker threads to fetch queries from the queue and send them. * We start with the start QPS, and keep adding delta QPS to the start QPS during the test. The main thread is * responsible for collecting the statistic information and log them periodically. * * @param conf perf benchmark driver config. * @param queryFile query file.//from w ww. jav a 2s. co m * @param numThreads number of threads sending queries. * @param startQPS start QPS * @param deltaQPS delta QPS * @throws Exception */ @SuppressWarnings("InfiniteLoopStatement") public static void targetQPSQueryRunner(PerfBenchmarkDriverConf conf, String queryFile, int numThreads, double startQPS, double deltaQPS) throws Exception { final long randomSeed = 123456789L; final Random random = new Random(randomSeed); final int timePerTargetQPSMillis = 60000; final int queueLengthThreshold = Math.max(20, (int) startQPS); final List<String> queries; try (FileInputStream input = new FileInputStream(new File(queryFile))) { queries = IOUtils.readLines(input); } final int numQueries = queries.size(); final PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf); final AtomicInteger counter = new AtomicInteger(0); final AtomicLong totalResponseTime = new AtomicLong(0L); final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); final ConcurrentLinkedQueue<String> queryQueue = new ConcurrentLinkedQueue<>(); double currentQPS = startQPS; int intervalMillis = (int) (MILLIS_PER_SECOND / currentQPS); for (int i = 0; i < numThreads; i++) { executorService.submit(new Runnable() { @Override public void run() { while (true) { String query = queryQueue.poll(); if (query == null) { try { Thread.sleep(1); continue; } catch (InterruptedException e) { LOGGER.error("Interrupted.", e); return; } } long startTime = System.currentTimeMillis(); try { driver.postQuery(query); counter.getAndIncrement(); totalResponseTime.getAndAdd(System.currentTimeMillis() - startTime); } catch (Exception e) { LOGGER.error("Caught exception while running query: {}", query, e); return; } } } }); } LOGGER.info("Start with QPS: {}, delta QPS: {}", startQPS, deltaQPS); while (true) { long startTime = System.currentTimeMillis(); while (System.currentTimeMillis() - startTime <= timePerTargetQPSMillis) { if (queryQueue.size() > queueLengthThreshold) { executorService.shutdownNow(); throw new RuntimeException("Cannot achieve target QPS of: " + currentQPS); } queryQueue.add(queries.get(random.nextInt(numQueries))); Thread.sleep(intervalMillis); } double timePassedSeconds = ((double) (System.currentTimeMillis() - startTime)) / MILLIS_PER_SECOND; int count = counter.getAndSet(0); double avgResponseTime = ((double) totalResponseTime.getAndSet(0)) / count; LOGGER.info("Target QPS: {}, Interval: {}ms, Actual QPS: {}, Avg Response Time: {}ms", currentQPS, intervalMillis, count / timePassedSeconds, avgResponseTime); // Find a new interval int newIntervalMillis; do { currentQPS += deltaQPS; newIntervalMillis = (int) (MILLIS_PER_SECOND / currentQPS); } while (newIntervalMillis == intervalMillis); intervalMillis = newIntervalMillis; } }
From source file:ca.sqlpower.architect.transformation.VelocityTransformationTest.java
public void testTransform() throws Exception { File tempdir = new File(System.getProperty("java.io.tmpdir")); File vm = new File(tempdir, "test_template.vm"); OutputStream out = new FileOutputStream(vm); PrintWriter writer = new PrintWriter(new OutputStreamWriter(out, ENCODING)); writer.print("#foreach ($table in $tables) \n" + "Table: $table.name\n" + "#foreach ($col in $table.columns)\n" + " $col.shortDisplayName\n" + "#end\n" + "#end\n"); writer.close();//from w ww . j ava 2s . c o m File output = new File(tempdir, "test-output.txt"); VelocityTransformation vt = new VelocityTransformation(); vt.transform(vm, output, session); assertTrue(output.exists()); FileInputStream fi = new FileInputStream(output); InputStreamReader in = new InputStreamReader(fi, "UTF-8"); List<String> lines = IOUtils.readLines(in); assertEquals("Table: Customers", lines.get(0)); assertEquals(" id: VARCHAR(10)", lines.get(1)); assertEquals(" name: VARCHAR(10)", lines.get(2)); assertEquals("Table: Orders", lines.get(3)); assertEquals(" id: VARCHAR(10)", lines.get(4)); assertEquals(" customer_id: INTEGER", lines.get(5)); assertEquals(" purchase_date: DATE", lines.get(6)); assertEquals("Table: mm_project", lines.get(7)); assertEquals(" project_oid: INTEGER", lines.get(8)); assertEquals(" FOLDER_OID: INTEGER", lines.get(9)); assertEquals(" project_name: VARCHAR(80)", lines.get(10)); assertEquals(" vision: DECIMAL(20, 20)", lines.get(11)); }
From source file:edu.ucsd.crbs.cws.cluster.submission.TestJobCmdScriptCreatorImpl.java
@Test public void testCreateWithJobWithNoArgs() throws Exception { File tempDirectory = Folder.newFolder(); File outputsDir = new File(tempDirectory + File.separator + Constants.OUTPUTS_DIR_NAME); assertTrue(outputsDir.mkdirs());// www.j a v a 2s. c om JobEmailNotificationData emailNotifyData = createJobEmailNotificationData(); JobBinaries jb = new JobBinaries(); jb.setKeplerScript("kepler.sh"); jb.setRegisterUpdateJar("register.jar"); jb.setRetryCount(1); JobCmdScriptCreatorImpl scriptCreator = new JobCmdScriptCreatorImpl("/workflowsdir", jb, emailNotifyData); Job j = new Job(); Workflow w = new Workflow(); w.setId(new Long(5)); j.setWorkflow(w); String jobCmd = scriptCreator.create(tempDirectory.getAbsolutePath(), j, new Long(10)); assertTrue(jobCmd != null); assertTrue( jobCmd.equals(outputsDir.getAbsolutePath() + File.separator + JobCmdScriptCreatorImpl.JOB_CMD_SH)); File checkCmdFile = new File(jobCmd); assertTrue(checkCmdFile.canExecute()); List<String> lines = IOUtils.readLines(new FileReader(jobCmd)); assertTrue(lines != null); boolean emailFound = false; boolean errorEmailFound = false; boolean javaFound = false; boolean keplerFound = false; for (String line : lines) { if (line.startsWith("kepler.sh")) { assertTrue(":" + line + ":", line.equals("kepler.sh -runwf -redirectgui " + outputsDir.getAbsolutePath() + " /workflowsdir/5/5.kar &")); keplerFound = true; } if (line.startsWith("EMAIL_ADDR=")) { assertTrue(line.equals("EMAIL_ADDR=\"\"")); emailFound = true; } if (line.startsWith("ERROR_EMAIL_ADDR")) { assertTrue(line.equals("ERROR_EMAIL_ADDR=\"\"")); errorEmailFound = true; } if (line.startsWith(" java")) { assertTrue(line, line.equals(" java -jar register.jar --updatepath \"10\" --path \"" + outputsDir.getAbsolutePath() + "\" --size `du " + outputsDir.getAbsolutePath() + " -bs | sed \"s/\\W*\\/.*//\"` $workspaceStatusFlag >> " + tempDirectory.getAbsolutePath() + "/updateworkspacefile.out 2>&1")); javaFound = true; } } assertTrue(emailFound); assertTrue(errorEmailFound); assertTrue(javaFound); assertTrue(keplerFound); }