List of usage examples for org.apache.commons.lang3 StringEscapeUtils escapeJava
public static final String escapeJava(final String input)
Escapes the characters in a String using Java String rules.
Deals correctly with quotes and control-chars (tab, backslash, cr, ff, etc.)
So a tab becomes the characters '\\' and 't' .
The only difference between Java strings and JavaScript strings is that in JavaScript, a single quote and forward-slash (/) are escaped.
Example:
input string: He didn't say, "Stop!"Usage
From source file:org.ut.biolab.medsavant.server.db.util.DBUtils.java
public static void loadTable(String sessID, File src, String dst) throws SQLException, SessionExpiredException { String query = "LOAD DATA LOCAL INFILE '" + src.getAbsolutePath().replaceAll("\\\\", "/") + "' " + "INTO TABLE " + dst + " " + "FIELDS TERMINATED BY '" + VariantManagerUtils.FIELD_DELIMITER + "' ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "' " + " LINES TERMINATED BY '\\r\\n'" + ";"; LOG.info(query);//from w w w .jav a 2 s .co m ConnectionController.executeQuery(sessID, query); }From source file:org.ut.biolab.medsavant.server.db.variants.VariantManager.java
@Override public int exportVariants(String sessID, int projID, int refID, Condition[][] conditions, boolean orderedByPosition, boolean zipOutputFile) throws SQLException, RemoteException, SessionExpiredException, IOException, InterruptedException { //generate directory File baseDir = DirectorySettings.generateDateStampDirectory(DirectorySettings.getTmpDirectory()); Process p = Runtime.getRuntime().exec("chmod -R o+w " + baseDir.getCanonicalPath()); p.waitFor();//w w w. j a v a 2s. c o m String filename = ProjectManager.getInstance().getProjectName(sessID, projID).replace(" ", "") + "-varexport-" + System.currentTimeMillis() + ".tdf"; File file = new File(baseDir, filename); LOG.info("Exporting variants to " + file.getAbsolutePath()); long start = System.currentTimeMillis(); TableSchema table = CustomTables.getInstance().getCustomTableSchema(sessID, ProjectManager.getInstance().getVariantTableName(sessID, projID, refID, true)); SelectQuery query = new SelectQuery(); query.addFromTable(table.getTable()); query.addAllColumns(); addConditionsToQuery(query, conditions); if (orderedByPosition) { query.addOrderings(table.getDBColumn(POSITION)); } String intoString = "INTO OUTFILE \"" + file.getAbsolutePath().replaceAll("\\\\", "/") + "\" " + "FIELDS TERMINATED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.FIELD_DELIMITER) + "' " + "ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "'" //+ " LINES TERMINATED BY '\\r\\n' "; ; String queryString = query.toString().replace("FROM", intoString + "FROM"); LOG.info(queryString); ConnectionController.executeQuery(sessID, queryString); LOG.info("Done exporting variants to " + file.getAbsolutePath()); LOG.info("Export took " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); if (zipOutputFile) { LOG.info("Zipping export..."); File zipFile = new File(file.getAbsoluteFile() + ".zip"); IOUtils.zipFile(file, zipFile); boolean deleted = file.delete(); LOG.info("Deleting " + file.getAbsolutePath() + " - " + (deleted ? "successful" : "failed")); file = zipFile; LOG.info("Done zipping"); } // add file to map and send the id back int fileID = NetworkManager.getInstance().openReaderOnServer(sessID, file); return fileID; }From source file:org.ut.biolab.medsavant.server.db.variants.VariantManagerUtils.java
public static int uploadTSVFileToVariantTable(String sid, File file, String tableName) throws SQLException, IOException, SessionExpiredException { file = cleanVariantFile(file, tableName, sid); BufferedReader br = new BufferedReader(new FileReader(file)); // TODO: for some reason the connection is closed going into this function Connection c = null;//from w w w. j a va 2 s.c om int lineNumber = 0; try { c = ConnectionController.connectPooled(sid); c.setAutoCommit(false); int chunkSize = 100000; // number of lines per chunk (100K lines = ~50MB for a standard VCF file) String parentDirectory = file.getParentFile().getAbsolutePath(); BufferedWriter bw = null; //BufferedWriter sw = null; String subsetFileName = parentDirectory + "/" + MiscUtils.extractFileName(file.getAbsolutePath()) + "_subset"; /*if (step > 0) { //assert sw = new BufferedWriter(new FileWriter(subsetFileName)); } else { throw new IllegalArgumentException("Can't upload TSV file " + file + " to variant table, invalid step size=" + step); }*/ String currentOutputPath = null; boolean stateOpen = false; String line; while ((line = br.readLine()) != null) { lineNumber++; // start a new output file if (lineNumber % chunkSize == 1) { currentOutputPath = parentDirectory + "/" + MiscUtils.extractFileName(file.getAbsolutePath()) + "_" + (lineNumber / chunkSize); LOG.info("Opening new partial file " + currentOutputPath); bw = new BufferedWriter(new FileWriter(currentOutputPath)); stateOpen = true; } /*if (sw != null && ((lineNumber - 1) % step == 0)) { sw.write(line + "\r\n"); }*/ // write line to chunk file bw.write(line + "\r\n"); // close and upload this output file if (lineNumber % chunkSize == 0) { bw.close(); LOG.info("Closing and uploading final partial file " + currentOutputPath); String query = "LOAD DATA LOCAL INFILE '" + currentOutputPath.replaceAll("\\\\", "/") + "' " + "INTO TABLE " + tableName + " " + "FIELDS TERMINATED BY '" + VariantManagerUtils.FIELD_DELIMITER + "' ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "' " + " LINES TERMINATED BY '\\r\\n'" + ";"; // LOG.info(query); Statement s = null; try { s = c.createStatement(); s.setQueryTimeout(30 * 60); // 30 minutes s.execute(query); } finally { s.close(); } /*if (VariantManager.REMOVE_TMP_FILES) { boolean deleted = new File(currentOutputPath).delete(); LOG.info("Deleting " + currentOutputPath + " - " + (deleted ? "successful" : "failed")); }*/ stateOpen = false; (new File(currentOutputPath)).delete(); } } // write the remaining open file if (bw != null && stateOpen) { bw.close(); String query = "LOAD DATA LOCAL INFILE '" + currentOutputPath.replaceAll("\\\\", "/") + "' " + "INTO TABLE " + tableName + " " + "FIELDS TERMINATED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.FIELD_DELIMITER) + "' ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "'" + " LINES TERMINATED BY '\\r\\n'" + ";"; LOG.info("Closing and uploading last partial file " + currentOutputPath); LOG.info(query); Statement s = null; try { s = c.createStatement(); s.setQueryTimeout(60 * 60); // 1 hour s.execute(query); } finally { s.close(); } (new File(currentOutputPath)).delete(); /*if (VariantManager.REMOVE_TMP_FILES) { boolean deleted = new File(currentOutputPath).delete(); LOG.info("Deleting " + currentOutputPath + " - " + (deleted ? "successful" : "failed")); }*/ } LOG.info("Imported " + lineNumber + " lines of variants in total"); /* if (sw != null) { sw.close(); String query = "LOAD DATA LOCAL INFILE '" + subsetFileName.replaceAll("\\\\", "/") + "' " + "INTO TABLE " + subTableName + " " + "FIELDS TERMINATED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.FIELD_DELIMITER) + "' ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "'" + " LINES TERMINATED BY '\\r\\n'" + ";"; LOG.info("Closing and uploading subset file " + subsetFileName); LOG.info(query); Statement s = c.createStatement(); s.setQueryTimeout(60 * 60); // 1 hour s.execute(query); LOG.info("Subset table import done"); (new File(subsetFileName)).delete(); } */ c.commit(); c.setAutoCommit(true); } finally { c.close(); } /*if (VariantManager.REMOVE_TMP_FILES) { boolean deleted = file.delete(); LOG.info("Deleting " + file.getAbsolutePath() + " - " + (deleted ? "successful" : "failed")); }*/ return lineNumber; }From source file:org.ut.biolab.medsavant.server.db.variants.VariantManagerUtils.java
@Deprecated public static int uploadTSVFileToVariantTableOld(String sid, File file, String tableName) throws SQLException, IOException, SessionExpiredException { file = cleanVariantFile(file, tableName, sid); BufferedReader br = new BufferedReader(new FileReader(file)); // TODO: for some reason the connection is closed going into this function Connection c = ConnectionController.connectPooled(sid); c.setAutoCommit(false);/* w ww. j av a 2 s .c o m*/ int chunkSize = 100000; // number of lines per chunk (100K lines = ~50MB for a standard VCF file) int lineNumber = 0; BufferedWriter bw = null; String currentOutputPath = null; boolean stateOpen = false; String parentDirectory = file.getParentFile().getAbsolutePath(); String line; while ((line = br.readLine()) != null) { lineNumber++; // start a new output file if (lineNumber % chunkSize == 1) { currentOutputPath = parentDirectory + "/" + MiscUtils.extractFileName(file.getAbsolutePath()) + "_" + (lineNumber / chunkSize); LOG.info("Opening new partial file " + currentOutputPath); bw = new BufferedWriter(new FileWriter(currentOutputPath)); stateOpen = true; } // write line to chunk file bw.write(line + "\r\n"); // close and upload this output file if (lineNumber % chunkSize == 0) { bw.close(); LOG.info("Closing and uploading partial file " + currentOutputPath); String query = "LOAD DATA LOCAL INFILE '" + currentOutputPath.replaceAll("\\\\", "/") + "' " + "INTO TABLE " + tableName + " " + "FIELDS TERMINATED BY '" + VariantManagerUtils.FIELD_DELIMITER + "' ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "' " + " LINES TERMINATED BY '\\r\\n'" + ";"; // LOG.info(query); Statement s = c.createStatement(); s.setQueryTimeout(30 * 60); // 30 minutes s.execute(query); /*if (VariantManager.REMOVE_TMP_FILES) { boolean deleted = new File(currentOutputPath).delete(); LOG.info("Deleting " + currentOutputPath + " - " + (deleted ? "successful" : "failed")); }*/ stateOpen = false; } } // write the remaining open file if (bw != null && stateOpen) { bw.close(); String query = "LOAD DATA LOCAL INFILE '" + currentOutputPath.replaceAll("\\\\", "/") + "' " + "INTO TABLE " + tableName + " " + "FIELDS TERMINATED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.FIELD_DELIMITER) + "' ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "'" + " LINES TERMINATED BY '\\r\\n'" + ";"; LOG.info("Closing and uploading last partial file " + currentOutputPath); LOG.info(query); Statement s = c.createStatement(); s.setQueryTimeout(60 * 60); // 1 hour s.execute(query); /*if (VariantManager.REMOVE_TMP_FILES) { boolean deleted = new File(currentOutputPath).delete(); LOG.info("Deleting " + currentOutputPath + " - " + (deleted ? "successful" : "failed")); }*/ } LOG.info("Imported " + lineNumber + " lines of variants in total"); c.commit(); c.setAutoCommit(true); c.close(); /*if (VariantManager.REMOVE_TMP_FILES) { boolean deleted = file.delete(); LOG.info("Deleting " + file.getAbsolutePath() + " - " + (deleted ? "successful" : "failed")); }*/ return lineNumber; }From source file:org.ut.biolab.medsavant.server.db.variants.VariantManagerUtils.java
public static void variantTableToTSVFile(String sid, String tableName, File file, String conditions) throws SQLException, SessionExpiredException { String query = "SELECT `upload_id`, `file_id`, `variant_id`, `dna_id`, `chrom`, `position`, `end`, `dbsnp_id`, `ref`, `alt`, `alt_number`, `qual`, `filter`, `variant_type`, `zygosity`, `gt`, `custom_info`"; query += " INTO OUTFILE \"" + file.getAbsolutePath().replaceAll("\\\\", "/") + "\" " + "FIELDS TERMINATED BY '" + StringEscapeUtils.escapeJava(FIELD_DELIMITER) + "' ENCLOSED BY '\"' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "' " //+ " LINES TERMINATED BY '\\r\\n'" + "FROM " + tableName; if (conditions != null && conditions.length() > 1) { query += " WHERE " + conditions; }//from w ww . j a v a 2 s. co m LOG.info(query); ConnectionController.executeQuery(sid, query); }From source file:org.ut.biolab.medsavant.server.serverapi.VariantManager.java
@Override public int exportVariants(String userSessionID, int projID, int refID, Condition[][] conditions, boolean orderedByPosition, boolean zipOutputFile) throws SQLException, RemoteException, SessionExpiredException, IOException, InterruptedException { String backgroundSessionID = SessionManager.getInstance().createBackgroundSessionFromSession(userSessionID); //generate directory File baseDir = DirectorySettings.generateDateStampDirectory(DirectorySettings.getTmpDirectory()); Process p = Runtime.getRuntime().exec("chmod -R o+w " + baseDir.getCanonicalPath()); p.waitFor();/*from w ww. j a va2 s. c o m*/ String filename = ProjectManager.getInstance().getProjectName(backgroundSessionID, projID).replace(" ", "") + "-varexport-" + System.currentTimeMillis() + ".tdf"; File file = new File(baseDir, filename); File zipFile = null; try { LOG.info("Exporting variants to " + file.getAbsolutePath()); long start = System.currentTimeMillis(); TableSchema table = CustomTables.getInstance().getCustomTableSchema(backgroundSessionID, ProjectManager.getInstance().getVariantTableName(backgroundSessionID, projID, refID, true)); SelectQuery query = new SelectQuery(); query.addFromTable(table.getTable()); query.addAllColumns(); addConditionsToQuery(query, conditions); if (orderedByPosition) { query.addOrderings(table.getDBColumn(START_POSITION), table.getDBColumn(END_POSITION)); } String intoString = "INTO OUTFILE \"" + file.getAbsolutePath().replaceAll("\\\\", "/") + "\" " + "FIELDS TERMINATED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.FIELD_DELIMITER) + "' " + "ENCLOSED BY '" + VariantManagerUtils.ENCLOSED_BY + "' " + "ESCAPED BY '" + StringEscapeUtils.escapeJava(VariantManagerUtils.ESCAPE_CHAR) + "' " //+ " LINES TERMINATED BY '\\r\\n' "; ; String queryString = query.toString().replace("FROM", intoString + "FROM"); LOG.info(queryString); ConnectionController.executeQuery(backgroundSessionID, queryString); if (zipOutputFile) { LOG.info("Zipping export..."); zipFile = new File(file.getAbsoluteFile() + ".zip"); IOUtils.zipFile(file, zipFile); } LOG.info("Done exporting variants to " + file.getAbsolutePath()); LOG.info("Export took " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); } finally { if (zipOutputFile) { boolean deleted = file.delete(); LOG.info("Deleting " + file.getAbsolutePath() + " - " + (deleted ? "successful" : "failed")); file = zipFile; LOG.info("Done zipping"); } } // add file to map and send the id back //int fileID = NetworkManager.getInstance().openReaderOnServer(backgroundSessionID, file); int fileID = NetworkManager.getInstance().openReaderOnServer(userSessionID, file); return fileID; }From source file:org.ut.biolab.medsavant.shared.vcf.VariantRecord.java
public static String createTabString(Object[] values) { String s = ""; if (values.length == 0) { return s; }/* w w w . j a v a 2 s .c om*/ for (Object o : values) { s += "\"" + StringEscapeUtils.escapeJava(getString(o)) + "\"" + delim; } return s.substring(0, s.length() - 1); }From source file:password.pwm.util.java.StringUtil.java
public static String escapeJava(final String input) { return StringEscapeUtils.escapeJava(input); }From source file:profiles.FilesThreader.java
public static void main(String[] args) throws InterruptedException { // Check how many arguments were passed in if ((args == null) || (args.length < 4)) { System.err.println("4 Parameters are required to launch a Job."); System.err.println("First: String 'INPUT: /path/to/files/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) Number of seconds to pause"); System.err.println("Example: fileToRun /input/path/ /output/path/ " + "10 2"); System.exit(-1);//from w w w . ja va2 s. co m } // to write output in a file ThreadPrintStream.replaceSystemOut(); try { INPUT = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } try { OUTPUT = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } try { TOTAL_JOBS_STR = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(-1); } try { PAUSE_STR = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } try { PAUSE = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } int TOTAL_JOBS_INT = 0; try { TOTAL_JOBS_INT = Integer.parseInt(TOTAL_JOBS_STR); } catch (NumberFormatException e) { System.err.println("Argument" + TOTAL_JOBS_STR + " must be an integer."); System.exit(-1); } System.out.println("Going to launch jobs. " + "Please see logs files to track jobs."); ExecutorService threadPool = Executors.newFixedThreadPool(TOTAL_JOBS_INT); for (int i = 0; i < TOTAL_JOBS_INT; i++) { final String JOB_NO = Integer.toString(i); threadPool.submit(new Runnable() { public void run() { try { // Create a text file where System.out.println() // will send its data for this thread. String name = Thread.currentThread().getName(); FileOutputStream fos = null; try { fos = new FileOutputStream(name + "-logs.txt"); } catch (Exception e) { System.err.println(e.getMessage()); System.exit(0); } // Create a PrintStream that will write to the new file. PrintStream stream = new PrintStream(new BufferedOutputStream(fos)); // Install the PrintStream to be used // as System.out for this thread. ((ThreadPrintStream) System.out).setThreadOut(stream); // Output three messages to System.out. System.out.println(name); System.out.println(); System.out.println(); FilesThreader.execTask(INPUT, OUTPUT, TOTAL_JOBS_STR, JOB_NO, PAUSE_STR); } catch (IOException | ClassNotFoundException | SQLException | JSONException e) { // e.printStackTrace(); System.out.println(e.getMessage()); } } }); helpers.pause(PAUSE); } threadPool.shutdown(); }From source file:profiles.FilesThreaderParser.java
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required to launch a Job."); System.err.println("First: String 'INPUT: /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Example: fileToRun /input/path/ /output/path/ " + "10 1 3"); System.exit(-1);//from w w w . ja va 2s . c o m } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/users/lookup"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No targeted user file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println("My offSet is " + offSet); System.out.println("My chunkSizeToGet is " + chunkSizeToGet); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); System.out.println("Going to get Profiles."); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); // to write output in a file System.out.flush(); List<String> fileNamesShare = allFiles.subList(offSet, chunkSizeToGet); for (String fileName : fileNamesShare) { System.out.println("Going to parse file: " + fileName); try { // open file to write all profiles String filesPath = outputPath + "/"; PrintWriter writer = new PrintWriter(filesPath + "/" + fileName, "UTF-8"); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + fileName))) { String jsonString; while ((jsonString = br.readLine()) != null) { // process the line. List<String> fileContent = new ArrayList<>(); fileContent.add(jsonString); String[] strarray = fileContent.toArray(new String[0]); String ss = Arrays.toString(strarray); JSONArray obj = new JSONArray(ss); JSONObject obj4 = (JSONObject) obj.get(0); JSONArray idsS = (JSONArray) obj4.get("ids"); ArrayList<String> Idslist = new ArrayList<String>(); if (idsS != null) { int len = idsS.length(); for (int i = 0; i < len; i++) { Idslist.add(idsS.get(i).toString()); } } for (int start = 0; start < Idslist.size(); start += 100) { int end = Math.min(start + 100, Idslist.size()); List<String> sublist = Idslist.subList(start, end); long[] idsdata = new long[sublist.size()]; for (int i = 0; i < sublist.size(); i++) { idsdata[i] = Long.valueOf(sublist.get(i)); } ResponseList<User> profiles = null; while (true) { try { profiles = twitter.lookupUsers(idsdata); if (profiles.size() > 0) { for (User user : profiles) { String rawJSON = TwitterObjectFactory.getRawJSON(user); // put profilesJSON in a file try { writer.println(rawJSON); } catch (Exception e) { System.err.println(e.getMessage()); System.exit(0); } } break; } } catch (TwitterException te) { // If rate limit reached then switch Auth // user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded" + " OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } } // while get profiles // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } } } writer.close(); } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + fileName); } // delete file if processed File fileToDelete = new File(inputPath + "/" + "/" + fileName); fileToDelete.delete(); // to write output in a file System.out.flush(); } // all my files // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get Profiles: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }