List of usage examples for twitter4j IDs getNextCursor
@Override
long getNextCursor();
From source file:friendsandfollowers.FilesThreaderFollowersIDsParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);// w ww . j a va 2s.c om } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/followers/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get followers ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get followers ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFollowersIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Followers Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:friendsandfollowers.FilesThreaderFriendsIDsParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, FileNotFoundException, UnsupportedEncodingException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: DB or /input/path/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Number of seconds to pause"); System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000"); System.exit(-1);/*from w ww . ja v a 2s . c om*/ } // TODO documentation for command line AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/friends/ids"; String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String outputPath = null; try { outputPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument " + args[1] + " must be an String."); System.exit(-1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument " + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument " + args[3] + " must be an integer."); System.exit(1); } int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } int IDS_TO_FETCH_INT = -1; if (args.length == 6) { try { IDS_TO_FETCH_INT = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(-1); } } int IDS_TO_FETCH = 0; if (IDS_TO_FETCH_INT > 5000) { float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000; IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F); } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) { IDS_TO_FETCH = 1; } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No screen names file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println(); // Load OAuh User TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls; int RemainingCallsCounter = 0; System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println(); IDs ids; System.out.println("Going to get friends ids."); // to write output in a file System.out.flush(); if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { // process the line. System.out.println("Going to get friends ids of Screen-name / user_id: " + line); System.out.println(); String targetedUser = line.trim(); // tmp long cursor = -1; int idsLoopCounter = 0; int totalIDs = 0; PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8"); // call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); do { ids = null; try { if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor); } else { ids = twitter.getFriendsIDs(targetedUser, cursor); } } catch (TwitterException te) { // do not throw if user has protected tweets, or // if they deleted their account if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED || te.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Friends Get Exception: " + te.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); // If error then switch to next user continue OUTERMOST; } if (ids.getIDs().length > 0) { idsLoopCounter++; totalIDs += ids.getIDs().length; System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length); JSONObject responseDetailsJson = new JSONObject(); JSONArray jsonArray = new JSONArray(); for (long id : ids.getIDs()) { jsonArray.put(id); } Object idsJSON = responseDetailsJson.put("ids", jsonArray); writer.println(idsJSON); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // load auth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls; RemainingCallsCounter = 0; System.out.println("New OAuth Remianing Calls: " + RemainingCalls); } if (IDS_TO_FETCH_INT != -1) { if (idsLoopCounter == IDS_TO_FETCH) { break; } } } while ((cursor = ids.getNextCursor()) != 0); writer.close(); System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs); // Remove file if ids not found if (totalIDs == 0) { System.out.println("No ids fetched so removing " + "file " + targetedUser); File fileToDelete = new File(outputPath + "/" + targetedUser); fileToDelete.delete(); } System.out.println(); } // while get records from single file } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.err.println("Failed to get friends' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:net.lacolaco.smileessence.twitter.task.BlockIDsTask.java
License:Open Source License
@Override protected Long[] doInBackground(Void... params) { try {//from w w w . ja v a2 s .c o m List<Long> idList = new ArrayList<>(); long cursor = -1; do { IDs blocksIDs = twitter.getBlocksIDs(cursor); cursor = blocksIDs.getNextCursor(); for (long id : blocksIDs.getIDs()) { idList.add(id); } } while (cursor != 0); return idList.toArray(new Long[idList.size()]); } catch (TwitterException e) { Logger.error(e); return new Long[0]; } }
From source file:net.lacolaco.smileessence.twitter.task.MutesIDsTask.java
License:Open Source License
@Override protected Long[] doInBackground(Void... params) { try {/*from w w w.j a va 2 s .co m*/ List<Long> idList = new ArrayList<>(); long cursor = -1; do { IDs mutesIDs = twitter.getMutesIDs(cursor); cursor = mutesIDs.getNextCursor(); for (long id : mutesIDs.getIDs()) { idList.add(id); } } while (cursor != 0); return idList.toArray(new Long[idList.size()]); } catch (TwitterException e) { Logger.error(e); return new Long[0]; } }
From source file:org.apache.streams.twitter.provider.TwitterFollowingProviderTask.java
License:Apache License
private void collectIds(Long id) { int keepTrying = 0; long curser = -1; twitter4j.User user;// w w w . ja v a2 s . co m String userJson; try { user = client.users().showUser(id); userJson = TwitterObjectFactory.getRawJSON(user); } catch (TwitterException ex) { LOGGER.error("Failure looking up " + id); return; } do { try { twitter4j.IDs ids = null; if (provider.getConfig().getEndpoint().equals("followers")) { ids = client.friendsFollowers().getFollowersIDs(id, curser, provider.getConfig().getMaxItems().intValue()); } else if (provider.getConfig().getEndpoint().equals("friends")) { ids = client.friendsFollowers().getFriendsIDs(id, curser, provider.getConfig().getMaxItems().intValue()); } Objects.requireNonNull(ids); Preconditions.checkArgument(ids.getIDs().length > 0); for (long otherId : ids.getIDs()) { try { Follow follow = null; if (provider.getConfig().getEndpoint().equals("followers")) { follow = new Follow().withFollowee(new User().withId(id)) .withFollower(new User().withId(otherId)); } else if (provider.getConfig().getEndpoint().equals("friends")) { follow = new Follow().withFollowee(new User().withId(otherId)) .withFollower(new User().withId(id)); } Objects.requireNonNull(follow); if (item_count < provider.getConfig().getMaxItems()) { ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue); item_count++; } } catch (Exception ex) { LOGGER.warn("Exception: {}", ex); } } if (!ids.hasNext()) { break; } if (ids.getNextCursor() == 0) { break; } curser = ids.getNextCursor(); page_count++; } catch (TwitterException twitterException) { keepTrying += TwitterErrorHandler.handleTwitterError(client, id, twitterException); } catch (Exception ex) { keepTrying += TwitterErrorHandler.handleTwitterError(client, null, ex); } } while (shouldContinuePulling() && curser != 0 && keepTrying < provider.getConfig().getRetryMax()); }
From source file:org.loklak.harvester.TwitterAPI.java
License:Open Source License
public static JSONObject getNetworkerNames(final String screen_name, final int max_count, final Networker networkRelation) throws IOException, TwitterException { if (max_count == 0) return new JSONObject(); boolean complete = true; Set<Number> networkingIDs = new LinkedHashSet<>(); Set<Number> unnetworkingIDs = new LinkedHashSet<>(); JsonFactory mapcapsule = (networkRelation == Networker.FOLLOWERS ? DAO.followers_dump : DAO.following_dump) .get("screen_name", screen_name); if (mapcapsule == null) { JsonDataset ds = networkRelation == Networker.FOLLOWERS ? DAO.followers_dump : DAO.following_dump; mapcapsule = ds.get("screen_name", screen_name); }/* ww w. j a v a2 s. c o m*/ if (mapcapsule != null) { JSONObject json = mapcapsule.getJSON(); // check date and completeness complete = json.has("complete") ? (Boolean) json.get("complete") : Boolean.FALSE; String retrieval_date_string = json.has("retrieval_date") ? (String) json.get("retrieval_date") : null; DateTime retrieval_date = retrieval_date_string == null ? null : AbstractObjectEntry.utcFormatter.parseDateTime(retrieval_date_string); if (complete && System.currentTimeMillis() - retrieval_date.getMillis() < DateParser.DAY_MILLIS) return json; // load networking ids for incomplete retrievals (untested) String nr = networkRelation == Networker.FOLLOWERS ? "follower" : "following"; if (json.has(nr)) { JSONArray fro = json.getJSONArray(nr); for (Object f : fro) networkingIDs.add((Number) f); } } TwitterFactory tf = getUserTwitterFactory(screen_name); if (tf == null) tf = getAppTwitterFactory(); if (tf == null) return new JSONObject(); Twitter twitter = tf.getInstance(); long cursor = -1; collect: while (cursor != 0) { try { IDs ids = networkRelation == Networker.FOLLOWERS ? twitter.getFollowersIDs(screen_name, cursor) : twitter.getFriendsIDs(screen_name, cursor); RateLimitStatus rateStatus = ids.getRateLimitStatus(); if (networkRelation == Networker.FOLLOWERS) { getFollowerIdRemaining = rateStatus.getRemaining(); getFollowerIdResetTime = System.currentTimeMillis() + rateStatus.getSecondsUntilReset() * 1000; } else { getFollowingIdRemaining = rateStatus.getRemaining(); getFollowingIdResetTime = System.currentTimeMillis() + rateStatus.getSecondsUntilReset() * 1000; } //System.out.println("got: " + ids.getIDs().length + " ids"); //System.out.println("Rate Status: " + rateStatus.toString() + "; time=" + System.currentTimeMillis()); boolean dd = false; for (long id : ids.getIDs()) { if (networkingIDs.contains(id)) dd = true; // don't break loop here networkingIDs.add(id); } if (dd) break collect; // this is complete! if (rateStatus.getRemaining() == 0) { complete = false; break collect; } if (networkingIDs.size() >= Math.min(10000, max_count >= 0 ? max_count : 10000)) { complete = false; break collect; } cursor = ids.getNextCursor(); } catch (TwitterException e) { complete = false; break collect; } } // create result JSONObject json = new JSONObject(true); json.put("screen_name", screen_name); json.put("retrieval_date", AbstractObjectEntry.utcFormatter.print(System.currentTimeMillis())); json.put("complete", complete); Map<String, Number> networking = getScreenName(networkingIDs, max_count, true); Map<String, Number> unnetworking = getScreenName(unnetworkingIDs, max_count, true); if (networkRelation == Networker.FOLLOWERS) { json.put("followers_count", networking.size()); json.put("unfollowers_count", unnetworking.size()); json.put("followers_names", networking); json.put("unfollowers_names", unnetworking); if (complete) DAO.followers_dump.putUnique(json); // currently we write only complete data sets. In the future the update of datasets shall be supported } else { json.put("following_count", networking.size()); json.put("unfollowing_count", unnetworking.size()); json.put("following_names", networking); json.put("unfollowing_names", unnetworking); if (complete) DAO.following_dump.putUnique(json); } return json; }
From source file:org.sociotech.communitymashup.source.twitter.TwitterSourceService.java
License:Open Source License
private void addFollowing(User twitterUser, boolean createNewUsers) { if (twitterUser == null) { return;//from w w w. j a v a 2s. com } IDs following; long cursor = -1; // look up person for twitter user and connect Person connectToPerson = getPersonForTwitterUser(twitterUser); do { try { following = twitterAPI.getFriendsIDs(twitterUser.getId(), cursor); log("Retrieving following with cursor: " + cursor, LogService.LOG_INFO); } catch (TwitterException e) { log("Could not get more following Users from twitter. (Cursor: " + cursor + "): " + e.getMessage(), LogService.LOG_DEBUG); break; } if (following == null) { return; } if (connectToPerson != null) { connectPersonsForTwitterUserIds(following.getIDs(), connectToPerson, true, createNewUsers); } // next cursor cursor = following.getNextCursor(); } while (following.hasNext()); }
From source file:org.sociotech.communitymashup.source.twitter.TwitterSourceService.java
License:Open Source License
private void addFollower(User twitterUser, boolean createNewUsers) { if (twitterUser == null) { return;//from w w w.java 2s . c o m } int followersMax = 1000; int followersCount = twitterUser.getFollowersCount(); if (followersCount > followersMax) { // ignore users with to much followers log("Ignoring the " + followersCount + " Followers of " + twitterUser.getName()); return; } IDs followers; long cursor = -1; do { try { followers = twitterAPI.getFollowersIDs(twitterUser.getId(), cursor); log("Retrieving followers with cursor: " + cursor, LogService.LOG_INFO); } catch (TwitterException e) { log("Could not get more followers from twitter. (Cursor: " + cursor + "): " + e.getMessage(), LogService.LOG_DEBUG); break; } if (followers == null) { return; } // look up person for twitter user and connect Person connectToPerson = getPersonForTwitterUser(twitterUser); if (connectToPerson != null) { connectPersonsForTwitterUserIds(followers.getIDs(), connectToPerson, false, createNewUsers); } // next cursor cursor = followers.getNextCursor(); } while (followers.hasNext()); }
From source file:org.yukung.following2ldr.command.impl.FindFeedUrlCommand.java
License:Apache License
@Override public void run() throws Throwable { // Twitter??/*from ww w. j a v a2s .c o m*/ long start = System.currentTimeMillis(); String consumerKey = config.getProperty(Constants.CONSUMER_KEY); String consumerSecret = config.getProperty(Constants.CONSUMER_SECRET); String userName = params.get(0); Twitter twitter = new TwitterFactory().getInstance(); twitter.setOAuthConsumer(consumerKey, consumerSecret); RequestToken requestToken = twitter.getOAuthRequestToken(); String authorizationURL = requestToken.getAuthorizationURL(); System.out.println(":" + authorizationURL); BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); System.out.print("Please enter the PIN:"); String pin = br.readLine(); AccessToken accessToken = twitter.getOAuthAccessToken(requestToken, pin); twitter.setOAuthAccessToken(accessToken); long cursor = -1L; IDs friendIDs; List<Long> iDsList = new ArrayList<Long>(5000); do { friendIDs = twitter.getFriendsIDs(userName, cursor); long[] iDs = friendIDs.getIDs(); for (long iD : iDs) { iDsList.add(iD); } cursor = friendIDs.getNextCursor(); } while (friendIDs.hasNext()); List<long[]> list = new ArrayList<long[]>(); int offset = 0; long[] tmp = new long[100]; for (Long id : iDsList) { if (offset < 100) { tmp[offset] = id; offset++; } else { list.add(tmp); offset = 0; tmp = new long[100]; } } list.add(tmp); List<URL> urlList = new ArrayList<URL>(); for (long[] array : list) { ResponseList<User> lookupUsers = twitter.lookupUsers(array); for (User user : lookupUsers) { log.info("URL:" + user.getURL()); urlList.add(user.getURL()); } } String path = "C:\\Users\\ikeda_yusuke\\Documents\\sandbox\\java\\data\\" + userName + ".txt"; FileWriter writer = new FileWriter(path); BufferedWriter out = new BufferedWriter(writer); // PrintWriter pw = new PrintWriter(writer); for (URL url : urlList) { if (url != null) { out.write(url.toString() + "\n"); } } out.flush(); out.close(); long end = System.currentTimeMillis(); log.info("?:" + (end - start) + " ms"); // ??IDor?? // Twitter API??????ID? // ?????ID?URL100??? // ??URL? }
From source file:peoplesearch.FindFriendsAndFollowers.java
public void GetFollowersIDs() { try {// w w w . j av a 2s. co m // I need to pass the Person name and the TwitterID. //String targetname="Philip Bergkvist"; Twitter twitter = new TwitterFactory().getInstance(); long cursor = -1; IDs ids; ResponseList<User> users1 = null; ResponseList<User> users2 = null; System.out.println("Listing followers's ids."); GraphManager mgr = EmbeddedGraphManager.getInstance(); mgr.init(new File("/usr/local/Cellar/neo4j/2.1.7/libexec/data/forlang1.db")); //mgr.addTwitterAccount(new TwitterAccountImpl(new Date(), "I am Studying", 13, 82, true,"Aalborg", "Philiptwoshoes", 2730631792L)); List<TwitterAccount> twitteraccountslist; twitteraccountslist = null; twitteraccountslist = mgr.listTwitterAccounts(); System.out.println("the number of twitter account in the neo4J DB is" + twitteraccountslist.size()); for (TwitterAccount Twit : twitteraccountslist) { do { TwitterLimitWait tlw = new TwitterLimitWait(); tlw.CheckLimit(); if (0 < twitteraccountslist.size()) { ids = twitter.getFollowersIDs(Twit.getScreenName(), cursor); //.getFollowersIDs(pep[0], cursor); //ids = twitter.getFollowersIDs("Philiptwoshoes", cursor); //.getFollowersIDs(pep[0], cursor); tlw.CheckLimit(); users1 = twitter.getFollowersList(Twit.getScreenName(), cursor); tlw.CheckLimit(); users2 = twitter.getFriendsList(Twit.getScreenName(), cursor); } else { tlw.CheckLimit(); ids = twitter.getFollowersIDs(cursor); } for (User user : users1) { tlw.CheckLimit(); System.out.println("the follower called " + user.getName() + " with twitter handler " + user.getScreenName()); String username = user.getName(); //mgr.addPerson(new PersonImpl(username)); Date Creation = user.getCreatedAt(); tlw.CheckLimit(); String descript = user.getDescription(); boolean empty1 = user.getDescription().isEmpty(); if (empty1 == true) { descript = " "; } tlw.CheckLimit(); int followers = user.getFollowersCount(); tlw.CheckLimit(); int following = user.getFriendsCount(); boolean geo = user.isGeoEnabled(); String loc = user.getLocation(); boolean empty2 = user.getLocation().isEmpty(); if (empty2 == true) { loc = " "; } String screenname = user.getScreenName(); boolean empty3 = user.getScreenName().isEmpty(); if (empty3 == true) { screenname = " "; } tlw.CheckLimit(); long twitID = user.getId(); mgr.linkPersonToTwitterAccount(new PersonImpl(username), new TwitterAccountImpl(Creation, descript, followers, following, geo, loc, screenname, twitID)); mgr.linkTwitterAccounts(new TwitterAccountImpl(Creation, descript, followers, following, geo, loc, screenname, twitID), Twit); } System.out.println("The total number of followers is: " + users1.size()); // the same procedure for the Following for (User user : users2) { tlw.CheckLimit(); System.out.println("the following called " + user.getName() + " with twitter handler " + user.getScreenName()); String username1 = user.getName(); //mgr.addPerson(new PersonImpl(username1)); Date Creation = user.getCreatedAt(); String descript = user.getDescription(); boolean empty1 = user.getDescription().isEmpty(); if (empty1 == true) { descript = " "; } int followers = user.getFollowersCount(); int following = user.getFriendsCount(); boolean geo = user.isGeoEnabled(); String loc = user.getLocation(); boolean empty2 = user.getLocation().isEmpty(); if (empty2 == true) { loc = " "; } String screenname = user.getScreenName(); tlw.CheckLimit(); boolean empty3 = user.getScreenName().isEmpty(); if (empty3 == true) { screenname = " "; } tlw.CheckLimit(); long twitID = user.getId(); //mgr.addTwitterAccount(new TwitterAccountImpl(Creation,descript,followers,following,geo,loc,screenname,twitID)); mgr.linkPersonToTwitterAccount(new PersonImpl(username1), new TwitterAccountImpl(Creation, descript, followers, following, geo, loc, screenname, twitID)); mgr.linkTwitterAccounts(Twit, new TwitterAccountImpl(Creation, descript, followers, following, geo, loc, screenname, twitID)); } System.out.println("The total number of friend is: " + users2.size()); //} } while ((cursor = ids.getNextCursor()) != 0); } mgr.destroy(); // I have to check that the second iteration works fine, because i could not test that. System.exit(0); } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } }