List of usage examples for twitter4j TwitterFactory getInstance
public Twitter getInstance()
From source file:StringMatching.GetTweet.java
/** * @param args the command line arguments */// w ww. j a va 2 s . c o m public static void main(String[] args) throws JSONException, IOException { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("Gkr9iZwYDALZ16OdxMp5rubBH") .setOAuthConsumerSecret("nhEwYFfiX5qp90sLLwO2eeYMxLwb3WC120lgihrocZDPWRNcUK") .setOAuthAccessToken("94107100-572UpcOkkz9kMWGaJS8YFsIGdlmJAd2cDw8y9rOnA") .setOAuthAccessTokenSecret("ST0XtXUjYgYWKHryL2feNM0VcDQQAgrov2V7nB7hq1xBC") .setHttpProxyHost("cache.itb.ac.id").setHttpProxyPort(8080).setHttpProxyUser("jonathan.benedict") .setHttpProxyPassword("rollingonthefloor"); TwitterFactory tf = new TwitterFactory(cb.build()); Twitter twitter = tf.getInstance(); JSONObject obj = new JSONObject(); int counterTweet = 0; FileWriter file = new FileWriter("C:\\Users\\user\\IdeaProjects\\TwitterStringMatching\\input.txt"); file.flush(); try { Query query = new Query("barca".toLowerCase()); QueryResult result; do { result = twitter.search(query); List<Status> tweets = result.getTweets(); for (Status tweet : tweets) { counterTweet++; System.out.println("@" + tweet.getUser().getScreenName() + " - " + tweet.getText()); obj.put("user", tweet.getUser().getScreenName()); obj.put("tweets", tweet.getText()); //Tulis file ke dalam txt try { file.write(obj.toString()); System.out.println("Successfully Copied JSON Object to File..."); System.out.println("\nJSON Object: " + obj); } catch (IOException e) { e.printStackTrace(); } } } while (counterTweet < 1000); file.close(); System.exit(0); } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); System.exit(-1); } System.exit(0); }
From source file:summarizer.NewApplication.java
private void HashtagSearchButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_HashtagSearchButtonActionPerformed // TODO add your handling code here: ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("uEeExq1FHARMvAGTJAY0dGxPh") .setOAuthConsumerSecret("jMcofTKFsayd4bpA6HFNlgkMfiveoS3ffRSCy9FCSs9pWYqdAD") .setOAuthAccessToken("2443437752-PrbVsDrQCxthX3T7lV3dtHs3FCXFXfBBHsdNrOS") .setOAuthAccessTokenSecret("MbauBu7R6vUUKqFD9ogK7VTIfce4nmvewYsgYqBOQ2ZbC"); TwitterFactory tf = new TwitterFactory(cb.build()); Twitter twitter = tf.getInstance(); String hashtag;/*w w w. jav a 2s . co m*/ hashtag = HashtagTextField.getText(); Query query = new Query(hashtag); query.count(100); if (TwitterComboBox.getSelectedIndex() == 0) { query.resultType(Query.ResultType.mixed); } else if (TwitterComboBox.getSelectedIndex() == 0) { query.resultType(Query.ResultType.recent); } else { query.resultType(Query.ResultType.popular); } QueryResult result = null; try { result = twitter.search(query); } catch (TwitterException ex) { Logger.getLogger(NewApplication.class.getName()).log(Level.SEVERE, null, ex); } String alltweets = "", temp = ""; for (Status status : result.getTweets()) { temp = status.getText(); temp = temp.replaceAll("[\\t\\n\\r]", " "); if (!"".equals(temp)) { if ("RT".equals(temp.substring(0, 2))) { temp = temp.split(":", 2)[1]; } alltweets += (temp + "\n"); } } InputTextArea.setText(alltweets); }
From source file:testtweet.TweetUsingTwitter4jExample.java
/** * @param args the command line arguments *///from ww w. j av a2s .c o m public static void main(String[] args) throws IOException, TwitterException { //Instantiate a re-usable and thread-safe factory TwitterFactory twitterFactory = new TwitterFactory(Data.getConf().build()); //Instantiate a new Twitter instance Twitter twitter = twitterFactory.getInstance(); /* //setup OAuth Consumer Credentials twitter.setOAuthConsumer(consumerKey, consumerSecret); //setup OAuth Access Token twitter.setOAuthAccessToken(new AccessToken(accessToken, accessTokenSecret)); */ //Instantiate and initialize a new twitter status update StatusUpdate statusUpdate = new StatusUpdate( //your tweet or status message "Twitter API #Hacked"); //attach any media, if you want to /* statusUpdate.setMedia( //title of media "http://h1b-work-visa-usa.blogspot.com" , new URL("http://lh6.ggpht.com/-NiYLR6SkOmc/Uen_M8CpB7I/AAAAAAAAEQ8/tO7fufmK0Zg/h-1b%252520transfer%252520jobs%25255B4%25255D.png?imgmax=800").openStream()); */ //tweet or update status Status status = twitter.updateStatus(statusUpdate); //response from twitter server System.out.println("status.toString() = " + status.toString()); System.out.println("status.getInReplyToScreenName() = " + status.getInReplyToScreenName()); System.out.println("status.getSource() = " + status.getSource()); System.out.println("status.getText() = " + status.getText()); System.out.println("status.getContributors() = " + Arrays.toString(status.getContributors())); System.out.println("status.getCreatedAt() = " + status.getCreatedAt()); System.out.println("status.getCurrentUserRetweetId() = " + status.getCurrentUserRetweetId()); System.out.println("status.getGeoLocation() = " + status.getGeoLocation()); System.out.println("status.getId() = " + status.getId()); System.out.println("status.getInReplyToStatusId() = " + status.getInReplyToStatusId()); System.out.println("status.getInReplyToUserId() = " + status.getInReplyToUserId()); System.out.println("status.getPlace() = " + status.getPlace()); System.out.println("status.getRetweetCount() = " + status.getRetweetCount()); System.out.println("status.getRetweetedStatus() = " + status.getRetweetedStatus()); System.out.println("status.getUser() = " + status.getUser()); System.out.println("status.getAccessLevel() = " + status.getAccessLevel()); System.out.println("status.getHashtagEntities() = " + Arrays.toString(status.getHashtagEntities())); System.out.println("status.getMediaEntities() = " + Arrays.toString(status.getMediaEntities())); if (status.getRateLimitStatus() != null) { System.out .println("status.getRateLimitStatus().getLimit() = " + status.getRateLimitStatus().getLimit()); System.out.println( "status.getRateLimitStatus().getRemaining() = " + status.getRateLimitStatus().getRemaining()); System.out.println("status.getRateLimitStatus().getResetTimeInSeconds() = " + status.getRateLimitStatus().getResetTimeInSeconds()); System.out.println("status.getRateLimitStatus().getSecondsUntilReset() = " + status.getRateLimitStatus().getSecondsUntilReset()); System.out.println("status.getRateLimitStatus().getRemainingHits() = " + status.getRateLimitStatus().getRemaining()); } System.out.println("status.getURLEntities() = " + Arrays.toString(status.getURLEntities())); System.out.println("status.getUserMentionEntities() = " + Arrays.toString(status.getUserMentionEntities())); }
From source file:the_vaps_project.The_vaps_project.java
public static void main(String[] args)/* throws TwitterException */ { // TODO code application logic here /*/* ww w. jav a 2s . c o m*/ List<Status> status = twitter.getHomeTimeline(); for(Status st : status){ System.out.println(st.getUser().getName()+"--------"+st.getText()); System.out.println(st.getCreatedAt()); }*/ // The factory instance is re-useable and thread safe. /*Twitter twitter2 = tf.getSingleton(); Query query = new Query("source:twitter4j yusukey"); QueryResult result = twitter.search(query); for (Status statuss : result.getTweets()) { System.out.println("@" + statuss.getUser().getName() + ":" + statuss.getText()); }*/ TwitterCriteria criteria = null; Tweet t = null; criteria = TwitterCriteria.create() .setMaxTweets(10).setUntil("2016-11-08").setQuerySearch("#NeverTrump"); /*Scribe s = new Scribe(); s.detruireFichier("tweets.xml"); s.ouvrir("tweets.xml"); s.ecrire("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); s.ecrire("<tweets>\n"); for (int i = 0; i < TweetManager.getTweets(criteria).size(); i++) { t = TweetManager.getTweets(criteria).get(i); s.ecrire("\t<tweet>\n"); s.ecrire("\t\t<text><![CDATA["+t.getText()+"]]></text>\n"); s.ecrire("\t\t<hashtags>"+t.getHashtags()+"</hashtags>\n"); s.ecrire("\t\t<date>"+t.getDate()+"</date>\n"); s.ecrire("\t\t<username>"+t.getUsername()+"</username>\n"); s.ecrire("\t\t<retweets>"+t.getRetweets()+"</retweets>\n"); s.ecrire("\t\t<favorites>"+t.getFavorites()+"</favorites>\n"); s.ecrire("\t\t<id>"+t.getId()+"</id>\n"); s.ecrire("\t\t<geo>"+t.getGeo()+"</geo>\n"); s.ecrire("\t\t<mentions>"+t.getMentions()+"</mentions>\n"); s.ecrire("\t\t<permalink>"+t.getPermalink()+"</permalink>\n"); s.ecrire("\t\t<smileys>"+t.getSmileys()+"</smileys>\n"); s.ecrire("\t\t<polarite>"+t.getPolarite()+"</polarite>\n"); s.ecrire("\t</tweet>\n"); corpus.ecrire(s.filtrerTexte(t.getText() + "\n")); } s.ecrire("</tweets>"); s.fermer();*/ /* SUPER SITE D'EXEMPLES https://github.com/yusuke/twitter4j/tree/master/twitter4j-examples/src/main/java/twitter4j/examples*/ /*PARTIE GRAPHE DES RETWEETS*/ Set<String> nameOfUsers = new HashSet<>(); for (int i = 0; i < TweetManager.getTweets(criteria).size(); i++) { t = TweetManager.getTweets(criteria).get(i); System.out.println(t.getUsername()); nameOfUsers.add(t.getUsername()); } ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("gVaO166IGfSCun5Esj1sZ5bX3") .setOAuthConsumerSecret("KhN1L4pXCG5U6d6tee1IGUnIV1bqxieQJDxE2HOx3AGI5npQRE") .setOAuthAccessToken("822018210178469888-4Pr1KRE2rmFvof0O5IUdTlxtkGKamU4") .setOAuthAccessTokenSecret("D1thDl7k8v2s1SupU5ZGhK3U7Mphb69fpUW4JuDou5O65"); TwitterFactory tf = new TwitterFactory(cb.build()); twitter4j.Twitter twitter = tf.getInstance(); /*String[] screenNames = nameOfUsers.toArray(new String[0]); ResponseList<User> usersNames; long cursor = -1; IDs ids; try { usersNames = twitter.lookupUsers(screenNames); for(User u: usersNames){ System.out.println(u.getId()+" "+u.getName()+" "+u.getName()); ids = twitter.getFollowersIDs("username", cursor); twitter.getFollowers } } catch (TwitterException ex) { Exceptions.printStackTrace(ex); }*/ Set<DuoKey> utilisateurs = new HashSet<>(); Set<DuoKey> liens = new HashSet<>(); //User us; try { int nbpages = 0; long cursor = -1; PagableResponseList<User> listeFollowersTemp; ArrayList<User> listeFollowers = new ArrayList<User>(); for (String s : nameOfUsers) { if (!utilisateurs.contains(new DuoKey(s, "auteurTweet"))) { utilisateurs.add(new DuoKey(s, "auteurTweet")); //listeFollowersTemp = twitter.getFollowersList(s, -1); do { TimeUnit.MINUTES.sleep(1); listeFollowersTemp = twitter.getFollowersList(s, cursor); for (User u : listeFollowersTemp) { listeFollowers.add(u); } nbpages++; } while (((cursor = listeFollowersTemp.getNextCursor()) != 0) && nbpages <= 9); //4=100 followers par auteur de tweet recherch } System.out.println(s + ", mes followers sont :"); for (User u : listeFollowers) { utilisateurs.add(new DuoKey(u.getName(), "auteurTweet")); liens.add(new DuoKey(u.getName(), s)); System.out.println("\t" + u.getName()); } listeFollowers.clear(); nbpages = 0; } } catch (TwitterException | InterruptedException ex) { Exceptions.printStackTrace(ex); } Scribe sc = new Scribe(); sc.detruireFichier("graphFollowers.gexf"); sc.ouvrir("graphFollowers.gexf"); sc.ecrire("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); sc.ecrire( "<gexf xmlns=\"http://www.gexf.net/1.2draft/gexf.xsd\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.1draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\n"); sc.ecrire("\t<meta lastmodifieddate=\"2017-02-17\">\n"); sc.ecrire("\t\t<creator>Pirboss</creator>\n"); sc.ecrire("\t\t<description>Test v2</description>\n"); sc.ecrire("\t</meta>\n"); sc.ecrire("\t<graph defaultedgetype=\"directed\" mode=\"static\">\n"); sc.ecrire("\t\t<nodes>\n"); ArrayList<DuoKey> distinctListOfUsers = new ArrayList(utilisateurs); System.out.println(distinctListOfUsers.size()); for (int i = 0; i < distinctListOfUsers.size(); i++) { sc.ecrire("\t\t\t<node id=\"" + distinctListOfUsers.get(i).getX().replace("&", "&") + "\" label=\"" + distinctListOfUsers.get(i).getX().replace("&", "&") + "\"/>\n"); } sc.ecrire("\t\t</nodes>\n"); sc.ecrire("\t\t<edges>\n"); ArrayList<DuoKey> distinctListOfLinks = new ArrayList(liens); System.out.println(distinctListOfLinks.size()); for (int i = 0; i < liens.size(); i++) { sc.ecrire("\t\t\t<edge id=\"" + i + "\" source=\"" + distinctListOfLinks.get(i).getX().replace("&", "&") + "\" target=\"" + distinctListOfLinks.get(i).getY().replace("&", "&") + "\"/>\n"); } sc.ecrire("\t\t</edges>\n"); sc.ecrire("\t</graph>\n"); sc.ecrire("</gexf>"); sc.fermer(); PreviewJFrame p = new PreviewJFrame(); p.script(); /*PARTIE GRAPHE DES MENTIONS*/ //NE PAS OUBLIE LE COUP DE REMPLACER LES .replace("&", "&") POUR LE XML /*Set<String> users = new HashSet<>(); HashMap<DuoKey, Integer> map; map = new HashMap<DuoKey, Integer>(); DuoKey duokey; for (int i = 0; i < TweetManager.getTweets(criteria).size(); i++) { System.out.println(i); t = TweetManager.getTweets(criteria).get(i); users.add(t.getUsername()); if(!t.getMentions().isEmpty()){ for(String target : t.getMentions().split(" ")){ target = target.replaceFirst("@", ""); users.add(target); duokey = new DuoKey(t.getUsername(), target); if(map.get(duokey) != null){ map.put(duokey, map.get(duokey)+1); }else{ map.put(duokey, 1); } } } } Scribe s = new Scribe(); s.detruireFichier("tweets.gexf"); s.ouvrir("tweets.gexf"); s.ecrire("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); s.ecrire("<gexf xmlns=\"http://www.gexf.net/1.2draft/gexf.xsd\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.1draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\n"); s.ecrire("\t<meta lastmodifieddate=\"2017-02-17\">\n"); s.ecrire("\t\t<creator>Pirboss</creator>\n"); s.ecrire("\t\t<description>Test v1</description>\n"); s.ecrire("\t</meta>\n"); s.ecrire("\t<graph defaultedgetype=\"directed\" mode=\"static\">\n"); s.ecrire("\t\t<nodes>\n"); ArrayList distinctList = new ArrayList(users); for(int i=0; i<distinctList.size(); i++){ s.ecrire("\t\t\t<node id=\""+i+"\" label=\""+distinctList.get(i)+"\"/>\n"); } s.ecrire("\t\t</nodes>\n"); s.ecrire("\t\t<edges>\n"); Iterator ite = map.entrySet().iterator(); int jj=0; while (ite.hasNext()) { Map.Entry pair = (Map.Entry)ite.next(); s.ecrire("\t\t\t<edge id=\""+(jj++)+"\" source=\""+distinctList.indexOf(((DuoKey)pair.getKey()).getX())+"\" target=\""+distinctList.indexOf(((DuoKey)pair.getKey()).getY())+"\" weight=\""+pair.getValue()+"\"/>\n"); ite.remove(); // avoids a ConcurrentModificationException } s.ecrire("\t\t</edges>\n"); s.ecrire("\t</graph>\n"); s.ecrire("</gexf>"); s.fermer(); PreviewJFrame pr = new PreviewJFrame(); pr.script();*/ }
From source file:tientx.supercode.myproejectdemov3.connection.TwitterConnection.java
private TwitterConnection() { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey(TwitterConfig.API_KEY) .setOAuthConsumerSecret(TwitterConfig.API_SECRET).setOAuthAccessToken(TwitterConfig.ACCES_TOKEN) // .setOAuthAccessTokenSecret(TwitterConfig.ACCES_TOKEN_SECRET).setHttpConnectionTimeout(100000) .setOAuthAccessTokenSecret(TwitterConfig.ACCES_TOKEN_SECRET).setHttpConnectionTimeout(100000); // .setHttpProxyHost(TwitterConfig.PROXY_HOST) // .setHttpProxyPort(TwitterConfig.PROXY_PORT); TwitterFactory factory = new TwitterFactory(cb.build()); twitter = factory.getInstance(); }
From source file:timeline.CmdSearchTerm.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length != 6)) { System.err.println("Please provide command as following."); System.err.println("java -cp twitter4j-multi-oauth-0.5.jar " + "timeline.CmdSearchTerm consumer_key consumer_secret" + " user_token user_secret output_path " + "term "); System.exit(-1);// w ww . jav a 2 s .c o m } AppOAuth AppOAuths = new AppOAuth(); String endpoint = "/search/tweets"; String consumer_key = null; try { consumer_key = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } String consumer_secret = null; try { consumer_secret = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } String user_token = null; try { user_token = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } String user_secret = null; try { user_secret = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an String."); System.exit(-1); } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an String."); System.exit(-1); } String term = ""; try { term = StringEscapeUtils.escapeJava(args[5]); } catch (Exception e) { System.err.println("Argument" + args[5] + " must be an String."); System.exit(-1); } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, consumer_key, consumer_secret, user_token, user_secret); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("Remianing Calls: " + RemainingCalls); // screen_name / user_id provided by arguments System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } String fileName = filesPath + term.replace(" ", ""); PrintWriter writer = new PrintWriter(fileName, "UTF-8"); Query query = new Query(term); QueryResult result; List<Status> statuses = new ArrayList<>(); int totalTweets = 0; int numberOfTweetsToGet = 5000; long lastID = Long.MAX_VALUE; while (totalTweets < numberOfTweetsToGet) { if (numberOfTweetsToGet - totalTweets > 100) { query.setCount(100); } else { query.setCount(numberOfTweetsToGet - totalTweets); } try { result = twitter.search(query); statuses.addAll(result.getTweets()); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (status.getId() < lastID) { lastID = status.getId(); } } } else { break; } System.out.println("totalTweets: " + totalTweets); statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); System.out.println("Tweets Get Exception: " + e.getMessage()); // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { System.out.println("No more remianing calls"); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); break; } } query.setMaxId(lastID - 1); // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { System.out.println("No more remianing calls"); break; } } if (totalTweets > 0) { System.out.println("Total dumped tweets of " + term + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } writer.close(); } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get term results because: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:timeline.CmdUserTimeline.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length != 7)) { System.err.println("Please provide command as following."); System.err.println("java -cp twitter4j-multi-oauth-0.3.jar " + "timeline.CmdUserTimeline consumer_key consumer_secret" + " user_token user_secret output_path " + "screen_name_or_userid " + "number_of_tweets_to_get_max_is_3200"); System.exit(-1);/*from www .j a va 2 s . c o m*/ } AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; String consumer_key = null; try { consumer_key = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } String consumer_secret = null; try { consumer_secret = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } String user_token = null; try { user_token = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } String user_secret = null; try { user_secret = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an String."); System.exit(-1); } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an String."); System.exit(-1); } String targetedUser = ""; try { targetedUser = StringEscapeUtils.escapeJava(args[5]); } catch (Exception e) { System.err.println("Argument" + args[5] + " must be an String."); System.exit(-1); } int NUMBER_OF_TWEETS = 0; try { NUMBER_OF_TWEETS = Integer.parseInt(args[6]); } catch (Exception e) { System.err.println("Argument" + args[6] + " must be an integer."); System.exit(-1); } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, consumer_key, consumer_secret, user_token, user_secret); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("Remianing Calls: " + RemainingCalls); // screen_name / user_id provided by arguments System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } String fileName = filesPath + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed then exit if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, or if they // deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { System.out.println("No more remianing calls"); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } break; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { System.out.println("No more remianing calls"); break; } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:timeline.DBNonThreader.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length == 0)) { System.err.println("First: 'OUTPUT_PATH' is mendatory."); System.err.println("Second: (int) Number of Tweets to get. Max 3200"); System.err.println("Third: 'screen_name / id_str'" + " is optional."); System.err.println("If 3rd argument not provided then provide" + " Twitter users through database."); System.exit(-1);//from w w w .jav a 2 s. co m } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } int NUMBER_OF_TWEETS = 3200; try { NUMBER_OF_TWEETS = Integer.parseInt(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(-1); } String targetedUser = ""; if (args.length == 3) { try { targetedUser = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an String."); System.exit(-1); } } try { TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println("User's Tweets"); // If targetedUser not provided by argument, then look into // database. if (StringUtils.isEmpty(targetedUser)) { String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE " + "`tweets_dumped_all` = 0"; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println("No User in database to get Tweets"); System.exit(-1); } System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } OUTERMOST: while (results.next()) { int targetedUserID = results.getInt("id"); targetedUser = results.getString("targeteduser"); System.out.println("Targeted User: " + targetedUser); String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less than 195 // then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed // then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, // or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // Update stats String fieldValues = "`tweets_dumped_all` = 2"; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); continue OUTERMOST; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); // Update stats String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); } else { // Update stats String fieldValues = "`tweets_dumped_all` = 2"; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get users from database } else { // screen_name / user_id provided by arguments System.out.println("screen_name / user_id provided by arguments"); System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } System.out.println("Targeted User: " + targetedUser); String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } } // screen_name / user_id provided by arguments } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get followers' ids: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); }
From source file:timeline.DBThreaderParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length < 5)) { System.err.println("5 Parameters are required to launch a Job."); System.err.println("First: String 'OUTPUT_PATH'"); System.err.println("Second: (int) Total Number Of Jobs"); System.err.println("Third: (int) This Job Number"); System.err.println("Fourth: (int) Seconds to pause between next launch"); System.err.println("Fifth: (int) Number of Tweets to get. Max 3200"); System.err.println("Example: fileName.class /output/path 10 1 2 3200"); System.exit(-1);/*from www .j av a2 s .c o m*/ } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[1]); } catch (NumberFormatException e) { System.err.println("Argument" + args[1] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(1); } int NUMBER_OF_TWEETS = 0; try { NUMBER_OF_TWEETS = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(1); } MysqlDB DB = new MysqlDB(); AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; try { String totalRowsQuery = "SELECT count( * ) as wLoad FROM `twitter_users`" + " WHERE `tweets_dumped_all` = 0"; ResultSet totalRowsResults = DB.selectQ(totalRowsQuery); int TotalWorkLoad = 0; while (totalRowsResults.next()) { TotalWorkLoad = totalRowsResults.getInt("wLoad"); } if (TotalWorkLoad < 1) { System.err.println("No User in database to get Tweets"); System.exit(-1); } // Free memmory totalRowsResults = null; if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load Please reduce Number of jobs to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE " + "`tweets_dumped_all` = 0 LIMIT " + offSet + "," + chunkSize; ResultSet results = DB.selectQ(selectQuery); int numRows = DB.numRows(results); if (numRows < 1) { System.err.println( "No User in database to get Tweets" + " with offset " + offSet + " and limit " + chunkSize); System.exit(-1); } /** * wait before launching actual job */ int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); /** * wait before launching actual job */ TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println("User's Tweets"); System.out.println("Trying to create output directory"); String filesPath = OutputDirPath + "/"; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath); System.err.println(se.getMessage()); System.exit(-1); } } System.out.flush(); OUTERMOST: while (results.next()) { int targetedUserID = results.getInt("id"); String targetedUser = results.getString("targeteduser"); System.out.println("Targeted User: " + targetedUser); // Create User file to push tweets in it String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less // than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed // then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, // or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // Update stats String fieldValues = "`tweets_dumped_all` = 2"; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); continue OUTERMOST; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); // Update stats String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); } else { // Update stats String fieldValues = "`tweets_dumped_all` = 2, " + " `tweets_all_count` = " + totalTweets; String where = "id = " + targetedUserID; DB.Update("`twitter_users`", fieldValues, where); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } System.out.flush(); } // while get users from database } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get tweets: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }
From source file:timeline.FilesThreaderParser.java
License:Apache License
public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException { // Check how many arguments were passed in if ((args == null) || (args.length < 7)) { System.err.println("7 Parameters are required to launch a Job."); System.err.println("First: String 'INPUT: " + "/path/to/perline/screen_names/files/'"); System.err.println("Second: String 'OUTPUT_PATH'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) This Job Number"); System.err.println("Fifth: (int) Seconds to pause between " + "next launch"); System.err.println("Sixth: (int) Number of Tweets to get. Max 3200"); System.err.println("Name of current threader."); System.err.println("Example: fileName.class /output/path " + "10 2 3200 pool-1-thread-1"); System.exit(-1);/*w w w . j a v a 2 s . co m*/ } String inputPath = null; try { inputPath = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument " + args[0] + " must be an String."); System.exit(-1); } String OutputDirPath = null; try { OutputDirPath = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(1); } int TOTAL_JOBS = 0; try { TOTAL_JOBS = Integer.parseInt(args[2]); } catch (NumberFormatException e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(1); } int JOB_NO = 0; try { JOB_NO = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(1); } int NUMBER_OF_TWEETS = 0; try { NUMBER_OF_TWEETS = Integer.parseInt(args[5]); } catch (NumberFormatException e) { System.err.println("Argument" + args[5] + " must be an integer."); System.exit(1); } String ThreadName = null; try { ThreadName = StringEscapeUtils.escapeJava(args[6]); } catch (Exception e) { System.err.println("Argument" + args[6] + " must be an String."); System.exit(1); } AppOAuth AppOAuths = new AppOAuth(); Misc helpers = new Misc(); String endpoint = "/statuses/user_timeline"; try { int TotalWorkLoad = 0; ArrayList<String> allFiles = null; try { final File folder = new File(inputPath); allFiles = helpers.listFilesForSingleFolder(folder); TotalWorkLoad = allFiles.size(); } catch (Exception e) { System.err.println("Input folder is not exists: " + e.getMessage()); System.exit(-1); } System.out.println("Total Workload is: " + TotalWorkLoad); if (TotalWorkLoad < 1) { System.err.println("No input file exists in: " + inputPath); System.exit(-1); } if (TOTAL_JOBS > TotalWorkLoad) { System.err.println("Number of jobs are more than total work" + " load. Please reduce 'Number of jobs' to launch."); System.exit(-1); } float TotalWorkLoadf = TotalWorkLoad; float TOTAL_JOBSf = TOTAL_JOBS; float res = (TotalWorkLoadf / TOTAL_JOBSf); int chunkSize = (int) Math.ceil(res); int offSet = JOB_NO * chunkSize; int chunkSizeToGet = (JOB_NO + 1) * chunkSize; System.out.println("My Share is " + chunkSize); System.out.println("My offSet is " + offSet); System.out.println("My chunkSizeToGet is " + chunkSizeToGet); System.out.println(); /** * wait before launching actual job */ int secondsToPause = 0; try { secondsToPause = Integer.parseInt(args[4]); } catch (NumberFormatException e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause); System.out.println("secondsToPause: " + secondsToPause); helpers.pause(secondsToPause); /** * wait before launching actual job */ TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); Twitter twitter = tf.getInstance(); int RemainingCalls = AppOAuths.RemainingCalls - 2; int RemainingCallsCounter = 0; System.out.println("First Time Remianing Calls: " + RemainingCalls); String Screen_name = AppOAuths.screen_name; System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name); System.out.println("User's Tweets"); System.out.println("Trying to create output directory"); int dirCounter = 1; int fileCounter = 0; String filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter; File theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter); System.err.println(se.getMessage()); System.exit(-1); } } if (JOB_NO + 1 == TOTAL_JOBS) { chunkSizeToGet = TotalWorkLoad; } // to write output in a file System.out.flush(); List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet); for (String myFile : myFilesShare) { System.out.println("Going to parse file: " + myFile); System.out.println(); try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) { String line; OUTERMOST: while ((line = br.readLine()) != null) { if (fileCounter >= 30000) { dirCounter++; fileCounter = 0; filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter; theDir = new File(filesPath); // If the directory does not exist, create it if (!theDir.exists()) { try { theDir.mkdirs(); } catch (SecurityException se) { System.err.println("Could not create output " + "directory: " + OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter); System.err.println(se.getMessage()); System.exit(-1); } } } System.out.println("Going to get tweets of " + "Screen-name / user_id: " + line); String targetedUser = line.trim(); System.out.println("Targeted User: " + targetedUser); // Create User file to push tweets in it String fileName = filesPath + "/" + targetedUser; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); // Call different functions for screen_name and id_str Boolean chckedNumaric = helpers.isNumeric(targetedUser); List<Status> statuses = new ArrayList<>(); int size = statuses.size(); int pageno = 1; int totalTweets = 0; boolean tweetCounterReached = false; System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS); System.out.println(); while (true) { try { Paging page = new Paging(pageno++, 200); if (chckedNumaric) { long LongValueTargetedUser = Long.valueOf(targetedUser).longValue(); statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } else { statuses.addAll(twitter.getUserTimeline(targetedUser, page)); if (statuses.size() > 0) { for (Status status : statuses) { String rawJSON = TwitterObjectFactory.getRawJSON(status); writer.println(rawJSON); totalTweets += 1; if (totalTweets >= NUMBER_OF_TWEETS) { tweetCounterReached = true; break; } } if (tweetCounterReached) { break; } } } // If user's total tweet are less // than 195 then no next call if (size == 0) { if (totalTweets < 195) { break; } } // If user's all tweets parsed // then proceed to next user if (totalTweets == size) { break; } size = totalTweets; statuses.clear(); } catch (TwitterException e) { // e.printStackTrace(); // do not throw if user has protected tweets, // or if they deleted their account if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED || e.getStatusCode() == HttpResponseCode.NOT_FOUND) { System.out.println(targetedUser + " is protected or account is deleted"); } else { System.out.println("Tweets Get Exception: " + e.getMessage()); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println( "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } if (totalTweets < 1) { writer.close(); // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } continue OUTERMOST; } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get tweets writer.close(); if (totalTweets > 0) { fileCounter++; System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets); } else { // Remove file if tweets not found File fileToDelete = new File(fileName); fileToDelete.delete(); } // If rate limit reached then switch Auth user RemainingCallsCounter++; if (RemainingCallsCounter >= RemainingCalls) { // Load OAuth user tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO); twitter = tf.getInstance(); System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name); RemainingCalls = AppOAuths.RemainingCalls - 2; RemainingCallsCounter = 0; System.out.println("New Remianing Calls: " + RemainingCalls); } } // while get users } // read my single file catch (IOException e) { System.err.println("Failed to read lines from " + myFile); } File currentFile = new File(inputPath + "/" + myFile); currentFile.delete(); // to write output in a file System.out.flush(); } // all my files share } catch (TwitterException te) { // te.printStackTrace(); System.out.println("Failed to get tweets: " + te.getMessage()); System.exit(-1); } System.out.println("!!!! DONE !!!!"); // Close System.out for this thread which will // flush and close this thread. System.out.close(); }