Example usage for twitter4j TwitterFactory getInstance

List of usage examples for twitter4j TwitterFactory getInstance

Introduction

In this page you can find the example usage for twitter4j TwitterFactory getInstance.

Prototype

public Twitter getInstance() 

Source Link

Document

Returns a instance associated with the configuration bound to this factory.

Usage

From source file:StringMatching.GetTweet.java

/**
 * @param args the command line arguments
 *///  w ww.  j a  va 2 s  . c o  m
public static void main(String[] args) throws JSONException, IOException {

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey("Gkr9iZwYDALZ16OdxMp5rubBH")
            .setOAuthConsumerSecret("nhEwYFfiX5qp90sLLwO2eeYMxLwb3WC120lgihrocZDPWRNcUK")
            .setOAuthAccessToken("94107100-572UpcOkkz9kMWGaJS8YFsIGdlmJAd2cDw8y9rOnA")
            .setOAuthAccessTokenSecret("ST0XtXUjYgYWKHryL2feNM0VcDQQAgrov2V7nB7hq1xBC")
            .setHttpProxyHost("cache.itb.ac.id").setHttpProxyPort(8080).setHttpProxyUser("jonathan.benedict")
            .setHttpProxyPassword("rollingonthefloor");
    TwitterFactory tf = new TwitterFactory(cb.build());
    Twitter twitter = tf.getInstance();
    JSONObject obj = new JSONObject();
    int counterTweet = 0;
    FileWriter file = new FileWriter("C:\\Users\\user\\IdeaProjects\\TwitterStringMatching\\input.txt");
    file.flush();
    try {
        Query query = new Query("barca".toLowerCase());
        QueryResult result;
        do {
            result = twitter.search(query);
            List<Status> tweets = result.getTweets();
            for (Status tweet : tweets) {
                counterTweet++;
                System.out.println("@" + tweet.getUser().getScreenName() + " - " + tweet.getText());
                obj.put("user", tweet.getUser().getScreenName());
                obj.put("tweets", tweet.getText());

                //Tulis file ke dalam txt

                try {
                    file.write(obj.toString());
                    System.out.println("Successfully Copied JSON Object to File...");
                    System.out.println("\nJSON Object: " + obj);

                } catch (IOException e) {
                    e.printStackTrace();

                }
            }
        } while (counterTweet < 1000);

        file.close();
        System.exit(0);
    } catch (TwitterException te) {
        te.printStackTrace();
        System.out.println("Failed to search tweets: " + te.getMessage());
        System.exit(-1);
    }
    System.exit(0);
}

From source file:summarizer.NewApplication.java

private void HashtagSearchButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_HashtagSearchButtonActionPerformed
    // TODO add your handling code here:
    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey("uEeExq1FHARMvAGTJAY0dGxPh")
            .setOAuthConsumerSecret("jMcofTKFsayd4bpA6HFNlgkMfiveoS3ffRSCy9FCSs9pWYqdAD")
            .setOAuthAccessToken("2443437752-PrbVsDrQCxthX3T7lV3dtHs3FCXFXfBBHsdNrOS")
            .setOAuthAccessTokenSecret("MbauBu7R6vUUKqFD9ogK7VTIfce4nmvewYsgYqBOQ2ZbC");
    TwitterFactory tf = new TwitterFactory(cb.build());
    Twitter twitter = tf.getInstance();

    String hashtag;/*w w  w.  jav a  2s  . co m*/
    hashtag = HashtagTextField.getText();
    Query query = new Query(hashtag);
    query.count(100);
    if (TwitterComboBox.getSelectedIndex() == 0) {
        query.resultType(Query.ResultType.mixed);
    } else if (TwitterComboBox.getSelectedIndex() == 0) {
        query.resultType(Query.ResultType.recent);
    } else {
        query.resultType(Query.ResultType.popular);
    }
    QueryResult result = null;
    try {
        result = twitter.search(query);
    } catch (TwitterException ex) {
        Logger.getLogger(NewApplication.class.getName()).log(Level.SEVERE, null, ex);
    }
    String alltweets = "", temp = "";

    for (Status status : result.getTweets()) {
        temp = status.getText();
        temp = temp.replaceAll("[\\t\\n\\r]", " ");
        if (!"".equals(temp)) {
            if ("RT".equals(temp.substring(0, 2))) {
                temp = temp.split(":", 2)[1];
            }
            alltweets += (temp + "\n");
        }
    }
    InputTextArea.setText(alltweets);
}

From source file:testtweet.TweetUsingTwitter4jExample.java

/**
 * @param args the command line arguments
 *///from   ww  w.  j av  a2s .c  o  m
public static void main(String[] args) throws IOException, TwitterException {

    //Instantiate a re-usable and thread-safe factory
    TwitterFactory twitterFactory = new TwitterFactory(Data.getConf().build());

    //Instantiate a new Twitter instance
    Twitter twitter = twitterFactory.getInstance();

    /*
    //setup OAuth Consumer Credentials
    twitter.setOAuthConsumer(consumerKey, consumerSecret);
            
    //setup OAuth Access Token
    twitter.setOAuthAccessToken(new AccessToken(accessToken, accessTokenSecret));
    */

    //Instantiate and initialize a new twitter status update
    StatusUpdate statusUpdate = new StatusUpdate(
            //your tweet or status message
            "Twitter API #Hacked");
    //attach any media, if you want to
    /*
    statusUpdate.setMedia(
    //title of media
    "http://h1b-work-visa-usa.blogspot.com"
    , new URL("http://lh6.ggpht.com/-NiYLR6SkOmc/Uen_M8CpB7I/AAAAAAAAEQ8/tO7fufmK0Zg/h-1b%252520transfer%252520jobs%25255B4%25255D.png?imgmax=800").openStream());
    */
    //tweet or update status
    Status status = twitter.updateStatus(statusUpdate);

    //response from twitter server
    System.out.println("status.toString() = " + status.toString());
    System.out.println("status.getInReplyToScreenName() = " + status.getInReplyToScreenName());
    System.out.println("status.getSource() = " + status.getSource());
    System.out.println("status.getText() = " + status.getText());
    System.out.println("status.getContributors() = " + Arrays.toString(status.getContributors()));
    System.out.println("status.getCreatedAt() = " + status.getCreatedAt());
    System.out.println("status.getCurrentUserRetweetId() = " + status.getCurrentUserRetweetId());
    System.out.println("status.getGeoLocation() = " + status.getGeoLocation());
    System.out.println("status.getId() = " + status.getId());
    System.out.println("status.getInReplyToStatusId() = " + status.getInReplyToStatusId());
    System.out.println("status.getInReplyToUserId() = " + status.getInReplyToUserId());
    System.out.println("status.getPlace() = " + status.getPlace());
    System.out.println("status.getRetweetCount() = " + status.getRetweetCount());
    System.out.println("status.getRetweetedStatus() = " + status.getRetweetedStatus());
    System.out.println("status.getUser() = " + status.getUser());
    System.out.println("status.getAccessLevel() = " + status.getAccessLevel());
    System.out.println("status.getHashtagEntities() = " + Arrays.toString(status.getHashtagEntities()));
    System.out.println("status.getMediaEntities() = " + Arrays.toString(status.getMediaEntities()));
    if (status.getRateLimitStatus() != null) {
        System.out
                .println("status.getRateLimitStatus().getLimit() = " + status.getRateLimitStatus().getLimit());
        System.out.println(
                "status.getRateLimitStatus().getRemaining() = " + status.getRateLimitStatus().getRemaining());
        System.out.println("status.getRateLimitStatus().getResetTimeInSeconds() = "
                + status.getRateLimitStatus().getResetTimeInSeconds());
        System.out.println("status.getRateLimitStatus().getSecondsUntilReset() = "
                + status.getRateLimitStatus().getSecondsUntilReset());
        System.out.println("status.getRateLimitStatus().getRemainingHits() = "
                + status.getRateLimitStatus().getRemaining());
    }
    System.out.println("status.getURLEntities() = " + Arrays.toString(status.getURLEntities()));
    System.out.println("status.getUserMentionEntities() = " + Arrays.toString(status.getUserMentionEntities()));
}

From source file:the_vaps_project.The_vaps_project.java

public static void main(String[] args)/* throws TwitterException */ {
    // TODO code application logic here

    /*/*  ww w. jav a  2s  . c  o m*/
            
    List<Status> status = twitter.getHomeTimeline();
    for(Status st : status){
    System.out.println(st.getUser().getName()+"--------"+st.getText());
    System.out.println(st.getCreatedAt());
    }*/

    // The factory instance is re-useable and thread safe.
    /*Twitter twitter2 = tf.getSingleton();
     Query query = new Query("source:twitter4j yusukey");
     QueryResult result = twitter.search(query);
     for (Status statuss : result.getTweets()) {
     System.out.println("@" + statuss.getUser().getName() + ":" + statuss.getText());
     }*/
    TwitterCriteria criteria = null;
    Tweet t = null;
    criteria = TwitterCriteria.create()

            .setMaxTweets(10).setUntil("2016-11-08").setQuerySearch("#NeverTrump");

    /*Scribe s = new Scribe();
    s.detruireFichier("tweets.xml");
    s.ouvrir("tweets.xml");
    s.ecrire("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
    s.ecrire("<tweets>\n");
    for (int i = 0; i < TweetManager.getTweets(criteria).size(); i++) {
    t = TweetManager.getTweets(criteria).get(i);
    s.ecrire("\t<tweet>\n");
    s.ecrire("\t\t<text><![CDATA["+t.getText()+"]]></text>\n");
    s.ecrire("\t\t<hashtags>"+t.getHashtags()+"</hashtags>\n");
    s.ecrire("\t\t<date>"+t.getDate()+"</date>\n");
    s.ecrire("\t\t<username>"+t.getUsername()+"</username>\n");
    s.ecrire("\t\t<retweets>"+t.getRetweets()+"</retweets>\n");
    s.ecrire("\t\t<favorites>"+t.getFavorites()+"</favorites>\n");
    s.ecrire("\t\t<id>"+t.getId()+"</id>\n");
    s.ecrire("\t\t<geo>"+t.getGeo()+"</geo>\n");
    s.ecrire("\t\t<mentions>"+t.getMentions()+"</mentions>\n");
    s.ecrire("\t\t<permalink>"+t.getPermalink()+"</permalink>\n");
    s.ecrire("\t\t<smileys>"+t.getSmileys()+"</smileys>\n");
    s.ecrire("\t\t<polarite>"+t.getPolarite()+"</polarite>\n");
    s.ecrire("\t</tweet>\n");
            
    corpus.ecrire(s.filtrerTexte(t.getText() + "\n"));
    }
    s.ecrire("</tweets>");
            
    s.fermer();*/

    /* SUPER SITE D'EXEMPLES
    https://github.com/yusuke/twitter4j/tree/master/twitter4j-examples/src/main/java/twitter4j/examples*/

    /*PARTIE GRAPHE DES RETWEETS*/
    Set<String> nameOfUsers = new HashSet<>();
    for (int i = 0; i < TweetManager.getTweets(criteria).size(); i++) {
        t = TweetManager.getTweets(criteria).get(i);
        System.out.println(t.getUsername());
        nameOfUsers.add(t.getUsername());
    }

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey("gVaO166IGfSCun5Esj1sZ5bX3")
            .setOAuthConsumerSecret("KhN1L4pXCG5U6d6tee1IGUnIV1bqxieQJDxE2HOx3AGI5npQRE")
            .setOAuthAccessToken("822018210178469888-4Pr1KRE2rmFvof0O5IUdTlxtkGKamU4")
            .setOAuthAccessTokenSecret("D1thDl7k8v2s1SupU5ZGhK3U7Mphb69fpUW4JuDou5O65");
    TwitterFactory tf = new TwitterFactory(cb.build());
    twitter4j.Twitter twitter = tf.getInstance();

    /*String[] screenNames = nameOfUsers.toArray(new String[0]);
    ResponseList<User> usersNames;
    long cursor = -1;
    IDs ids;
    try {
    usersNames = twitter.lookupUsers(screenNames);
    for(User u: usersNames){
        System.out.println(u.getId()+" "+u.getName()+" "+u.getName());
        ids = twitter.getFollowersIDs("username", cursor);
        twitter.getFollowers
    }
                        
            
    } catch (TwitterException ex) {
    Exceptions.printStackTrace(ex);
    }*/

    Set<DuoKey> utilisateurs = new HashSet<>();
    Set<DuoKey> liens = new HashSet<>();

    //User us;

    try {
        int nbpages = 0;
        long cursor = -1;
        PagableResponseList<User> listeFollowersTemp;
        ArrayList<User> listeFollowers = new ArrayList<User>();
        for (String s : nameOfUsers) {
            if (!utilisateurs.contains(new DuoKey(s, "auteurTweet"))) {

                utilisateurs.add(new DuoKey(s, "auteurTweet"));

                //listeFollowersTemp = twitter.getFollowersList(s, -1);
                do {
                    TimeUnit.MINUTES.sleep(1);
                    listeFollowersTemp = twitter.getFollowersList(s, cursor);

                    for (User u : listeFollowersTemp) {
                        listeFollowers.add(u);
                    }
                    nbpages++;
                } while (((cursor = listeFollowersTemp.getNextCursor()) != 0) && nbpages <= 9); //4=100 followers par auteur de tweet recherch
            }
            System.out.println(s + ", mes followers sont :");
            for (User u : listeFollowers) {
                utilisateurs.add(new DuoKey(u.getName(), "auteurTweet"));
                liens.add(new DuoKey(u.getName(), s));
                System.out.println("\t" + u.getName());
            }
            listeFollowers.clear();
            nbpages = 0;
        }
    } catch (TwitterException | InterruptedException ex) {
        Exceptions.printStackTrace(ex);
    }

    Scribe sc = new Scribe();
    sc.detruireFichier("graphFollowers.gexf");
    sc.ouvrir("graphFollowers.gexf");
    sc.ecrire("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
    sc.ecrire(
            "<gexf xmlns=\"http://www.gexf.net/1.2draft/gexf.xsd\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.1draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\n");
    sc.ecrire("\t<meta lastmodifieddate=\"2017-02-17\">\n");
    sc.ecrire("\t\t<creator>Pirboss</creator>\n");
    sc.ecrire("\t\t<description>Test v2</description>\n");
    sc.ecrire("\t</meta>\n");
    sc.ecrire("\t<graph defaultedgetype=\"directed\" mode=\"static\">\n");
    sc.ecrire("\t\t<nodes>\n");
    ArrayList<DuoKey> distinctListOfUsers = new ArrayList(utilisateurs);
    System.out.println(distinctListOfUsers.size());
    for (int i = 0; i < distinctListOfUsers.size(); i++) {
        sc.ecrire("\t\t\t<node id=\"" + distinctListOfUsers.get(i).getX().replace("&", "&amp;") + "\" label=\""
                + distinctListOfUsers.get(i).getX().replace("&", "&amp;") + "\"/>\n");
    }
    sc.ecrire("\t\t</nodes>\n");
    sc.ecrire("\t\t<edges>\n");
    ArrayList<DuoKey> distinctListOfLinks = new ArrayList(liens);
    System.out.println(distinctListOfLinks.size());
    for (int i = 0; i < liens.size(); i++) {
        sc.ecrire("\t\t\t<edge id=\"" + i + "\" source=\""
                + distinctListOfLinks.get(i).getX().replace("&", "&amp;") + "\" target=\""
                + distinctListOfLinks.get(i).getY().replace("&", "&amp;") + "\"/>\n");
    }

    sc.ecrire("\t\t</edges>\n");
    sc.ecrire("\t</graph>\n");
    sc.ecrire("</gexf>");
    sc.fermer();

    PreviewJFrame p = new PreviewJFrame();
    p.script();

    /*PARTIE GRAPHE DES MENTIONS*/
    //NE PAS OUBLIE LE COUP DE REMPLACER LES .replace("&", "&amp;") POUR LE XML 
    /*Set<String> users = new HashSet<>();
    HashMap<DuoKey, Integer> map;
    map = new HashMap<DuoKey, Integer>();
    DuoKey duokey;
    for (int i = 0; i < TweetManager.getTweets(criteria).size(); i++) {
    System.out.println(i);
    t = TweetManager.getTweets(criteria).get(i);
    users.add(t.getUsername());
    if(!t.getMentions().isEmpty()){
        for(String target : t.getMentions().split(" ")){
            target = target.replaceFirst("@", "");
            users.add(target);
            duokey = new DuoKey(t.getUsername(), target);
            if(map.get(duokey) != null){
                map.put(duokey, map.get(duokey)+1);
            }else{
                map.put(duokey, 1);
            }
        }
    }
    }
            
    Scribe s = new Scribe();
    s.detruireFichier("tweets.gexf");
    s.ouvrir("tweets.gexf");
    s.ecrire("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
    s.ecrire("<gexf xmlns=\"http://www.gexf.net/1.2draft/gexf.xsd\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.1draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\n");
    s.ecrire("\t<meta lastmodifieddate=\"2017-02-17\">\n");
    s.ecrire("\t\t<creator>Pirboss</creator>\n");
    s.ecrire("\t\t<description>Test v1</description>\n");
    s.ecrire("\t</meta>\n");
    s.ecrire("\t<graph defaultedgetype=\"directed\" mode=\"static\">\n");
    s.ecrire("\t\t<nodes>\n");
    ArrayList distinctList = new ArrayList(users);
    for(int i=0; i<distinctList.size(); i++){
    s.ecrire("\t\t\t<node id=\""+i+"\" label=\""+distinctList.get(i)+"\"/>\n");
    }
    s.ecrire("\t\t</nodes>\n");
    s.ecrire("\t\t<edges>\n");
    Iterator ite = map.entrySet().iterator();
    int jj=0;
    while (ite.hasNext()) {
    Map.Entry pair = (Map.Entry)ite.next();
    s.ecrire("\t\t\t<edge id=\""+(jj++)+"\" source=\""+distinctList.indexOf(((DuoKey)pair.getKey()).getX())+"\" target=\""+distinctList.indexOf(((DuoKey)pair.getKey()).getY())+"\" weight=\""+pair.getValue()+"\"/>\n");
    ite.remove(); // avoids a ConcurrentModificationException
    }
    s.ecrire("\t\t</edges>\n");
    s.ecrire("\t</graph>\n");
    s.ecrire("</gexf>");
    s.fermer();
            
    PreviewJFrame pr = new PreviewJFrame();
    pr.script();*/
}

From source file:tientx.supercode.myproejectdemov3.connection.TwitterConnection.java

private TwitterConnection() {
    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey(TwitterConfig.API_KEY)
            .setOAuthConsumerSecret(TwitterConfig.API_SECRET).setOAuthAccessToken(TwitterConfig.ACCES_TOKEN)
            //                .setOAuthAccessTokenSecret(TwitterConfig.ACCES_TOKEN_SECRET).setHttpConnectionTimeout(100000)
            .setOAuthAccessTokenSecret(TwitterConfig.ACCES_TOKEN_SECRET).setHttpConnectionTimeout(100000);
    //                .setHttpProxyHost(TwitterConfig.PROXY_HOST)
    //                .setHttpProxyPort(TwitterConfig.PROXY_PORT);
    TwitterFactory factory = new TwitterFactory(cb.build());
    twitter = factory.getInstance();
}

From source file:timeline.CmdSearchTerm.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length != 6)) {

        System.err.println("Please provide command as following.");
        System.err.println("java -cp twitter4j-multi-oauth-0.5.jar "
                + "timeline.CmdSearchTerm consumer_key consumer_secret" + " user_token user_secret output_path "
                + "term ");
        System.exit(-1);// w ww  . jav a 2 s .c o m
    }

    AppOAuth AppOAuths = new AppOAuth();
    String endpoint = "/search/tweets";

    String consumer_key = null;
    try {
        consumer_key = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    String consumer_secret = null;
    try {
        consumer_secret = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(-1);
    }
    String user_token = null;
    try {
        user_token = StringEscapeUtils.escapeJava(args[2]);
    } catch (Exception e) {
        System.err.println("Argument" + args[2] + " must be an String.");
        System.exit(-1);
    }
    String user_secret = null;
    try {
        user_secret = StringEscapeUtils.escapeJava(args[3]);
    } catch (Exception e) {
        System.err.println("Argument" + args[3] + " must be an String.");
        System.exit(-1);
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[4]);
    } catch (Exception e) {
        System.err.println("Argument" + args[4] + " must be an String.");
        System.exit(-1);
    }

    String term = "";
    try {
        term = StringEscapeUtils.escapeJava(args[5]);
    } catch (Exception e) {
        System.err.println("Argument" + args[5] + " must be an String.");
        System.exit(-1);
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, consumer_key, consumer_secret, user_token,
                user_secret);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("Remianing Calls: " + RemainingCalls);

        // screen_name / user_id provided by arguments
        System.out.println("Trying to create output directory");
        String filesPath = OutputDirPath + "/";
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {

            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        String fileName = filesPath + term.replace(" ", "");
        PrintWriter writer = new PrintWriter(fileName, "UTF-8");

        Query query = new Query(term);
        QueryResult result;

        List<Status> statuses = new ArrayList<>();
        int totalTweets = 0;
        int numberOfTweetsToGet = 5000;
        long lastID = Long.MAX_VALUE;

        while (totalTweets < numberOfTweetsToGet) {
            if (numberOfTweetsToGet - totalTweets > 100) {
                query.setCount(100);
            } else {
                query.setCount(numberOfTweetsToGet - totalTweets);
            }
            try {
                result = twitter.search(query);
                statuses.addAll(result.getTweets());

                if (statuses.size() > 0) {
                    for (Status status : statuses) {
                        String rawJSON = TwitterObjectFactory.getRawJSON(status);
                        writer.println(rawJSON);

                        totalTweets += 1;

                        if (status.getId() < lastID) {
                            lastID = status.getId();
                        }
                    }
                } else {
                    break;
                }
                System.out.println("totalTweets: " + totalTweets);
                statuses.clear();

            } catch (TwitterException e) {
                // e.printStackTrace();

                System.out.println("Tweets Get Exception: " + e.getMessage());

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    System.out.println("No more remianing calls");
                }

                if (totalTweets < 1) {
                    writer.close();
                    // Remove file if tweets not found
                    File fileToDelete = new File(fileName);
                    fileToDelete.delete();
                    break;
                }
            }
            query.setMaxId(lastID - 1);

            // If rate limit reached then switch Auth user
            RemainingCallsCounter++;
            if (RemainingCallsCounter >= RemainingCalls) {

                System.out.println("No more remianing calls");
                break;
            }
        }

        if (totalTweets > 0) {
            System.out.println("Total dumped tweets of " + term + " are: " + totalTweets);
        } else {

            // Remove file if tweets not found
            File fileToDelete = new File(fileName);
            fileToDelete.delete();
        }
        writer.close();
    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get term results because: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:timeline.CmdUserTimeline.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length != 7)) {

        System.err.println("Please provide command as following.");
        System.err.println("java -cp twitter4j-multi-oauth-0.3.jar "
                + "timeline.CmdUserTimeline consumer_key consumer_secret"
                + " user_token user_secret output_path " + "screen_name_or_userid "
                + "number_of_tweets_to_get_max_is_3200");
        System.exit(-1);/*from  www  .j  a va  2  s  .  c o m*/
    }

    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    String consumer_key = null;
    try {
        consumer_key = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    String consumer_secret = null;
    try {
        consumer_secret = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(-1);
    }
    String user_token = null;
    try {
        user_token = StringEscapeUtils.escapeJava(args[2]);
    } catch (Exception e) {
        System.err.println("Argument" + args[2] + " must be an String.");
        System.exit(-1);
    }
    String user_secret = null;
    try {
        user_secret = StringEscapeUtils.escapeJava(args[3]);
    } catch (Exception e) {
        System.err.println("Argument" + args[3] + " must be an String.");
        System.exit(-1);
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[4]);
    } catch (Exception e) {
        System.err.println("Argument" + args[4] + " must be an String.");
        System.exit(-1);
    }

    String targetedUser = "";
    try {
        targetedUser = StringEscapeUtils.escapeJava(args[5]);
    } catch (Exception e) {
        System.err.println("Argument" + args[5] + " must be an String.");
        System.exit(-1);
    }

    int NUMBER_OF_TWEETS = 0;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[6]);
    } catch (Exception e) {
        System.err.println("Argument" + args[6] + " must be an integer.");
        System.exit(-1);
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, consumer_key, consumer_secret, user_token,
                user_secret);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("Remianing Calls: " + RemainingCalls);

        // screen_name / user_id provided by arguments
        System.out.println("Trying to create output directory");
        String filesPath = OutputDirPath + "/";
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {

            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        String fileName = filesPath + targetedUser;
        PrintWriter writer = new PrintWriter(fileName, "UTF-8");

        // Call different functions for screen_name and id_str
        Boolean chckedNumaric = helpers.isNumeric(targetedUser);

        List<Status> statuses = new ArrayList<>();
        int size = statuses.size();
        int pageno = 1;
        int totalTweets = 0;
        boolean tweetCounterReached = false;
        System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
        while (true) {

            try {

                Paging page = new Paging(pageno++, 200);

                if (chckedNumaric) {

                    long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                    statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                    if (statuses.size() > 0) {
                        for (Status status : statuses) {
                            String rawJSON = TwitterObjectFactory.getRawJSON(status);
                            writer.println(rawJSON);

                            totalTweets += 1;
                            if (totalTweets >= NUMBER_OF_TWEETS) {
                                tweetCounterReached = true;
                                break;
                            }
                        }
                        if (tweetCounterReached) {
                            break;
                        }
                    }
                } else {

                    statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                    if (statuses.size() > 0) {
                        for (Status status : statuses) {
                            String rawJSON = TwitterObjectFactory.getRawJSON(status);
                            writer.println(rawJSON);

                            totalTweets += 1;
                            if (totalTweets >= NUMBER_OF_TWEETS) {
                                tweetCounterReached = true;
                                break;
                            }
                        }
                        if (tweetCounterReached) {
                            break;
                        }
                    }
                }

                // If user's total tweet are less than 195 then no next call
                if (size == 0) {
                    if (totalTweets < 195) {
                        break;
                    }
                }

                // If user's all tweets parsed then exit
                if (totalTweets == size) {
                    break;
                }

                size = totalTweets;

                statuses.clear();

            } catch (TwitterException e) {

                // e.printStackTrace();
                // do not throw if user has protected tweets, or if they
                // deleted their account
                if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                        || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                    System.out.println(targetedUser + " is protected or account is deleted");
                } else {
                    System.out.println("Tweets Get Exception: " + e.getMessage());
                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    System.out.println("No more remianing calls");
                }

                if (totalTweets < 1) {
                    writer.close();
                    // Remove file if tweets not found
                    File fileToDelete = new File(fileName);
                    fileToDelete.delete();
                }
                break;
            }

            // If rate limit reached then switch Auth user
            RemainingCallsCounter++;
            if (RemainingCallsCounter >= RemainingCalls) {

                System.out.println("No more remianing calls");
                break;
            }

        } // while get tweets
        writer.close();

        if (totalTweets > 0) {
            System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);
        } else {

            // Remove file if tweets not found
            File fileToDelete = new File(fileName);
            fileToDelete.delete();
        }

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:timeline.DBNonThreader.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length == 0)) {
        System.err.println("First: 'OUTPUT_PATH' is mendatory.");
        System.err.println("Second: (int) Number of Tweets to get. Max 3200");
        System.err.println("Third: 'screen_name / id_str'" + " is optional.");
        System.err.println("If 3rd argument not provided then provide" + " Twitter users through database.");

        System.exit(-1);//from w  w w .jav a  2 s. co m
    }

    MysqlDB DB = new MysqlDB();
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    int NUMBER_OF_TWEETS = 3200;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an integer.");
        System.exit(-1);
    }

    String targetedUser = "";
    if (args.length == 3) {
        try {
            targetedUser = StringEscapeUtils.escapeJava(args[2]);
        } catch (Exception e) {
            System.err.println("Argument" + args[2] + " must be an String.");
            System.exit(-1);
        }
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        System.out.println("User's Tweets");

        // If targetedUser not provided by argument, then look into
        // database.
        if (StringUtils.isEmpty(targetedUser)) {

            String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE "
                    + "`tweets_dumped_all` = 0";

            ResultSet results = DB.selectQ(selectQuery);

            int numRows = DB.numRows(results);
            if (numRows < 1) {
                System.err.println("No User in database to get Tweets");
                System.exit(-1);
            }

            System.out.println("Trying to create output directory");
            String filesPath = OutputDirPath + "/";
            File theDir = new File(filesPath);

            // If the directory does not exist, create it
            if (!theDir.exists()) {

                try {
                    theDir.mkdirs();

                } catch (SecurityException se) {

                    System.err.println("Could not create output " + "directory: " + OutputDirPath);
                    System.err.println(se.getMessage());
                    System.exit(-1);
                }
            }

            OUTERMOST: while (results.next()) {

                int targetedUserID = results.getInt("id");
                targetedUser = results.getString("targeteduser");
                System.out.println("Targeted User: " + targetedUser);

                String fileName = filesPath + "/" + targetedUser;
                PrintWriter writer = new PrintWriter(fileName, "UTF-8");

                // Call different functions for screen_name and id_str
                Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                List<Status> statuses = new ArrayList<>();
                int size = statuses.size();
                int pageno = 1;
                int totalTweets = 0;
                boolean tweetCounterReached = false;
                System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
                while (true) {

                    try {

                        Paging page = new Paging(pageno++, 200);

                        if (chckedNumaric) {

                            long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                            statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                            if (statuses.size() > 0) {
                                for (Status status : statuses) {
                                    String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                    writer.println(rawJSON);

                                    totalTweets += 1;
                                    if (totalTweets >= NUMBER_OF_TWEETS) {
                                        tweetCounterReached = true;
                                        break;
                                    }
                                }
                                if (tweetCounterReached) {
                                    break;
                                }
                            }
                        } else {

                            statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                            if (statuses.size() > 0) {
                                for (Status status : statuses) {
                                    String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                    writer.println(rawJSON);

                                    totalTweets += 1;
                                    if (totalTweets >= NUMBER_OF_TWEETS) {
                                        tweetCounterReached = true;
                                        break;
                                    }
                                }
                                if (tweetCounterReached) {
                                    break;
                                }
                            }
                        }

                        // If user's total tweet are less than 195 
                        // then no next call
                        if (size == 0) {
                            if (totalTweets < 195) {
                                break;
                            }
                        }

                        // If user's all tweets parsed 
                        // then proceed to next user 
                        if (totalTweets == size) {
                            break;
                        }

                        size = totalTweets;

                        statuses.clear();

                    } catch (TwitterException e) {

                        // e.printStackTrace();
                        // do not throw if user has protected tweets, 
                        // or if they deleted their account
                        if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                            System.out.println(targetedUser + " is protected or account is deleted");
                        } else {
                            System.out.println("Tweets Get Exception: " + e.getMessage());
                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // Load OAuth user
                            tf = AppOAuths.loadOAuthUser(endpoint);
                            twitter = tf.getInstance();

                            System.out.println(
                                    "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls - 2;
                            RemainingCallsCounter = 0;

                            System.out.println("New Remianing Calls: " + RemainingCalls);
                        }

                        if (totalTweets < 1) {
                            writer.close();
                            // Remove file if tweets not found
                            File fileToDelete = new File(fileName);
                            fileToDelete.delete();
                        }

                        // Update stats
                        String fieldValues = "`tweets_dumped_all` = 2";
                        String where = "id = " + targetedUserID;
                        DB.Update("`twitter_users`", fieldValues, where);
                        continue OUTERMOST;

                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }
                } // while get tweets
                writer.close();

                if (totalTweets > 0) {

                    System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);

                    // Update stats
                    String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets;
                    String where = "id = " + targetedUserID;
                    DB.Update("`twitter_users`", fieldValues, where);
                } else {

                    // Update stats
                    String fieldValues = "`tweets_dumped_all` = 2";
                    String where = "id = " + targetedUserID;
                    DB.Update("`twitter_users`", fieldValues, where);

                    // Remove file if tweets not found
                    File fileToDelete = new File(fileName);
                    fileToDelete.delete();
                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // Load OAuth user
                    tf = AppOAuths.loadOAuthUser(endpoint);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls - 2;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

            } // while get users from database

        } else {
            // screen_name / user_id provided by arguments
            System.out.println("screen_name / user_id provided by arguments");

            System.out.println("Trying to create output directory");
            String filesPath = OutputDirPath + "/";
            File theDir = new File(filesPath);

            // If the directory does not exist, create it
            if (!theDir.exists()) {

                try {
                    theDir.mkdirs();

                } catch (SecurityException se) {

                    System.err.println("Could not create output " + "directory: " + OutputDirPath);
                    System.err.println(se.getMessage());
                    System.exit(-1);
                }
            }

            System.out.println("Targeted User: " + targetedUser);

            String fileName = filesPath + "/" + targetedUser;
            PrintWriter writer = new PrintWriter(fileName, "UTF-8");

            // Call different functions for screen_name and id_str
            Boolean chckedNumaric = helpers.isNumeric(targetedUser);

            List<Status> statuses = new ArrayList<>();
            int size = statuses.size();
            int pageno = 1;
            int totalTweets = 0;
            boolean tweetCounterReached = false;
            System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
            while (true) {

                try {

                    Paging page = new Paging(pageno++, 200);

                    if (chckedNumaric) {

                        long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                        statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    } else {

                        statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    }

                    // If user's total tweet are less than 195 then no next call
                    if (size == 0) {
                        if (totalTweets < 195) {
                            break;
                        }
                    }

                    // If user's all tweets parsed then proceed to next user 
                    if (totalTweets == size) {
                        break;
                    }

                    size = totalTweets;

                    statuses.clear();

                } catch (TwitterException e) {

                    // e.printStackTrace();
                    // do not throw if user has protected tweets, or if they deleted their account
                    if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                            || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                        System.out.println(targetedUser + " is protected or account is deleted");
                    } else {
                        System.out.println("Tweets Get Exception: " + e.getMessage());
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                    if (totalTweets < 1) {
                        writer.close();
                        // Remove file if tweets not found
                        File fileToDelete = new File(fileName);
                        fileToDelete.delete();
                    }
                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // Load OAuth user
                    tf = AppOAuths.loadOAuthUser(endpoint);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls - 2;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

            } // while get tweets
            writer.close();

            if (totalTweets > 0) {
                System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);
            } else {

                // Remove file if tweets not found
                File fileToDelete = new File(fileName);
                fileToDelete.delete();
            }

        } // screen_name / user_id provided by arguments

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:timeline.DBThreaderParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 5)) {
        System.err.println("5 Parameters are required to launch a Job.");
        System.err.println("First: String 'OUTPUT_PATH'");
        System.err.println("Second: (int) Total Number Of Jobs");
        System.err.println("Third: (int) This Job Number");
        System.err.println("Fourth: (int) Seconds to pause between next launch");
        System.err.println("Fifth: (int) Number of Tweets to get. Max 3200");
        System.err.println("Example: fileName.class /output/path 10 1 2 3200");
        System.exit(-1);/*from   www  .j  av  a2 s .c o  m*/
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[1]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[1] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[2] + " must be an integer.");
        System.exit(1);
    }

    int NUMBER_OF_TWEETS = 0;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[4]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[4] + " must be an integer.");
        System.exit(1);
    }

    MysqlDB DB = new MysqlDB();
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    try {

        String totalRowsQuery = "SELECT count( * ) as wLoad FROM `twitter_users`"
                + " WHERE `tweets_dumped_all` = 0";
        ResultSet totalRowsResults = DB.selectQ(totalRowsQuery);

        int TotalWorkLoad = 0;
        while (totalRowsResults.next()) {
            TotalWorkLoad = totalRowsResults.getInt("wLoad");
        }

        if (TotalWorkLoad < 1) {
            System.err.println("No User in database to get Tweets");
            System.exit(-1);
        }

        // Free memmory
        totalRowsResults = null;

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load Please reduce Number of jobs to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;

        String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE "
                + "`tweets_dumped_all` = 0 LIMIT " + offSet + "," + chunkSize;

        ResultSet results = DB.selectQ(selectQuery);

        int numRows = DB.numRows(results);
        if (numRows < 1) {
            System.err.println(
                    "No User in database to get Tweets" + " with offset " + offSet + " and limit " + chunkSize);
            System.exit(-1);
        }

        /**
         * wait before launching actual job
         */
        int secondsToPause = 0;
        try {
            secondsToPause = Integer.parseInt(args[3]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[3] + " must be an integer.");
            System.exit(-1);
        }

        secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
        System.out.println("secondsToPause: " + secondsToPause);
        helpers.pause(secondsToPause);
        /**
         * wait before launching actual job
         */

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        System.out.println("User's Tweets");

        System.out.println("Trying to create output directory");
        String filesPath = OutputDirPath + "/";
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {
            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        System.out.flush();

        OUTERMOST: while (results.next()) {

            int targetedUserID = results.getInt("id");
            String targetedUser = results.getString("targeteduser");
            System.out.println("Targeted User: " + targetedUser);

            // Create User file to push tweets in it
            String fileName = filesPath + "/" + targetedUser;
            PrintWriter writer = new PrintWriter(fileName, "UTF-8");

            // Call different functions for screen_name and id_str
            Boolean chckedNumaric = helpers.isNumeric(targetedUser);

            List<Status> statuses = new ArrayList<>();
            int size = statuses.size();
            int pageno = 1;
            int totalTweets = 0;
            boolean tweetCounterReached = false;
            System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
            while (true) {
                try {

                    Paging page = new Paging(pageno++, 200);

                    if (chckedNumaric) {

                        long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                        statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    } else {

                        statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    }

                    // If user's total tweet are less 
                    // than 195 then no next call
                    if (size == 0) {
                        if (totalTweets < 195) {
                            break;
                        }
                    }

                    // If user's all tweets parsed 
                    // then proceed to next user 
                    if (totalTweets == size) {
                        break;
                    }

                    size = totalTweets;

                    statuses.clear();

                } catch (TwitterException e) {

                    // e.printStackTrace();
                    // do not throw if user has protected tweets, 
                    // or if they deleted their account
                    if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                            || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                        System.out.println(targetedUser + " is protected or account is deleted");
                    } else {
                        System.out.println("Tweets Get Exception: " + e.getMessage());
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                    if (totalTweets < 1) {
                        writer.close();
                        // Remove file if tweets not found
                        File fileToDelete = new File(fileName);
                        fileToDelete.delete();
                    }

                    // Update stats
                    String fieldValues = "`tweets_dumped_all` = 2";
                    String where = "id = " + targetedUserID;
                    DB.Update("`twitter_users`", fieldValues, where);
                    continue OUTERMOST;

                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // Load OAuth user
                    tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls - 2;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

            } // while get tweets
            writer.close();

            if (totalTweets > 0) {

                System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);
                // Update stats
                String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets;
                String where = "id = " + targetedUserID;
                DB.Update("`twitter_users`", fieldValues, where);
            } else {

                // Update stats
                String fieldValues = "`tweets_dumped_all` = 2, " + " `tweets_all_count` = " + totalTweets;
                String where = "id = " + targetedUserID;
                DB.Update("`twitter_users`", fieldValues, where);

                // Remove file if tweets not found
                File fileToDelete = new File(fileName);
                fileToDelete.delete();
            }

            // If rate limit reached then switch Auth user
            RemainingCallsCounter++;
            if (RemainingCallsCounter >= RemainingCalls) {

                // Load OAuth user
                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                twitter = tf.getInstance();

                System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                RemainingCalls = AppOAuths.RemainingCalls - 2;
                RemainingCallsCounter = 0;

                System.out.println("New Remianing Calls: " + RemainingCalls);
            }

            System.out.flush();
        } // while get users from database
    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get tweets: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}

From source file:timeline.FilesThreaderParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 7)) {
        System.err.println("7 Parameters are required to launch a Job.");
        System.err.println("First: String 'INPUT: " + "/path/to/perline/screen_names/files/'");
        System.err.println("Second: String 'OUTPUT_PATH'");
        System.err.println("Third: (int) Total Number Of Jobs");
        System.err.println("Fourth: (int) This Job Number");
        System.err.println("Fifth: (int) Seconds to pause between " + "next launch");
        System.err.println("Sixth: (int) Number of Tweets to get. Max 3200");
        System.err.println("Name of current threader.");
        System.err.println("Example: fileName.class /output/path " + "10 2 3200 pool-1-thread-1");
        System.exit(-1);/*w w  w  . j a  v  a  2 s  . co m*/
    }

    String inputPath = null;
    try {
        inputPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument " + args[0] + " must be an String.");
        System.exit(-1);
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[2] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[3] + " must be an integer.");
        System.exit(1);
    }

    int NUMBER_OF_TWEETS = 0;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[5]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[5] + " must be an integer.");
        System.exit(1);
    }

    String ThreadName = null;
    try {
        ThreadName = StringEscapeUtils.escapeJava(args[6]);
    } catch (Exception e) {
        System.err.println("Argument" + args[6] + " must be an String.");
        System.exit(1);
    }

    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    try {

        int TotalWorkLoad = 0;
        ArrayList<String> allFiles = null;
        try {
            final File folder = new File(inputPath);
            allFiles = helpers.listFilesForSingleFolder(folder);
            TotalWorkLoad = allFiles.size();
        } catch (Exception e) {

            System.err.println("Input folder is not exists: " + e.getMessage());
            System.exit(-1);
        }

        System.out.println("Total Workload is: " + TotalWorkLoad);

        if (TotalWorkLoad < 1) {
            System.err.println("No input file exists in: " + inputPath);
            System.exit(-1);
        }

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load. Please reduce 'Number of jobs' to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;
        int chunkSizeToGet = (JOB_NO + 1) * chunkSize;

        System.out.println("My Share is " + chunkSize);
        System.out.println("My offSet is " + offSet);
        System.out.println("My chunkSizeToGet is " + chunkSizeToGet);
        System.out.println();

        /**
         * wait before launching actual job
         */
        int secondsToPause = 0;
        try {
            secondsToPause = Integer.parseInt(args[4]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[4] + " must be an integer.");
            System.exit(-1);
        }

        secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
        System.out.println("secondsToPause: " + secondsToPause);
        helpers.pause(secondsToPause);
        /**
         * wait before launching actual job
         */

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        System.out.println("User's Tweets");

        System.out.println("Trying to create output directory");

        int dirCounter = 1;
        int fileCounter = 0;

        String filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter;
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {
            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath + "/" + ThreadName
                        + "-tweets-" + dirCounter);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        if (JOB_NO + 1 == TOTAL_JOBS) {
            chunkSizeToGet = TotalWorkLoad;
        }

        // to write output in a file
        System.out.flush();

        List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet);

        for (String myFile : myFilesShare) {
            System.out.println("Going to parse file: " + myFile);
            System.out.println();

            try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) {
                String line;

                OUTERMOST: while ((line = br.readLine()) != null) {

                    if (fileCounter >= 30000) {

                        dirCounter++;
                        fileCounter = 0;
                        filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter;
                        theDir = new File(filesPath);

                        // If the directory does not exist, create it
                        if (!theDir.exists()) {
                            try {
                                theDir.mkdirs();

                            } catch (SecurityException se) {

                                System.err.println("Could not create output " + "directory: " + OutputDirPath
                                        + "/" + ThreadName + "-tweets-" + dirCounter);
                                System.err.println(se.getMessage());
                                System.exit(-1);
                            }
                        }
                    }

                    System.out.println("Going to get tweets of " + "Screen-name / user_id: " + line);

                    String targetedUser = line.trim();
                    System.out.println("Targeted User: " + targetedUser);

                    // Create User file to push tweets in it
                    String fileName = filesPath + "/" + targetedUser;
                    PrintWriter writer = new PrintWriter(fileName, "UTF-8");

                    // Call different functions for screen_name and id_str
                    Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                    List<Status> statuses = new ArrayList<>();
                    int size = statuses.size();
                    int pageno = 1;
                    int totalTweets = 0;
                    boolean tweetCounterReached = false;
                    System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
                    System.out.println();

                    while (true) {
                        try {

                            Paging page = new Paging(pageno++, 200);

                            if (chckedNumaric) {

                                long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                                statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                                if (statuses.size() > 0) {
                                    for (Status status : statuses) {
                                        String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                        writer.println(rawJSON);

                                        totalTweets += 1;
                                        if (totalTweets >= NUMBER_OF_TWEETS) {
                                            tweetCounterReached = true;
                                            break;
                                        }
                                    }
                                    if (tweetCounterReached) {
                                        break;
                                    }
                                }
                            } else {

                                statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                                if (statuses.size() > 0) {
                                    for (Status status : statuses) {
                                        String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                        writer.println(rawJSON);

                                        totalTweets += 1;
                                        if (totalTweets >= NUMBER_OF_TWEETS) {
                                            tweetCounterReached = true;
                                            break;
                                        }
                                    }
                                    if (tweetCounterReached) {
                                        break;
                                    }
                                }
                            }

                            // If user's total tweet are less 
                            // than 195 then no next call
                            if (size == 0) {
                                if (totalTweets < 195) {
                                    break;
                                }
                            }

                            // If user's all tweets parsed 
                            // then proceed to next user 
                            if (totalTweets == size) {
                                break;
                            }

                            size = totalTweets;

                            statuses.clear();

                        } catch (TwitterException e) {

                            // e.printStackTrace();
                            // do not throw if user has protected tweets, 
                            // or if they deleted their account
                            if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                    || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                                System.out.println(targetedUser + " is protected or account is deleted");
                            } else {
                                System.out.println("Tweets Get Exception: " + e.getMessage());
                            }

                            // If rate limit reached then switch Auth user
                            RemainingCallsCounter++;
                            if (RemainingCallsCounter >= RemainingCalls) {

                                // Load OAuth user
                                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                                twitter = tf.getInstance();

                                System.out.println(
                                        "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                                RemainingCalls = AppOAuths.RemainingCalls - 2;
                                RemainingCallsCounter = 0;

                                System.out.println("New Remianing Calls: " + RemainingCalls);
                            }

                            if (totalTweets < 1) {
                                writer.close();
                                // Remove file if tweets not found
                                File fileToDelete = new File(fileName);
                                fileToDelete.delete();
                            }

                            continue OUTERMOST;

                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // Load OAuth user
                            tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                            twitter = tf.getInstance();

                            System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls - 2;
                            RemainingCallsCounter = 0;

                            System.out.println("New Remianing Calls: " + RemainingCalls);
                        }

                    } // while get tweets
                    writer.close();

                    if (totalTweets > 0) {
                        fileCounter++;
                        System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);

                    } else {

                        // Remove file if tweets not found
                        File fileToDelete = new File(fileName);
                        fileToDelete.delete();
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                } // while get users

            } // read my single file
            catch (IOException e) {
                System.err.println("Failed to read lines from " + myFile);
            }

            File currentFile = new File(inputPath + "/" + myFile);
            currentFile.delete();

            // to write output in a file
            System.out.flush();
        } // all my files share
    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get tweets: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}