de.jetwick.tw.NewClass.java Source code

Java tutorial

Introduction

Here is the source code for de.jetwick.tw.NewClass.java

Source

/*
 *  Copyright 2011 Peter Karich, jetwick_@_pannous_._info.
 * 
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 * 
 *       http://www.apache.org/licenses/LICENSE-2.0
 * 
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *  under the License.
 */
package de.jetwick.tw;

import java.util.Collections;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import twitter4j.FilterQuery;
import twitter4j.Query;
import twitter4j.QueryResult;
import twitter4j.Status;
import twitter4j.StatusDeletionNotice;
import twitter4j.StatusListener;
import twitter4j.Tweet;
import twitter4j.Twitter;
import twitter4j.TwitterFactory;
import twitter4j.TwitterStream;
import twitter4j.TwitterStreamFactory;
import twitter4j.http.AccessToken;

/**
 *
 * @author Peter Karich, jetwick_@_pannous_._info
 */
public class NewClass {

    public static void main(String[] args) throws Exception {
        if (args.length != 3) {
            System.out.println("usage\njava -Dtwitter4j.oauth.consumerKey=key -Dtwitter4j.oauth.consumerSecret=val "
                    + "de.jetwick.tw.NewClass \"term1 term2\" token tokenSecure");
            return;
        }
        final String queryTerms = args[0];
        final String token = args[1];
        final String tokenSecret = args[2];
        System.out.println("query terms are " + queryTerms);
        final NewClass test = new NewClass(queryTerms);
        Runnable runOnExit = new Runnable() {

            @Override
            public void run() {
                test.calc();
            }
        };
        Runtime.getRuntime().addShutdownHook(new Thread(runOnExit));
        Thread t = test.search();
        Thread t2 = test.streaming(token, tokenSecret);

        t.start();
        t2.start();

        t.join();
        t2.join();
    }

    private String queryTerms;

    public NewClass(String query) {
        this.queryTerms = query;
    }

    private Map<Long, String> searchMap = new TreeMap<Long, String>();
    private Map<Long, String> asyncMap = new TreeMap<Long, String>();

    private void error(String str) {
        System.out.println(new Date() + " ERROR:" + str);
    }

    private void log(String str) {
        System.out.println(new Date() + "  INFO:" + str);
    }

    /** A thread using the search API */
    public Thread search() {
        return new Thread() {

            @Override
            public void run() {
                int MINUTES = 2;
                Twitter twitter = new TwitterFactory().getInstance();
                try {
                    while (!isInterrupted()) {
                        Query query = new Query(queryTerms);
                        // RECENT or POPULAR or MIXED
                        // doesn't make a difference if MIXED or RECENT
                        query.setResultType(Query.MIXED);
                        query.setPage(1);
                        query.setRpp(100);
                        QueryResult res = twitter.search(query);
                        for (Tweet tw : res.getTweets()) {
                            searchMap.put(tw.getId(), tw.getText());
                        }
                        Thread.sleep(MINUTES * 60 * 1000L);
                    }
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
        };

    }

    /**
     * A thread using the streaming API.
     * Maximum tweets/sec == 50 !! we'll lose even infrequent tweets for a lot keywords!
     */
    public Thread streaming(final String token, final String tokenSecret) {
        return new Thread() {

            StatusListener listener = new StatusListener() {

                int counter = 0;

                public void onStatus(Status status) {
                    if (++counter % 20 == 0)
                        log("async counter=" + counter);
                    asyncMap.put(status.getId(), status.getText());
                }

                public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
                    log("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
                }

                public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
                    log("Got track limitation notice:" + numberOfLimitedStatuses);
                }

                public void onScrubGeo(int userId, long upToStatusId) {
                    log("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
                }

                public void onException(Exception ex) {
                    ex.printStackTrace();
                }
            };

            public void run() {
                TwitterStream twitterStream = new TwitterStreamFactory()
                        .getInstance(new AccessToken(token, tokenSecret));
                twitterStream.addListener(listener);
                twitterStream.filter(new FilterQuery(0, null, new String[] { queryTerms }, null));
            }
        };
    }

    /**
     * Calculates which tweets are missing from search and which are missing from streaming API
     */
    public void calc() {
        if (asyncMap.isEmpty()) {
            log("async is empty");
            return;
        }
        long minId = Collections.min(asyncMap.keySet());
        Map<Long, String> all = new TreeMap<Long, String>(asyncMap);
        Map<Long, String> searchMapWithoutHistoric = new TreeMap<Long, String>();
        for (Entry<Long, String> searchEntry : searchMap.entrySet()) {
            // streaming does not catch historic tweets, so do not include them
            if (searchEntry.getKey() < minId)
                continue;

            searchMapWithoutHistoric.put(searchEntry.getKey(), searchEntry.getValue());
            String asyncStr = all.put(searchEntry.getKey(), searchEntry.getValue());
            if (asyncStr != null && !asyncStr.equals(searchEntry.getValue()))
                error(searchEntry.getKey() + " strings different:" + asyncStr + " != " + searchEntry.getValue());
        }

        Map<Long, String> onlyInSearch = new TreeMap<Long, String>(searchMapWithoutHistoric);
        for (Long id : asyncMap.keySet()) {
            onlyInSearch.remove(id);
        }
        Map<Long, String> onlyInAsync = new TreeMap<Long, String>(asyncMap);
        for (Long id : searchMapWithoutHistoric.keySet()) {
            onlyInAsync.remove(id);
        }

        log("async :" + asyncMap.size());
        log("search:" + searchMapWithoutHistoric.size());
        log("all   :" + all.size());

        String str = "";
        for (Entry<Long, String> e : onlyInSearch.entrySet()) {
            str += toString(e);
        }
        log("### only in search ###\n" + str);

        str = "";
        for (Entry<Long, String> e : onlyInAsync.entrySet()) {
            str += toString(e);
        }
        log("### only in async ###\n" + str);
    }

    private String toString(Entry<Long, String> entry) {
        return entry.getKey() + " " + entry.getValue().replaceAll("\n", " ") + "\n";
    }
}