List of usage examples for twitter4j Twitter search
QueryResult search(Query query) throws TwitterException;
From source file:MapDemo.java
public static void main(String[] args) throws TwitterException, IOException, Exception { TwitterFactory tf = new TwitterFactory(); Twitter twitter = tf.getInstance(); boolean flag = true; // Ask the user for a search string. String searchStr = "ParkandGoUNL"; // Create a Query object. Query query = new Query(searchStr); // Send API request to execute a search with the given query. QueryResult result = twitter.search(query); // Display search results. result.getTweets().stream().forEach((Status status) -> { System.out.println("\n" + status.getText() + status.getCreatedAt()); });//from w ww . ja v a2 s . c o m if (flag) { GeoApiContext context = new GeoApiContext().setApiKey("AIzaSyArz1NljiDpuOriFWalOerYEdHOyi8ow8Y"); List<Marker> markers = new ArrayList<>(); Marker e = new Marker(GeocodingApi.geocode(context, "616 North 16th Street, Lincoln, Nebraska, USA") .await()[0].geometry.location, "marker1", "KKG", new Date()); markers.add(e); StreetExtractor se = new StreetExtractor(); List<Address> address = se.find("Park and Go 601 North 16th Street"); address.forEach(((Address a) -> { try { Marker m = new Marker( GeocodingApi.geocode(context, a.getAddress()).await()[0].geometry.location, "marker0", "A Tweet", new Date()); markers.add(m); } catch (Exception ex) { Logger.getLogger(MapDemo.class.getName()).log(Level.SEVERE, null, ex); } })); Map map = new Map(markers); map.create(); } }
From source file:TweetMapManager.java
public static void main(String[] args) { TwitterFactory tf = new TwitterFactory(); Twitter twitter = tf.getInstance(); String searchStr = "#ParkandGoUNL"; Query query = new Query(searchStr); GeoApiContext context = new GeoApiContext().setApiKey("AIzaSyArz1NljiDpuOriFWalOerYEdHOyi8ow8Y"); List<Marker> markers = new LinkedList<>(); StreetExtractor se = new StreetExtractor(); LocalTime now = LocalTime.now(); while (true) { DayOfWeek today = LocalDate.now().getDayOfWeek(); if (today.getValue() < WEEKEND) { if (now.isAfter(STARTTIME) && now.isBefore(ENDTIME)) { // Send API request to execute a search with the given query. QueryResult results = null; try { results = twitter.search(query); } catch (TwitterException ex) { LOGGER.warn(ex.getMessage()); }/*w w w . j av a2s .c om*/ // Display search results. if (results != null) { results.getTweets().stream().forEach((Status status) -> { try { Date created = status.getCreatedAt(); String text = status.getText(); LOGGER.info(status.getText()); LatLng location; if (status.getGeoLocation() != null) { location = new LatLng(status.getGeoLocation().getLatitude(), status.getGeoLocation().getLongitude()); } else { String modified = text.replace("#ParkAndGoUNL", ""); List<Address> address = se.find(modified); location = GeocodingApi.geocode(context, address.get(0).getAddress()) .await()[0].geometry.location; } String id = UUID.randomUUID().toString().substring(0, 8); Marker m = new Marker(location, "m" + id, text, created); markers.add(m); } catch (Exception ex) { LOGGER.warn(ex.getMessage()); } }); } if (!markers.isEmpty()) { Marker m = markers.get(markers.size() - 1); if (m.getTimestamp().getTime() < Time.valueOf(now.minusMinutes(30)).getTime()) { DailyLogs.addMarkerToLog(m, today); markers.remove(m); } } Map map = new Map(markers); map.create(); } else { //wait 5 hours try { LOGGER.info("Sleeping 5 hours"); Thread.sleep((long) 1.8e+7); } catch (InterruptedException ex) { LOGGER.warn(ex.getMessage()); } } now = LocalTime.now(); } else { try { //Wait a day LOGGER.info("Sleeping 1 day"); Thread.sleep((long) 8.64e+7); } catch (InterruptedException ex) { LOGGER.warn(ex.getMessage()); } } try { LOGGER.info("Successful Loop, Resting"); Thread.sleep(1000); } catch (InterruptedException ex) { LOGGER.warn(ex.getMessage()); } } }
From source file:AwsConsoleApp.java
License:Open Source License
public static void searchTweets(String searchString) throws Exception { Twitter twitter = new TwitterFactory().getInstance(); QueryResult result;// w w w . java2 s. c o m Query query = new Query(searchString); query.setRpp(100); for (int i = 1; i < 16; i++) { System.out.println("CURRENT PAGE: " + i); query.setPage(i); result = twitter.search(query); for (Tweet tweet : result.getTweets()) { System.out.println(tweet.getFromUser() + ":" + tweet.getText()); } } }
From source file:Pencarian.java
private void btSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btSearchActionPerformed // TODO add your handling code here: txtSearchTimeline.setText(""); TwitterFactory tf = new TwitterFactory(); Twitter twitter = tf.getInstance(); Query query = new Query(txSearch.getText()); QueryResult result = null;/* w w w.jav a2 s . c om*/ try { result = twitter.search(query); result.getTweets().stream().forEach((status) -> { txtSearchTimeline.append("@" + status.getUser().getScreenName() + ":" + status.getText() + " - " + status.getCreatedAt() + " \n Via : " + status.getSource() + "\n\n"); }); } catch (TwitterException ex) { Logger.getLogger(Pencarian.class.getName()).log(Level.SEVERE, null, ex); JOptionPane.showMessageDialog(rootPane, "Failed !! " + ex.getMessage()); } }
From source file:Demo.java
/** * Main method.//from ww w .ja v a 2s . c o m * * @param args * @throws TwitterException */ public static void main(String[] args) throws TwitterException, IOException { // The TwitterFactory object provides an instance of a Twitter object // via the getInstance() method. The Twitter object is the API consumer. // It has the methods for interacting with the Twitter API. TwitterFactory tf = new TwitterFactory(); Twitter twitter = tf.getInstance(); boolean keepItGoinFullSteam = true; do { // Main menu Scanner input = new Scanner(System.in); System.out.print("\n--------------------" + "\nH. Home Timeline\nS. Search\nT. Tweet" + "\n--------------------" + "\nA. Get Access Token\nQ. Quit" + "\n--------------------\n> "); String choice = input.nextLine(); try { // Home Timeline if (choice.equalsIgnoreCase("H")) { // Display the user's screen name. User user = twitter.verifyCredentials(); System.out.println("\n@" + user.getScreenName() + "'s timeline:"); // Display recent tweets from the Home Timeline. for (Status status : twitter.getHomeTimeline()) { System.out.println("\n@" + status.getUser().getScreenName() + ": " + status.getText()); } } // Search else if (choice.equalsIgnoreCase("S")) { // Ask the user for a search string. System.out.print("\nSearch: "); String searchStr = input.nextLine(); // Create a Query object. Query query = new Query(searchStr); // Send API request to execute a search with the given query. QueryResult result = twitter.search(query); // Display search results. result.getTweets().stream().forEach((Status status) -> { System.out.println("\n@" + status.getUser().getName() + ": " + status.getText()); }); } // Tweet else if (choice.equalsIgnoreCase("T")) { boolean isOkayLength = true; String tweet; do { // Ask the user for a tweet. System.out.print("\nTweet: "); tweet = input.nextLine(); // Ensure the tweet length is okay. if (tweet.length() > 140) { System.out.println("Too long! Keep it under 140."); isOkayLength = false; } } while (isOkayLength == false); // Send API request to create a new tweet. Status status = twitter.updateStatus(tweet); System.out.println("Just tweeted: \"" + status.getText() + "\""); } // Get Access Token else if (choice.equalsIgnoreCase("A")) { // First, we ask Twitter for a request token. RequestToken reqToken = twitter.getOAuthRequestToken(); System.out.println("\nRequest token: " + reqToken.getToken() + "\nRequest token secret: " + reqToken.getTokenSecret()); AccessToken accessToken = null; while (accessToken == null) { // The authorization URL sends the request token to Twitter in order // to request an access token. At this point, Twitter asks the user // to authorize the request. If the user authorizes, then Twitter // provides a PIN. System.out .print("\nOpen this URL in a browser: " + "\n " + reqToken.getAuthorizationURL() + "\n" + "\nAuthorize the app, then enter the PIN here: "); String pin = input.nextLine(); try { // We use the provided PIN to get the access token. The access // token allows this app to access the user's account without // knowing his/her password. accessToken = twitter.getOAuthAccessToken(reqToken, pin); } catch (TwitterException te) { System.out.println(te.getMessage()); } } System.out.println("\nAccess token: " + accessToken.getToken() + "\nAccess token secret: " + accessToken.getTokenSecret() + "\nSuccess!"); } // Quit else if (choice.equalsIgnoreCase("Q")) { keepItGoinFullSteam = false; GeoApiContext context = new GeoApiContext() .setApiKey("AIzaSyArz1NljiDpuOriFWalOerYEdHOyi8ow8Y"); System.out.println(GeocodingApi.geocode(context, "Sigma Phi Epsilon, Lincoln, Nebraska, USA") .await()[0].geometry.location.toString()); System.out.println(GeocodingApi.geocode(context, "16th and R, Lincoln, Nebraska, USA") .await()[0].geometry.location.toString()); File htmlFile = new File("map.html"); Desktop.getDesktop().browse(htmlFile.toURI()); } // Bad choice else { System.out.println("Invalid option."); } } catch (IllegalStateException ex) { System.out.println(ex.getMessage()); } catch (Exception ex) { System.out.println(ex.getMessage()); } } while (keepItGoinFullSteam == true); }
From source file:TwitterRetrieval.java
License:Apache License
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { request.setCharacterEncoding("UTF-8"); response.setCharacterEncoding("UTF-8"); PrintWriter out;/*from ww w .java 2 s . c o m*/ try { // img stuff not req'd for source code html showing // all links relative // XXX // making these absolute till we work out the // addition of a PathInfo issue ConfigurationBuilder cb = new ConfigurationBuilder(); System.setProperty("twitter4j.http.httpClient", "twitter4j.internal.http.HttpClientImpl"); cb.setOAuthConsumerKey("56NAE9lQHSOZIGXRktd5Qw") .setOAuthConsumerSecret("zJjJrUUs1ubwKjtPOyYzrwBJzpwq7ud8Aryq1VhYH2E") .setOAuthAccessTokenURL("https://api.twitter.com/oauth/access_token") .setOAuthRequestTokenURL("https://api.twitter.com/oauth/request_token") .setOAuthAuthorizationURL("https://api.twitter.com/oauth/authorize") .setOAuthAccessToken("234742739-I1l0VGTTjRUbZrfH1jvKnTVFU9ZEvkxxUDpvsAJ2") .setOAuthAccessTokenSecret("jLe3imI3JiPgmHCatt6SqYgRAcX5q8s6z38oUrqMc"); TwitterFactory tf = new TwitterFactory(cb.build()); Twitter twitter = tf.getInstance(); Query query = new Query(request.getParameter("q")); String tags = request.getParameter("tags"); String zone = request.getParameter("zone"); String users = request.getParameter("users"); Map<String, LatLon> latslongs = new HashMap(); String[] tagsArray = null; if (tags != null) { tagsArray = tags.split(","); } String[] userArray = null; if (users != null) { userArray = users.split(","); int count = userArray.length; usuarios = new String[count]; latitud = new String[count]; longitud = new String[count]; for (int i = 0; i < userArray.length; i++) { temp = userArray[i]; if (temp != null) { int hit1 = temp.indexOf("["); int hit2 = temp.indexOf(";"); int hit3 = temp.indexOf("]"); latslongs.put(temp.substring(0, hit1), new LatLon(temp.substring(hit1 + 1, hit2), temp.substring(hit2 + 1, hit3))); /* * usuarios[i] = temp.substring(0, hit1); latitud[i] = * temp.substring(hit1 + 1, hit2); longitud[i] = * temp.substring(hit2 + 1, hit3); */ } } } QueryResult result; result = twitter.search(query); List<Post> postList = new ArrayList(); List<PostType> postsList = new ArrayList(); Post solrPost; PostType post; //List<LinkType> links = new ArrayList(); List<ActionType> actions; ArrayList<User> toUsers = new ArrayList(); ArrayList<LinkType> links; int d; InputStream stream = getServletContext().getResourceAsStream("/WEB-INF/servlet.properties"); Properties props = null; if (props == null) { props = new Properties(); props.load(stream); } ZoneDao zoneDao = new ZoneDao(props.getProperty("db_host"), Integer.valueOf(props.getProperty("db_port")), props.getProperty("db_name")); PlaceDao placeDao = new PlaceDao(props.getProperty("db_host"), Integer.valueOf(props.getProperty("db_port")), props.getProperty("db_name")); Place place = null; org.zonales.tagsAndZones.objects.Zone zoneObj = zoneDao .retrieveByExtendedString(Utils.normalizeZone(zone)); for (Tweet tweet : (List<Tweet>) result.getTweets()) { d = MAX_TITLE_LENGTH; actions = new ArrayList(); try { actions.add(new ActionType("retweets", twitter.getRetweets(tweet.getId()).size())); actions.add(new ActionType("replies", twitter.getRelatedResults(tweet.getId()).getTweetsWithReply().size())); } catch (TwitterException ex) { Logger.getLogger(TwitterRetrieval.class.getName()).log(Level.SEVERE, "Error intentando obtener retweets o replies: {0}", new Object[] { ex }); } solrPost = new Post(); solrPost.setZone(new Zone(String.valueOf(zoneObj.getId()), zoneObj.getName(), zoneObj.getType().getName(), zoneObj.getExtendedString())); solrPost.setSource("Twitter"); solrPost.setId(String.valueOf(tweet.getId())); if (request.getParameter(tweet.getFromUser() + "Place") != null) { place = placeDao.retrieveByExtendedString(request.getParameter(tweet.getFromUser() + "Place")); } else { place = null; } User usersolr = new User(String.valueOf(tweet.getFromUserId()), tweet.getFromUser(), "http://twitter.com/#!/" + tweet.getFromUser(), tweet.getSource(), place != null ? new org.zonales.entities.Place(String.valueOf(place.getId()), place.getName(), place.getType().getName()) : null); if (users != null) { /* * for (int i = 0; i < usuarios.length; i++) { if * (tweet.getFromUser().equals(usuarios[i])) { * usersolr.setLatitude(Double.parseDouble(latitud[i])); * usersolr.setLongitude(Double.parseDouble(longitud[i])); } } */ //usersolr.setLatitude(Double.parseDouble(latslongs.get(tweet.getFromUser()).latitud)); //usersolr.setLongitude(Double.parseDouble(latslongs.get(tweet.getFromUser()).longitud)); } solrPost.setFromUser(usersolr); if (tweet.getToUser() != null) { toUsers.add(new User(String.valueOf(tweet.getToUserId()), tweet.getToUser(), null, tweet.getSource(), null)); solrPost.setToUsers(toUsers); } if (tweet.getText().length() > d) { while (d > 0 && tweet.getText().charAt(d - 1) != ' ') { d--; } } else { d = tweet.getText().length() - 1; } solrPost.setTitle(tweet.getText().substring(0, d) + (tweet.getText().length() > MAX_TITLE_LENGTH ? "..." : "")); solrPost.setText(tweet.getText()); //post.setLinks(new LinksType(links)); solrPost.setActions((ArrayList<ActionType>) actions); solrPost.setCreated(tweet.getCreatedAt().getTime()); solrPost.setModified(tweet.getCreatedAt().getTime()); solrPost.setRelevance( actions.size() == 2 ? actions.get(0).getCant() * 3 + actions.get(1).getCant() : 0); solrPost.setPostLatitude( tweet.getGeoLocation() != null ? tweet.getGeoLocation().getLatitude() : null); solrPost.setPostLongitude( tweet.getGeoLocation() != null ? tweet.getGeoLocation().getLongitude() : null); links = new ArrayList<LinkType>(); links.add(new LinkType("avatar", tweet.getProfileImageUrl())); if (tweet.getText() != null && getLinks(tweet.getText()) != null) { links.addAll(getLinks(tweet.getText())); } if (solrPost.getLinks() == null) { solrPost.setLinks(new ArrayList<LinkType>()); } solrPost.setLinks(links); if (tagsArray != null && tagsArray.length > 0) { solrPost.setTags(new ArrayList<String>(Arrays.asList(tagsArray))); } solrPost.setExtendedString(WordUtils.capitalize((solrPost.getFromUser().getPlace() != null ? solrPost.getFromUser().getPlace().getName() + ", " : "") + solrPost.getZone().getExtendedString().replace("_", " "))); postList.add(solrPost); post = new PostType(); post.setZone(new Zone(String.valueOf(zoneObj.getId()), zoneObj.getName(), zoneObj.getType().getName(), zoneObj.getExtendedString())); post.setSource("Twitter"); post.setId(String.valueOf(tweet.getId())); User user = new User(String.valueOf(tweet.getFromUserId()), tweet.getFromUser(), "http://twitter.com/#!/" + tweet.getFromUser(), tweet.getSource(), place != null ? new org.zonales.entities.Place(String.valueOf(place.getId()), place.getName(), place.getType().getName()) : null); if (users != null) { /* * for (int i = 0; i < usuarios.length; i++) { if * (tweet.getFromUser().equals(usuarios[i])) { * user.setLatitude(Double.parseDouble(latitud[i])); * user.setLongitude(Double.parseDouble(longitud[i])); } } */ //user.setLatitude(Double.parseDouble(latslongs.get(tweet.getFromUser()).latitud)); //user.setLongitude(Double.parseDouble(latslongs.get(tweet.getFromUser()).longitud)); } post.setFromUser(user); if (tweet.getToUser() != null) { toUsers.add(new User(String.valueOf(tweet.getToUserId()), tweet.getToUser(), null, tweet.getSource(), null)); post.setToUsers(new ToUsersType(toUsers)); } post.setTitle(tweet.getText().substring(0, d) + (tweet.getText().length() > MAX_TITLE_LENGTH ? "..." : "")); post.setText(tweet.getText()); //post.setLinks(new LinksType(links)); post.setActions(new ActionsType(actions)); post.setCreated(String.valueOf(tweet.getCreatedAt().getTime())); post.setModified(String.valueOf(tweet.getCreatedAt().getTime())); post.setRelevance( actions.size() == 2 ? actions.get(0).getCant() * 3 + actions.get(1).getCant() : 0); post.setPostLatitude(tweet.getGeoLocation() != null ? tweet.getGeoLocation().getLatitude() : null); post.setPostLongitude( tweet.getGeoLocation() != null ? tweet.getGeoLocation().getLongitude() : null); links = new ArrayList<LinkType>(); links.add(new LinkType("avatar", tweet.getProfileImageUrl())); post.setLinks(new LinksType(getLinks(tweet.getText()))); if (tagsArray != null && tagsArray.length > 0) { post.setTags(new TagsType(Arrays.asList(tagsArray))); } postsList.add(post); } PostsType posts = new PostsType(postsList); Gson gson = new Gson(); if ("xml".equalsIgnoreCase(request.getParameter("format"))) { response.setContentType("application/xml"); out = response.getWriter(); try { for (PostType postIt : posts.getPost()) { postIt.setVerbatim(gson.toJson(postIt)); } Twitter2XML(posts, out); } catch (Exception ex) { Logger.getLogger(TwitterRetrieval.class.getName()).log(Level.SEVERE, null, ex); } } else { response.setContentType("text/javascript"); out = response.getWriter(); out.println("{post: " + gson.toJson(postList) + "}"); } } catch (TwitterException ex) { Logger.getLogger(TwitterRetrieval.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:SearchTweets.java
License:Apache License
/** * Usage: java twitter4j.examples.search.SearchTweets [query] * * @param args// www . ja v a 2 s.c om */ public static void main(String[] args) { if (args.length < 1) { System.out.println("java twitter4j.examples.search.SearchTweets [query]"); System.exit(-1); } Twitter twitter = new TwitterFactory().getInstance(); try { Query query = new Query(args[0]); QueryResult result; do { result = twitter.search(query); List<Status> tweets = result.getTweets(); for (Status tweet : tweets) { System.out.println("@" + tweet.getUser().getScreenName() + " - " + tweet.getText()); } } while ((query = result.nextQuery()) != null); System.exit(0); } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); System.exit(-1); } }
From source file:android.stickynotes.StickyNotesActivity.java
License:Apache License
private void getTweets(String twit) { wifi.setWifiEnabled(true);//ww w .jav a 2 s . c o m ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("TvywVhWx7r7QQev2UGfA4g") .setOAuthConsumerSecret("Nv22zsyf1VS0vvi6hwAMyvJk9LUtSXwRUB4xwp2gRs"); TwitterFactory tf = new TwitterFactory(cb.build()); Twitter twitter = tf.getInstance(); try { QueryResult result = twitter.search(new Query(twit)); List<Tweet> tweets = result.getTweets(); textStatus.setText(""); textStatus.append("Recent tweets about '" + twit + "':\n"); for (Tweet tweet : tweets) { textStatus.append("@" + tweet.getFromUser() + " - " + tweet.getText() + "\n\n"); } } catch (TwitterException te) { te.printStackTrace(); textStatus.append("Failed to search tweets: " + te.getMessage() + " " + twit); } }
From source file:au.com.infiniterecursion.hashqanda.MainActivity.java
public boolean loadTweets() { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("Q2DfeCNOxprbDp66fWlw") .setOAuthConsumerSecret("TgyJ26CKXz1SxGiEJx4HG9rytFsqiMZlEPeCO7S9g"); Twitter twitter = new TwitterFactory(cb.build()).getInstance(); // clear tweet list. MainActivity.this.runOnUiThread(new Runnable() { public void run() { // add to it , via the runOnUIThread app.getTweetList().clear();// ww w . j av a 2 s . c om } }); try { Log.d(TAG, " starting Tweets loading "); Query q = new Query("#qanda"); q.setRpp(numberTweets); QueryResult result = twitter.search(q); List<Tweet> tweets = result.getTweets(); for (Tweet tweet : tweets) { // Log.d(TAG, "@" + tweet.getFromUser() + " - " + // tweet.getText()); // Log.d(TAG, " img url " + tweet.getProfileImageUrl()); final CachedBitmapAndTweet cachedbmtwt = new CachedBitmapAndTweet(); cachedbmtwt.twt = tweet; try { URL url = new URL(tweet.getProfileImageUrl()); InputStream is = (InputStream) url.getContent(); if (is != null) { Bitmap bitmap = BitmapFactory.decodeStream(is); cachedbmtwt.bm = bitmap; } else { cachedbmtwt.bm = null; } } catch (MalformedURLException e) { e.printStackTrace(); cachedbmtwt.bm = null; } catch (IOException e) { e.printStackTrace(); cachedbmtwt.bm = null; } catch (NullPointerException npe) { npe.printStackTrace(); cachedbmtwt.bm = null; } MainActivity.this.runOnUiThread(new Runnable() { public void run() { // add to it , via the runOnUIThread app.getTweetList().add(cachedbmtwt); } }); } Log.d(TAG, " finished Tweets loading "); return true; } catch (TwitterException te) { te.printStackTrace(); Log.d(TAG, "Failed to search tweets: " + te.getMessage()); return false; } }
From source file:au.net.moon.tSearchArchiver.SearchArchiver.java
License:Open Source License
SearchArchiver() {
Twitter twitter;
int waitBetweenRequests = 2000;
// 2 sec delay between requests to avoid maxing out the API.
Status theTweet;// w w w .j ava 2 s.co m
Query query;
QueryResult result;
// String[] searches;
ArrayList<String> searchQuery = new ArrayList<String>();
ArrayList<Integer> searchId = new ArrayList<Integer>();
int searchIndex;
int totalTweets;
SimpleDateFormat myFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z");
System.out.println("tSearchArchiver: Loading search queries...");
// Set timezone to UTC for the Twitter created at dates
myFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
twitterAuthorise twitterAuth = new twitterAuthorise(false);
twitter = twitterAuth.getTwitter();
// Open the old twitter_archive database
openSQLDataBase();
if (isDatabaseReady()) {
// probably should have these in an object not separate arrays?
try {
rs = stmt.executeQuery("select * from searches where active = true");
// perform each search
while (rs.next()) {
// if (searchQuery
searchQuery.add(rs.getString("query"));
searchId.add(rs.getInt("id"));
}
if (rs.wasNull()) {
System.out.println("tSearchArchiver: No searches in the table \"searches\"");
System.exit(30);
} else {
System.out.println("tSearchArchiver: Found " + searchQuery.size() + " searches.");
}
} catch (SQLException e) {
System.out.println("tSearchArchiver: e:" + e.toString());
}
searchIndex = 0;
totalTweets = 0;
// set initial value of i to start from middle of search set
while (searchIndex < searchQuery.size()) {
query = new Query();
query.setQuery(searchQuery.get(searchIndex));
// check to see if their are any tweets already in the database for
// this search
//TODO: Change this to look in new raw data files for each search instead
long max_tw_id = 0;
try {
rs = stmt.executeQuery("select max(tweet_id) as max_id from archive where search_id = "
+ searchId.get(searchIndex));
if (rs.next()) {
max_tw_id = rs.getLong("max_id");
// System.out.println("MaxID: " + max_tw_id);
query.setSinceId(max_tw_id);
}
} catch (SQLException e1) {
System.err.println("tSearchArchiver: Error looking for maximum tweet_id for " + query.getQuery()
+ " in archive");
e1.printStackTrace();
}
// System.out.println("Starting searching for tweets for: " +
// query.getQuery());
// new style replacement for pagination
// Query query = new Query("whatEverYouWantToSearch");
// do {
// result = twitter.search(query);
// System.out.println(result);
// do something
// } while ((query = result.nextQuery()) != null);
// TODO: check if twitter4j is doing all the backing off handling already
int tweetCount = 0;
Boolean searching = true;
do {
// delay waitBetweenRequests milliseconds before making request
// to make sure not overloading API
try {
Thread.sleep(waitBetweenRequests);
} catch (InterruptedException e1) {
System.err.println("tSearchArchiver: Sleep between requests failed.");
e1.printStackTrace();
}
try {
result = twitter.search(query);
} catch (TwitterException e) {
System.out.println(e.getStatusCode());
System.out.println(e.toString());
if (e.getStatusCode() == 503) {
// TODO use the Retry-After header value to delay & then
// retry the request
System.out
.println("tSearchArchiver: Delaying for 10 minutes before making new request");
try {
Thread.sleep(600000);
} catch (InterruptedException e1) {
System.err.println(
"tSearchArchiver: Sleep for 10 minutes because of API load failed.");
e1.printStackTrace();
}
}
result = null;
}
if (result != null) {
List<Status> results = result.getTweets();
if (results.size() == 0) {
searching = false;
} else {
tweetCount += results.size();
for (int j = 0; j < results.size(); j++) {
theTweet = (Status) results.get(j);
String cleanText = theTweet.getText();
cleanText = cleanText.replaceAll("'", "'");
cleanText = cleanText.replaceAll("\"", """);
try {
stmt.executeUpdate("insert into archive values (0, " + searchId.get(searchIndex)
+ ", '" + theTweet.getId() + "', now())");
} catch (SQLException e) {
System.err.println("tSearchArchiver: Insert into archive failed.");
System.err.println(searchId.get(searchIndex) + ", " + theTweet.getId());
e.printStackTrace();
}
// TODO: change to storing in file instead of database
try {
rs = stmt.executeQuery("select id from tweets where id = " + theTweet.getId());
} catch (SQLException e) {
System.err.println(
"tSearchArchiver: checking for tweet in tweets archive failed.");
e.printStackTrace();
}
Boolean tweetNotInArchive = false;
try {
tweetNotInArchive = !rs.next();
} catch (SQLException e) {
System.err.println(
"tSearchArchiver: checking for tweet in archive failed at rs.next().");
e.printStackTrace();
}
if (tweetNotInArchive) {
String tempLangCode = "";
// getIsoLanguageCode() has been removed from twitter4j
// looks like it might be added back in in the next version
// if (tweet.getIsoLanguageCode() != null) {
// if (tweet.getIsoLanguageCode().length() > 2) {
// System.out
// .println("tSearchArchiver Error: IsoLanguageCode too long: >"
// + tweet.getIsoLanguageCode()
// + "<");
// tempLangCode = tweet
// .getIsoLanguageCode()
// .substring(0, 2);
// } else {
// tempLangCode = tweet
// .getIsoLanguageCode();
// }
// }
double myLatitude = 0;
double myLongitude = 0;
int hasGeoCode = 0;
if (theTweet.getGeoLocation() != null) {
System.out.println("GeoLocation: " + theTweet.getGeoLocation().toString());
myLatitude = theTweet.getGeoLocation().getLatitude();
myLongitude = theTweet.getGeoLocation().getLongitude();
hasGeoCode = 1;
}
Date tempCreatedAt = theTweet.getCreatedAt();
String myDate2 = myFormatter
.format(tempCreatedAt, new StringBuffer(), new FieldPosition(0))
.toString();
totalTweets++;
try {
stmt.executeUpdate("insert into tweets values (" + theTweet.getId() + ", '"
+ tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText
+ "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '"
+ theTweet.getInReplyToScreenName() + "', '"
+ theTweet.getUser().getId() + "', '"
+ theTweet.getUser().getScreenName() + "', '" + hasGeoCode + "',"
+ myLatitude + ", " + myLongitude + ", now())");
} catch (SQLException e) {
System.err.println("tSearchArchiver: Insert into tweets failed.");
System.err.println(theTweet.getId() + ", '" + tempLangCode + "', '"
+ theTweet.getSource() + "', '" + cleanText + "', '" + myDate2
+ "', '" + theTweet.getInReplyToUserId() + "', '"
+ theTweet.getInReplyToScreenName() + "', '"
+ theTweet.getUser().getId() + "', '"
+ theTweet.getUser().getScreenName());
e.printStackTrace();
}
}
}
}
}
} while ((query = result.nextQuery()) != null && searching);
if (tweetCount > 0) {
System.out.println("tSearchArchiver: New Tweets Found for \"" + searchQuery.get(searchIndex)
+ "\" = " + tweetCount);
} else {
// System.out.println("tSearchArchiver: No Tweets Found for \""
// + searchQuery.get(searchIndex) + "\" = " + tweetCount);
}
try {
stmt.executeUpdate("update searches SET lastFoundCount=" + tweetCount
+ ", lastSearchDate=now() where id=" + searchId.get(searchIndex));
} catch (SQLException e) {
System.err.println("tSearchArchiver: failed to update searches with lastFoundCount="
+ tweetCount + " and datetime for search: " + searchId.get(searchIndex));
e.printStackTrace();
}
searchIndex++;
}
System.out.println("tSearchArchiver: Completed all " + searchQuery.size() + " searches");
System.out.println("tSearchArchiver: Archived " + totalTweets + " new tweets");
}
}