List of usage examples for twitter4j Query Query
public Query()
From source file:au.net.moon.tSearchArchiver.SearchArchiver.java
License:Open Source License
SearchArchiver() { Twitter twitter;//from w w w . j a v a 2 s .c o m int waitBetweenRequests = 2000; // 2 sec delay between requests to avoid maxing out the API. Status theTweet; Query query; QueryResult result; // String[] searches; ArrayList<String> searchQuery = new ArrayList<String>(); ArrayList<Integer> searchId = new ArrayList<Integer>(); int searchIndex; int totalTweets; SimpleDateFormat myFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z"); System.out.println("tSearchArchiver: Loading search queries..."); // Set timezone to UTC for the Twitter created at dates myFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); twitterAuthorise twitterAuth = new twitterAuthorise(false); twitter = twitterAuth.getTwitter(); // Open the old twitter_archive database openSQLDataBase(); if (isDatabaseReady()) { // probably should have these in an object not separate arrays? try { rs = stmt.executeQuery("select * from searches where active = true"); // perform each search while (rs.next()) { // if (searchQuery searchQuery.add(rs.getString("query")); searchId.add(rs.getInt("id")); } if (rs.wasNull()) { System.out.println("tSearchArchiver: No searches in the table \"searches\""); System.exit(30); } else { System.out.println("tSearchArchiver: Found " + searchQuery.size() + " searches."); } } catch (SQLException e) { System.out.println("tSearchArchiver: e:" + e.toString()); } searchIndex = 0; totalTweets = 0; // set initial value of i to start from middle of search set while (searchIndex < searchQuery.size()) { query = new Query(); query.setQuery(searchQuery.get(searchIndex)); // check to see if their are any tweets already in the database for // this search //TODO: Change this to look in new raw data files for each search instead long max_tw_id = 0; try { rs = stmt.executeQuery("select max(tweet_id) as max_id from archive where search_id = " + searchId.get(searchIndex)); if (rs.next()) { max_tw_id = rs.getLong("max_id"); // System.out.println("MaxID: " + max_tw_id); query.setSinceId(max_tw_id); } } catch (SQLException e1) { System.err.println("tSearchArchiver: Error looking for maximum tweet_id for " + query.getQuery() + " in archive"); e1.printStackTrace(); } // System.out.println("Starting searching for tweets for: " + // query.getQuery()); // new style replacement for pagination // Query query = new Query("whatEverYouWantToSearch"); // do { // result = twitter.search(query); // System.out.println(result); // do something // } while ((query = result.nextQuery()) != null); // TODO: check if twitter4j is doing all the backing off handling already int tweetCount = 0; Boolean searching = true; do { // delay waitBetweenRequests milliseconds before making request // to make sure not overloading API try { Thread.sleep(waitBetweenRequests); } catch (InterruptedException e1) { System.err.println("tSearchArchiver: Sleep between requests failed."); e1.printStackTrace(); } try { result = twitter.search(query); } catch (TwitterException e) { System.out.println(e.getStatusCode()); System.out.println(e.toString()); if (e.getStatusCode() == 503) { // TODO use the Retry-After header value to delay & then // retry the request System.out .println("tSearchArchiver: Delaying for 10 minutes before making new request"); try { Thread.sleep(600000); } catch (InterruptedException e1) { System.err.println( "tSearchArchiver: Sleep for 10 minutes because of API load failed."); e1.printStackTrace(); } } result = null; } if (result != null) { List<Status> results = result.getTweets(); if (results.size() == 0) { searching = false; } else { tweetCount += results.size(); for (int j = 0; j < results.size(); j++) { theTweet = (Status) results.get(j); String cleanText = theTweet.getText(); cleanText = cleanText.replaceAll("'", "'"); cleanText = cleanText.replaceAll("\"", """); try { stmt.executeUpdate("insert into archive values (0, " + searchId.get(searchIndex) + ", '" + theTweet.getId() + "', now())"); } catch (SQLException e) { System.err.println("tSearchArchiver: Insert into archive failed."); System.err.println(searchId.get(searchIndex) + ", " + theTweet.getId()); e.printStackTrace(); } // TODO: change to storing in file instead of database try { rs = stmt.executeQuery("select id from tweets where id = " + theTweet.getId()); } catch (SQLException e) { System.err.println( "tSearchArchiver: checking for tweet in tweets archive failed."); e.printStackTrace(); } Boolean tweetNotInArchive = false; try { tweetNotInArchive = !rs.next(); } catch (SQLException e) { System.err.println( "tSearchArchiver: checking for tweet in archive failed at rs.next()."); e.printStackTrace(); } if (tweetNotInArchive) { String tempLangCode = ""; // getIsoLanguageCode() has been removed from twitter4j // looks like it might be added back in in the next version // if (tweet.getIsoLanguageCode() != null) { // if (tweet.getIsoLanguageCode().length() > 2) { // System.out // .println("tSearchArchiver Error: IsoLanguageCode too long: >" // + tweet.getIsoLanguageCode() // + "<"); // tempLangCode = tweet // .getIsoLanguageCode() // .substring(0, 2); // } else { // tempLangCode = tweet // .getIsoLanguageCode(); // } // } double myLatitude = 0; double myLongitude = 0; int hasGeoCode = 0; if (theTweet.getGeoLocation() != null) { System.out.println("GeoLocation: " + theTweet.getGeoLocation().toString()); myLatitude = theTweet.getGeoLocation().getLatitude(); myLongitude = theTweet.getGeoLocation().getLongitude(); hasGeoCode = 1; } Date tempCreatedAt = theTweet.getCreatedAt(); String myDate2 = myFormatter .format(tempCreatedAt, new StringBuffer(), new FieldPosition(0)) .toString(); totalTweets++; try { stmt.executeUpdate("insert into tweets values (" + theTweet.getId() + ", '" + tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText + "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '" + theTweet.getInReplyToScreenName() + "', '" + theTweet.getUser().getId() + "', '" + theTweet.getUser().getScreenName() + "', '" + hasGeoCode + "'," + myLatitude + ", " + myLongitude + ", now())"); } catch (SQLException e) { System.err.println("tSearchArchiver: Insert into tweets failed."); System.err.println(theTweet.getId() + ", '" + tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText + "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '" + theTweet.getInReplyToScreenName() + "', '" + theTweet.getUser().getId() + "', '" + theTweet.getUser().getScreenName()); e.printStackTrace(); } } } } } } while ((query = result.nextQuery()) != null && searching); if (tweetCount > 0) { System.out.println("tSearchArchiver: New Tweets Found for \"" + searchQuery.get(searchIndex) + "\" = " + tweetCount); } else { // System.out.println("tSearchArchiver: No Tweets Found for \"" // + searchQuery.get(searchIndex) + "\" = " + tweetCount); } try { stmt.executeUpdate("update searches SET lastFoundCount=" + tweetCount + ", lastSearchDate=now() where id=" + searchId.get(searchIndex)); } catch (SQLException e) { System.err.println("tSearchArchiver: failed to update searches with lastFoundCount=" + tweetCount + " and datetime for search: " + searchId.get(searchIndex)); e.printStackTrace(); } searchIndex++; } System.out.println("tSearchArchiver: Completed all " + searchQuery.size() + " searches"); System.out.println("tSearchArchiver: Archived " + totalTweets + " new tweets"); } }
From source file:com.aremaitch.codestock2010.library.TwitterLib.java
License:Apache License
public QueryResult search(long sinceId, String[] hashTags) throws TwitterException { Query q = new Query(); if (sinceId >= 0) { q.setSinceId(sinceId);/*from w w w. j ava2 s .c o m*/ } q.setQuery(TextUtils.join(" OR ", hashTags)); try { return t.search(q); } catch (NumberFormatException nfe) { ACLogger.error(CSConstants.LOG_TAG, "number format exception was thrown by twitter4j"); } return null; }
From source file:com.daiv.android.twitter.ui.drawer_activities.discover.NearbyTweets.java
License:Apache License
public void getTweets() { canRefresh = false;// w w w .j av a 2 s. co m new Thread(new Runnable() { @Override public void run() { try { Twitter twitter = Utils.getTwitter(context, DrawerActivity.settings); boolean manualLoc = sharedPrefs.getBoolean("manually_config_location", false); int i = 0; while (!connected && i < 5 && !manualLoc) { try { Thread.sleep(1500); } catch (Exception e) { } i++; } double latitude = -1; double longitude = -1; if (manualLoc) { // need to query yahoos api for the location... double[] loc = getLocationFromYahoo(sharedPrefs.getInt("woeid", 2379574)); latitude = loc[0]; longitude = loc[1]; } else { // set it from the location client Location location = mLastLocation; latitude = location.getLatitude(); longitude = location.getLongitude(); } query = new Query(); query.setGeoCode(new GeoLocation(latitude, longitude), 10, Query.MILES); QueryResult result = twitter.search(query); if (result.hasNext()) { hasMore = true; query = result.nextQuery(); } else { hasMore = false; } for (Status s : result.getTweets()) { statuses.add(s); } ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { adapter = new TimelineArrayAdapter(context, statuses); listView.setAdapter(adapter); listView.setVisibility(View.VISIBLE); LinearLayout spinner = (LinearLayout) layout.findViewById(R.id.list_progress); spinner.setVisibility(View.GONE); } }); } catch (Throwable e) { e.printStackTrace(); ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { try { Toast.makeText(context, getString(R.string.error), Toast.LENGTH_SHORT).show(); } catch (IllegalStateException e) { // not attached to activity } } }); } canRefresh = true; } }).start(); }
From source file:com.klinker.android.twitter.activities.drawer_activities.discover.trends.SearchedTrendsActivity.java
License:Apache License
public void doSearch(final String mQuery) { spinner.setVisibility(View.VISIBLE); new Thread(new Runnable() { @Override/*from w w w. ja v a2s. c om*/ public void run() { try { Twitter twitter = Utils.getTwitter(context, settings); Log.v("talon_search", "search query: " + mQuery); query = new Query(); if (mQuery.contains(" TOP")) { query.setResultType(Query.ResultType.popular); } query.setQuery(mQuery.replace(" TOP", "")); QueryResult result; try { result = twitter.search(query); } catch (OutOfMemoryError e) { return; } tweets.clear(); for (twitter4j.Status status : result.getTweets()) { tweets.add(status); } if (result.hasNext()) { query = result.nextQuery(); hasMore = true; } else { hasMore = false; } ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { adapter = new TimelineArrayAdapter(context, tweets); listView.setAdapter(adapter); listView.setVisibility(View.VISIBLE); spinner.setVisibility(View.GONE); } }); } catch (Exception e) { e.printStackTrace(); ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { spinner.setVisibility(View.GONE); } }); } } }).start(); }
From source file:com.klinker.android.twitter.ui.drawer_activities.discover.NearbyTweets.java
License:Apache License
public void getTweets() { canRefresh = false;//from w w w . ja v a 2 s . c o m new Thread(new Runnable() { @Override public void run() { try { Twitter twitter = Utils.getTwitter(context, DrawerActivity.settings); boolean manualLoc = sharedPrefs.getBoolean("manually_config_location", false); int i = 0; while (!connected && i < 5 && !manualLoc) { try { Thread.sleep(1500); } catch (Exception e) { } i++; } double latitude = -1; double longitude = -1; if (manualLoc) { // need to query yahoos api for the location... double[] loc = getLocationFromYahoo(sharedPrefs.getInt("woeid", 2379574)); latitude = loc[0]; longitude = loc[1]; } else { // set it from the location client Location location = mLocationClient.getLastLocation(); latitude = location.getLatitude(); longitude = location.getLongitude(); } query = new Query(); query.setGeoCode(new GeoLocation(latitude, longitude), 10, Query.MILES); QueryResult result = twitter.search(query); if (result.hasNext()) { hasMore = true; query = result.nextQuery(); } else { hasMore = false; } for (Status s : result.getTweets()) { statuses.add(s); } ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { adapter = new TimelineArrayAdapter(context, statuses); listView.setAdapter(adapter); listView.setVisibility(View.VISIBLE); LinearLayout spinner = (LinearLayout) layout.findViewById(R.id.list_progress); spinner.setVisibility(View.GONE); } }); } catch (Throwable e) { e.printStackTrace(); ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { try { Toast.makeText(context, getString(R.string.error), Toast.LENGTH_SHORT).show(); } catch (IllegalStateException e) { // not attached to activity } } }); } canRefresh = true; } }).start(); }
From source file:com.klinker.android.twitter.ui.drawer_activities.discover.trends.SearchedTrendsActivity.java
License:Apache License
public void doSearch(final String mQuery) { spinner.setVisibility(View.VISIBLE); new Thread(new Runnable() { @Override/*from w w w . j ava 2 s .c o m*/ public void run() { try { Twitter twitter = Utils.getTwitter(context, settings); query = new Query(); if (mQuery.contains(" TOP")) { query.setResultType(Query.ResultType.popular); } query.setQuery(mQuery.replace(" TOP", "")); QueryResult result; try { result = twitter.search(query); } catch (OutOfMemoryError e) { return; } tweets.clear(); for (twitter4j.Status status : result.getTweets()) { tweets.add(status); } if (result.hasNext()) { query = result.nextQuery(); hasMore = true; } else { hasMore = false; } ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { adapter = new TimelineArrayAdapter(context, tweets); listView.setAdapter(adapter); listView.setVisibility(View.VISIBLE); spinner.setVisibility(View.GONE); } }); } catch (Exception e) { e.printStackTrace(); ((Activity) context).runOnUiThread(new Runnable() { @Override public void run() { spinner.setVisibility(View.GONE); } }); } } }).start(); }
From source file:crawling.FoundUsersBySearchGeo.java
License:Apache License
public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { /* my shutdown code here */ // Record the user status out.println("Final status -------------------:"); for (Long id : discoveredUsers.keySet()) { out.println(id + "," + discoveredUsers.get(id)); }/* ww w . j a v a 2 s.c o m*/ out.close(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); date = new Date(); System.out.println("End at: " + dateFormat.format(date)); } }); if (args.length < 5) { System.out.println("Usage: java twitter4j.examples.PrintFilterStreamGeo lati long radius unit CITY"); System.exit(-1); } if (args.length == 6) { // Preload the collected user IDs preloadID(args[5]); } try { FileWriter outFile = new FileWriter("discoveredUser" + args[4] + ".txt", true); out = new PrintWriter(outFile); //out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } avgUsers = new int[histCount]; Query query = new Query(); query.setGeoCode(new GeoLocation(Double.parseDouble(args[0]), Double.parseDouble(args[1])), Double.parseDouble(args[2]), args[3]); // New York Metropolitan //query.setGeoCode(new GeoLocation(40.7, -74), 100, "km"); // Bay Area MetroPolitan //query.setGeoCode(new GeoLocation(37.78, -122.4), 100, "km"); // LA MetroPolitan //query.setGeoCode(new GeoLocation(34.05, -118.24), 200, "km"); query.setCount(100); //query.setPage(20); //Twitter twitter = new TwitterFactory().getInstance(); Twitter twitter = getOAuthTwitter(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); //System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); while (true) { //for (int i = 1; i <= 15; i ++){ // query.setPage(i); try { doASearch(twitter, query); } catch (TwitterException te) { // TODO Auto-generated catch block te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); // back-off Thread.currentThread(); try { Thread.sleep(60 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } Thread.currentThread(); try { Thread.sleep(5 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:crawling.FoundUsersBySearchHashtag.java
License:Apache License
public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { /* my shutdown code here */ // Record the user status out.println("Final status -------------------:"); for (Long id : discoveredUsers.keySet()) { out.println(id + "," + discoveredUsers.get(id)); }/*w w w . java 2 s .com*/ out.close(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); date = new Date(); System.out.println("End at: " + dateFormat.format(date)); } }); if (args.length < 1) { System.out.println("Usage: java twitter4j.examples.PrintFilterStreamHashtag hashtag"); System.exit(-1); } if (args.length == 2) { // Preload the collected user IDs preloadID(args[1]); } try { FileWriter outFile = new FileWriter("discoveredUser" + args[0] + ".txt", true); out = new PrintWriter(outFile); hashtag = args[0]; //out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } avgUsers = new int[histCount]; Query query = new Query(); //query.setGeoCode(new GeoLocation(Double.parseDouble(args[0]) // , Double.parseDouble(args[1])), Double.parseDouble(args[2]), args[3]); query.setQuery(hashtag); // New York Metropolitan //query.setGeoCode(new GeoLocation(40.7, -74), 100, "km"); // Bay Area MetroPolitan //query.setGeoCode(new GeoLocation(37.78, -122.4), 100, "km"); // LA MetroPolitan //query.setGeoCode(new GeoLocation(34.05, -118.24), 200, "km"); query.setCount(100); //query.setPage(20); //Twitter twitter = new TwitterFactory().getInstance(); Twitter twitter = getOAuthTwitter(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); //System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); while (true) { //for (int i = 1; i <= 15; i ++){ // query.setPage(i); try { doASearch(twitter, query); } catch (TwitterException te) { // TODO Auto-generated catch block te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); // back-off Thread.currentThread(); try { Thread.sleep(60 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } Thread.currentThread(); try { Thread.sleep(5 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:crawling.SearchTweetsHashtag.java
License:Apache License
public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { /* my shutdown code here */ // Record the user status out.println("Final status -------------------:"); for (Long id : discoveredTweets.keySet()) { out.println(id + "," + discoveredTweets.get(id)); }/* w ww . j av a2 s . c om*/ out.close(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); date = new Date(); System.out.println("End at: " + dateFormat.format(date)); } }); if (args.length < 1) { System.out.println("Usage: java twitter4j.examples.PrintFilterStreamHashtag hashtag"); System.exit(-1); } if (args.length == 2) { // Preload the collected user IDs //preloadID(args[1]); } buildStartTime(); File file = new File(fileName); InputStream is = null; try { if (file.exists()) { is = new FileInputStream(file); prop.load(is); } else { System.out.println(fileName + " doesn't exist!"); System.exit(-1); } } catch (IOException ioe) { ioe.printStackTrace(); System.exit(-1); } try { FileWriter outFile = new FileWriter(args[0] + ".txt", true); out = new PrintWriter(outFile); hashtag = args[0]; //out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } avgUsers = new int[histCount]; Query query = new Query(); query.setQuery(hashtag); //query.setCount(100); query.since("2015-03-15"); //query.setPage(20); //Twitter twitter = new TwitterFactory().getInstance(); Twitter twitter = getOAuthTwitter(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); //System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); while (true) { //for (int i = 1; i <= 15; i ++){ // query.setPage(i); try { doASearch(twitter, query); } catch (TwitterException te) { // TODO Auto-generated catch block te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); // back-off Thread.currentThread(); try { Thread.sleep(60 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } Thread.currentThread(); try { Thread.sleep(5 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:crawltweets2mongo.MonoThread.java
void getNewTweets(GeoLocation myLoc, double radius) { try {// w w w . j a v a 2 s . c o m Query query = new Query(); Query.Unit unit = Query.KILOMETERS; // or Query.MILES; query.setGeoCode(myLoc, radius, unit); if (radius > 200) query.setCount(20000); else query.setCount(20000); QueryResult result; result = this.twitter.search(query); //System.out.println("Getting Tweets. by Geo.."); List<Status> tweets = result.getTweets(); for (Status tweet : tweets) { BasicDBObject basicObj = new BasicDBObject(); basicObj.put("user_Rname", tweet.getUser().getName()); basicObj.put("user_name", tweet.getUser().getScreenName()); basicObj.put("retweet_count", tweet.getRetweetCount()); basicObj.put("tweet_followers_count", tweet.getUser().getFollowersCount()); UserMentionEntity[] mentioned = tweet.getUserMentionEntities(); basicObj.put("tweet_mentioned_count", mentioned.length); basicObj.put("tweet_ID", tweet.getId()); basicObj.put("tweet_text", tweet.getText()); Status temp1 = tweet.getRetweetedStatus(); if (temp1 != null) basicObj.put("Re_tweet_ID", temp1.getUser().getId()); GeoLocation loc = tweet.getGeoLocation(); if (loc != null) { basicObj.put("Latitude", loc.getLatitude()); basicObj.put("Longitude", loc.getLongitude()); } basicObj.put("CreateTime", tweet.getCreatedAt()); basicObj.put("FavoriteCount", tweet.getFavoriteCount()); basicObj.put("user_Id", tweet.getUser().getId()); if (tweet.getUser().getTimeZone() != null) basicObj.put("UsertimeZone", tweet.getUser().getTimeZone()); if (tweet.getUser().getStatus() != null) basicObj.put("UserStatus", tweet.getUser().getStatus()); //basicObj.put("tweetLocation", tweet.getPlace().getGeometryCoordinates()); String U_Loc = tweet.getUser().getLocation(); if (U_Loc != null) basicObj.put("userLocation", U_Loc); basicObj.put("number_of_rt", tweet.getRetweetCount()); //basicObj.put("isRetweet", tweet.getPlace().getGeometryCoordinates()); //basicObj.put("POS", tweet.getWithheldInCountries()); if (mentioned.length > 0) { basicObj.append("mentions", pickMentions(mentioned)); } try { //items.insert(basicObj); collection.insert(basicObj); } catch (Exception e) { // System.out.println("MongoDB Connection Error : " + e.getMessage()); // loadMenu(); } } collection.ensureIndex(new BasicDBObject("tweet_ID", 1), new BasicDBObject("unique", true)); } catch (TwitterException ex) { java.util.logging.Logger.getLogger(MonoThread.class.getName()).log(Level.SEVERE, null, ex); } }