List of usage examples for twitter4j Query setQuery
public void setQuery(String query)
From source file:au.net.moon.tSearchArchiver.SearchArchiver.java
License:Open Source License
SearchArchiver() { Twitter twitter;// w ww. ja v a 2 s . c o m int waitBetweenRequests = 2000; // 2 sec delay between requests to avoid maxing out the API. Status theTweet; Query query; QueryResult result; // String[] searches; ArrayList<String> searchQuery = new ArrayList<String>(); ArrayList<Integer> searchId = new ArrayList<Integer>(); int searchIndex; int totalTweets; SimpleDateFormat myFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z"); System.out.println("tSearchArchiver: Loading search queries..."); // Set timezone to UTC for the Twitter created at dates myFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); twitterAuthorise twitterAuth = new twitterAuthorise(false); twitter = twitterAuth.getTwitter(); // Open the old twitter_archive database openSQLDataBase(); if (isDatabaseReady()) { // probably should have these in an object not separate arrays? try { rs = stmt.executeQuery("select * from searches where active = true"); // perform each search while (rs.next()) { // if (searchQuery searchQuery.add(rs.getString("query")); searchId.add(rs.getInt("id")); } if (rs.wasNull()) { System.out.println("tSearchArchiver: No searches in the table \"searches\""); System.exit(30); } else { System.out.println("tSearchArchiver: Found " + searchQuery.size() + " searches."); } } catch (SQLException e) { System.out.println("tSearchArchiver: e:" + e.toString()); } searchIndex = 0; totalTweets = 0; // set initial value of i to start from middle of search set while (searchIndex < searchQuery.size()) { query = new Query(); query.setQuery(searchQuery.get(searchIndex)); // check to see if their are any tweets already in the database for // this search //TODO: Change this to look in new raw data files for each search instead long max_tw_id = 0; try { rs = stmt.executeQuery("select max(tweet_id) as max_id from archive where search_id = " + searchId.get(searchIndex)); if (rs.next()) { max_tw_id = rs.getLong("max_id"); // System.out.println("MaxID: " + max_tw_id); query.setSinceId(max_tw_id); } } catch (SQLException e1) { System.err.println("tSearchArchiver: Error looking for maximum tweet_id for " + query.getQuery() + " in archive"); e1.printStackTrace(); } // System.out.println("Starting searching for tweets for: " + // query.getQuery()); // new style replacement for pagination // Query query = new Query("whatEverYouWantToSearch"); // do { // result = twitter.search(query); // System.out.println(result); // do something // } while ((query = result.nextQuery()) != null); // TODO: check if twitter4j is doing all the backing off handling already int tweetCount = 0; Boolean searching = true; do { // delay waitBetweenRequests milliseconds before making request // to make sure not overloading API try { Thread.sleep(waitBetweenRequests); } catch (InterruptedException e1) { System.err.println("tSearchArchiver: Sleep between requests failed."); e1.printStackTrace(); } try { result = twitter.search(query); } catch (TwitterException e) { System.out.println(e.getStatusCode()); System.out.println(e.toString()); if (e.getStatusCode() == 503) { // TODO use the Retry-After header value to delay & then // retry the request System.out .println("tSearchArchiver: Delaying for 10 minutes before making new request"); try { Thread.sleep(600000); } catch (InterruptedException e1) { System.err.println( "tSearchArchiver: Sleep for 10 minutes because of API load failed."); e1.printStackTrace(); } } result = null; } if (result != null) { List<Status> results = result.getTweets(); if (results.size() == 0) { searching = false; } else { tweetCount += results.size(); for (int j = 0; j < results.size(); j++) { theTweet = (Status) results.get(j); String cleanText = theTweet.getText(); cleanText = cleanText.replaceAll("'", "'"); cleanText = cleanText.replaceAll("\"", """); try { stmt.executeUpdate("insert into archive values (0, " + searchId.get(searchIndex) + ", '" + theTweet.getId() + "', now())"); } catch (SQLException e) { System.err.println("tSearchArchiver: Insert into archive failed."); System.err.println(searchId.get(searchIndex) + ", " + theTweet.getId()); e.printStackTrace(); } // TODO: change to storing in file instead of database try { rs = stmt.executeQuery("select id from tweets where id = " + theTweet.getId()); } catch (SQLException e) { System.err.println( "tSearchArchiver: checking for tweet in tweets archive failed."); e.printStackTrace(); } Boolean tweetNotInArchive = false; try { tweetNotInArchive = !rs.next(); } catch (SQLException e) { System.err.println( "tSearchArchiver: checking for tweet in archive failed at rs.next()."); e.printStackTrace(); } if (tweetNotInArchive) { String tempLangCode = ""; // getIsoLanguageCode() has been removed from twitter4j // looks like it might be added back in in the next version // if (tweet.getIsoLanguageCode() != null) { // if (tweet.getIsoLanguageCode().length() > 2) { // System.out // .println("tSearchArchiver Error: IsoLanguageCode too long: >" // + tweet.getIsoLanguageCode() // + "<"); // tempLangCode = tweet // .getIsoLanguageCode() // .substring(0, 2); // } else { // tempLangCode = tweet // .getIsoLanguageCode(); // } // } double myLatitude = 0; double myLongitude = 0; int hasGeoCode = 0; if (theTweet.getGeoLocation() != null) { System.out.println("GeoLocation: " + theTweet.getGeoLocation().toString()); myLatitude = theTweet.getGeoLocation().getLatitude(); myLongitude = theTweet.getGeoLocation().getLongitude(); hasGeoCode = 1; } Date tempCreatedAt = theTweet.getCreatedAt(); String myDate2 = myFormatter .format(tempCreatedAt, new StringBuffer(), new FieldPosition(0)) .toString(); totalTweets++; try { stmt.executeUpdate("insert into tweets values (" + theTweet.getId() + ", '" + tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText + "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '" + theTweet.getInReplyToScreenName() + "', '" + theTweet.getUser().getId() + "', '" + theTweet.getUser().getScreenName() + "', '" + hasGeoCode + "'," + myLatitude + ", " + myLongitude + ", now())"); } catch (SQLException e) { System.err.println("tSearchArchiver: Insert into tweets failed."); System.err.println(theTweet.getId() + ", '" + tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText + "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '" + theTweet.getInReplyToScreenName() + "', '" + theTweet.getUser().getId() + "', '" + theTweet.getUser().getScreenName()); e.printStackTrace(); } } } } } } while ((query = result.nextQuery()) != null && searching); if (tweetCount > 0) { System.out.println("tSearchArchiver: New Tweets Found for \"" + searchQuery.get(searchIndex) + "\" = " + tweetCount); } else { // System.out.println("tSearchArchiver: No Tweets Found for \"" // + searchQuery.get(searchIndex) + "\" = " + tweetCount); } try { stmt.executeUpdate("update searches SET lastFoundCount=" + tweetCount + ", lastSearchDate=now() where id=" + searchId.get(searchIndex)); } catch (SQLException e) { System.err.println("tSearchArchiver: failed to update searches with lastFoundCount=" + tweetCount + " and datetime for search: " + searchId.get(searchIndex)); e.printStackTrace(); } searchIndex++; } System.out.println("tSearchArchiver: Completed all " + searchQuery.size() + " searches"); System.out.println("tSearchArchiver: Archived " + totalTweets + " new tweets"); } }
From source file:com.aremaitch.codestock2010.library.TwitterLib.java
License:Apache License
public QueryResult search(long sinceId, String[] hashTags) throws TwitterException { Query q = new Query(); if (sinceId >= 0) { q.setSinceId(sinceId);//from w w w. j av a2s. c o m } q.setQuery(TextUtils.join(" OR ", hashTags)); try { return t.search(q); } catch (NumberFormatException nfe) { ACLogger.error(CSConstants.LOG_TAG, "number format exception was thrown by twitter4j"); } return null; }
From source file:com.thesmartweb.swebrank.TwitterAnalysis.java
License:Apache License
/** * Method to get tweets regarding a string * @param query_string the string to search for * @param config_path the directory with the twitter api key * @return the tweets in a string//from w w w . j a va2s. c o m */ public String perform(String query_string, String config_path) { try { List<String> twitterkeys = GetKeys(config_path); //configuration builder in order to set the keys of twitter ConfigurationBuilder cb = new ConfigurationBuilder(); String consumerkey = twitterkeys.get(0); String consumersecret = twitterkeys.get(1); String accesstoken = twitterkeys.get(2); String accesstokensecret = twitterkeys.get(3); cb.setDebugEnabled(true).setOAuthConsumerKey(consumerkey).setOAuthConsumerSecret(consumersecret) .setOAuthAccessToken(accesstoken).setOAuthAccessTokenSecret(accesstokensecret); TwitterFactory tf = new TwitterFactory(cb.build()); AccessToken acc = new AccessToken(accesstoken, accesstokensecret); Twitter twitter = tf.getInstance(acc); //query the twitter Query query = new Query(query_string); int rpp = 100; query.count(rpp); query.setQuery(query_string); //----------get the tweets------------ QueryResult result = twitter.search(query); List<Status> tweets = result.getTweets(); RateLimitStatus rls = result.getRateLimitStatus(); String tweet_txt = ""; for (Status tweet : tweets) { tweet_txt = tweet_txt + " " + tweet.getText(); } DataManipulation txtpro = new DataManipulation(); Stopwords st = new Stopwords(); tweet_txt = txtpro.removeChars(tweet_txt); tweet_txt = st.stop(tweet_txt); tweet_txt = txtpro.removeChars(tweet_txt); return tweet_txt; } catch (TwitterException ex) { String tweet_txt = ""; Logger.getLogger(TwitterAnalysis.class.getName()).log(Level.SEVERE, null, ex); return tweet_txt = "fail"; } }
From source file:crawling.FoundUsersBySearchHashtag.java
License:Apache License
public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { /* my shutdown code here */ // Record the user status out.println("Final status -------------------:"); for (Long id : discoveredUsers.keySet()) { out.println(id + "," + discoveredUsers.get(id)); }/* w w w . j av a 2 s.c om*/ out.close(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); date = new Date(); System.out.println("End at: " + dateFormat.format(date)); } }); if (args.length < 1) { System.out.println("Usage: java twitter4j.examples.PrintFilterStreamHashtag hashtag"); System.exit(-1); } if (args.length == 2) { // Preload the collected user IDs preloadID(args[1]); } try { FileWriter outFile = new FileWriter("discoveredUser" + args[0] + ".txt", true); out = new PrintWriter(outFile); hashtag = args[0]; //out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } avgUsers = new int[histCount]; Query query = new Query(); //query.setGeoCode(new GeoLocation(Double.parseDouble(args[0]) // , Double.parseDouble(args[1])), Double.parseDouble(args[2]), args[3]); query.setQuery(hashtag); // New York Metropolitan //query.setGeoCode(new GeoLocation(40.7, -74), 100, "km"); // Bay Area MetroPolitan //query.setGeoCode(new GeoLocation(37.78, -122.4), 100, "km"); // LA MetroPolitan //query.setGeoCode(new GeoLocation(34.05, -118.24), 200, "km"); query.setCount(100); //query.setPage(20); //Twitter twitter = new TwitterFactory().getInstance(); Twitter twitter = getOAuthTwitter(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); //System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); while (true) { //for (int i = 1; i <= 15; i ++){ // query.setPage(i); try { doASearch(twitter, query); } catch (TwitterException te) { // TODO Auto-generated catch block te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); // back-off Thread.currentThread(); try { Thread.sleep(60 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } Thread.currentThread(); try { Thread.sleep(5 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:crawling.SearchTweetsHashtag.java
License:Apache License
public static void main(String[] args) { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { /* my shutdown code here */ // Record the user status out.println("Final status -------------------:"); for (Long id : discoveredTweets.keySet()) { out.println(id + "," + discoveredTweets.get(id)); }/* ww w .j av a 2 s . c o m*/ out.close(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); date = new Date(); System.out.println("End at: " + dateFormat.format(date)); } }); if (args.length < 1) { System.out.println("Usage: java twitter4j.examples.PrintFilterStreamHashtag hashtag"); System.exit(-1); } if (args.length == 2) { // Preload the collected user IDs //preloadID(args[1]); } buildStartTime(); File file = new File(fileName); InputStream is = null; try { if (file.exists()) { is = new FileInputStream(file); prop.load(is); } else { System.out.println(fileName + " doesn't exist!"); System.exit(-1); } } catch (IOException ioe) { ioe.printStackTrace(); System.exit(-1); } try { FileWriter outFile = new FileWriter(args[0] + ".txt", true); out = new PrintWriter(outFile); hashtag = args[0]; //out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } avgUsers = new int[histCount]; Query query = new Query(); query.setQuery(hashtag); //query.setCount(100); query.since("2015-03-15"); //query.setPage(20); //Twitter twitter = new TwitterFactory().getInstance(); Twitter twitter = getOAuthTwitter(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); //System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); while (true) { //for (int i = 1; i <= 15; i ++){ // query.setPage(i); try { doASearch(twitter, query); } catch (TwitterException te) { // TODO Auto-generated catch block te.printStackTrace(); System.out.println("Failed to search tweets: " + te.getMessage()); // back-off Thread.currentThread(); try { Thread.sleep(60 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } Thread.currentThread(); try { Thread.sleep(5 * 1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:dk.netarkivet.harvester.tools.TwitterDecidingScope.java
License:Open Source License
/** * This routine makes any necessary Twitter API calls and queues the content discovered. * * @param controller The controller for this crawl. *//* www. ja va 2s . c om*/ @Override public void initialize(CrawlController controller) { super.initialize(controller); twitter = (new TwitterFactory()).getInstance(); keywords = null; try { keywords = (StringList) super.getAttribute(ATTR_KEYWORDS); pages = ((Integer) super.getAttribute(ATTR_PAGES)).intValue(); geoLocations = (StringList) super.getAttribute(ATTR_GEOLOCATIONS); language = (String) super.getAttribute(ATTR_LANG); if (language == null) { language = "all"; } resultsPerPage = (Integer) super.getAttribute(ATTR_RESULTS_PER_PAGE); queueLinks = (Boolean) super.getAttribute(ATTR_QUEUE_LINKS); queueUserStatus = (Boolean) super.getAttribute(ATTR_QUEUE_USER_STATUS); queueUserStatusLinks = (Boolean) super.getAttribute(ATTR_QUEUE_USER_STATUS_LINKS); queueKeywordLinks = (Boolean) super.getAttribute(ATTR_QUEUE_KEYWORD_LINKS); } catch (AttributeNotFoundException e1) { e1.printStackTrace(); throw new RuntimeException(e1); } catch (MBeanException e1) { e1.printStackTrace(); throw new RuntimeException(e1); } catch (ReflectionException e1) { e1.printStackTrace(); throw new RuntimeException(e1); } for (Object keyword : keywords) { log.info("Twitter Scope keyword: {}", keyword); } // If keywords or geoLocations is missing, add a list with a single empty string so that the main loop is // executed at least once. if (keywords == null || keywords.isEmpty()) { keywords = new StringList("keywords", "empty keyword list", new String[] { "" }); } if (geoLocations == null || geoLocations.isEmpty()) { geoLocations = new StringList("geolocations", "empty geolocation list", new String[] { "" }); } log.info("Twitter Scope will queue {} page(s) of results.", pages); // Nested loop over keywords, geo_locations and pages. for (Object keyword : keywords) { String keywordString = (String) keyword; for (Object geoLocation : geoLocations) { String urlQuery = (String) keyword; Query query = new Query(); query.setRpp(resultsPerPage); if (language != null && !language.equals("")) { query.setLang(language); urlQuery += " lang:" + language; keywordString += " lang:" + language; } urlQuery = "http://twitter.com/search/" + URLEncoder.encode(urlQuery); if (queueKeywordLinks) { addSeedIfLegal(urlQuery); } for (int page = 1; page <= pages; page++) { query.setPage(page); if (!keyword.equals("")) { query.setQuery(keywordString); } if (!geoLocation.equals("")) { String[] locationArray = ((String) geoLocation).split(","); try { GeoLocation location = new GeoLocation(Double.parseDouble(locationArray[0]), Double.parseDouble(locationArray[1])); query.setGeoCode(location, Double.parseDouble(locationArray[2]), locationArray[3]); } catch (NumberFormatException e) { e.printStackTrace(); } } try { final QueryResult result = twitter.search(query); List<Tweet> tweets = result.getTweets(); for (Tweet tweet : tweets) { long id = tweet.getId(); String fromUser = tweet.getFromUser(); String tweetUrl = "http://www.twitter.com/" + fromUser + "/status/" + id; addSeedIfLegal(tweetUrl); tweetCount++; if (queueLinks) { extractEmbeddedLinks(tweet); } if (queueUserStatus) { String statusUrl = "http://twitter.com/" + tweet.getFromUser() + "/"; addSeedIfLegal(statusUrl); linkCount++; if (queueUserStatusLinks) { queueUserStatusLinks(tweet.getFromUser()); } } } } catch (TwitterException e1) { log.error(e1.getMessage()); } } } } System.out.println( TwitterDecidingScope.class + " added " + tweetCount + " tweets and " + linkCount + " other links."); }
From source file:dk.netarkivet.harvester.tools.TwitterDecidingScope.java
License:Open Source License
/** * Searches for a given users recent tweets and queues and embedded material found. * * @param user The twitter username (without the @ prefix). */// w w w .j av a 2 s . com private void queueUserStatusLinks(String user) { Query query = new Query(); query.setQuery("@" + user); query.setRpp(20); if (!language.equals("")) { query.setLang(language); } try { List<Tweet> results = twitter.search(query).getTweets(); if (results != null && !results.isEmpty()) { System.out.println("Extracting embedded links for user " + user); } for (Tweet result : results) { if (result.getIsoLanguageCode().equals(language) || language.equals("")) { extractEmbeddedLinks(result); } } } catch (TwitterException e) { e.printStackTrace(); } }
From source file:it.greenvulcano.gvesb.social.twitter.directcall.TwitterOperationSearch.java
License:Open Source License
@Override public void execute(SocialAdapterAccount account) throws SocialAdapterException { try {/*from www . j ava2 s . c o m*/ Twitter twitter = (Twitter) account.getProxyObject(); Query q = new Query(); if ((query != null) && !"".equals(query)) { q.setQuery(query); } if ((sinceId != null) && !"".equals(sinceId)) { q.setSinceId(Long.parseLong(sinceId)); } if ((maxId != null) && !"".equals(maxId)) { q.setMaxId(Long.parseLong(maxId)); } if ((since != null) && !"".equals(since)) { q.setSince(since); } if ((until != null) && !"".equals(until)) { q.setUntil(until); } if ((count != null) && !"".equals(count)) { q.setCount(Integer.parseInt(count)); } XMLUtils parser = null; try { parser = XMLUtils.getParserInstance(); doc = parser.newDocument("TwitterQuery"); Element root = doc.getDocumentElement(); parser.setAttribute(root, "user", twitter.getScreenName()); parser.setAttribute(root, "userId", String.valueOf(twitter.getId())); parser.setAttribute(root, "createdAt", DateUtils.nowToString(DateUtils.FORMAT_ISO_DATETIME_UTC)); QueryResult result; do { result = twitter.search(q); List<Status> tweets = result.getTweets(); for (Status tweet : tweets) { dumpTweet(parser, root, tweet); } } while ((q = result.nextQuery()) != null); } catch (Exception exc) { logger.error("Error formatting TwitterOperationSearch query[" + query + "], sinceId[" + sinceId + "], maxId[" + maxId + "], since[" + since + "], until[" + until + "] and count[" + count + "] response.", exc); throw new SocialAdapterException("Error formatting TwitterOperationSearch query[" + query + "], sinceId[" + sinceId + "], maxId[" + maxId + "], since[" + since + "], until[" + until + "] and count[" + count + "] response.", exc); } finally { XMLUtils.releaseParserInstance(parser); } } catch (NumberFormatException exc) { logger.error("Call to TwitterOperationSearch failed. Check query[" + query + "], sinceId[" + sinceId + "], maxId[" + maxId + "], since[" + since + "], until[" + until + "] and count[" + count + "] format.", exc); throw new SocialAdapterException("Call to TwitterOperationSearch failed. Check query[" + query + "], sinceId[" + sinceId + "], maxId[" + maxId + "], since[" + since + "], until[" + until + "] and count[" + count + "] format.", exc); } }
From source file:net.lacolaco.smileessence.twitter.task.SearchTask.java
License:Open Source License
public static Query getBaseQuery(MainActivity activity, String queryString) { Configuration config = activity.getResources().getConfiguration(); Query query = new Query(); query.setQuery(queryString); query.setLang(config.locale.getLanguage()); query.setCount(TwitterUtils.getPagingCount(activity)); query.setResultType(Query.RECENT); return query; }
From source file:org.codice.ddf.catalog.twitter.source.TwitterSource.java
License:Open Source License
@Override public SourceResponse query(QueryRequest request) throws UnsupportedQueryException { Twitter instance = twitterFactory.getInstance(); try {//w w w. ja v a2 s .c om instance.getOAuth2Token(); } catch (TwitterException e) { throw new UnsupportedQueryException("Unable to get OAuth2 token.", e); } TwitterFilterVisitor visitor = new TwitterFilterVisitor(); request.getQuery().accept(visitor, null); Query query = new Query(); query.setCount(request.getQuery().getPageSize()); if (visitor.hasSpatial()) { GeoLocation geoLocation = new GeoLocation(visitor.getLatitude(), visitor.getLongitude()); query.setGeoCode(geoLocation, visitor.getRadius(), Query.Unit.km); } if (visitor.getContextualSearch() != null) { query.setQuery(visitor.getContextualSearch().getSearchPhrase()); } if (visitor.getTemporalSearch() != null) { Calendar.Builder builder = new Calendar.Builder(); builder.setInstant(visitor.getTemporalSearch().getStartDate()); Calendar calendar = builder.build(); query.setSince(calendar.get(Calendar.YEAR) + "-" + calendar.get(Calendar.MONTH) + "-" + calendar.get(Calendar.DAY_OF_MONTH)); builder = new Calendar.Builder(); builder.setInstant(visitor.getTemporalSearch().getEndDate()); calendar = builder.build(); query.setUntil(calendar.get(Calendar.YEAR) + "-" + calendar.get(Calendar.MONTH) + "-" + calendar.get(Calendar.DAY_OF_MONTH)); } QueryResult queryResult; try { queryResult = instance.search().search(query); } catch (TwitterException e) { throw new UnsupportedQueryException(e); } List<Result> resultList = new ArrayList<>(queryResult.getCount()); resultList.addAll(queryResult.getTweets().stream().map(status -> new ResultImpl(getMetacard(status))) .collect(Collectors.toList())); return new SourceResponseImpl(request, resultList); }