List of usage examples for twitter4j Query getQuery
public String getQuery()
From source file:au.net.moon.tSearchArchiver.SearchArchiver.java
License:Open Source License
SearchArchiver() {
Twitter twitter;//from w w w . j a v a 2s . c o m
int waitBetweenRequests = 2000;
// 2 sec delay between requests to avoid maxing out the API.
Status theTweet;
Query query;
QueryResult result;
// String[] searches;
ArrayList<String> searchQuery = new ArrayList<String>();
ArrayList<Integer> searchId = new ArrayList<Integer>();
int searchIndex;
int totalTweets;
SimpleDateFormat myFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z");
System.out.println("tSearchArchiver: Loading search queries...");
// Set timezone to UTC for the Twitter created at dates
myFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
twitterAuthorise twitterAuth = new twitterAuthorise(false);
twitter = twitterAuth.getTwitter();
// Open the old twitter_archive database
openSQLDataBase();
if (isDatabaseReady()) {
// probably should have these in an object not separate arrays?
try {
rs = stmt.executeQuery("select * from searches where active = true");
// perform each search
while (rs.next()) {
// if (searchQuery
searchQuery.add(rs.getString("query"));
searchId.add(rs.getInt("id"));
}
if (rs.wasNull()) {
System.out.println("tSearchArchiver: No searches in the table \"searches\"");
System.exit(30);
} else {
System.out.println("tSearchArchiver: Found " + searchQuery.size() + " searches.");
}
} catch (SQLException e) {
System.out.println("tSearchArchiver: e:" + e.toString());
}
searchIndex = 0;
totalTweets = 0;
// set initial value of i to start from middle of search set
while (searchIndex < searchQuery.size()) {
query = new Query();
query.setQuery(searchQuery.get(searchIndex));
// check to see if their are any tweets already in the database for
// this search
//TODO: Change this to look in new raw data files for each search instead
long max_tw_id = 0;
try {
rs = stmt.executeQuery("select max(tweet_id) as max_id from archive where search_id = "
+ searchId.get(searchIndex));
if (rs.next()) {
max_tw_id = rs.getLong("max_id");
// System.out.println("MaxID: " + max_tw_id);
query.setSinceId(max_tw_id);
}
} catch (SQLException e1) {
System.err.println("tSearchArchiver: Error looking for maximum tweet_id for " + query.getQuery()
+ " in archive");
e1.printStackTrace();
}
// System.out.println("Starting searching for tweets for: " +
// query.getQuery());
// new style replacement for pagination
// Query query = new Query("whatEverYouWantToSearch");
// do {
// result = twitter.search(query);
// System.out.println(result);
// do something
// } while ((query = result.nextQuery()) != null);
// TODO: check if twitter4j is doing all the backing off handling already
int tweetCount = 0;
Boolean searching = true;
do {
// delay waitBetweenRequests milliseconds before making request
// to make sure not overloading API
try {
Thread.sleep(waitBetweenRequests);
} catch (InterruptedException e1) {
System.err.println("tSearchArchiver: Sleep between requests failed.");
e1.printStackTrace();
}
try {
result = twitter.search(query);
} catch (TwitterException e) {
System.out.println(e.getStatusCode());
System.out.println(e.toString());
if (e.getStatusCode() == 503) {
// TODO use the Retry-After header value to delay & then
// retry the request
System.out
.println("tSearchArchiver: Delaying for 10 minutes before making new request");
try {
Thread.sleep(600000);
} catch (InterruptedException e1) {
System.err.println(
"tSearchArchiver: Sleep for 10 minutes because of API load failed.");
e1.printStackTrace();
}
}
result = null;
}
if (result != null) {
List<Status> results = result.getTweets();
if (results.size() == 0) {
searching = false;
} else {
tweetCount += results.size();
for (int j = 0; j < results.size(); j++) {
theTweet = (Status) results.get(j);
String cleanText = theTweet.getText();
cleanText = cleanText.replaceAll("'", "'");
cleanText = cleanText.replaceAll("\"", """);
try {
stmt.executeUpdate("insert into archive values (0, " + searchId.get(searchIndex)
+ ", '" + theTweet.getId() + "', now())");
} catch (SQLException e) {
System.err.println("tSearchArchiver: Insert into archive failed.");
System.err.println(searchId.get(searchIndex) + ", " + theTweet.getId());
e.printStackTrace();
}
// TODO: change to storing in file instead of database
try {
rs = stmt.executeQuery("select id from tweets where id = " + theTweet.getId());
} catch (SQLException e) {
System.err.println(
"tSearchArchiver: checking for tweet in tweets archive failed.");
e.printStackTrace();
}
Boolean tweetNotInArchive = false;
try {
tweetNotInArchive = !rs.next();
} catch (SQLException e) {
System.err.println(
"tSearchArchiver: checking for tweet in archive failed at rs.next().");
e.printStackTrace();
}
if (tweetNotInArchive) {
String tempLangCode = "";
// getIsoLanguageCode() has been removed from twitter4j
// looks like it might be added back in in the next version
// if (tweet.getIsoLanguageCode() != null) {
// if (tweet.getIsoLanguageCode().length() > 2) {
// System.out
// .println("tSearchArchiver Error: IsoLanguageCode too long: >"
// + tweet.getIsoLanguageCode()
// + "<");
// tempLangCode = tweet
// .getIsoLanguageCode()
// .substring(0, 2);
// } else {
// tempLangCode = tweet
// .getIsoLanguageCode();
// }
// }
double myLatitude = 0;
double myLongitude = 0;
int hasGeoCode = 0;
if (theTweet.getGeoLocation() != null) {
System.out.println("GeoLocation: " + theTweet.getGeoLocation().toString());
myLatitude = theTweet.getGeoLocation().getLatitude();
myLongitude = theTweet.getGeoLocation().getLongitude();
hasGeoCode = 1;
}
Date tempCreatedAt = theTweet.getCreatedAt();
String myDate2 = myFormatter
.format(tempCreatedAt, new StringBuffer(), new FieldPosition(0))
.toString();
totalTweets++;
try {
stmt.executeUpdate("insert into tweets values (" + theTweet.getId() + ", '"
+ tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText
+ "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '"
+ theTweet.getInReplyToScreenName() + "', '"
+ theTweet.getUser().getId() + "', '"
+ theTweet.getUser().getScreenName() + "', '" + hasGeoCode + "',"
+ myLatitude + ", " + myLongitude + ", now())");
} catch (SQLException e) {
System.err.println("tSearchArchiver: Insert into tweets failed.");
System.err.println(theTweet.getId() + ", '" + tempLangCode + "', '"
+ theTweet.getSource() + "', '" + cleanText + "', '" + myDate2
+ "', '" + theTweet.getInReplyToUserId() + "', '"
+ theTweet.getInReplyToScreenName() + "', '"
+ theTweet.getUser().getId() + "', '"
+ theTweet.getUser().getScreenName());
e.printStackTrace();
}
}
}
}
}
} while ((query = result.nextQuery()) != null && searching);
if (tweetCount > 0) {
System.out.println("tSearchArchiver: New Tweets Found for \"" + searchQuery.get(searchIndex)
+ "\" = " + tweetCount);
} else {
// System.out.println("tSearchArchiver: No Tweets Found for \""
// + searchQuery.get(searchIndex) + "\" = " + tweetCount);
}
try {
stmt.executeUpdate("update searches SET lastFoundCount=" + tweetCount
+ ", lastSearchDate=now() where id=" + searchId.get(searchIndex));
} catch (SQLException e) {
System.err.println("tSearchArchiver: failed to update searches with lastFoundCount="
+ tweetCount + " and datetime for search: " + searchId.get(searchIndex));
e.printStackTrace();
}
searchIndex++;
}
System.out.println("tSearchArchiver: Completed all " + searchQuery.size() + " searches");
System.out.println("tSearchArchiver: Archived " + totalTweets + " new tweets");
}
}
From source file:com.dhamacher.tweetsentimentanalysis.Main.java
License:Open Source License
public static void main(String args[]) { try {/*from w w w. j av a 2 s .c o m*/ license(); /* Prompt for Search token */ Scanner in = new Scanner(System.in); System.out.print("Type in the search query: "); Query searchToken = new Query(in.nextLine()); /* Set maximum count */ searchToken.setCount(1500); /* Send search request to Twitter and receive List<> of results */ QueryResult result = twitter.search(searchToken); /* Notify user of the result */ System.out.println( "Amount of tweets pulled for \"" + searchToken.getQuery() + "\": " + result.getTweets().size()); persistTweet(result, searchToken.getQuery()); end(); } catch (TwitterException ex) { Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:com.github.moko256.mastodon.MastodonTwitterImpl.java
License:Apache License
@Override public QueryResult search(Query query) { Pageable<com.sys1yagi.mastodon4j.api.entity.Status> pageable = null; try {/*from w w w . j a va 2s. c o m*/ pageable = new com.sys1yagi.mastodon4j.api.method.Timelines(client) .getHashtagTimeline(query.getQuery(), false, false, MTRangeConverter.convert(query)).execute(); } catch (Mastodon4jRequestException e) { e.printStackTrace(); } long previous; long next; if (pageable == null) return null; if (pageable.getLink() != null) { next = pageable.getLink().getMaxId(); previous = pageable.getLink().getSinceId(); } else { next = -1; previous = -1; } List<com.sys1yagi.mastodon4j.api.entity.Status> part = pageable.getPart(); Pageable<com.sys1yagi.mastodon4j.api.entity.Status> finalPageable = pageable; return new QueryResult() { @Override public long getSinceId() { return previous; } @Override public long getMaxId() { return next; } @Override public String getRefreshURL() { return null; } @Override public int getCount() { return part.size(); } @Override public double getCompletedIn() { return 0; } @Override public String getQuery() { return query.getQuery(); } @Override public List<Status> getTweets() { return MTResponseList.convert(finalPageable); } @Override public Query nextQuery() { return new Query(query.getQuery()).sinceId(next); } @Override public boolean hasNext() { return next != -1; } @Override public RateLimitStatus getRateLimitStatus() { return null; } @Override public int getAccessLevel() { return 0; } }; }
From source file:org.ipccenter.newsagg.impl.twitterapi.TwitterPuller.java
public void findPosts() throws TwitterException { StringBuilder searchURL = new StringBuilder(); searchURL.append("https://api.twitter.com/1.1/search/tweets.json&q="); QueryResult searchResult = null;// www . j ava 2 s . c o m List<Status> posts = new ArrayList<Status>(); Query q = new Query(); LOG.info("Query: {}", q.getQuery()); List<String> requests = new ArrayList<String>(); requests.add(""); requests.add(""); requests.add(""); for (String request : requests) { q.query(request.toString()); LOG.info("New query: {}", q.getQuery()); searchResult = twitter.search(q); posts.addAll(searchResult.getTweets()); } LOG.info("Posts amount: {}", posts.size()); for (Status status : posts) { parsePost(status); } }
From source file:org.wandora.application.tools.extractors.twitter.AbstractTwitterExtractor.java
License:Open Source License
public void searchTwitter(Query[] queries, int pages, TopicMap tm) { if (tm == null || queries == null || queries.length == 0) return;/*from w ww .j a v a 2 s .c om*/ initializeTwitter(); if (pin == null || pin.length() == 0) { try { AccessToken accessToken = null; String authorizationURL = requestToken.getAuthorizationURL(); TwitterAuthorizationDialog authorizer = new TwitterAuthorizationDialog(); authorizer.open(authorizationURL); if (authorizer.wasAccepted()) { pin = authorizer.getPin(); } else { return; } if (pin != null && pin.length() > 0) { accessToken = twitter.getOAuthAccessToken(requestToken, pin); } else { accessToken = twitter.getOAuthAccessToken(); } } catch (TwitterException ex) { log("Invalid Twitter authorization. Please, check the PIN code '" + pin + "' and try again."); //log(ex); pin = null; return; } } try { Topic twitterSearchTopic = tm.createTopic(); if (twitterSearchTopic != null) { long stamp = System.currentTimeMillis(); twitterSearchTopic.addSubjectIdentifier(new Locator(TWITTER_SEARCH_BODY + "/" + stamp)); twitterSearchTopic.setBaseName("Twitter search " + stamp); twitterSearchTopic.addType(getTwitterSearchType(tm)); Topic twitterSearchDateType = getTwitterSearchDateType(tm); if (twitterSearchDateType != null) { Date d = new Date(stamp); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); String dStr = sdf.format(d); twitterSearchTopic.setData(twitterSearchDateType, TMBox.getLangTopic(twitterSearchTopic, DEFAULT_LANG), dStr); } for (Query currentQuery : queries) { if (currentQuery == null) continue; int currentPage = 0; log("Processing Twitter query '" + currentQuery.getQuery() + "'"); while (currentQuery != null) { if (forceStop()) { log("Aborting..."); break; } currentPage++; hlog("Requesting search result page " + currentPage + (pages == 0 ? "" : " of " + pages)); QueryResult result = twitter.search(currentQuery); Topic twitterSearchQueryType = getTwitterSearchQueryType(tm); if (twitterSearchQueryType != null) { twitterSearchTopic.setData(twitterSearchQueryType, TMBox.getLangTopic(twitterSearchTopic, DEFAULT_LANG), currentQuery.getQuery()); } ArrayList tweets = (ArrayList) result.getTweets(); for (Object tweet : tweets) { if (tweet != null && tweet instanceof Status) { Status t = (Status) tweet; Topic topic = reifyTweet(t, tm); topic.addType(twitterSearchTopic); } } try { Thread.currentThread().sleep(SLEEP_TIME_BETWEEN_SEARCHES); } catch (Exception e) { // ...WAKE UP... } if (pages == 0 || currentPage < pages) { currentQuery = result.nextQuery(); } else { currentQuery = null; } } log("Number of processed search result pages is " + currentPage); } } } catch (TopicMapException tme) { log("A topic map exception " + tme.getMessage() + " occurred while searching Twitter messages."); tme.printStackTrace(); } catch (TwitterException te) { log("A Twitter exception " + te.getMessage() + " occurred while searching messages."); te.printStackTrace(); } ; }