List of usage examples for twitter4j Status getGeoLocation
GeoLocation getGeoLocation();
From source file:TweetMapManager.java
public static void main(String[] args) { TwitterFactory tf = new TwitterFactory(); Twitter twitter = tf.getInstance();/*www. j a va 2 s. co m*/ String searchStr = "#ParkandGoUNL"; Query query = new Query(searchStr); GeoApiContext context = new GeoApiContext().setApiKey("AIzaSyArz1NljiDpuOriFWalOerYEdHOyi8ow8Y"); List<Marker> markers = new LinkedList<>(); StreetExtractor se = new StreetExtractor(); LocalTime now = LocalTime.now(); while (true) { DayOfWeek today = LocalDate.now().getDayOfWeek(); if (today.getValue() < WEEKEND) { if (now.isAfter(STARTTIME) && now.isBefore(ENDTIME)) { // Send API request to execute a search with the given query. QueryResult results = null; try { results = twitter.search(query); } catch (TwitterException ex) { LOGGER.warn(ex.getMessage()); } // Display search results. if (results != null) { results.getTweets().stream().forEach((Status status) -> { try { Date created = status.getCreatedAt(); String text = status.getText(); LOGGER.info(status.getText()); LatLng location; if (status.getGeoLocation() != null) { location = new LatLng(status.getGeoLocation().getLatitude(), status.getGeoLocation().getLongitude()); } else { String modified = text.replace("#ParkAndGoUNL", ""); List<Address> address = se.find(modified); location = GeocodingApi.geocode(context, address.get(0).getAddress()) .await()[0].geometry.location; } String id = UUID.randomUUID().toString().substring(0, 8); Marker m = new Marker(location, "m" + id, text, created); markers.add(m); } catch (Exception ex) { LOGGER.warn(ex.getMessage()); } }); } if (!markers.isEmpty()) { Marker m = markers.get(markers.size() - 1); if (m.getTimestamp().getTime() < Time.valueOf(now.minusMinutes(30)).getTime()) { DailyLogs.addMarkerToLog(m, today); markers.remove(m); } } Map map = new Map(markers); map.create(); } else { //wait 5 hours try { LOGGER.info("Sleeping 5 hours"); Thread.sleep((long) 1.8e+7); } catch (InterruptedException ex) { LOGGER.warn(ex.getMessage()); } } now = LocalTime.now(); } else { try { //Wait a day LOGGER.info("Sleeping 1 day"); Thread.sleep((long) 8.64e+7); } catch (InterruptedException ex) { LOGGER.warn(ex.getMessage()); } } try { LOGGER.info("Successful Loop, Resting"); Thread.sleep(1000); } catch (InterruptedException ex) { LOGGER.warn(ex.getMessage()); } } }
From source file:twitterGateway_v2_06.java
License:Creative Commons License
public void SetupTwitter() { //twitterIn = new TwitterConnectStream(); //accessToken = new AccessToken(TwitterAccessToken, TwitterAccessTokenSecret); //TwitterOAuthAuthorization.setOAuthAccessToken(accessToken); //TwitterOAuthAuthorization = new OAuthAuthorization(conf); //TwitterOAuthAuthorization.setOAuthConsumer(TwitterConsumerKey, TwitterConsumerSecret); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey(TwitterConsumerKey) .setOAuthConsumerSecret(TwitterConsumerSecret).setOAuthAccessToken(TwitterAccessToken) .setOAuthAccessTokenSecret(TwitterAccessTokenSecret); TwitterFactory tf = new TwitterFactory(cb.build()); twitterOut = tf.getInstance();/*from ww w .jav a 2 s .c o m*/ // try { // twitterOut.updateStatus("Hello World!"); // } // catch (TwitterException ex) { // println(ex); // } ActivityLogAddLine("twitter connector ready"); output = createWriter("log.txt"); StatusListener twitterIn = new StatusListener() { public void onStatus(Status status) { double Longitude; double Latitude; GeoLocation GeoLoc = status.getGeoLocation(); if (GeoLoc != null) { //println("YES got a location"); Longitude = GeoLoc.getLongitude(); Latitude = GeoLoc.getLatitude(); } else { Longitude = 0; Latitude = 0; } println(TimeStamp() + "\t" + Latitude + "\t" + Longitude + "\t" + status.getUser().getScreenName() + "\t" + status.getText()); output.println(TimeStamp() + "\t" + Latitude + "\t" + Longitude + "\t" + status.getUser().getScreenName() + "\t" + status.getText()); output.flush(); TwitterToOsc(status.getUser().getScreenName(), status.getText()); } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } public void onException(Exception ex) { println("CAUGHT in the ACT: " + ex); } }; ConfigurationBuilder cbIn = new ConfigurationBuilder(); cbIn.setDebugEnabled(true).setOAuthConsumerKey(TwitterConsumerKey) .setOAuthConsumerSecret(TwitterConsumerSecret).setOAuthAccessToken(TwitterAccessToken) .setOAuthAccessTokenSecret(TwitterAccessTokenSecret); TwitterStreamFactory ts = new TwitterStreamFactory(cbIn.build()); TwitterStream twitterStream = ts.getInstance(); twitterStream.addListener(twitterIn); // filter() method internally creates a thread which manipulates TwitterStream and calls these adequate listener methods continuously. FilterQuery twitterFilter = new FilterQuery(0, TwitterFollowIDs, TwitterTrackWords); twitterStream.filter(twitterFilter); }
From source file:GetLiveMedia.java
License:Open Source License
public void TweetCollect(String searchString) throws TwitterException { tXCoord = 0;/* w ww . j av a2 s . c o m*/ tYCoord = 0; ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("0c1v1wqqNATaVcWpINaHoW3o5") .setOAuthConsumerSecret("a6sEAJ1NLY3cXhkMTrMZF62bIfww6qLvlEetDAU9O09Mp04lyh") .setOAuthAccessToken("330427831-RJig6B6YycJL3xQVoEjjyRaujwnr1xHEQGtSHOOc") .setOAuthAccessTokenSecret("Da71qk4SN6Nu88mLkugDDR6CftVcgOcz3QB8dHNw4hVbD"); TwitterFactory tf = new TwitterFactory(cb.build()); twitter = tf.getInstance(); query = new Query("#" + searchString); result = twitter.search(query); for (Status status : result.getTweets()) { try { System.out.println(status.getGeoLocation()); System.out.println(status.getCreatedAt()); tXCoord = status.getGeoLocation().getLatitude(); tYCoord = status.getGeoLocation().getLongitude(); twitterString = ("@" + status.getUser().getScreenName() + ": " + status.getText() + " : " + "(" + tXCoord + ", " + tYCoord + ")"); twitterURL = "https://twitter.com/" + status.getUser().getScreenName() + "/status/" + status.getId(); } catch (Exception e) { } } }
From source file:CollectAndPredict.java
public static void main(String[] args) { //StreamingExamples.setStreamingLogLevels(); // Set logging level if log4j not configured (override by adding log4j.properties to classpath) if (!Logger.getRootLogger().getAllAppenders().hasMoreElements()) { Logger.getRootLogger().setLevel(Level.WARN); }/* ww w. ja va 2s . c om*/ String OAuthConsumerKey = "QxeynbXmN93DuNiZKkYfZcS2o"; String OAuthConsumerSecret = "2rAzjerHeW6sIgeDim0A77iGaRn9O683m0DrTbBhaoIuRRq7oU"; String OAuthAccessToken = "825094416935297025-jCegwA25yj3QxF2rHeJ5hRoVu86AfaY"; String OAuthAccessTokenSecret = "CwfNmGcWHoL8qvr5dWDdknYM4k4KvAZc7XlGZuYl2DcR8"; String[] filters = Arrays.copyOfRange(args, 0, args.length); // Set the system properties so that Twitter4j library used by Twitter stream // can use them to generate OAuth credentials System.setProperty("twitter4j.oauth.consumerKey", OAuthConsumerKey); System.setProperty("twitter4j.oauth.consumerSecret", OAuthConsumerSecret); System.setProperty("twitter4j.oauth.accessToken", OAuthAccessToken); System.setProperty("twitter4j.oauth.accessTokenSecret", OAuthAccessTokenSecret); SparkConf sparkConf = new SparkConf().setAppName("JavaTwitterHashTagJoinSentiments"); // check Spark configuration for master URL, set it to local if not configured if (!sparkConf.contains("spark.master")) { sparkConf.setMaster("local[2]"); } SparkSession spark = SparkSession.builder().appName("teste2").config(sparkConf).getOrCreate(); JavaStreamingContext jssc = new JavaStreamingContext(new JavaSparkContext(spark.sparkContext()), Seconds.apply(30)); TokenizerFactory tokFactory = TwitterTokenizerFactory.getTokFactory(); NaiveBayesModel model = NaiveBayesModel.load(spark.sparkContext(), "Docker/myNaiveBayesModel"); HashingTF hashingTF = new HashingTF(1000); JavaReceiverInputDStream<Status> stream = TwitterUtils.createStream(jssc, filters); JavaDStream<Tweet> statuses = stream.map((Status status) -> new Tweet() .addUser(new TwitterUser().addID(status.getUser().getId()).addName(status.getUser().getName()) .addLocation(status.getUser().getLocation()).addDateSignin(status.getUser().getCreatedAt()) .addCountTweets(status.getUser().getStatusesCount()) .addCountFavorites(status.getUser().getFavouritesCount()) .addCountFriends(status.getUser().getFriendsCount()) .addCountFollowers(status.getUser().getFollowersCount())) .addText(status.getText()).addID(status.getId()).addDate(status.getCreatedAt()) .addLatitude( status.getGeoLocation() != null ? status.getGeoLocation().getLatitude() : Double.MAX_VALUE) .addLongitude(status.getGeoLocation() != null ? status.getGeoLocation().getLongitude() : Double.MAX_VALUE)); statuses.foreachRDD(new VoidFunction2<JavaRDD<Tweet>, Time>() { long numTweetsCollected = 0; long numTweetsToCollect = 200; @Override public void call(JavaRDD<Tweet> t1, Time t2) throws Exception { List<Tweet> collect = t1.collect(); long count = collect.size(); if (count > 0) { for (Tweet tweet : collect) { String textoSemUrl = URLRemove.remove(tweet.getText()); Vector v = hashingTF.transform(Arrays.asList(tokFactory .tokenizer(textoSemUrl.toCharArray(), 0, textoSemUrl.length()).tokenize())); double predict = model.predict(v); if (predict == 1) { tweet.setClassifier("POSITIVE"); } else { tweet.setClassifier("NEGATIVE"); } } ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); try { ow.writeValue( new FileOutputStream(new File("Docker/Twitter" + t2.milliseconds() + ".json")), collect); } catch (Exception ex) { spark.log().error(ex.getMessage(), ex); } numTweetsCollected += count; spark.log().info("coletou :" + numTweetsCollected + " tweets"); if (numTweetsCollected > numTweetsToCollect) { System.exit(0); } } } }); // statuses.foreachRDD(new VoidFunction2<JavaRDD<String>, Time>() { // long numTweetsCollected = 0; // long numTweetsToCollect = 200; // // @Override // public void call(JavaRDD<String> rdd, Time time) throws Exception { // long count = rdd.count(); // if (count > 0) { // JavaRDD<String> outputRDD = rdd.repartition(10); // outputRDD.saveAsTextFile("/Docker/tweets_" + time.milliseconds()); // numTweetsCollected += count; // if (numTweetsCollected > numTweetsToCollect) { // System.exit(0); // } // } // } // }); // JavaDStream<String> words = stream.flatMap(new FlatMapFunction<Status, String>() { // // @Override // public Iterable<String> call(Status t) throws Exception { // return Arrays.asList(t.getText().split(" ")); // } // }); // // JavaDStream<String> hashTags = words.filter(new Function<String, Boolean>() { // @Override // public Boolean call(String word) { // return word.startsWith("#"); // } // }); // // // Read in the word-sentiment list and create a static RDD from it // String wordSentimentFilePath = "streaming-twitter/examples/data/AFINN-111.txt"; // final JavaPairRDD<String, Double> wordSentiments = jssc.sparkContext() // .textFile(wordSentimentFilePath) // .mapToPair(new PairFunction<String, String, Double>() { // @Override // public Tuple2<String, Double> call(String line) { // String[] columns = line.split("\t"); // return new Tuple2<>(columns[0], Double.parseDouble(columns[1])); // } // }); // // JavaPairDStream<String, Integer> hashTagCount = hashTags.mapToPair( // new PairFunction<String, String, Integer>() { // @Override // public Tuple2<String, Integer> call(String s) { // // leave out the # character // return new Tuple2<>(s.substring(1), 1); // } // }); // // JavaPairDStream<String, Integer> hashTagTotals = hashTagCount.reduceByKeyAndWindow( // new Function2<Integer, Integer, Integer>() { // @Override // public Integer call(Integer a, Integer b) { // return a + b; // } // }, new Duration(10000)); // // // Determine the hash tags with the highest sentiment values by joining the streaming RDD // // with the static RDD inside the transform() method and then multiplying // // the frequency of the hash tag by its sentiment value // JavaPairDStream<String, Tuple2<Double, Integer>> joinedTuples // = hashTagTotals.transformToPair(new Function<JavaPairRDD<String, Integer>, JavaPairRDD<String, Tuple2<Double, Integer>>>() { // @Override // public JavaPairRDD<String, Tuple2<Double, Integer>> call( // JavaPairRDD<String, Integer> topicCount) { // return wordSentiments.join(topicCount); // } // }); // // JavaPairDStream<String, Double> topicHappiness = joinedTuples.mapToPair( // new PairFunction<Tuple2<String, Tuple2<Double, Integer>>, String, Double>() { // @Override // public Tuple2<String, Double> call(Tuple2<String, Tuple2<Double, Integer>> topicAndTuplePair) { // Tuple2<Double, Integer> happinessAndCount = topicAndTuplePair._2(); // return new Tuple2<>(topicAndTuplePair._1(), // happinessAndCount._1() * happinessAndCount._2()); // } // }); // // JavaPairDStream<Double, String> happinessTopicPairs = topicHappiness.mapToPair( // new PairFunction<Tuple2<String, Double>, Double, String>() { // @Override // public Tuple2<Double, String> call(Tuple2<String, Double> topicHappiness) { // return new Tuple2<>(topicHappiness._2(), // topicHappiness._1()); // } // }); // // JavaPairDStream<Double, String> happiest10 = happinessTopicPairs.transformToPair( // new Function<JavaPairRDD<Double, String>, JavaPairRDD<Double, String>>() { // @Override // public JavaPairRDD<Double, String> call( // JavaPairRDD<Double, String> happinessAndTopics) { // return happinessAndTopics.sortByKey(false); // } // } // ); // // // Print hash tags with the most positive sentiment values // happiest10.foreachRDD(new VoidFunction<JavaPairRDD<Double, String>>() { // @Override // public void call(JavaPairRDD<Double, String> happinessTopicPairs) { // List<Tuple2<Double, String>> topList = happinessTopicPairs.take(10); // System.out.println( // String.format("\nHappiest topics in last 10 seconds (%s total):", // happinessTopicPairs.count())); // for (Tuple2<Double, String> pair : topList) { // System.out.println( // String.format("%s (%s happiness)", pair._2(), pair._1())); // } // } // }); jssc.start(); try { jssc.awaitTermination(); } catch (InterruptedException e) { e.printStackTrace(); } }
From source file:ac.simons.tweetarchive.tweets.TweetStorageService.java
License:Apache License
@Transactional public TweetEntity store(final Status status, final String rawContent) { final Optional<TweetEntity> existingTweet = this.tweetRepository.findOne(status.getId()); if (existingTweet.isPresent()) { final TweetEntity rv = existingTweet.get(); log.warn("Tweet with status {} already existed...", rv.getId()); return rv; }/*from ww w. j a va2 s . c om*/ final TweetEntity tweet = new TweetEntity(status.getId(), status.getUser().getId(), status.getUser().getScreenName(), status.getCreatedAt().toInstant().atZone(ZoneId.of("UTC")), extractContent(status), extractSource(status), rawContent); tweet.setCountryCode(Optional.ofNullable(status.getPlace()).map(Place::getCountryCode).orElse(null)); if (status.getInReplyToStatusId() != -1L && status.getInReplyToUserId() != -1L && status.getInReplyToScreenName() != null) { tweet.setInReplyTo(new InReplyTo(status.getInReplyToStatusId(), status.getInReplyToScreenName(), status.getInReplyToUserId())); } tweet.setLang(status.getLang()); tweet.setLocation(Optional.ofNullable(status.getGeoLocation()) .map(g -> new TweetEntity.Location(g.getLatitude(), g.getLongitude())).orElse(null)); // TODO Handle quoted tweets return this.tweetRepository.save(tweet); }
From source file:adapter.TwitterAllAdapter.java
License:Apache License
@Override public void run() { while (true) { try {/*w w w . ja v a 2 s . c o m*/ Status status = this.messageQueue.take(); Event event = new Event(); int rtCount = 0; int fvCount = 0; if (status.isRetweet()) { rtCount = status.getRetweetedStatus().getRetweetCount(); fvCount = status.getRetweetedStatus().getFavoriteCount(); } Tweet tweet = new Tweet(status.getId(), status.getText(), status.getCreatedAt(), status.getPlace(), status.getUser().getScreenName(), status.getUser().getLang(), status.getUser().getFollowersCount(), status.getUser().getFriendsCount(), status.getHashtagEntities(), fvCount, rtCount, status.getGeoLocation()); //status.getRetweetedStatus().getRetweetCount(), status.getGeoLocation()); eventCount++; // cantReplicas: Cantidad de PEs que se quieren generar para el proximo operador // Nota: recuerden que la topologa no necesariamente deba ser de este estilo // tambin poda ser por un hash int cantReplicas = 10; event.put("levelTweet", Integer.class, eventCount % cantReplicas); event.put("id", Integer.class, eventCount); event.put("tweet", Tweet.class, tweet); getRemoteStream().put(event); } catch (Exception e) { logger.error("Error: " + e); logger.error("Error al crear evento"); } } }
From source file:adapter.TwitterKeywordsAdapter.java
License:Apache License
@Override public void run() { while (true) { try {/*from www . j a va 2 s. co m*/ Status status = this.messageQueue.take(); Event event = new Event(); Tweet tweet = new Tweet(status.getId(), status.getText(), status.getCreatedAt(), status.getPlace(), status.getUser().getScreenName(), status.getUser().getLang(), status.getUser().getFollowersCount(), status.getUser().getFriendsCount(), status.getHashtagEntities(), status.getFavoriteCount(), status.getRetweetCount(), status.getGeoLocation()); eventCount++; // cantReplicas: Cantidad de PEs que se quieren generar para el proximo operador // Nota: recuerden que la topologa no necesariamente deba ser de este estilo // tambin poda ser por un hash int cantReplicas = 10; event.put("levelTweet", Integer.class, eventCount % cantReplicas); event.put("id", Integer.class, eventCount); event.put("tweet", Tweet.class, tweet); getRemoteStream().put(event); } catch (Exception e) { logger.error("Error: " + e); logger.error("Error al crear evento"); } } }
From source file:adapter.TwitterLanguageAdapter.java
License:Apache License
@Override public void run() { while (true) { try {// ww w .j a va 2 s . c o m Status status = this.messageQueue.take(); Event event = new Event(); Tweet tweet = new Tweet(status.getId(), status.getText(), status.getCreatedAt(), status.getPlace(), status.getUser().getScreenName(), status.getUser().getLang(), status.getUser().getFollowersCount(), status.getUser().getFriendsCount(), status.getHashtagEntities(), status.getFavoriteCount(), status.getRetweetCount(), status.getGeoLocation()); eventCount++; // cantReplicas: Cantidad de PEs que se quieren generar para el proximo operador // Nota: recuerden que la topologa no necesariamente deba ser de este estilo // tambin poda ser por un hash int cantReplicas = 10; event.put("levelTweet", Integer.class, eventCount % cantReplicas); event.put("id", Integer.class, eventCount); event.put("tweet", Tweet.class, tweet); getRemoteStream().put(event); } catch (Exception e) { logger.error("Error: " + e); logger.error("Error al crear evento"); } } }
From source file:adapter.TwitterLocationAdapter.java
License:Apache License
@Override public void run() { while (true) { try {//www. jav a 2 s .com Status status = this.messageQueue.take(); Event event = new Event(); Tweet tweet = new Tweet(status.getId(), status.getText(), status.getCreatedAt(), status.getPlace(), status.getUser().getScreenName(), status.getUser().getLang(), status.getUser().getFollowersCount(), status.getUser().getFriendsCount(), status.getHashtagEntities(), status.getFavoriteCount(), status.getRetweetCount(), status.getGeoLocation()); eventCount++; // cantReplicas: Cantidad de PEs que se quieren generar para el // proximo operador // Nota: recuerden que la topologa no necesariamente deba ser // de este estilo // tambin poda ser por un hash int cantReplicas = 10; event.put("levelTweet", Integer.class, eventCount % cantReplicas); event.put("id", Integer.class, eventCount); event.put("tweet", Tweet.class, tweet); getRemoteStream().put(event); } catch (Exception e) { logger.error("Error: " + e); logger.error("Error al crear evento"); } } }
From source file:au.net.moon.tSearchArchiver.SearchArchiver.java
License:Open Source License
SearchArchiver() {
Twitter twitter;/*from w w w. jav a 2 s .c o m*/
int waitBetweenRequests = 2000;
// 2 sec delay between requests to avoid maxing out the API.
Status theTweet;
Query query;
QueryResult result;
// String[] searches;
ArrayList<String> searchQuery = new ArrayList<String>();
ArrayList<Integer> searchId = new ArrayList<Integer>();
int searchIndex;
int totalTweets;
SimpleDateFormat myFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z");
System.out.println("tSearchArchiver: Loading search queries...");
// Set timezone to UTC for the Twitter created at dates
myFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
twitterAuthorise twitterAuth = new twitterAuthorise(false);
twitter = twitterAuth.getTwitter();
// Open the old twitter_archive database
openSQLDataBase();
if (isDatabaseReady()) {
// probably should have these in an object not separate arrays?
try {
rs = stmt.executeQuery("select * from searches where active = true");
// perform each search
while (rs.next()) {
// if (searchQuery
searchQuery.add(rs.getString("query"));
searchId.add(rs.getInt("id"));
}
if (rs.wasNull()) {
System.out.println("tSearchArchiver: No searches in the table \"searches\"");
System.exit(30);
} else {
System.out.println("tSearchArchiver: Found " + searchQuery.size() + " searches.");
}
} catch (SQLException e) {
System.out.println("tSearchArchiver: e:" + e.toString());
}
searchIndex = 0;
totalTweets = 0;
// set initial value of i to start from middle of search set
while (searchIndex < searchQuery.size()) {
query = new Query();
query.setQuery(searchQuery.get(searchIndex));
// check to see if their are any tweets already in the database for
// this search
//TODO: Change this to look in new raw data files for each search instead
long max_tw_id = 0;
try {
rs = stmt.executeQuery("select max(tweet_id) as max_id from archive where search_id = "
+ searchId.get(searchIndex));
if (rs.next()) {
max_tw_id = rs.getLong("max_id");
// System.out.println("MaxID: " + max_tw_id);
query.setSinceId(max_tw_id);
}
} catch (SQLException e1) {
System.err.println("tSearchArchiver: Error looking for maximum tweet_id for " + query.getQuery()
+ " in archive");
e1.printStackTrace();
}
// System.out.println("Starting searching for tweets for: " +
// query.getQuery());
// new style replacement for pagination
// Query query = new Query("whatEverYouWantToSearch");
// do {
// result = twitter.search(query);
// System.out.println(result);
// do something
// } while ((query = result.nextQuery()) != null);
// TODO: check if twitter4j is doing all the backing off handling already
int tweetCount = 0;
Boolean searching = true;
do {
// delay waitBetweenRequests milliseconds before making request
// to make sure not overloading API
try {
Thread.sleep(waitBetweenRequests);
} catch (InterruptedException e1) {
System.err.println("tSearchArchiver: Sleep between requests failed.");
e1.printStackTrace();
}
try {
result = twitter.search(query);
} catch (TwitterException e) {
System.out.println(e.getStatusCode());
System.out.println(e.toString());
if (e.getStatusCode() == 503) {
// TODO use the Retry-After header value to delay & then
// retry the request
System.out
.println("tSearchArchiver: Delaying for 10 minutes before making new request");
try {
Thread.sleep(600000);
} catch (InterruptedException e1) {
System.err.println(
"tSearchArchiver: Sleep for 10 minutes because of API load failed.");
e1.printStackTrace();
}
}
result = null;
}
if (result != null) {
List<Status> results = result.getTweets();
if (results.size() == 0) {
searching = false;
} else {
tweetCount += results.size();
for (int j = 0; j < results.size(); j++) {
theTweet = (Status) results.get(j);
String cleanText = theTweet.getText();
cleanText = cleanText.replaceAll("'", "'");
cleanText = cleanText.replaceAll("\"", """);
try {
stmt.executeUpdate("insert into archive values (0, " + searchId.get(searchIndex)
+ ", '" + theTweet.getId() + "', now())");
} catch (SQLException e) {
System.err.println("tSearchArchiver: Insert into archive failed.");
System.err.println(searchId.get(searchIndex) + ", " + theTweet.getId());
e.printStackTrace();
}
// TODO: change to storing in file instead of database
try {
rs = stmt.executeQuery("select id from tweets where id = " + theTweet.getId());
} catch (SQLException e) {
System.err.println(
"tSearchArchiver: checking for tweet in tweets archive failed.");
e.printStackTrace();
}
Boolean tweetNotInArchive = false;
try {
tweetNotInArchive = !rs.next();
} catch (SQLException e) {
System.err.println(
"tSearchArchiver: checking for tweet in archive failed at rs.next().");
e.printStackTrace();
}
if (tweetNotInArchive) {
String tempLangCode = "";
// getIsoLanguageCode() has been removed from twitter4j
// looks like it might be added back in in the next version
// if (tweet.getIsoLanguageCode() != null) {
// if (tweet.getIsoLanguageCode().length() > 2) {
// System.out
// .println("tSearchArchiver Error: IsoLanguageCode too long: >"
// + tweet.getIsoLanguageCode()
// + "<");
// tempLangCode = tweet
// .getIsoLanguageCode()
// .substring(0, 2);
// } else {
// tempLangCode = tweet
// .getIsoLanguageCode();
// }
// }
double myLatitude = 0;
double myLongitude = 0;
int hasGeoCode = 0;
if (theTweet.getGeoLocation() != null) {
System.out.println("GeoLocation: " + theTweet.getGeoLocation().toString());
myLatitude = theTweet.getGeoLocation().getLatitude();
myLongitude = theTweet.getGeoLocation().getLongitude();
hasGeoCode = 1;
}
Date tempCreatedAt = theTweet.getCreatedAt();
String myDate2 = myFormatter
.format(tempCreatedAt, new StringBuffer(), new FieldPosition(0))
.toString();
totalTweets++;
try {
stmt.executeUpdate("insert into tweets values (" + theTweet.getId() + ", '"
+ tempLangCode + "', '" + theTweet.getSource() + "', '" + cleanText
+ "', '" + myDate2 + "', '" + theTweet.getInReplyToUserId() + "', '"
+ theTweet.getInReplyToScreenName() + "', '"
+ theTweet.getUser().getId() + "', '"
+ theTweet.getUser().getScreenName() + "', '" + hasGeoCode + "',"
+ myLatitude + ", " + myLongitude + ", now())");
} catch (SQLException e) {
System.err.println("tSearchArchiver: Insert into tweets failed.");
System.err.println(theTweet.getId() + ", '" + tempLangCode + "', '"
+ theTweet.getSource() + "', '" + cleanText + "', '" + myDate2
+ "', '" + theTweet.getInReplyToUserId() + "', '"
+ theTweet.getInReplyToScreenName() + "', '"
+ theTweet.getUser().getId() + "', '"
+ theTweet.getUser().getScreenName());
e.printStackTrace();
}
}
}
}
}
} while ((query = result.nextQuery()) != null && searching);
if (tweetCount > 0) {
System.out.println("tSearchArchiver: New Tweets Found for \"" + searchQuery.get(searchIndex)
+ "\" = " + tweetCount);
} else {
// System.out.println("tSearchArchiver: No Tweets Found for \""
// + searchQuery.get(searchIndex) + "\" = " + tweetCount);
}
try {
stmt.executeUpdate("update searches SET lastFoundCount=" + tweetCount
+ ", lastSearchDate=now() where id=" + searchId.get(searchIndex));
} catch (SQLException e) {
System.err.println("tSearchArchiver: failed to update searches with lastFoundCount="
+ tweetCount + " and datetime for search: " + searchId.get(searchIndex));
e.printStackTrace();
}
searchIndex++;
}
System.out.println("tSearchArchiver: Completed all " + searchQuery.size() + " searches");
System.out.println("tSearchArchiver: Archived " + totalTweets + " new tweets");
}
}