List of usage examples for twitter4j Status getCreatedAt
Date getCreatedAt();
From source file:org.apache.asterix.external.parser.TweetParser.java
License:Apache License
@Override public void parse(IRawRecord<? extends Status> record, DataOutput out) throws HyracksDataException { Status tweet = record.get(); User user = tweet.getUser();// ww w. j a v a2 s .c o m // Tweet user data ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.SCREEN_NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getScreenName())); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.LANGUAGE)]) .setValue(JObjectUtil.getNormalizedString(user.getLang())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FRIENDS_COUNT)]) .setValue(user.getFriendsCount()); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.STATUS_COUNT)]) .setValue(user.getStatusesCount()); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getName())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FOLLOWERS_COUNT)]) .setValue(user.getFollowersCount()); // Tweet data ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.ID)]) .setValue(String.valueOf(tweet.getId())); int userPos = tweetFieldNameMap.get(Tweet.USER); for (int i = 0; i < mutableUserFields.length; i++) { ((AMutableRecord) mutableTweetFields[userPos]).setValueAtPos(i, mutableUserFields[i]); } if (tweet.getGeoLocation() != null) { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]) .setValue(tweet.getGeoLocation().getLatitude()); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]) .setValue(tweet.getGeoLocation().getLongitude()); } else { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(0); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(0); } ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.CREATED_AT)]) .setValue(JObjectUtil.getNormalizedString(tweet.getCreatedAt().toString())); ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.MESSAGE)]) .setValue(JObjectUtil.getNormalizedString(tweet.getText())); for (int i = 0; i < mutableTweetFields.length; i++) { mutableRecord.setValueAtPos(i, mutableTweetFields[i]); } recordBuilder.reset(mutableRecord.getType()); recordBuilder.init(); IDataParser.writeRecord(mutableRecord, out, recordBuilder); }
From source file:org.apache.asterix.external.util.TweetProcessor.java
License:Apache License
public AMutableRecord processNextTweet(Status tweet) { User user = tweet.getUser();//from w w w . j av a2s . co m // Tweet user data ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.SCREEN_NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getScreenName())); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.LANGUAGE)]) .setValue(JObjectUtil.getNormalizedString(user.getLang())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FRIENDS_COUNT)]) .setValue(user.getFriendsCount()); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.STATUS_COUNT)]) .setValue(user.getStatusesCount()); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getName())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FOLLOWERS_COUNT)]) .setValue(user.getFollowersCount()); // Tweet data ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.ID)]) .setValue(String.valueOf(tweet.getId())); int userPos = tweetFieldNameMap.get(Tweet.USER); for (int i = 0; i < mutableUserFields.length; i++) { ((AMutableRecord) mutableTweetFields[userPos]).setValueAtPos(i, mutableUserFields[i]); } if (tweet.getGeoLocation() != null) { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]) .setValue(tweet.getGeoLocation().getLatitude()); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]) .setValue(tweet.getGeoLocation().getLongitude()); } else { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(0); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(0); } ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.CREATED_AT)]) .setValue(JObjectUtil.getNormalizedString(tweet.getCreatedAt().toString())); ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.MESSAGE)]) .setValue(JObjectUtil.getNormalizedString(tweet.getText())); for (int i = 0; i < mutableTweetFields.length; i++) { mutableRecord.setValueAtPos(i, mutableTweetFields[i]); } return mutableRecord; }
From source file:org.apache.camel.component.twitter.util.TwitterConverter.java
License:Apache License
@Converter public static String toString(Status status) throws ParseException { StringBuilder s = new StringBuilder(); s.append(status.getCreatedAt()).append(" (").append(status.getUser().getScreenName()).append(") "); s.append(status.getText());//from ww w. j a v a2 s . co m return s.toString(); }
From source file:org.apache.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser, File temporaryDirectory) { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override/* w ww. ja v a2s .c o m*/ public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { log.error(ex, "Got exception"); } @Override public void onStallWarning(StallWarning warning) { log.warn("Got stall warning: %s", warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { @Override public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Nullable @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); Thread.sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(StringUtils.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? StringUtils.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:org.apache.flume.sink.solr.morphline.TwitterSource.java
License:Apache License
private Record extractRecord(String idPrefix, Schema avroSchema, Status status) { User user = status.getUser();//from www . j a va 2 s .c o m Record doc = new Record(avroSchema); doc.put("id", idPrefix + status.getId()); doc.put("created_at", formatterTo.format(status.getCreatedAt())); doc.put("retweet_count", status.getRetweetCount()); doc.put("retweeted", status.isRetweet()); doc.put("in_reply_to_user_id", status.getInReplyToUserId()); doc.put("in_reply_to_status_id", status.getInReplyToStatusId()); addString(doc, "source", status.getSource()); addString(doc, "text", status.getText()); MediaEntity[] mediaEntities = status.getMediaEntities(); if (mediaEntities.length > 0) { addString(doc, "media_url_https", mediaEntities[0].getMediaURLHttps()); addString(doc, "expanded_url", mediaEntities[0].getExpandedURL()); } doc.put("user_friends_count", user.getFriendsCount()); doc.put("user_statuses_count", user.getStatusesCount()); doc.put("user_followers_count", user.getFollowersCount()); addString(doc, "user_location", user.getLocation()); addString(doc, "user_description", user.getDescription()); addString(doc, "user_screen_name", user.getScreenName()); addString(doc, "user_name", user.getName()); return doc; }
From source file:org.apache.solr.handler.dataimport.TwitterEntityProcessor.java
License:Apache License
@Override public Map<String, Object> nextRow() { Map<String, Object> row = new HashMap<>(); if (twitter == null || query == null) return null; try {/*from ww w. j a va 2s . c om*/ if (results == null) { results = twitter.search(query); if (results == null || results.getCount() == 0) return null; } if (tweets == null) tweets = (ArrayList<Status>) results.getTweets(); Status tweet = null; if (index < tweets.size()) { tweet = (Status) tweets.get(index++); } else { query = results.nextQuery(); if (query != null) { results = twitter.search(query); if (results == null || results.getCount() == 0) return null; tweets = (ArrayList<Status>) results.getTweets(); index = 0; tweet = (Status) tweets.get(index++); } } if (tweet == null) return null; // id row.put(MESSAGE_ID, tweet.getId()); // lang row.put(MESSAGE_LANG, tweet.getLang()); // user User user = tweet.getUser(); // name row.put(MESSAGE_USER, user.getName()); // pseudo row.put(MESSAGE_PSEUDO, tweet.getUser().getScreenName()); // text row.put(MESSAGE_TEXT, tweet.getText()); // date Date date = tweet.getCreatedAt(); row.put(MESSAGE_DATE, date.toString()); } catch (TwitterException e) { e.printStackTrace(); return null; } return row; }
From source file:org.bireme.interop.toJson.Twitter2Json.java
License:Open Source License
private JSONObject getDocument(final Status status) { assert status != null; final JSONObject obj = new JSONObject(); final GeoLocation geo = status.getGeoLocation(); final Place place = status.getPlace(); final User user = status.getUser(); obj.put("createdAt", status.getCreatedAt()).put("id", status.getId()).put("lang", status.getLang()); if (geo != null) { obj.put("location_latitude", geo.getLatitude()).put("location_longitude", geo.getLongitude()); }/*from w w w .j a v a2 s. com*/ if (place != null) { obj.put("place_country", place.getCountry()).put("place_fullName", place.getFullName()) .put("place_id", place.getId()).put("place_name", place.getName()) .put("place_type", place.getPlaceType()).put("place_streetAddress", place.getStreetAddress()) .put("place_url", place.getURL()); } obj.put("source", status.getSource()).put("text", status.getText()); if (user != null) { obj.put("user_description", user.getDescription()).put("user_id", user.getId()) .put("user_lang", user.getLang()).put("user_location", user.getLocation()) .put("user_name", user.getName()).put("user_url", user.getURL()); } obj.put("isTruncated", status.isTruncated()).put("isRetweet", status.isRetweet()); return obj; }
From source file:org.botlibre.sense.twitter.Twitter.java
License:Open Source License
/** * Check status.//from w w w.jav a 2s .com */ public void checkStatus() { if (!getProcessStatus()) { return; } log("Checking status", Level.FINE); try { Network memory = getBot().memory().newMemory(); Vertex twitter = memory.createVertex(getPrimitive()); Vertex vertex = twitter.getRelationship(Primitive.LASTTIMELINE); long last = 0; if (vertex != null) { last = ((Number) vertex.getData()).longValue(); } long max = 0; ResponseList<Status> timeline = null; boolean more = true; int page = 1; int count = 0; this.errors = 0; while (more && (count <= this.maxStatus) && page <= this.maxPage) { if (last == 0) { timeline = getConnection().getHomeTimeline(); more = false; } else { Paging paging = new Paging(page, last); timeline = getConnection().getHomeTimeline(paging); if ((timeline == null) || (timeline.size() < 20)) { more = false; } page++; } if ((timeline == null) || timeline.isEmpty()) { break; } log("Processing status", Level.INFO, timeline.size()); for (int index = timeline.size() - 1; index >= 0; index--) { if (count >= this.maxStatus) { break; } if (this.errors > this.maxErrors) { break; } Status status = timeline.get(index); String name = status.getUser().getScreenName(); if (!name.equals(this.userName)) { long statusTime = status.getCreatedAt().getTime(); long statusId = status.getId(); if (statusId > max) { max = statusId; } if ((System.currentTimeMillis() - statusTime) > DAY) { log("Day old status", Level.INFO, statusId, statusTime); more = false; continue; } if (statusId > last) { if (Utils.checkProfanity(status.getText())) { continue; } boolean match = false; List<String> statusWords = new TextStream(status.getText().toLowerCase()).allWords(); if (getListenStatus()) { this.languageState = LanguageState.Listening; match = true; } else { for (String text : getStatusKeywords()) { List<String> keywords = new TextStream(text.toLowerCase()).allWords(); if (!keywords.isEmpty() && statusWords.containsAll(keywords)) { match = true; break; } } } if (getLearn()) { learnTweet(status, true, true, memory); } if (match) { count++; input(status); Utils.sleep(500); } else { log("Skipping status, missing keywords", Level.FINE, status.getText()); if (!status.isRetweet() && !status.getUser().isProtected() && !status.isRetweetedByMe()) { boolean retweeted = false; // Check retweet. for (String keywords : getRetweet()) { List<String> keyWords = new TextStream(keywords.toLowerCase()).allWords(); if (!keyWords.isEmpty()) { if (statusWords.containsAll(keyWords)) { retweeted = true; count++; retweet(status); Utils.sleep(500); break; } } } if (!retweeted) { log("Skipping rewteet, missing keywords", Level.FINE, status.getText()); } } else if (!getRetweet().isEmpty()) { if (status.isRetweet()) { log("Skipping rewteet", Level.FINE, status.getText()); } else if (status.getUser().isProtected()) { log("Skipping protected user", Level.FINE, status.getText()); } else if (status.isRetweetedByMe()) { log("Skipping already retweeted", Level.FINE, status.getText()); } } } } else { log("Old status", Level.INFO, statusId, statusTime); } } } } if (max != 0) { twitter.setRelationship(Primitive.LASTTIMELINE, memory.createVertex(max)); memory.save(); } } catch (Exception exception) { log(exception); } // Wait for language processing. int count = 0; while (count < 60 && !getBot().memory().getActiveMemory().isEmpty()) { Utils.sleep(1000); } }
From source file:org.botlibre.sense.twitter.Twitter.java
License:Open Source License
/** * Learn from the profiles posts.//from ww w . java 2s .c o m */ public void checkLearning() { if (!getLearnFromSelf()) { return; } log("Checking learning", Level.FINE); try { Network memory = getBot().memory().newMemory(); Vertex twitter = memory.createVertex(getPrimitive()); Vertex vertex = twitter.getRelationship(Primitive.LASTLEARN); long last = 0; if (vertex != null) { last = ((Number) vertex.getData()).longValue(); } long max = 0; ResponseList<Status> timeline = getConnection().getUserTimeline(); if ((timeline == null) || timeline.isEmpty()) { return; } log("Processing status", Level.INFO, timeline.size()); for (int index = timeline.size() - 1; index >= 0; index--) { Status tweet = timeline.get(index); long statusTime = tweet.getCreatedAt().getTime(); long statusId = tweet.getId(); if (statusId > max) { max = statusId; } if ((System.currentTimeMillis() - statusTime) > DAY) { log("Day old status", Level.INFO, statusId, statusTime); continue; } if (statusId > last) { learnTweet(tweet, true, true, memory); } else { log("Old status", Level.INFO, statusId, statusTime); } } if (max != 0) { twitter.setRelationship(Primitive.LASTTIMELINE, memory.createVertex(max)); memory.save(); } } catch (Exception exception) { log(exception); } }
From source file:org.botlibre.sense.twitter.Twitter.java
License:Open Source License
/** * Check messages to this user.// w w w . j av a 2s . c om */ public void checkMentions() { if (!getReplyToMentions()) { return; } try { log("Checking mentions", Level.FINE); Network memory = getBot().memory().newMemory(); Vertex twitter = memory.createVertex(getPrimitive()); Vertex vertex = twitter.getRelationship(Primitive.LASTMENTION); long last = 0; if (vertex != null) { last = ((Number) vertex.getData()).longValue(); } long max = 0; ResponseList<Status> mentions = null; boolean more = true; int page = 1; while (more) { if (last == 0) { mentions = getConnection().getMentionsTimeline(); more = false; } else { Paging paging = new Paging(page, last); mentions = getConnection().getMentionsTimeline(paging); if ((mentions == null) || (mentions.size() < 20)) { more = false; } page++; } if ((mentions == null) || mentions.isEmpty()) { break; } log("Processing mentions", Level.FINE, mentions.size()); for (int index = mentions.size() - 1; index >= 0; index--) { Status tweet = mentions.get(index); long statusTime = tweet.getCreatedAt().getTime(); long statusId = tweet.getId(); if (statusId > max) { max = statusId; } if ((System.currentTimeMillis() - statusTime) > DAY) { log("Day old mention", Level.INFO, statusId, statusTime); more = false; continue; } // Exclude self if (tweet.getUser().getScreenName().equals(getUserName())) { continue; } if (statusId > last) { log("Processing mention", Level.INFO, tweet.getText(), tweet.getUser().getScreenName()); input(tweet); Utils.sleep(100); } else { log("Old mention", Level.INFO, statusId, statusTime); } } } if (max != 0) { twitter.setRelationship(Primitive.LASTMENTION, memory.createVertex(max)); memory.save(); } } catch (Exception exception) { log(exception); } // Wait for language processing. int count = 0; while (count < 60 && !getBot().memory().getActiveMemory().isEmpty()) { Utils.sleep(1000); } this.languageState = LanguageState.Discussion; }