List of usage examples for twitter4j Status isRetweet
boolean isRetweet();
From source file:edu.csupomona.nlp.tool.crawler.Twitter.java
/** * Check whether retweet status mathces requirement * @param status Status from Twitter * @return True: match, False: mismatch *//* w ww.jav a2 s. c o m*/ private boolean isRetweetMatch(Status status) { return (status.isRetweet() == includeRetweet_); }
From source file:edu.uml.TwitterDataMining.TwitterConsumer.java
/** * FYI!!!!!!!!!!!!!!!!!!!!!! Requests to twitter are rate limited. Only 180 * requests per registered app every 15 minutes. */// w w w.j a v a 2s . c om public static void main(String[] args) { // query for a specific user example // String path = getPathToCorpus(); // System.out.println(path); // queryTwitterUser("@ActuallyNPH", getPathToCorpus().concat("test.txt")); // my custom terms List<String> myTerms = new ArrayList<>(); myTerms.add("#WTF"); myTerms.add("#noob"); myTerms.add("#sucks"); myTerms.add("#shit"); myTerms.add("#immature"); myTerms.add("#loser"); myTerms.add("#hater"); myTerms.add("#douchebag"); try { // Parse lexicon file for bad sentiment hashtags String lexiconPath = SearchTermFilter.getPathToLexicon(); lexiconPath = lexiconPath.concat("unigrams-pmilexicon.txt"); queryTerms = SearchTermFilter.parseSentimentLexicons(lexiconPath); // Add my own terms if they do not exist for (String myTerm : myTerms) { if (!queryTerms.contains(myTerm)) { queryTerms.add(myTerm); } } // Query each of the terms int queryCount = 0; List<Status> tweets = null; for (String term : queryTerms) { queryCount++; System.out.println("Searching: " + term); System.out.println("Query Count: " + queryCount + " of " + queryTerms.size()); tweets = queryTwitter(term); // Creates an array of tweets // resolve null pointer issue (cause: rate timeout error) if (tweets != null) { // Remove retweets. List<Status> retweets = new ArrayList<>(); for (Status tweet : tweets) { if (tweet.isRetweet()) { retweets.add(tweet); } } tweets.removeAll(retweets); saveResults(tweets, getPathToCorpus().concat("rawTweets.txt")); // Save array of tweets } } } catch (IOException ioe) { ioe.printStackTrace(); } }
From source file:ezbake.training.TweetIngestParser.java
License:Apache License
private Tweet convertToTweet(Status status) { Tweet tweet = new Tweet(); tweet.setTimestamp(status.getCreatedAt().getTime()); tweet.setId(status.getId());// w w w . j a v a 2 s . c o m tweet.setText(status.getText()); tweet.setUserId(status.getUser().getId()); tweet.setUserName(status.getUser().getName()); tweet.setIsFavorite(status.isFavorited()); tweet.setIsRetweet(status.isRetweet()); if (status.getGeoLocation() != null) { tweet.setGeoLocation( new Coordinate(status.getGeoLocation().getLatitude(), status.getGeoLocation().getLongitude())); } return tweet; }
From source file:h2weibo.Twitter2Weibo.java
License:Open Source License
public void syncTwitter(String userId) { T2WUser user = helper.findOneByUser(userId); weibo.setToken(user.getToken(), user.getTokenSecret()); Twitter twitter = new TwitterFactory().getInstance(); if (user.getTwitterToken() != null) { twitter.setOAuthAccessToken(new AccessToken(user.getTwitterToken(), user.getTwitterTokenSecret())); log.debug(String.format("Using OAuth for %s", user.getUserId())); }//from w w w . j a va 2 s . co m StatusFilters filters = new StatusFilters(); filters.use(new NoSyncFilter()); // should be used first filters.use(new TcoStatusFilter()).use(new URLStatusFilter()).use(new TagStatusFilter()) .use(new FlickrImageFilter()); NoMentionFilter mentionFilter = new NoMentionFilter(); UserMappingFilter mappingFilter = new UserMappingFilter(helper); if (!user.ready()) { log.debug(String.format("Skipping @%s ...", user.getUserId())); return; } // gets Twitter instance with default credentials String screenName = user.getUserId(); long latestId = user.getLatestId(); log.debug(String.format("Checking @%s's timeline, latest ID = %d.", userId, latestId)); try { if (latestId == 0) { List<Status> statuses = twitter.getUserTimeline(screenName); if (statuses.size() > 0) { user.setLatestId(statuses.get(0).getId()); // Record latestId, and sync next time } log.info(String.format("First time use for @%s. Set latest ID to %d.", userId, latestId)); } else { Paging paging = new Paging(latestId); List<Status> statuses = twitter.getUserTimeline(screenName, paging); // sync from the oldest one for (int i = statuses.size() - 1; i >= 0; i--) { Status status = statuses.get(i); if (status.getId() < user.getLatestId()) continue; // safe keeper String name = status.getUser().getScreenName(); String statusText = status.getText(); log.info(String.format("%s - %s", name, statusText)); try { if (status.isRetweet()) { if (user.isDropRetweets()) { user.setLatestId(status.getId()); log.debug("Skipped " + statusText + " because status is a retweet."); continue; } else { filters.remove(mentionFilter); filters.use(mappingFilter); } } else { if (user.isDropMentions()) { filters.remove(mappingFilter); filters.use(mentionFilter); } else { filters.remove(mentionFilter); filters.use(mappingFilter); } } statusText = filters.filter(statusText); if (statusText == null) { user.setLatestId(status.getId()); log.info(String.format("Skipped %s because of the filter.", statusText)); continue; } if (!user.isNoImage()) { // add twitter images to status text MediaEntity[] mediaEntities = status.getMediaEntities(); if (mediaEntities != null) { for (MediaEntity entity : mediaEntities) { statusText += " " + entity.getMediaURL(); } log.info("with media url: " + statusText); } StatusImageExtractor ex = new StatusImageExtractor(); StringBuffer buf = new StringBuffer(statusText); byte[] image = ex.extract(buf); if (image != null) { user.setLatestId(status.getId()); try { statusText = buf.toString(); // with image urls removed weibo.uploadStatus(statusText, new ImageItem(image)); log.info(String.format("@%s - %s sent with image.", name, statusText)); } catch (WeiboException e) { log.error("Faile to update image.", e); } continue; } } GeoLocation location = status.getGeoLocation(); if (user.isWithGeo() && location != null) { weibo.updateStatus(statusText, location.getLatitude(), location.getLongitude()); log.info(String.format("@%s - %s sent with geo locations.", name, statusText)); } else { weibo.updateStatus(statusText); log.info(String.format("@%s - %s sent.", name, statusText)); } } catch (WeiboException e) { if (e.getStatusCode() != 400) { // resending same tweet log.error("Failed to update Weibo", e); break; } else { log.error("Sending same message", e); } } log.info(String.format("Sent: by %s - %s", name, statusText)); user.setLatestId(status.getId()); } } helper.saveUser(user); } catch (Exception e) { if (!(e instanceof TwitterException)) { log.error("Failed to update.", e); } } }
From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser) throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override//w w w . j a va2s . co m public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(String.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:io.rakam.datasource.twitter.TweetProcessor.java
License:Apache License
@Override public void onStatus(Status status) { Map<String, Object> map = new HashMap<>(); GeoLocation geoLocation = status.getGeoLocation(); if (geoLocation != null) { map.put("latitude", geoLocation.getLatitude()); map.put("longitude", geoLocation.getLongitude()); }/*from w ww. ja v a 2s . c o m*/ map.put("_time", status.getCreatedAt().getTime()); Place place = status.getPlace(); if (place != null) { map.put("country_code", place.getCountryCode()); map.put("place", place.getName()); map.put("place_type", place.getPlaceType()); map.put("place_id", place.getId()); } User user = status.getUser(); map.put("_user", user.getId()); map.put("user_lang", user.getLang()); map.put("user_created", user.getCreatedAt()); map.put("user_followers", user.getFollowersCount()); map.put("user_status_count", user.getStatusesCount()); map.put("user_verified", user.isVerified()); map.put("id", status.getId()); map.put("is_reply", status.getInReplyToUserId() > -1); map.put("is_retweet", status.isRetweet()); map.put("has_media", status.getMediaEntities().length > 0); map.put("urls", Arrays.stream(status.getURLEntities()).map(URLEntity::getText).collect(Collectors.toList())); map.put("hashtags", Arrays.stream(status.getHashtagEntities()).map(HashtagEntity::getText) .collect(Collectors.toList())); map.put("user_mentions", Arrays.stream(status.getUserMentionEntities()).map(UserMentionEntity::getText) .collect(Collectors.toList())); map.put("language", "und".equals(status.getLang()) ? null : status.getLang()); map.put("is_positive", classifier.isPositive(status.getText())); Event event = new Event().properties(map).collection(collection); buffer.add(event); commitIfNecessary(); }
From source file:jp.wda.azeda.core.daemons.impl.RetweetsCrawlerImpl.java
License:Apache License
/** * *///from w ww . ja v a2 s. c o m @Aspect("aop.requiredTx") public void crawl() throws TwitterException { log.debug("crawl retweets!!" + tweets.size()); if (tweets.size() == 0) { loadTweets(); } if (tweets.size() == 0) { return; } RetweetedTweet tweet = tweets.remove(0); log.debug("check tweet " + tweet.getTweetID()); Twitter twitter = new TwitterFactory().getInstance(); AccessToken accessToken = new AccessToken(tweet.getAccessToken().getAccessToken(), tweet.getAccessToken().getAccessTokenSecret()); twitter.setOAuthAccessToken(accessToken); ResponseList<Status> rt = getRetweets(twitter, tweet.getTweetID()); if (!alive) { return; } if (rt == null || rt.size() == 0) { return; } HashMap<Long, Status> retweets = new HashMap<Long, Status>(); for (Status status : rt) { retweets.put(status.getUser().getId(), status); } List<Retweeter> registered = retweetersDao.getRetweeterIDs(tweet.getTweetID()); for (Retweeter retweeter : registered) { long registeredUserID = retweeter.getRetweeterID(); if (retweets.containsKey(registeredUserID)) { retweets.remove(registeredUserID); continue; } } if (retweets.size() == 0) { return; } for (Status status : retweets.values()) { long retweeter = status.getUser().getId(); long oldest = System.currentTimeMillis() - retweetExpired * 60 * 1000L; if (status.getCreatedAt().getTime() < oldest) { log.info(tweet.getTweetID() + " retweet by " + status.getUser().getScreenName() + " ? " + retweetExpired + "?????????"); insertRetweeter(twitter.getId(), tweet.getTweetID(), retweeter); continue; } List<Status> userTimeline = loadRetweetersTweets(twitter, tweet.getTweetID(), retweeter, 1); if (!alive) { return; } if (userTimeline == null) { continue; } int skips = 0; for (int i = userTimeline.size() - 1; i >= 0; i--) { Status s = userTimeline.get(i); UserMentionEntity[] mentions = s.getUserMentionEntities(); if (s.isRetweet()) { skips++; continue; } if (mentions == null || mentions.length > 0) { skips++; continue; } log.debug(s.getId() + ":" + s.getCreatedAt() + "(" + s.isRetweet() + ") " + s.getText()); Retweet retweet = new Retweet(); retweet.setTweetID(s.getId()); retweet.setRetweetTo(tweet.getTweetID()); retweet.setRetweeterID(retweeter); retweet.setTweettext(s.getText()); retweet.setCreatedat(new Timestamp(s.getCreatedAt().getTime())); retweet.setSkiptweets(skips); retweet.setScreenName(status.getUser().getScreenName()); dtoRetweets.add(retweet); insertRetweeter(twitter.getId(), tweet.getTweetID(), retweeter); break; } } register(); }
From source file:jp.wda.azeda.core.daemons.impl.RetweetsCrawlerImpl.java
License:Apache License
/** * * @param twitter// w ww.j a v a2s . c o m * @param retweetTo * @param retweeter * @param page * @return * @throws TwitterException */ private List<Status> loadRetweetersTweets(Twitter twitter, long retweetTo, long retweeter, int page) throws TwitterException { if (!alive) { return null; } if (page > timelinePages) { log.info(retweetTo + " retweet by " + retweeter + " ? " + (timelinePages * 100) + "????????????"); insertRetweeter(twitter.getId(), retweetTo, retweeter); return null; } Paging paging = new Paging(retweetTo); paging.setPage(page); paging.setCount(100); ResponseList<Status> userTimeline = gtUserTimeline(twitter, retweeter, paging); if (userTimeline.size() == 0) { return null; } List<Status> result = new ArrayList<Status>(); for (Status s : userTimeline) { result.add(s); if (s.isRetweet()) { if (s.getRetweetedStatus().getId() != retweetTo) { continue; } log.debug("found!!"); return result; } } return loadRetweetersTweets(twitter, retweetTo, retweeter, page + 1); }
From source file:my.twittergui.TwitterUI.java
private void jButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton2ActionPerformed String text;/*from w w w .j av a 2 s .com*/ text = jTextField1.getText(); //jTextArea1.append(text+"\n"); jTextArea1.append("Searching for: "); String[] strarray = text.split(" "); for (int i = 0; i < strarray.length; i++) { jTextArea1.append(strarray[i] + "\n"); } File filed = null; filed = new File("C:\\Results"); if (!filed.exists()) { filed.mkdir(); } ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("").setOAuthConsumerSecret("").setOAuthAccessToken("") .setOAuthAccessTokenSecret(""); StatusListener listener = new StatusListener() { public void onStatus(Status status) { try { int fileno = 0; String strI = Integer.toString(fileno); String fname = "C:\\results\\DataCol" + strI + ".csv"; File file = new File(fname); FileWriter bw = new FileWriter(file, true); if (file.length() == 0) { bw.write("\"Screen Name\", text, \"created at\", geolocation, Retweet? \r\n"); bw.flush(); } while (file.length() > 10485760) { fileno += 1; strI = Integer.toString(fileno); fname = "DataCol" + strI + ".txt"; file = new File(fname); bw = new FileWriter(file, true); bw.write("\"Screen Name\", \"text\", \"created_at\", \"geolocation\" \r\n"); bw.flush(); } // if(!status.isRetweet()){ bw.write("\r\n"); bw.write("\"" + status.getUser().getScreenName() + "\","); String tweettxt = status.getText(); tweettxt = tweettxt.replace("\n", ""); tweettxt = tweettxt.replace(",", ""); tweettxt = tweettxt.replace("\"", ""); bw.write("\"" + tweettxt + "\","); bw.write("\"" + status.getCreatedAt() + "\","); if (status.getGeoLocation() != null) { bw.write("\"" + status.getGeoLocation() + "\""); } else bw.write("N/A,"); if (status.isRetweet()) bw.write("Yes"); else bw.write("No"); bw.flush(); bw.close(); //System.out.print("\n"); String str = "@" + status.getUser().getScreenName() + " " + status.getText() + " " + status.getCreatedAt() + " "; if (status.getGeoLocation() != null) str += status.getGeoLocation(); //System.out.print(str); jTextArea1.append(str + "\n"); // } bw.close(); } catch (IOException e) { //System.out.print("EXCEPTION"); e.printStackTrace(); } } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { } public void onStallWarning(StallWarning stall) { } public void onScrubGeo(long a, long b) { } public void onException(Exception ex) { ex.printStackTrace(); } }; TwitterStreamFactory tf = new TwitterStreamFactory(cb.build()); TwitterStream twitterStream = tf.getInstance(); twitterStream.addListener(listener); // Filter FilterQuery filtre = new FilterQuery(); Scanner in = new Scanner(System.in); filtre.track(strarray); twitterStream.filter(filtre); }
From source file:net.lacolaco.smileessence.activity.MainActivity.java
License:Open Source License
public void startNewSearch(final Twitter twitter, final String query) { setLastSearch(query);//from w ww .j av a 2s.c om if (!TextUtils.isEmpty(query)) { final SearchListAdapter adapter = (SearchListAdapter) getListAdapter(ADAPTER_SEARCH); adapter.initSearch(query); adapter.clear(); adapter.updateForce(); new SearchTask(twitter, query, this) { @Override protected void onPostExecute(QueryResult queryResult) { super.onPostExecute(queryResult); if (queryResult != null) { List<twitter4j.Status> tweets = queryResult.getTweets(); for (int i = tweets.size() - 1; i >= 0; i--) { twitter4j.Status status = tweets.get(i); if (!status.isRetweet()) { StatusViewModel viewModel = new StatusViewModel(status, getCurrentAccount()); adapter.addToTop(viewModel); StatusFilter.filter(MainActivity.this, viewModel); } } adapter.setTopID(queryResult.getMaxId()); adapter.updateForce(); } } }.execute(); } }