List of usage examples for twitter4j User getFriendsCount
int getFriendsCount();
From source file:demo.UserInfo.java
License:Apache License
public static void main(String[] args) throws IOException, TwitterException { //?//from w w w. jav a 2 s .c o m Configuration configuration = new ConfigurationBuilder().setOAuthConsumerKey(CONSUMER_KEY) .setOAuthConsumerSecret(CONSUMER_SECRET).setOAuthAccessToken(ACCESS_TOKEN) .setOAuthAccessTokenSecret(ACCESS_TOKEN_SECRET).build(); Twitter tw = new TwitterFactory(configuration).getInstance(); String screenName = ""; BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); System.out.print("????ScreenName???????!! ex)masason : "); screenName = br.readLine(); //String screenName = "masason";//masason try { //?& User user = tw.showUser(screenName); System.out.println("???"); System.out.println("User ID : " + user.getId()); System.out.println("ScreenName : " + user.getScreenName()); System.out.println("User's Name : " + user.getName()); System.out.println("Number of Followers : " + user.getFollowersCount()); System.out.println("Number of Friends : " + user.getFriendsCount()); System.out.println("Language : " + user.getLang()); //? Status status = user.getStatus(); System.out.println("???"); System.out.println("User Created : " + status.getCreatedAt()); System.out.println("Status ID : " + status.getId()); System.out.println(status.getSource()); System.out.println("Tweet" + status.getText()); } catch (Exception e) { e.printStackTrace(); } }
From source file:druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Open Source License
@Override public Firehose connect() throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override/*from w ww. j a va2 s . c o m*/ public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final LinkedList<String> dimensions = new LinkedList<String>(); final long startMsec = System.currentTimeMillis(); dimensions.add("htags"); dimensions.add("lang"); dimensions.add("utc_offset"); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (maxEventCount < 0L); private final Map<String, Object> theMap = new HashMap<String, Object>(2); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (maxRunMinutes <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= maxRunMinutes; } } private boolean maxCountReached() { return maxEventCount >= 0 && rowCount >= maxEventCount; } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } HashtagEntity[] hts = status.getHashtagEntities(); if (hts != null && hts.length > 0) { List<String> hashTags = Lists.newArrayListWithExpectedSize(hts.length); for (HashtagEntity ht : hts) { hashTags.add(ht.getText()); } theMap.put("htags", Arrays.asList(hashTags.get(0))); } long retweetCount = status.getRetweetCount(); theMap.put("retweet_count", retweetCount); User user = status.getUser(); if (user != null) { theMap.put("follower_count", user.getFollowersCount()); theMap.put("friends_count", user.getFriendsCount()); theMap.put("lang", user.getLang()); theMap.put("utc_offset", user.getUtcOffset()); // resolution in seconds, -1 if not available? theMap.put("statuses_count", user.getStatusesCount()); } return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:edu.uci.ics.asterix.external.util.TweetProcessor.java
License:Apache License
public AMutableRecord processNextTweet(Status tweet) { User user = tweet.getUser(); ((AMutableString) mutableUserFields[0]).setValue(getNormalizedString(user.getScreenName())); ((AMutableString) mutableUserFields[1]).setValue(getNormalizedString(user.getLang())); ((AMutableInt32) mutableUserFields[2]).setValue(user.getFriendsCount()); ((AMutableInt32) mutableUserFields[3]).setValue(user.getStatusesCount()); ((AMutableString) mutableUserFields[4]).setValue(getNormalizedString(user.getName())); ((AMutableInt32) mutableUserFields[5]).setValue(user.getFollowersCount()); ((AMutableString) mutableTweetFields[0]).setValue(tweet.getId() + ""); for (int i = 0; i < 6; i++) { ((AMutableRecord) mutableTweetFields[1]).setValueAtPos(i, mutableUserFields[i]); }// w w w. ja va2 s.c o m if (tweet.getGeoLocation() != null) { ((AMutableDouble) mutableTweetFields[2]).setValue(tweet.getGeoLocation().getLatitude()); ((AMutableDouble) mutableTweetFields[3]).setValue(tweet.getGeoLocation().getLongitude()); } else { ((AMutableDouble) mutableTweetFields[2]).setValue(0); ((AMutableDouble) mutableTweetFields[3]).setValue(0); } ((AMutableString) mutableTweetFields[4]).setValue(getNormalizedString(tweet.getCreatedAt().toString())); ((AMutableString) mutableTweetFields[5]).setValue(getNormalizedString(tweet.getText())); for (int i = 0; i < 6; i++) { mutableRecord.setValueAtPos(i, mutableTweetFields[i]); } return mutableRecord; }
From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser) throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override/*w w w .j a va 2 s . c om*/ public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(String.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:net.lacolaco.smileessence.view.dialog.UserDetailDialogFragment.java
License:Open Source License
private void initUserData(User user, final Account account) { textViewName.setText(user.getName()); textViewScreenName.setText(user.getScreenName()); if (TextUtils.isEmpty(user.getLocation())) { textViewLocate.setVisibility(View.GONE); } else {//from ww w .j ava 2s . c o m textViewLocate.setText(user.getLocation()); } if (TextUtils.isEmpty(user.getURL())) { textViewURL.setVisibility(View.GONE); } else { textViewURL.setText(user.getURL()); } textViewTweetCount.setText(String.valueOf(user.getStatusesCount())); textViewFriendCount.setText(String.valueOf(user.getFriendsCount())); textViewFollowerCount.setText(String.valueOf(user.getFollowersCount())); textViewFavoriteCount.setText(String.valueOf(user.getFavouritesCount())); textViewProtected.setVisibility(user.isProtected() ? View.VISIBLE : View.GONE); String htmlDescription = getHtmlDescription(user.getDescription()); textViewDescription.setText(Html.fromHtml(htmlDescription)); textViewDescription.setMovementMethod(LinkMovementMethod.getInstance()); ImageCache.getInstance().setImageToView(user.getBiggerProfileImageURL(), imageViewIcon); ImageCache.getInstance().setImageToView(user.getProfileBannerURL(), imageViewHeader); MainActivity activity = (MainActivity) getActivity(); final StatusListAdapter adapter = new StatusListAdapter(activity); listViewTimeline.setAdapter(adapter); listViewTimeline.setOnRefreshListener(this); activity.setListAdapter(ADAPTER_INDEX, adapter); executeUserTimelineTask(user, account, adapter); updateRelationship(activity, user.getId()); }
From source file:net.lacolaco.smileessence.viewmodel.UserViewModel.java
License:Open Source License
public UserViewModel(User user) { id = user.getId();/*from ww w. j a v a 2 s . c o m*/ screenName = user.getScreenName(); name = user.getName(); description = user.getDescription(); location = user.getLocation(); url = user.getURL(); iconURL = user.getBiggerProfileImageURL(); bannerURL = user.getProfileBannerURL(); statusesCount = user.getStatusesCount(); friendsCount = user.getFriendsCount(); followersCount = user.getFollowersCount(); favoritesCount = user.getFavouritesCount(); isProtected = user.isProtected(); isVerified = user.isVerified(); }
From source file:org.apache.asterix.external.parser.TweetParser.java
License:Apache License
@Override public void parse(IRawRecord<? extends Status> record, DataOutput out) throws HyracksDataException { Status tweet = record.get();//from ww w . j av a2 s . co m User user = tweet.getUser(); // Tweet user data ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.SCREEN_NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getScreenName())); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.LANGUAGE)]) .setValue(JObjectUtil.getNormalizedString(user.getLang())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FRIENDS_COUNT)]) .setValue(user.getFriendsCount()); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.STATUS_COUNT)]) .setValue(user.getStatusesCount()); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getName())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FOLLOWERS_COUNT)]) .setValue(user.getFollowersCount()); // Tweet data ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.ID)]) .setValue(String.valueOf(tweet.getId())); int userPos = tweetFieldNameMap.get(Tweet.USER); for (int i = 0; i < mutableUserFields.length; i++) { ((AMutableRecord) mutableTweetFields[userPos]).setValueAtPos(i, mutableUserFields[i]); } if (tweet.getGeoLocation() != null) { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]) .setValue(tweet.getGeoLocation().getLatitude()); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]) .setValue(tweet.getGeoLocation().getLongitude()); } else { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(0); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(0); } ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.CREATED_AT)]) .setValue(JObjectUtil.getNormalizedString(tweet.getCreatedAt().toString())); ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.MESSAGE)]) .setValue(JObjectUtil.getNormalizedString(tweet.getText())); for (int i = 0; i < mutableTweetFields.length; i++) { mutableRecord.setValueAtPos(i, mutableTweetFields[i]); } recordBuilder.reset(mutableRecord.getType()); recordBuilder.init(); IDataParser.writeRecord(mutableRecord, out, recordBuilder); }
From source file:org.apache.asterix.external.util.TweetProcessor.java
License:Apache License
public AMutableRecord processNextTweet(Status tweet) { User user = tweet.getUser(); // Tweet user data ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.SCREEN_NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getScreenName())); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.LANGUAGE)]) .setValue(JObjectUtil.getNormalizedString(user.getLang())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FRIENDS_COUNT)]) .setValue(user.getFriendsCount()); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.STATUS_COUNT)]) .setValue(user.getStatusesCount()); ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.NAME)]) .setValue(JObjectUtil.getNormalizedString(user.getName())); ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FOLLOWERS_COUNT)]) .setValue(user.getFollowersCount()); // Tweet data ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.ID)]) .setValue(String.valueOf(tweet.getId())); int userPos = tweetFieldNameMap.get(Tweet.USER); for (int i = 0; i < mutableUserFields.length; i++) { ((AMutableRecord) mutableTweetFields[userPos]).setValueAtPos(i, mutableUserFields[i]); }//from w w w . j a v a 2 s. c o m if (tweet.getGeoLocation() != null) { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]) .setValue(tweet.getGeoLocation().getLatitude()); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]) .setValue(tweet.getGeoLocation().getLongitude()); } else { ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(0); ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(0); } ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.CREATED_AT)]) .setValue(JObjectUtil.getNormalizedString(tweet.getCreatedAt().toString())); ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.MESSAGE)]) .setValue(JObjectUtil.getNormalizedString(tweet.getText())); for (int i = 0; i < mutableTweetFields.length; i++) { mutableRecord.setValueAtPos(i, mutableTweetFields[i]); } return mutableRecord; }
From source file:org.apache.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser, File temporaryDirectory) { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override// ww w . j a v a2s . co m public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { log.error(ex, "Got exception"); } @Override public void onStallWarning(StallWarning warning) { log.warn("Got stall warning: %s", warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { @Override public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Nullable @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); Thread.sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(StringUtils.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? StringUtils.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:org.apache.flume.sink.solr.morphline.TwitterSource.java
License:Apache License
private Record extractRecord(String idPrefix, Schema avroSchema, Status status) { User user = status.getUser(); Record doc = new Record(avroSchema); doc.put("id", idPrefix + status.getId()); doc.put("created_at", formatterTo.format(status.getCreatedAt())); doc.put("retweet_count", status.getRetweetCount()); doc.put("retweeted", status.isRetweet()); doc.put("in_reply_to_user_id", status.getInReplyToUserId()); doc.put("in_reply_to_status_id", status.getInReplyToStatusId()); addString(doc, "source", status.getSource()); addString(doc, "text", status.getText()); MediaEntity[] mediaEntities = status.getMediaEntities(); if (mediaEntities.length > 0) { addString(doc, "media_url_https", mediaEntities[0].getMediaURLHttps()); addString(doc, "expanded_url", mediaEntities[0].getExpandedURL()); }/*from w ww .j a v a 2 s. c o m*/ doc.put("user_friends_count", user.getFriendsCount()); doc.put("user_statuses_count", user.getStatusesCount()); doc.put("user_followers_count", user.getFollowersCount()); addString(doc, "user_location", user.getLocation()); addString(doc, "user_description", user.getDescription()); addString(doc, "user_screen_name", user.getScreenName()); addString(doc, "user_name", user.getName()); return doc; }