List of usage examples for twitter4j Status getUser
User getUser();
From source file:gh.polyu.user.TrackUsers.java
License:Apache License
public void track(final int no, final int p) { final TwitterDBHandle handle = new TwitterDBHandle(); handle.intialTwitterDBhandle();//www.ja v a2 s . c o m while (alive) { alive = false; StatusListener listener = new StatusListener() { ArrayList<_TweetLink> listlink = new ArrayList<_TweetLink>(); int cnt = 0; String oldmonth = "20138"; String table = "UserTweet20138"; String oldday = ""; String currentday = ""; String currentmonth = ""; long lastinsert = 0l; long nowinsert = 0l; int newday = 0; String newtime = ""; @Override public void onStatus(Status status) { if (status.getId() == 123 && status.getText().equals("YOU are WORNG!.")) { System.out.println("Connection Need to be rebuilt!!"); alive = true; } else if (status.getLang().equals("en")) { _TweetLink tweet = new _TweetLink(); String Test = status.getText(); tweet.setText(Test); Date time = status.getCreatedAt(); tweet.setTime(time); tweet.setUserName(status.getUser().getName()); HashtagEntity[] hashtagentity = status.getHashtagEntities(); StringBuffer hashen = new StringBuffer(); for (int i = 0; i < hashtagentity.length; i++) { hashen.append(hashtagentity[i].getText()); hashen.append(";"); } tweet.setHashtag(hashen.toString()); URLEntity[] URLEn = status.getURLEntities(); StringBuffer URL = new StringBuffer(); for (int i = 0; i < URLEn.length; i++) { URL.append(URLEn[i].getURL()); URL.append(";"); } tweet.setURL(URL.toString()); //user mention UserMentionEntity[] userEn = status.getUserMentionEntities(); StringBuffer mentuser = new StringBuffer(); for (int i = 0; i < userEn.length; i++) { mentuser.append(userEn[i].getId()); mentuser.append(";"); } tweet.setUerMention(mentuser.toString()); //if(mentuser.length()!=0); //System.out.println("mentuser: "+ mentuser); //tweetID tweet.setTweetID(status.getId()); //if(ID!=null) // original twitterID tweet.setOriginID(status.getInReplyToStatusId()); //original user ID tweet.setOriginUser(status.getInReplyToUserId()); // user ID User users = status.getUser(); tweet.setTweetUser(users.getId()); //places Place Pl = status.getPlace(); String place = ""; if (Pl != null) { place = Pl.getFullName(); //System.out.println("place "+place); } tweet.setPlace(place); // Retweetcoun long num = 0; if (status.getRetweetedStatus() != null) { num = status.getRetweetedStatus().getRetweetCount(); //System.out.println("retweetcount"+num); tweet.setRetweetCount(num); tweet.setRetweet(1); } else { tweet.setRetweetCount(0); tweet.setRetweet(0); } // if(Retweet!=null) //System.out.println("Retweetcount: "+ Retweet); //isfavourate boolean favourate = status.isFavorited(); /*if(favourate) { fav = 1; tweet.setFavourate(fav); fav =0; System.out.println("isf "+ fav); }*/ // is retweet //String other = status.toString(); // tweet.setOther(other); listlink.add(tweet); Calendar cal = Calendar.getInstance(); int year = cal.get(Calendar.YEAR); int month = cal.get(Calendar.MONTH) + 1; int day = cal.get(Calendar.DAY_OF_MONTH); currentmonth = String.valueOf(year) + String.valueOf(month); currentday = String.valueOf(day); if (currentmonth.equals(oldmonth)) { if (currentday.equals(oldday)) ; else { newday = 1; SimpleDateFormat formatter = new SimpleDateFormat("MMddHH:mm:ss "); Date curDate = new Date(System.currentTimeMillis());// newtime = formatter.format(curDate); } } else { try { handle.database_connection(); table = "UserTweet" + String.valueOf(year) + String.valueOf(month); System.out.println("create new table " + table); String CREATE_TABLE = "create table " + table + "(TweetID varchar(100), UserName varchar(200), TwitterUser varchar(145), OriginID varchar(100), OriginUser varchar(100), place varchar(100), RetweetCount varchar(100), isRetweet int(5), Text varchar(500), Time datetime," + "Hashtag varchar(200), URL varchar(200), UerMention varchar(200))"; Statement st = handle.conn.createStatement(); st.execute(CREATE_TABLE); String Create_Index = "alter table " + table + " add index time (Time)"; st.execute(Create_Index); String Create_Index2 = "alter table " + table + " add index userID (TwitterUser)"; st.execute(Create_Index2); String key = "ALTER TABLE " + table + " ADD PRIMARY KEY (TweetID)"; st.execute(key); } catch (SQLException e) { e.printStackTrace(); } handle.close_databasehandle(); try { TwitterDBHandle handle2 = new TwitterDBHandle(); handle2.intialTwitterDBhandle2(); handle2.database_connection(); table = "UserTweet" + String.valueOf(year) + String.valueOf(month); System.out.println("create new table " + table); String CREATE_TABLE = "create table " + table + "(TweetID varchar(100), UserName varchar(200), TwitterUser varchar(145), OriginID varchar(100), OriginUser varchar(100), place varchar(100), RetweetCount varchar(100), isRetweet int(5), Text varchar(500), Time datetime," + "Hashtag varchar(200), URL varchar(200), UerMention varchar(200))"; Statement st = handle2.conn.createStatement(); st.execute(CREATE_TABLE); String Create_Index = "alter table " + table + " add index time (Time)"; st.execute(Create_Index); String Create_Index2 = "alter table " + table + " add index userID (TwitterUser)"; st.execute(Create_Index2); String key = "ALTER TABLE " + table + " ADD PRIMARY KEY (TweetID)"; st.execute(key); handle2.close_databasehandle(); } catch (SQLException e) { e.printStackTrace(); } } //System.out.println("OTHER: "+ other); if ((cnt++) % 1000 == 0) { if (newday == 1) { newday = 0; oldday = currentday; GmailSend gs = new GmailSend("cscchenyoyo@gmail.com", "910316ccy"); gs.send("THREAD" + p + " :" + "program no" + no + "message" + newtime, "I am still alive"); newtime = ""; } try { handle.database_connection(); handle.userTweet(table, listlink); nowinsert = System.currentTimeMillis(); System.err.println( "No: " + no + "program " + "totally " + cnt + " tweets downloaded!\n" + new Date(nowinsert) + " " + new Date(lastinsert)); lastinsert = nowinsert; nowinsert = 0l; handle.close_databasehandle(); } catch (SQLException e) { handle.close_databasehandle(); e.printStackTrace(); // TODO Auto-generated catch block TwitterDBHandle handle2 = new TwitterDBHandle(); handle2.intialTwitterDBhandle2(); handle2.database_connection(); try { handle2.userTweet(table, listlink); nowinsert = System.currentTimeMillis(); System.err.println("New Database No: " + no + "program " + "totally " + cnt + " tweets downloaded!\n" + new Date(nowinsert) + new Date(lastinsert)); lastinsert = nowinsert; nowinsert = 0l; handle2.close_databasehandle(); } catch (SQLException e1) { // TODO Auto-generated catch block GmailSend gs = new GmailSend("cscchenyoyo@gmail.com", "910316ccy"); try { gs.SendSSLMessage("cscchenyoyo@gmail.com", "program error", "both databases are down"); } catch (MessagingException ee) { // TODO Auto-generated catch block e.printStackTrace(); } } GmailSend gs = new GmailSend("cscchenyoyo@gmail.com", "910316ccy"); try { gs.SendSSLMessage("cscchenyoyo@gmail.com", "program error", "change database to another one"); } catch (MessagingException ee) { // TODO Auto-generated catch block e.printStackTrace(); } } listlink.clear(); } } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { //System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { //System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); twitterOAuth twtOauth = new twitterOAuth(); twtOauth.AuthoritywithS(twitterStream, key); twitterStream.addListener(listener); twitterStream.filter(new FilterQuery(0, follow)); /* try { Thread.sleep(3000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); }*/ } }
From source file:gohai.simpletweet.SimpleTweet.java
License:Apache License
/** * Send a tweet//from w ww . ja v a 2 s.c om * @param message message to send * @return partial URL of the tweet (e.g. "mrgohai/status/841086396194529280") */ public String tweet(String message) { try { StatusUpdate update = new StatusUpdate(message); Status status = twitter.updateStatus(update); User user = status.getUser(); return user.getScreenName() + "/status/" + status.getId(); } catch (TwitterException e) { System.err.println(e.getMessage()); throw new RuntimeException("Error posting tweet"); } }
From source file:gohai.simpletweet.SimpleTweet.java
License:Apache License
/** * Tweet an image/*from w ww.ja v a 2 s.c o m*/ * @param img PImage instance to attach * @param update twitter4j StatusUpdate instance to use * @return partial URL of the tweet (e.g. "mrgohai/status/841086396194529280") */ public String tweetImage(PImage img, StatusUpdate update) { File temp; // write image into temporary file that gets deleted when the VM exits try { // make sure one pixel is somewhat transparent // to prevent Twitter from recompressing the image // to JPEG PImage copy = new PImage(img.width, img.height, PImage.ARGB); copy.copy(img, 0, 0, img.width, img.height, 0, 0, img.width, img.height); copy.set(0, 0, 254 << 24 | (copy.get(0, 0) & 0xffffff)); temp = File.createTempFile("out", ".png"); temp.deleteOnExit(); copy.save(temp.getCanonicalPath()); } catch (IOException e) { throw new RuntimeException("Error exporting image to PNG"); } try { update.setMedia(temp); Status status = twitter.updateStatus(update); temp.delete(); User user = status.getUser(); return user.getScreenName() + "/status/" + status.getId(); } catch (TwitterException e) { System.err.println(e.getMessage()); throw new RuntimeException("Error posting tweet"); } }
From source file:gov.nasa.jpl.memex.elwha.impl.ElwhaResource.java
License:Apache License
/** * Main entry of this application.//w w w . j a va 2 s. c om * * @param userIds follow(comma separated user ids) track(comma separated filter terms) * @throws TwitterException when Twitter service or network is unavailable * @return */ @Override public Elwha get(String userIds) { //return new Elwha().setMessage("Hello, Rest.li!"); // } //if (userIds.length < 1) { // System.out.println("Usage: java twitter4j.examples.PrintFilterStream [follow(comma separated numerical user ids)] [track(comma separated filter terms)]"); // System.exit(-1); //} StatusListener listener = new StatusListener() { @Override public void onStatus(Status status) { System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText()); } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addListener(listener); ArrayList<Long> follow = new ArrayList<Long>(); ArrayList<String> track = new ArrayList<String>(); if (isNumericalArgument(userIds)) { for (String id : userIds.split(",")) { follow.add(Long.parseLong(id)); } } else { track.addAll(Arrays.asList(userIds.split(","))); } long[] followArray = new long[follow.size()]; for (int i = 0; i < follow.size(); i++) { followArray[i] = follow.get(i); } String[] trackArray = track.toArray(new String[track.size()]); // filter() method internally creates a thread which manipulates TwitterStream and calls these adequate listener methods continuously. twitterStream.filter(new FilterQuery(0, followArray, trackArray)); return new Elwha().setUserIds("Mock stub!"); }
From source file:Group5.Model.java
public static List GetTimeline() throws TwitterException { //Your Twitter App's Consumer Key String consumerKey = "wK7lQLpl3t8xvIABqpgoJzYYd"; //Your Twitter App's Consumer Secret String consumerSecret = "4M5TgmNfS0EKeaSqna8eHTNaNi970Plq3dynX5gvYsh848j0mj"; //Your Twitter Access Token String accessToken = "829891753473892361-7jkKyXLYc6HOStzCPGjWOnVoAVNU7cd"; //Your Twitter Access Token Secret String accessTokenSecret = "ATidrzRzhVqAamuMbYiskcHBPSisB9MWsCsYYY2Bec4y9"; //Instantiate a re-usable and thread-safe factory TwitterFactory twitterFactory = new TwitterFactory(); //Instantiate a new Twitter instance Twitter twitter = twitterFactory.getInstance(); //setup OAuth Consumer Credentials twitter.setOAuthConsumer(consumerKey, consumerSecret); //setup OAuth Access Token twitter.setOAuthAccessToken(new AccessToken(accessToken, accessTokenSecret)); List<Status> statuses = twitter.getHomeTimeline(); System.out.println("Showing home timeline."); for (Status status : statuses) { System.out.println(status.getUser().getName() + ":" + status.getText()); }//from ww w . j a v a 2 s . co m return statuses; }
From source file:gui.project2.v1.FXMLDocumentController.java
@FXML public void searchloaction() throws InterruptedException { int maxFollowerCount = 0; userslocations = new ArrayList<>(); nodes = new ArrayList<>(); GraphicsContext gc = graph.getGraphicsContext2D(); gc.setFill(Color.GAINSBORO);// w w w . j av a 2 s . com gc.fillRect(0, 0, 308, 308); Twitter twitter; twitter = tf.getInstance(); ArrayList<User> users = new ArrayList<>(); try { Query query = new Query(""); GeoLocation location; location = new GeoLocation(parseDouble(latitude.getText()), parseDouble(longitude.getText())); Query.Unit unit = Query.KILOMETERS; query.setGeoCode(location, parseDouble(radius.getText()), unit); QueryResult result; do { result = twitter.search(query); List<Status> tweets = result.getTweets(); for (Status tweet : tweets) { System.out.println("@" + tweet.getUser().getScreenName() + " - " + tweet.getText()); boolean q = false; if (userslocations != null && !userslocations.isEmpty()) { for (int i = 0; i < userslocations.size(); i++) { if (userslocations.get(i).getName().equals(tweet.getUser().getScreenName())) { q = true; break; } } } if (!q && tweet.getGeoLocation() != null) { pair n; String latString = ""; String lonString = ""; int la = 0; int lo = 0; String geoString = tweet.getGeoLocation().toString(); for (int i = 0; i < geoString.length(); i++) { if (geoString.charAt(i) == '=') { if (la == 0) { la = 1; } else if (la == -1) { lo = 1; } } else if (geoString.charAt(i) == ',') { la = -1; } else if (geoString.charAt(i) == '}') { lo = -1; } else if (la == 1) { latString = latString + geoString.charAt(i); } else if (lo == 1) { lonString = lonString + geoString.charAt(i); } } User thisUser; thisUser = tweet.getUser(); double lat = parseDouble(latString); double lon = parseDouble(lonString); System.out.println(tweet.getGeoLocation().toString()); n = new pair(tweet.getUser().getScreenName(), lat, lon); userslocations.add(n); users.add(thisUser); if (thisUser.getFollowersCount() > maxFollowerCount) { maxFollowerCount = thisUser.getFollowersCount(); } } } } while ((query = result.nextQuery()) != null); for (int i = 0; i < users.size(); i++) { if (i % 14 == 0 && i != 0) { Thread.sleep(1000 * 60 * 15 + 30); } IDs friends; friends = twitter.getFriendsIDs(users.get(i).getId(), -1); for (long j : friends.getIDs()) { for (int k = i + 1; k < users.size(); k++) { if (users.get(k).getId() == j) { nodes.add(users.get(i).getScreenName() + ":" + users.get(k).getScreenName()); } } } } } catch (TwitterException te) { System.out.println("Failed to search tweets: " + te.getMessage()); System.exit(-1); } double xmin; double xmax; double ymin; double ymax; xmin = userslocations.get(0).getA(); xmax = userslocations.get(0).getA(); ymin = userslocations.get(0).getB(); ymax = userslocations.get(0).getB(); for (int i = 1; i < userslocations.size(); i++) { if (xmin > userslocations.get(i).getA()) { xmin = userslocations.get(i).getA(); } if (xmax < userslocations.get(i).getA()) { xmax = userslocations.get(i).getA(); } if (ymin > userslocations.get(i).getB()) { ymin = userslocations.get(i).getB(); } if (ymax < userslocations.get(i).getB()) { ymax = userslocations.get(i).getB(); } } for (int i = 0; i < userslocations.size(); i++) { if (userslocations.get(i).getA() - xmin >= 0 && userslocations.get(i).getB() - ymin >= 0) { gc.setLineWidth(users.get(i).getFollowersCount() / maxFollowerCount * 3 + 1); gc.strokeOval((userslocations.get(i).getA() - xmin) / (xmax - xmin) * 300 + 4, (userslocations.get(i).getB() - ymin) / (ymax - ymin) * 300 + 4, 4, 4); } } ObservableList<String> usersLeftList = FXCollections.observableArrayList(); for (int i = 0; i < users.size() - 1; i++) { User k = null; for (int j = i + 1; j < users.size(); j++) { if (users.get(j).getFollowersCount() > users.get(i).getFollowersCount()) { k = users.get(i); users.set(i, users.get(j)); users.set(j, k); } } } for (int i = 0; i < users.size() / 5; i++) { usersLeftList.add(users.get(i).getScreenName() + " " + users.get(i).getFollowersCount()); } listView.setItems(usersLeftList); gc.setLineWidth(1); gc.setFill(Color.BLUE); for (int i = 0; i < nodes.size(); i++) { String user1 = ""; String user2 = ""; int p = 0; double x1 = 0; double x2 = 0; double y1 = 0; double y2 = 0; for (int j = 0; j < nodes.get(i).length(); j++) { if (nodes.get(i).charAt(j) == ':') { p = 1; } else if (p == 0) { user1 = user1 + nodes.get(i).charAt(j); } else if (p == 1) { user2 = user2 + nodes.get(i).charAt(j); } } for (int j = 0; j < userslocations.size(); j++) { if (userslocations.get(j).getName().equals(user1)) { x1 = (userslocations.get(j).getA() - xmin) / (xmax - xmin) * 300 + 6; y1 = (userslocations.get(j).getB() - ymin) / (ymax - ymin) * 300 + 6; } else if (userslocations.get(j).getName().equals(user2)) { x2 = (userslocations.get(j).getA() - xmin) / (xmax - xmin) * 300 + 6; y2 = (userslocations.get(j).getB() - ymin) / (ymax - ymin) * 300 + 6; } } gc.strokeLine(x1, y1, x2, y2); gc.fillOval(x1 - 2, y1 - 2, 4, 4); gc.fillOval(x2 - 2, y2 - 2, 4, 4); } }
From source file:h2weibo.Twitter2Weibo.java
License:Open Source License
public void syncTwitter(String userId) { T2WUser user = helper.findOneByUser(userId); weibo.setToken(user.getToken(), user.getTokenSecret()); Twitter twitter = new TwitterFactory().getInstance(); if (user.getTwitterToken() != null) { twitter.setOAuthAccessToken(new AccessToken(user.getTwitterToken(), user.getTwitterTokenSecret())); log.debug(String.format("Using OAuth for %s", user.getUserId())); }/* ww w .jav a2 s. c o m*/ StatusFilters filters = new StatusFilters(); filters.use(new NoSyncFilter()); // should be used first filters.use(new TcoStatusFilter()).use(new URLStatusFilter()).use(new TagStatusFilter()) .use(new FlickrImageFilter()); NoMentionFilter mentionFilter = new NoMentionFilter(); UserMappingFilter mappingFilter = new UserMappingFilter(helper); if (!user.ready()) { log.debug(String.format("Skipping @%s ...", user.getUserId())); return; } // gets Twitter instance with default credentials String screenName = user.getUserId(); long latestId = user.getLatestId(); log.debug(String.format("Checking @%s's timeline, latest ID = %d.", userId, latestId)); try { if (latestId == 0) { List<Status> statuses = twitter.getUserTimeline(screenName); if (statuses.size() > 0) { user.setLatestId(statuses.get(0).getId()); // Record latestId, and sync next time } log.info(String.format("First time use for @%s. Set latest ID to %d.", userId, latestId)); } else { Paging paging = new Paging(latestId); List<Status> statuses = twitter.getUserTimeline(screenName, paging); // sync from the oldest one for (int i = statuses.size() - 1; i >= 0; i--) { Status status = statuses.get(i); if (status.getId() < user.getLatestId()) continue; // safe keeper String name = status.getUser().getScreenName(); String statusText = status.getText(); log.info(String.format("%s - %s", name, statusText)); try { if (status.isRetweet()) { if (user.isDropRetweets()) { user.setLatestId(status.getId()); log.debug("Skipped " + statusText + " because status is a retweet."); continue; } else { filters.remove(mentionFilter); filters.use(mappingFilter); } } else { if (user.isDropMentions()) { filters.remove(mappingFilter); filters.use(mentionFilter); } else { filters.remove(mentionFilter); filters.use(mappingFilter); } } statusText = filters.filter(statusText); if (statusText == null) { user.setLatestId(status.getId()); log.info(String.format("Skipped %s because of the filter.", statusText)); continue; } if (!user.isNoImage()) { // add twitter images to status text MediaEntity[] mediaEntities = status.getMediaEntities(); if (mediaEntities != null) { for (MediaEntity entity : mediaEntities) { statusText += " " + entity.getMediaURL(); } log.info("with media url: " + statusText); } StatusImageExtractor ex = new StatusImageExtractor(); StringBuffer buf = new StringBuffer(statusText); byte[] image = ex.extract(buf); if (image != null) { user.setLatestId(status.getId()); try { statusText = buf.toString(); // with image urls removed weibo.uploadStatus(statusText, new ImageItem(image)); log.info(String.format("@%s - %s sent with image.", name, statusText)); } catch (WeiboException e) { log.error("Faile to update image.", e); } continue; } } GeoLocation location = status.getGeoLocation(); if (user.isWithGeo() && location != null) { weibo.updateStatus(statusText, location.getLatitude(), location.getLongitude()); log.info(String.format("@%s - %s sent with geo locations.", name, statusText)); } else { weibo.updateStatus(statusText); log.info(String.format("@%s - %s sent.", name, statusText)); } } catch (WeiboException e) { if (e.getStatusCode() != 400) { // resending same tweet log.error("Failed to update Weibo", e); break; } else { log.error("Sending same message", e); } } log.info(String.format("Sent: by %s - %s", name, statusText)); user.setLatestId(status.getId()); } } helper.saveUser(user); } catch (Exception e) { if (!(e instanceof TwitterException)) { log.error("Failed to update.", e); } } }
From source file:info.usbo.skypetwitter.Run.java
License:Apache License
public static void main(String[] args) throws SkypeException, IOException { System.out.println("Working Directory = " + System.getProperty("user.dir")); Properties props = new Properties(); loadProperties(props, "twitter4j.properties"); loadProperties(props, "skype.properties"); loadProperties(props, "app.properties"); String chat_group_id = props.getProperty("skype.chat_group_id"); String twitter_user_id = props.getProperty("twitter.user"); data_dir = props.getProperty("data.dir"); Integer twitter_timeout = Integer.parseInt(props.getProperty("twitter.timeout")); System.out.println("Twitter user: " + twitter_user_id); System.out.println("Twitter timeout: " + twitter_timeout); if ("".equals(twitter_user_id)) { return;/*from www .j a v a 2 s. c om*/ } if (load_file() == 0) { System.out.println("File not found"); return; } while (true) { bChanged = 0; SimpleDateFormat sdf = new SimpleDateFormat("dd.MM.yyyy HH:mm"); System.out.println("Looking at " + sdf.format(Calendar.getInstance().getTime())); Chat ch = Skype.chat(chat_group_id); Twitter twitter = new TwitterFactory().getInstance(); try { List<Status> statuses; statuses = twitter.getUserTimeline(twitter_user_id); String sText; for (Status status : statuses) { Date d = status.getCreatedAt(); // ? Calendar cal = Calendar.getInstance(); cal.setTime(d); cal.add(Calendar.HOUR_OF_DAY, 7); d = cal.getTime(); sText = "@" + status.getUser().getScreenName() + " " + sdf.format(d) + " ( https://twitter.com/" + twitter_user_id + "/status/" + status.getId() + " ): \r\n" + status.getText() + "\r\n***"; for (URLEntity e : status.getURLEntities()) { sText = sText.replaceAll(e.getURL(), e.getExpandedURL()); } for (MediaEntity e : status.getMediaEntities()) { sText = sText.replaceAll(e.getURL(), e.getMediaURL()); } if (twitter_ids.indexOf(status.getId()) == -1) { System.out.println(sText); ch.send(sText); twitter_ids.add(status.getId()); bChanged = 1; } } } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to get timeline: " + te.getMessage()); // System.exit(-1); } catch (SkypeException ex) { ex.printStackTrace(); System.out.println("Failed to send message: " + ex.getMessage()); } // VK try { vk(); for (VK v : vk) { if (vk_ids.indexOf(v.getId()) == -1) { Date d = v.getDate(); // ? Calendar cal = Calendar.getInstance(); cal.setTime(d); cal.add(Calendar.HOUR_OF_DAY, 7); d = cal.getTime(); String sText = "@Depersonilized (VK) " + sdf.format(d) + " ( http://vk.com/Depersonilized?w=wall-0_" + v.getId() + " ): \r\n" + v.getText(); if (!"".equals(v.getAttachment())) { sText += "\r\n" + v.getAttachment(); } sText += "\r\n***"; System.out.println(sText); ch.send(sText); vk_ids.add(v.getId()); bChanged = 1; } } } catch (ParseException e) { e.printStackTrace(); System.out.println("Failed to get vk: " + e.getMessage()); // System.exit(-1); } catch (SkypeException ex) { ex.printStackTrace(); System.out.println("Failed to send message: " + ex.getMessage()); } if (bChanged == 1) { save_file(); } try { Thread.sleep(1000 * 60 * twitter_timeout); } catch (InterruptedException ex) { ex.printStackTrace(); System.out.println("Failed to sleep: " + ex.getMessage()); } } }
From source file:io.anserini.index.UpdateIndex.java
License:Apache License
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(//from w w w . j ava 2 s . c o m OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(UpdateIndex.class.getName(), options); System.exit(-1); } String indexPath = cmdline.getOptionValue(INDEX_OPTION); final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); textOptions.setStoreTermVectors(true); LOG.info("index: " + indexPath); File file = new File("PittsburghUserTimeline"); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } final StatusStream stream = new JsonStatusCorpusReader(file); Status status; String s; HashMap<Long, String> hm = new HashMap<Long, String>(); try { while ((s = stream.nextRaw()) != null) { try { status = DataObjectFactory.createStatus(s); if (status.getText() == null) { continue; } hm.put(status.getUser().getId(), hm.get(status.getUser().getId()) + status.getText().replaceAll("[\\r\\n]+", " ")); } catch (Exception e) { } } } catch (Exception e) { e.printStackTrace(); } finally { stream.close(); } ArrayList<String> userIDList = new ArrayList<String>(); try (BufferedReader br = new BufferedReader(new FileReader(new File("userID")))) { String line; while ((line = br.readLine()) != null) { userIDList.add(line.replaceAll("[\\r\\n]+", "")); // process the line. } } try { reader = DirectoryReader .open(FSDirectory.open(new File(cmdline.getOptionValue(INDEX_OPTION)).toPath())); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } final Directory dir = new SimpleFSDirectory(Paths.get(cmdline.getOptionValue(INDEX_OPTION))); final IndexWriterConfig config = new IndexWriterConfig(ANALYZER); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); final IndexWriter writer = new IndexWriter(dir, config); IndexSearcher searcher = new IndexSearcher(reader); System.out.println("The total number of docs indexed " + searcher.collectionStatistics(TweetStreamReader.StatusField.TEXT.name).docCount()); for (int city = 0; city < cityName.length; city++) { // Pittsburgh's coordinate -79.976389, 40.439722 Query q_long = NumericRangeQuery.newDoubleRange(TweetStreamReader.StatusField.LONGITUDE.name, new Double(longitude[city] - 0.05), new Double(longitude[city] + 0.05), true, true); Query q_lat = NumericRangeQuery.newDoubleRange(TweetStreamReader.StatusField.LATITUDE.name, new Double(latitude[city] - 0.05), new Double(latitude[city] + 0.05), true, true); BooleanQuery bqCityName = new BooleanQuery(); Term t = new Term("place", cityName[city]); TermQuery query = new TermQuery(t); bqCityName.add(query, BooleanClause.Occur.SHOULD); System.out.println(query.toString()); for (int i = 0; i < cityNameAlias[city].length; i++) { t = new Term("place", cityNameAlias[city][i]); query = new TermQuery(t); bqCityName.add(query, BooleanClause.Occur.SHOULD); System.out.println(query.toString()); } BooleanQuery bq = new BooleanQuery(); BooleanQuery finalQuery = new BooleanQuery(); // either a coordinate match bq.add(q_long, BooleanClause.Occur.MUST); bq.add(q_lat, BooleanClause.Occur.MUST); finalQuery.add(bq, BooleanClause.Occur.SHOULD); // or a place city name match finalQuery.add(bqCityName, BooleanClause.Occur.SHOULD); TotalHitCountCollector totalHitCollector = new TotalHitCountCollector(); // Query hasFieldQuery = new ConstantScoreQuery(new // FieldValueFilter("timeline")); // // searcher.search(hasFieldQuery, totalHitCollector); // // if (totalHitCollector.getTotalHits() > 0) { // TopScoreDocCollector collector = // TopScoreDocCollector.create(Math.max(0, // totalHitCollector.getTotalHits())); // searcher.search(finalQuery, collector); // ScoreDoc[] hits = collector.topDocs().scoreDocs; // // // HashMap<String, Integer> hasHit = new HashMap<String, Integer>(); // int dupcount = 0; // for (int i = 0; i < hits.length; ++i) { // int docId = hits[i].doc; // Document d; // // d = searcher.doc(docId); // // System.out.println(d.getFields()); // } // } // totalHitCollector = new TotalHitCountCollector(); searcher.search(finalQuery, totalHitCollector); if (totalHitCollector.getTotalHits() > 0) { TopScoreDocCollector collector = TopScoreDocCollector .create(Math.max(0, totalHitCollector.getTotalHits())); searcher.search(finalQuery, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; System.out.println("City " + cityName[city] + " " + collector.getTotalHits() + " hits."); HashMap<String, Integer> hasHit = new HashMap<String, Integer>(); int dupcount = 0; for (int i = 0; i < hits.length; ++i) { int docId = hits[i].doc; Document d; d = searcher.doc(docId); if (userIDList.contains(d.get(IndexTweets.StatusField.USER_ID.name)) && hm.containsKey(Long.parseLong(d.get(IndexTweets.StatusField.USER_ID.name)))) { // System.out.println("Has timeline field?" + (d.get("timeline") != null)); // System.out.println(reader.getDocCount("timeline")); // d.add(new Field("timeline", hm.get(Long.parseLong(d.get(IndexTweets.StatusField.USER_ID.name))), // textOptions)); System.out.println("Found a user hit"); BytesRefBuilder brb = new BytesRefBuilder(); NumericUtils.longToPrefixCodedBytes(Long.parseLong(d.get(IndexTweets.StatusField.ID.name)), 0, brb); Term term = new Term(IndexTweets.StatusField.ID.name, brb.get()); // System.out.println(reader.getDocCount("timeline")); Document d_new = new Document(); // for (IndexableField field : d.getFields()) { // d_new.add(field); // } // System.out.println(d_new.getFields()); d_new.add(new StringField("userBackground", d.get(IndexTweets.StatusField.USER_ID.name), Store.YES)); d_new.add(new Field("timeline", hm.get(Long.parseLong(d.get(IndexTweets.StatusField.USER_ID.name))), textOptions)); // System.out.println(d_new.get()); writer.addDocument(d_new); writer.commit(); // t = new Term("label", "why"); // TermQuery tqnew = new TermQuery(t); // // totalHitCollector = new TotalHitCountCollector(); // // searcher.search(tqnew, totalHitCollector); // // if (totalHitCollector.getTotalHits() > 0) { // collector = TopScoreDocCollector.create(Math.max(0, totalHitCollector.getTotalHits())); // searcher.search(tqnew, collector); // hits = collector.topDocs().scoreDocs; // // System.out.println("City " + cityName[city] + " " + collector.getTotalHits() + " hits."); // // for (int k = 0; k < hits.length; k++) { // docId = hits[k].doc; // d = searcher.doc(docId); // System.out.println(d.get(IndexTweets.StatusField.ID.name)); // System.out.println(d.get(IndexTweets.StatusField.PLACE.name)); // } // } // writer.deleteDocuments(term); // writer.commit(); // writer.addDocument(d); // writer.commit(); // System.out.println(reader.getDocCount("timeline")); // writer.updateDocument(term, d); // writer.commit(); } } } } reader.close(); writer.close(); }
From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser) throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override/*w w w .j a va 2s .c om*/ public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(String.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }