List of usage examples for twitter4j Status getCreatedAt
Date getCreatedAt();
From source file:cp.Demo.java
private void MoviesListValueChanged(javax.swing.event.ListSelectionEvent evt) {//GEN-FIRST:event_MoviesListValueChanged // Obtain the movie object Movie mov;/*w w w.j av a 2s .c o m*/ try { mov = (Movie) MoviesList.getSelectedValue(); System.out.println(mov.getApiQuery()); } catch (Exception ex) { PosterLabel.setIcon(null); PosterLabel.setText("Movie not found"); return; } // Display the movie poster try { // Load the movie poster URL url = new URL(mov.getPosterLink()); ImageIcon img = new ImageIcon(url); // Rescale the movie poster int w = img.getIconWidth(); int h = img.getIconHeight(); int labelH = MovieTabs.getHeight() - 36; img.setImage(img.getImage().getScaledInstance(labelH * w / h, labelH, Image.SCALE_SMOOTH)); // Put the movie poster into the label, and change the text PosterLabel.setIcon(img); PosterLabel.setText(""); } catch (Exception ex) { // If for some reason the poster isn't available, indicate this PosterLabel.setIcon(null); PosterLabel.setText("Poster could not be displayed"); } // Display tweets about the movie try { // clear out the tweets TweetsArea.setText(""); // The factory instance is re-useable and thread safe. Twitter twitter = TwitterFactory.getSingleton(); Query query = new Query(mov.getTitle()); QueryResult result = twitter.search(query); String tweet; for (Status status : result.getTweets()) { tweet = String.format("[%s] @%s tweeted %s\n", status.getCreatedAt(), status.getUser().getScreenName(), status.getText()); TweetsArea.append(tweet); } } catch (TwitterException ex) { TweetsArea.setText(ex.toString()); } // Display reviews of the movie try { ReviewsArea.setText(""); for (Review rev : (new ReviewRequest(mov.getMovie())).getReviews()) { String review = String.format("[%s] (by %s, %s)\n%s\n%s\n\n", rev.getDateOfReview(), rev.getCritic(), rev.getPublication(), rev.getQuote(), rev.getReviewLinkUrl()); ReviewsArea.append(review); } } catch (IOException ex) { Logger.getLogger(Demo.class.getName()).log(Level.SEVERE, null, ex); } catch (ReviewRequest.ReviewRequestException ex) { Logger.getLogger(Demo.class.getName()).log(Level.SEVERE, null, ex); } catch (ParseException ex) { Logger.getLogger(Demo.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:crawler.DataSearch.java
License:Apache License
public void collectNewTweets() { while (keywords.size() > 0) { String hash = keywords.get(0); Query query = new Query(hash); query.setCount(Settings.searchTweetNo); QueryResult result;//from w w w .jav a 2s . c o m try { result = twitter.search(query); List<Status> tweets = result.getTweets(); if (tweets.size() == 0) TwitterCrawler.TFFreq.remove(hash); for (Status status : tweets) { //System.out.println("{"+hash+"}["+status.getCreatedAt().toString()+"]- [USER: " + status.getUser().getScreenName() + "] - " + status.getText()); String temp; if (trash(status.getText())) { if (Settings.limitTime) { long currentTime = System.currentTimeMillis(); long tweetTime = status.getCreatedAt().getTime(); if (currentTime - tweetTime > Settings.timeLimit) continue; } if (hashtext.containsKey(hash)) temp = hashtext.get(hash) + status.getText(); else temp = status.getText(); hashtext.put(hash, temp); } } //System.out.println("{"+hash+"} with "+tweets.size()+"tweets"); keywords.remove(0); } catch (TwitterException e) { if (e.getErrorCode() == 88) { System.out.println("Using other keys???"); String ConsumerKey_temp = Settings.ConsumerKey.get(0); String ConsumerSecret_temp = Settings.ConsumerSecret.get(0); String AccessToken_temp = Settings.AccessToken.get(0); String AccessSecret_temp = Settings.AccessSecret.get(0); Settings.ConsumerKey.remove(ConsumerKey_temp); Settings.ConsumerSecret.remove(ConsumerSecret_temp); Settings.AccessToken.remove(AccessToken_temp); Settings.AccessSecret.remove(AccessSecret_temp); twitter = new TwitterFactory( OA.build(Settings.ConsumerKey.get(0), Settings.ConsumerSecret.get(0), Settings.AccessToken.get(0), Settings.AccessSecret.get(0))).getInstance(); Settings.ConsumerKey.add(Settings.ConsumerKey.size(), ConsumerKey_temp); Settings.ConsumerSecret.add(Settings.ConsumerSecret.size(), ConsumerSecret_temp); Settings.AccessToken.add(Settings.AccessToken.size(), AccessToken_temp); Settings.AccessSecret.add(Settings.AccessSecret.size(), AccessSecret_temp); System.out.println("**************************************************************"); System.out.println("[" + ConsumerKey_temp + "] to [" + Settings.ConsumerKey.get(0) + "]"); System.out.println("[" + ConsumerSecret_temp + "] to [" + Settings.ConsumerSecret.get(0) + "]"); System.out.println("[" + AccessToken_temp + "] to [" + Settings.AccessToken.get(0) + "]"); System.out.println("[" + AccessSecret_temp + "] to [" + Settings.AccessSecret.get(0) + "]"); System.out.println("**************************************************************"); continue; } e.printStackTrace(); } } }
From source file:crawler.DataStorage.java
License:Apache License
private static void sqlStore(Status status) throws SQLException { long sql_pid = Settings.pid; Settings.pid++;// ww w.j a va 2 s . c om SimpleDateFormat tempDate = new SimpleDateFormat("yyyy-MM-dd, HH:mm:ss, z"); String sqlCreateAt = tempDate.format(new java.util.Date(status.getCreatedAt().getTime())); double sqlGeoLocationLat = 0; double sqlGeoLocationLong = 0; if (status.getGeoLocation() != null) { sqlGeoLocationLat = status.getGeoLocation().getLatitude(); sqlGeoLocationLong = status.getGeoLocation().getLongitude(); } String sqlPlace = (status.getPlace() != null ? status.getPlace().getFullName() : ""); long sqlId = status.getId(); String sqlTweet = status.getText().replace("'", "''"); String sqlSource = status.getSource().replace("'", "''"); sqlSource = sqlSource.replace("\\", "\\\\"); String sqlLang = status.getUser().getLang(); String sqlScreenName = status.getUser().getScreenName(); String sqlReplyTo = status.getInReplyToScreenName(); long sqlRtCount = status.getRetweetCount(); HashtagEntity[] hashs = status.getHashtagEntities(); String sqlHashtags = ""; for (HashtagEntity hash : hashs) sqlHashtags += hash.getText() + " "; pstm.setLong(1, sql_pid); pstm.setString(2, sqlCreateAt); pstm.setDouble(3, sqlGeoLocationLat); pstm.setDouble(4, sqlGeoLocationLong); pstm.setString(5, sqlPlace); pstm.setLong(6, sqlId); pstm.setString(7, sqlTweet); pstm.setString(8, sqlSource); pstm.setString(9, sqlLang); pstm.setString(10, sqlScreenName); pstm.setString(11, sqlReplyTo); pstm.setLong(12, sqlRtCount); pstm.setString(13, sqlHashtags); pstm.addBatch(); }
From source file:crawler.TwitterFeed.java
/** * *//* w w w . j a v a 2s . c om*/ public void get_tweets() throws SQLException { CSVReader cr = new CSVReader(); ArrayList<String> names = new ArrayList<>(); Map<String, String> name_ticker; names = cr.get_company_names(tickers); System.out.println(names); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setOAuthConsumerKey("WNSycI2GS33ZAwHJ9Fb4A"); cb.setOAuthConsumerSecret("BBWeSThT3ZV2g9c5BDuUduDyNAWyzouMI0XjQy7KUc"); cb.setOAuthAccessToken("1852271029-SLfE061bImfcRxWQZpy1pAgpEkfFhg3757Q9JRf"); cb.setOAuthAccessTokenSecret("1uPSfYPbaENtXFYWsryBIAHnUsmG3pT6lGH0NzxTCGW7O"); Twitter twitter = new TwitterFactory(cb.build()).getInstance(); for (String s : names) { String[] temp; if (s.contains(",")) { temp = s.split(","); } else { temp = s.split(" "); } System.out.println( "--------------------------------------------------------------------------------------" + temp[0]); Query query = new Query("#" + temp[0]); query.lang("en"); int numberOfTweets = 500; long lastID = Long.MAX_VALUE; ArrayList<Status> tweets = new ArrayList<>(); while (tweets.size() < numberOfTweets) { if (numberOfTweets - tweets.size() > 100) { query.setCount(100); } else { query.setCount(numberOfTweets - tweets.size()); } try { QueryResult result = twitter.search(query); tweets.addAll(result.getTweets()); System.out.println("Gathered " + tweets.size() + " tweets"); for (Status t : tweets) { if (t.getId() < lastID) { lastID = t.getId(); } } } catch (TwitterException te) { System.out.println("Couldn't connect: " + te); } query.setMaxId(lastID - 1); } Connection con = DriverManager.getConnection("jdbc:mysql://localhost:3306/CrawledData", "root", ""); for (int i = 0; i < tweets.size(); i++) { Status t = (Status) tweets.get(i); //GeoLocation loc = t.getGeoLocation(); String user = t.getUser().getScreenName(); String msg = t.getText(); String time = t.getCreatedAt().toString(); //if (loc!=null) { // Double lat = t.getGeoLocation().getLatitude(); // Double lon = t.getGeoLocation().getLongitude(); // System.out.println(i + " USER: " + user + " wrote: " + msg + " located at " + lat + ", " + lon); //} //else{ System.out.println(i + " USER: " + user + " wrote: " + msg + " at: " + time); //} try { Class.forName("com.mysql.jdbc.Driver"); PreparedStatement ps = con .prepareStatement("INSERT IGNORE INTO TwitterFeed VALUES (?,?,?,?,?)"); ps.setString(1, s); ps.setString(2, cr.get_ticker_from_company(s)); ps.setString(3, user); ps.setString(4, msg); ps.setString(5, time); int k = ps.executeUpdate(); if (k <= 0) { System.out.println("Entry Unsuccessful"); } else { System.out.println("Entry Successful"); } } catch (ClassNotFoundException | SQLException e) { System.out.println(e); } } con.close(); } }
From source file:crawling.FoundUsersByStreamHashtag.java
License:Apache License
public static void main(String[] args) throws TwitterException { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { // my shutdown code here // Record the user status idOut.println("Final status -------------------:"); for (Long id : discoveredUsers.keySet()) { idOut.println(id + "," + discoveredUsers.get(id)); }/* w w w .ja v a 2 s. c o m*/ idOut.close(); } }); if (args.length != 1) { System.out.println("Usage: java twitter4j.examples.PrintFilterStreamHashtag hashtag"); System.exit(-1); } /*if(args.length == 2){ // Preload the collected user IDs preloadID(args[1]); }*/ //buildStartTime(); File file = new File(fileName); InputStream is = null; try { if (file.exists()) { is = new FileInputStream(file); prop.load(is); } else { System.out.println(fileName + " doesn't exist!"); System.exit(-1); } } catch (IOException ioe) { ioe.printStackTrace(); System.exit(-1); } StatusListener listener = new StatusListener() { public void onStatus(Status status) { //System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText()); CheckUser(status); /*storeATweet(status); tweets += 1; if (tweets % 1000 == 0){ System.out.println("We now have tweets: " + tweets); }*/ } private void CheckUser(Status status) { // TODO Auto-generated method stub Long id = status.getUser().getId(); String username = status.getUser().getScreenName(); String realname = status.getUser().getName(); String text = status.getText(); Date date = status.getCreatedAt(); if (discoveredUsers.containsKey(id)) { //System.out.println("Already found this user: " + id); long num = discoveredUsers.get(id); discoveredUsers.put(id, num + 1); } else { discoveredUsers.put(id, (long) 1); storeUserID(status); } } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning arg0) { // TODO Auto-generated method stub } }; try { FileWriter outFile = new FileWriter("discoveredUser" + args[0] + ".txt", true); idOut = new PrintWriter(outFile); //out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } //TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); TwitterStream twitterStream = getOAuthTwitterStream(); twitterStream.addListener(listener); FilterQuery query = new FilterQuery(); String[] track = { args[0] }; //String[] track = {"#BaltimoreRiots"}; query.track(track); twitterStream.filter(query); }
From source file:crawling.FoundUsersByStreamHashtag.java
License:Apache License
private static void storeUserID(Status status) { // TODO Auto-generated method stub Long id = status.getUser().getId(); String username = status.getUser().getScreenName(); String realname = status.getUser().getName(); String text = status.getText(); Date date = status.getCreatedAt(); //String line = id + "::" + username + "::" + realname + "::" + text + "::" + date + "::" + (new Date()); String line = id + "," + username; //System.out.println(line); idOut.println(line);/*from w w w .j av a 2s .c om*/ idOut.flush(); System.out.print("\r" + ++count); }
From source file:crawling.GetUserTimelineByFile.java
License:Apache License
public static void main(String[] args) { if (args.length < 1) { System.out.println("Usage: java twitter4j.examples.GetUserTimelineByFile UserFile"); System.exit(-1);/* ww w .j a va 2 s.c o m*/ } userFile = args[0]; try { FileWriter outFileWhole = new FileWriter("CrawlTweets" + ".txt", true); out = new PrintWriter(outFileWhole); // out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } File inFile = new File(userFile); if (!inFile.isFile()) { System.out.println("Parameter is not an existing file"); return; } BufferedReader br = null; try { br = new BufferedReader(new FileReader(userFile)); } catch (FileNotFoundException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } // gets Twitter instance with default credentials Twitter twitter = new TwitterFactory().getInstance(); int count = 0; int totalCount = 0; SimpleDateFormat sdf = new SimpleDateFormat("yyyy, MM, dd"); Date start = null; try { start = sdf.parse("2000, 1, 1"); } catch (ParseException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } List<Status> statuses = null; // String user = "Porter_Anderson"; // user = "pqtad"; // user = "paradunaa6"; // user = "palifarous"; ArrayList<Long> users = new ArrayList<Long>(); String line = null; //Read from the original file and write to the new //unless content matches data to be removed. try { while ((line = br.readLine()) != null) users.add(Long.parseLong(line)); br.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } //users.add(584928891L); //users.add(700425265L); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); Long sinceID = //218903304682471424L; // 2012/06/29 238893053518151680L; // 2012/08/24 Long maxID = 250409135600975873L; // 2012/09/24 //238893053518151680L; // 2012/08/24 // 227659323180974080L; // 2012/07/24 boolean overflow = false; for (Long usr : users) { System.out.println("%" + usr); out.println("%" + usr); out.println("%" + usr); Paging paging = null; count = 0; for (int i = 1; i < 21; i++) { //paging = new Paging(i, 200); paging = new Paging(i, 200, sinceID, maxID); overflow = false; try { // statuses = twitter.getUserTimeline(user, paging); statuses = twitter.getUserTimeline(usr, paging); Thread.currentThread(); try { Thread.sleep(10500); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to get timeline: " + te.getMessage()); //System.exit(-1); } if (statuses.isEmpty()) { if (i > 16) overflow = true; break; } for (Status status : statuses) { String record = ""; record += status.getId(); record += "::" + status.getInReplyToStatusId(); record += "::" + status.getInReplyToUserId(); record += "::" + status.getRetweetCount(); if (status.getRetweetedStatus() != null) record += "::" + status.getRetweetedStatus().getId(); else record += "::" + "-1"; //record += "::" + status.isRetweet(); int len = status.getUserMentionEntities().length; if (len > 0) { record += "::"; for (int l = 0; l < len; l++) { UserMentionEntity ent = status.getUserMentionEntities()[l]; record += "," + ent.getId(); } } else record += "::" + "-1"; len = status.getURLEntities().length; if (len > 0) { record += "::"; for (int l = 0; l < len; l++) { URLEntity ent = status.getURLEntities()[l]; record += "," + ent.getURL() + "|" //+ ent.getDisplayURL() + "|" + ent.getExpandedURL(); } } else record += "::" + "-1"; record += "::" + cleanText(status.getText()); record += "::" + // status.getCreatedAt(); (status.getCreatedAt().getTime() - start.getTime()) / 1000; record += "::" + getSource(status.getSource()); //System.out.println(record); out.println(record); } count += statuses.size(); out.flush(); } totalCount += count; out.println("%" + usr + ", " + count + ", " + overflow); System.out.println("%" + usr + ", " + count + ", " + overflow); out.println("------------------------------------------"); } System.out.println("Total status count is " + totalCount); out.println("#" + totalCount); out.close(); date = new Date(); System.out.println(); System.out.println("----------------------------------------------"); System.out.println("End at: " + dateFormat.format(date)); }
From source file:crawling.GetUserTimelineByFileDiff.java
License:Apache License
public static void main(String[] args) { if (args.length < 1) { System.out.println("Usage: java twitter4j.examples.GetUserTimelineByFile UserFile"); System.exit(-1);//w ww. java 2 s . co m } userFile = args[0]; try { FileWriter outFileWhole = new FileWriter("CrawlTweets" + ".txt", true); outWhole = new PrintWriter(outFileWhole); FileWriter outFileInter = new FileWriter("CrawlInter" + ".txt", true); outInter = new PrintWriter(outFileInter); // out.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } File inFile = new File(userFile); if (!inFile.isFile()) { System.out.println("Parameter is not an existing file"); return; } BufferedReader br = null; try { br = new BufferedReader(new FileReader(userFile)); } catch (FileNotFoundException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } // gets Twitter instance with default credentials Twitter twitter = new TwitterFactory().getInstance(); int count = 0; int countInter = 0; int totalCount = 0; int totalCountInter = 0; SimpleDateFormat sdf = new SimpleDateFormat("yyyy, MM, dd"); Date start = null; try { start = sdf.parse("2000, 1, 1"); } catch (ParseException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } List<Status> statuses = null; // String user = "Porter_Anderson"; // user = "pqtad"; // user = "paradunaa6"; // user = "palifarous"; // statuses = twitter.getUserTimeline(user); //String usersS = "490495670,210462629,731570426,174646182,381163846,308812526,595154838,47356666,590795627,466854690,560386951,406747522,767819150,193105498,725874775,381257304,486727765,39469575,68239634,187739283,27048882,172979072,16075589,618047540,841958365,707631710,76274554,334451278,573978436,782711790,498017142,764236789,88185343,293950091,726606902,221410697,566098451,32987449,73497385,223182159,571182929,769441794,16225570,404748449,249953519,713693096,727692930,178473099,366957548,255171527,411892240,370333418,753509292,765928471,49389074,422867487,185298515,112841821,524731824,132207254,431289858,35602356,747638244,76398937,276388822,221106768,608597994,532423674,509070257,235773945,263995811,61575477,48606725,518136068,450781132,631504150,842236698,840479664,306291201,59829276,541212815,264436593,262466303,302157661,135587748,399229428,265344815,225280446,540254332,218019401,260453139,187424025,177051847,351065900,406684867,293234971,219356963,280763734,59290083,214750688"; //String[] splits = usersS.split(","); ArrayList<Long> users = new ArrayList<Long>(); String line = null; //Read from the original file and write to the new //unless content matches data to be removed. try { while ((line = br.readLine()) != null) { users.add(Long.parseLong(line)); } br.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } //users.add(584928891L); //users.add(700425265L); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); System.out.println(); System.out.println("----------------------------------------------"); System.out.println("Start at: " + dateFormat.format(date)); Long sinceID = //218903304682471424L; // 2012/06/29 238893053518151680L; // 2012/08/24 Long maxID = 250409135600975873L; // 2012/09/24 //238893053518151680L; // 2012/08/24 // 227659323180974080L; // 2012/07/24 boolean overflow = false; for (Long usr : users) { System.out.println("%" + usr); outWhole.println("%" + usr); outInter.println("%" + usr); Paging paging = null; count = 0; countInter = 0; for (int i = 1; i < 21; i++) { //paging = new Paging(i, 200); paging = new Paging(i, 200, sinceID, maxID); overflow = false; try { // statuses = twitter.getUserTimeline(user, paging); statuses = twitter.getUserTimeline(usr, paging); Thread.currentThread(); try { Thread.sleep(10500); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to get timeline: " + te.getMessage()); //System.exit(-1); } if (statuses.isEmpty()) { if (i > 16) overflow = true; break; } for (Status status : statuses) { boolean inter = false; //countRT += status.getRetweetCount(); String record = ""; String recordInter = ""; record += status.getId(); // For reply Long replyId = status.getInReplyToStatusId(); if (replyId == -1) record += "::-1::-1"; else { record += "::" + replyId; record += "::" + status.getInReplyToUserId(); inter = true; } // For retweeting Long retweets = (long) status.getRetweetCount(); record += "::" + retweets; if (retweets > 0) { inter = true; if (status.getRetweetedStatus() != null) record += "::" + status.getRetweetedStatus().getId(); else record += "::" + "-1"; // The source of retweeting } //record += "::" + status.isRetweet(); // For mentions int len = status.getUserMentionEntities().length; if (len > 0) { record += "::"; for (int l = 0; l < len; l++) { UserMentionEntity ent = status.getUserMentionEntities()[l]; record += "," + ent.getId(); } inter = true; } else record += "::" + "-1"; recordInter += record; // For URL len = status.getURLEntities().length; if (len > 0) { record += "::"; for (int l = 0; l < len; l++) { URLEntity ent = status.getURLEntities()[l]; record += "," + ent.getURL() + "|" //+ ent.getDisplayURL() + "|" + ent.getExpandedURL(); } } else record += "::" + "-1"; // For text record += "::" + cleanText(status.getText()); // For creating time // record += "::" + status.getCreatedAt().toString(); Long seconds = // status.getCreatedAt(); (status.getCreatedAt().getTime() - start.getTime()) / 1000; record += "::" + seconds; recordInter += "::" + seconds; // For publishing source record += "::" + getSource(status.getSource()); //System.out.println(record); outWhole.println(record); if (inter) { outInter.println(recordInter); countInter++; } } count += statuses.size(); outWhole.flush(); outInter.flush(); } totalCount += count; totalCountInter += countInter; outWhole.println("%" + usr + "," + count + "," + countInter + "," + overflow); outInter.println("%" + usr + "," + count + "," + countInter + "," + overflow); System.out.println("%" + usr + "," + count + "," + countInter + "," + overflow); outWhole.println("------------------------------------------"); outInter.println("------------------------------------------"); } System.out.println("Total status count is " + totalCount + "," + totalCountInter); outWhole.println("#" + totalCount + "," + totalCountInter); outInter.println("#" + totalCount + "," + totalCountInter); outWhole.close(); outInter.close(); date = new Date(); System.out.println(); System.out.println("----------------------------------------------"); System.out.println("End at: " + dateFormat.format(date)); }
From source file:crawling.GetUserTimelineMulti.java
License:Apache License
public void doCrawl() { List<Status> statuses = null; int count = 0; int userIndex = 0; int userTotal = users.size(); boolean overflow = false; for (Long usr : users) { //System.out.println("%" + usr); out.println("%" + usr); Paging paging = null;//ww w .j a va2 s. co m count = 0; //for (int i = 1; i < 21; i++) { for (int i = 1; i < 4; i++) { paging = new Paging(i, 200); //paging = new Paging(i, 200, sinceId, maxId); overflow = false; try { // statuses = twitter.getUserTimeline(user, paging); statuses = twitters.get(currentCrawl).getUserTimeline(usr, paging); currentCrawl++; if (currentCrawl == this.numCrawl) currentCrawl = 0; /*Thread.currentThread(); try { Thread.sleep(updateFreq); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); }*/ } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to get timeline: " + te.getMessage()); //System.exit(-1); } if (statuses == null) { break; } if (statuses.isEmpty()) { if (i > 16) overflow = true; break; } for (Status status : statuses) { int len = status.getUserMentionEntities().length; /* * We only need the interactions */ if (len <= 0) continue; String record = ""; record += status.getId(); record += "::" + status.getInReplyToStatusId(); record += "::" + status.getInReplyToUserId(); record += "::" + status.getRetweetCount(); if (status.getRetweetedStatus() != null) record += "::" + status.getRetweetedStatus().getId(); else record += "::" + "-1"; //record += "::" + status.isRetweet(); //int len = status.getUserMentionEntities().length; if (len > 0) { record += "::"; for (int l = 0; l < len; l++) { UserMentionEntity ent = status.getUserMentionEntities()[l]; record += "," + ent.getId(); } } else { record += "::" + "-1"; } len = status.getURLEntities().length; if (len > 0) { record += "::"; for (int l = 0; l < len; l++) { URLEntity ent = status.getURLEntities()[l]; record += "," + ent.getURL() + "|" //+ ent.getDisplayURL() + "|" + ent.getExpandedURL(); } } else record += "::" + "-1"; //record += "::" + cleanText(status.getText()); // We just ignore the text content in this crawling record += "::-1"; record += "::" + // status.getCreatedAt(); (status.getCreatedAt().getTime() - start.getTime()) / 1000; record += "::" + getSource(status.getSource()); // Geo Location if (status.getGeoLocation() != null) record += "::" + status.getGeoLocation(); else record += "::" + "-1"; //System.out.println(record); out.println(record); count++; } //count += statuses.size(); } out.flush(); totalCount += count; out.println("%" + usr + ", " + count + ", " + overflow); outUserId.println(usr + "," + count); //System.out.println("%" + usr + ", " + count + ", " + overflow); System.out.println(userIndex + "/" + userTotal + ", " + count); //+ ", " + (overflow ? "overflow" : "not overflow")); userIndex++; out.println("------------------------------------------"); } System.out.println("Total status count is " + totalCount); outUserId.println("#" + totalCount); this.closeFile(); }
From source file:crawling.PrintFilterStreamGeo.java
License:Apache License
private static void storeUserID(Status status) { // TODO Auto-generated method stub Long id = status.getUser().getId(); String username = status.getUser().getScreenName(); String realname = status.getUser().getName(); String text = status.getText(); Date date = status.getCreatedAt(); String line = id + "::" + username + "::" + realname + "::" + text + "::" + date + "::" + (new Date()); //System.out.println(line); idOut.println(line);// w w w . j a v a2 s . co m idOut.flush(); System.out.print("\r" + ++count); }