List of usage examples for twitter4j StatusListener StatusListener
StatusListener
From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser) throws IOException { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override//from w w w .j av a 2s . co m public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(String.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() throws IOException { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:kafka.producer.PrintGeoStream.java
License:Apache License
/** * Main entry of this application.//from w ww . j av a 2 s. c o m * * @param args <kafka host> <kafka topic> * @throws twitter4j.TwitterException */ public static void main(String[] args) throws TwitterException { // if (args.length != 2) { // System.out.println("Usage: java twitter4j.examples.PrintFilterStream <kafka host> <kafka topic>"); // System.exit(-1); // } // final String kafkaZKHost = args[0]; // final String kafkaTopic = args[1]; final String kafkaZKHost = "localhost"; final String kafkaTopic = "flightDelay"; BasicConfigurator.configure(); StatusListener listener = new StatusListener() { Gson gson = new Gson(); Properties kafkaProps = new Properties(); //kafkaProps.put("serializer.class", "kafka.serializer.StringEncoder"); //kafkaProps.put("zk.connect", "localhost:2181"); Producer<Integer, String> kafkaProducer = null; int tweetCount = 0; int interimCount = 0; @Override public void onStatus(Status status) { if (kafkaProducer == null) { kafkaProps.put("serializer.class", "kafka.serializer.StringEncoder"); //kafkaProps.put("zk.connect", "localhost:2181"); kafkaProps.put("zk.connect", kafkaZKHost.concat(":2181")); kafkaProducer = new Producer<Integer, String>(new ProducerConfig(kafkaProps)); } tweetCount++; interimCount++; if (interimCount == 100) { LOG.info(" total tweets received: " + tweetCount); interimCount = 0; } //System.out.println("@" + status.getUser().getScreenName() + " - "); // + status.getText()); //if ( status.getPlace() != null ) { // System.out.println("coordinates = " + status.getPlace().getGeometryCoordinates()); //} TweetInfo tweetInfo = new TweetInfo(); tweetInfo.populate(status); String message = gson.toJson(tweetInfo, TweetInfo.class); kafkaProducer.send(new ProducerData<Integer, String>(kafkaTopic, message)); //kafkaProducer.send(new ProducerData<Integer, String>("live_tweets", message)); } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { //System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; /* * - use Redis to pass filter messages in - or ZK? * - Read in filter JSON with parms for: * - lat/lon boxj * - screen names to follow * - hashtags to track * - Convert screen names to userIds * - Twitter twitter = new TwitterFactory().getInstance(); * ResponseList<User> users = twitter.lookupUsers(args[0].split(",")); * - Kick off filter with new FilterQuery * - periodically check, and if filter updated, restart the TwitterStream: * .shutdown() * new * .addListener * * */ TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); // twitterStream.setOAuthAccessToken(new AccessToken("14098069-IgutahzmbaCubdoMU0MxsfsJT2iNVpRnoohCbV67k", "d1kFKqRSnrlA4bP1XWWLiHNsHfYDZQPWDHRVpGP4hM")); // twitterStream.setOAuthConsumer("G0SKTmMXjfMsunBh3Furg", "iVTYNLqVaVdUZz4xOn7H8ywylTaTuW3kH1dT21NUy4"); twitterStream.addListener(listener); /* * For CONUS only: * "lowerLat": "22.0", * "lowerLon": "-127.0", * "upperLat": "47.5", * "upperLon": "-67.0" */ FilterQuery fQuery = new FilterQuery(); String[] topics = new String[2]; topics[0] = "delayed flight"; topics[1] = "flight delay"; // double[][] us = {{-180, 24}, {-66, 50}}; double[][] lax = { { -118.5, 33.8 }, { -118.3, 34.01 } }; fQuery.locations(lax); fQuery.track(topics); twitterStream.filter(fQuery); }
From source file:my.twittergui.TwitterUI.java
private void jButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton2ActionPerformed String text;/*from w ww.j a va 2 s . c om*/ text = jTextField1.getText(); //jTextArea1.append(text+"\n"); jTextArea1.append("Searching for: "); String[] strarray = text.split(" "); for (int i = 0; i < strarray.length; i++) { jTextArea1.append(strarray[i] + "\n"); } File filed = null; filed = new File("C:\\Results"); if (!filed.exists()) { filed.mkdir(); } ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("").setOAuthConsumerSecret("").setOAuthAccessToken("") .setOAuthAccessTokenSecret(""); StatusListener listener = new StatusListener() { public void onStatus(Status status) { try { int fileno = 0; String strI = Integer.toString(fileno); String fname = "C:\\results\\DataCol" + strI + ".csv"; File file = new File(fname); FileWriter bw = new FileWriter(file, true); if (file.length() == 0) { bw.write("\"Screen Name\", text, \"created at\", geolocation, Retweet? \r\n"); bw.flush(); } while (file.length() > 10485760) { fileno += 1; strI = Integer.toString(fileno); fname = "DataCol" + strI + ".txt"; file = new File(fname); bw = new FileWriter(file, true); bw.write("\"Screen Name\", \"text\", \"created_at\", \"geolocation\" \r\n"); bw.flush(); } // if(!status.isRetweet()){ bw.write("\r\n"); bw.write("\"" + status.getUser().getScreenName() + "\","); String tweettxt = status.getText(); tweettxt = tweettxt.replace("\n", ""); tweettxt = tweettxt.replace(",", ""); tweettxt = tweettxt.replace("\"", ""); bw.write("\"" + tweettxt + "\","); bw.write("\"" + status.getCreatedAt() + "\","); if (status.getGeoLocation() != null) { bw.write("\"" + status.getGeoLocation() + "\""); } else bw.write("N/A,"); if (status.isRetweet()) bw.write("Yes"); else bw.write("No"); bw.flush(); bw.close(); //System.out.print("\n"); String str = "@" + status.getUser().getScreenName() + " " + status.getText() + " " + status.getCreatedAt() + " "; if (status.getGeoLocation() != null) str += status.getGeoLocation(); //System.out.print(str); jTextArea1.append(str + "\n"); // } bw.close(); } catch (IOException e) { //System.out.print("EXCEPTION"); e.printStackTrace(); } } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { } public void onStallWarning(StallWarning stall) { } public void onScrubGeo(long a, long b) { } public void onException(Exception ex) { ex.printStackTrace(); } }; TwitterStreamFactory tf = new TwitterStreamFactory(cb.build()); TwitterStream twitterStream = tf.getInstance(); twitterStream.addListener(listener); // Filter FilterQuery filtre = new FilterQuery(); Scanner in = new Scanner(System.in); filtre.track(strarray); twitterStream.filter(filtre); }
From source file:nlptexthatespeechdetection.dataCollection.GetTwitterDoc2VecTrainingData.java
public static void main(String[] args) throws FileNotFoundException, IOException { File dir = new File(folderName); if (!dir.exists()) dir.mkdir();//w w w .ja v a 2s. c om if (!dir.isDirectory()) { System.out.println(folderName + " is not a directory"); return; } System.out.println("number of tweets required: "); int numTweetsRequired = (new Scanner(System.in)).nextInt(); String path = folderName + "/" + fileName; File file = new File(path); if (!file.exists()) file.createNewFile(); FileWriter writer = new FileWriter(path, true); TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); StatusListener listener = new StatusListener() { int numTweets = 0; @Override public void onStatus(Status status) { if (status.getLang().equals("in")) { try { String statusText = status.getText(); writer.write("\n"); writer.write(statusText); numTweets++; System.out.println("numTweets: " + numTweets); if (numTweets >= numTweetsRequired) { writer.close(); System.exit(0); } } catch (IOException ex) { Logger.getLogger(GetTwitterDoc2VecTrainingData.class.getName()).log(Level.SEVERE, null, ex); } } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { // System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; twitterStream.addListener(listener); FilterQuery filterQuery = new FilterQuery(); filterQuery.track(new String[] { "a", "i", "u", "e", "o" }); filterQuery.language("in"); twitterStream.filter(filterQuery); }
From source file:nlptexthatespeechdetection.dataCollection.TwitterStreamingAnnotator.java
public static void main(String[] args) throws NotDirectoryException { Scanner sc = new Scanner(System.in); System.out.println("Nama Anda (sebagai anotator): "); String namaAnotator = sc.nextLine(); AnnotatedDataFolder annotatedDataFolder = new AnnotatedDataFolder(dataFolderName); TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); StatusListener listener = new StatusListener() { @Override/*from ww w . jav a 2s . c o m*/ public void onStatus(Status status) { if (status.getLang().equals("in")) { System.out.println(); System.out.println(); System.out.println("=======ANOTASI======="); System.out.println("status: " + status.getText()); System.out.println(); System.out.println("is this a hate speech?(y/n. any other if you do not know)"); String annotatorResponse = sc.nextLine().trim().toLowerCase(); Date date = new Date(); String dateString = dateFormat.format(date); try { if (annotatorResponse.equals("y")) { String filePath = annotatedDataFolder.saveHateSpeechString(namaAnotator, dateString, status.getText()); System.out.println("Saved data to: " + filePath); } else if (annotatorResponse.equals("n")) { String filePath = annotatedDataFolder.saveNotHateSpeechString(namaAnotator, dateString, status.getText()); System.out.println("Saved data to: " + filePath); } System.out.println("thank you!"); } catch (FileNotFoundException ex) { ex.printStackTrace(); } catch (IOException ex) { Logger.getLogger(TwitterStreamingAnnotator.class.getName()).log(Level.SEVERE, null, ex); } } else { System.out.println("ignoring non-indonesian tweet"); } // if (status.getGeoLocation() != null) { // System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " $ " + status.getGeoLocation().toString()); // } // if (status.getLang().equals("id")) { // System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString())); // } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { // System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; twitterStream.addListener(listener); FilterQuery filterQuery = new FilterQuery(); filterQuery.track(new String[] { "a", "i", "u", "e", "o" }); filterQuery.language("in"); twitterStream.filter(filterQuery); }
From source file:nlptexthatespeechdetection.NLPTextHateSpeechDetection.java
/** * @param args the command line arguments */// w w w .j a va2 s . c om public static void main(String[] args) throws TwitterException, NotDirectoryException, IOException { HateSpeechClassifier1 classifier = new HateSpeechClassifier1(); AnnotatedDataFolder data = new AnnotatedDataFolder("data"); boolean overSampling = false; classifier.train(data.getDateSortedLabeledData(overSampling)); TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); StatusListener listener = new StatusListener() { int numHateSpeech = 0; int numTweets = 0; @Override public void onStatus(Status status) { if (status.getLang().equals("in")) { numTweets++; if (classifier.isHateSpeech(status.getText(), 0.5)) { System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString())); System.out.println(); System.out.println("lang: " + status.getLang()); System.out.println("number of detected hate speech: " + numHateSpeech); System.out.println("total number of streamed tweets: " + numTweets); System.out.println(); System.out.println(); numHateSpeech++; } } else { System.out.println("ignoring non-Indonesian tweet"); } // if (status.getGeoLocation() != null) { // System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " $ " + status.getGeoLocation().toString()); // } // if (status.getLang().equals("id")) { // System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString())); // } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { // System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; twitterStream.addListener(listener); FilterQuery filterQuery = new FilterQuery(); filterQuery.track(new String[] { "a", "i", "u", "e", "o" }); filterQuery.language("in"); twitterStream.filter(filterQuery); twitterStream.sample(); }
From source file:nyu.twitter.lg.FentchTwitter.java
License:Open Source License
public static void invoke() throws Exception { init();/*from w w w . j a v a 2 s .c o m*/ // Create table if it does not exist yet if (Tables.doesTableExist(dynamoDB, tableName)) { System.out.println("Table " + tableName + " is already ACTIVE"); } else { // Create a table with a primary hash key named 'name', which holds // a string CreateTableRequest createTableRequest = new CreateTableRequest().withTableName(tableName) .withKeySchema(new KeySchemaElement().withAttributeName("id").withKeyType(KeyType.HASH)) .withAttributeDefinitions(new AttributeDefinition().withAttributeName("id") .withAttributeType(ScalarAttributeType.S)) .withProvisionedThroughput( new ProvisionedThroughput().withReadCapacityUnits(1L).withWriteCapacityUnits(1L)); TableDescription createdTableDescription = dynamoDB.createTable(createTableRequest) .getTableDescription(); System.out.println("Created Table: " + createdTableDescription); // Wait for it to become active System.out.println("Waiting for " + tableName + " to become ACTIVE..."); Tables.waitForTableToBecomeActive(dynamoDB, tableName); } ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("Emwo2pG").setOAuthConsumerSecret("RM9B7fske5T") .setOAuthAccessToken("19ubQOirq").setOAuthAccessTokenSecret("Lbg3C"); final TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener listener = new StatusListener() { @Override public void onStatus(Status status) { if (status.getGeoLocation() != null && status.getPlace() != null) { // if (count == 0) { // count++; // } // latitude = status.getGeoLocation().getLatitude(); longtitude = status.getGeoLocation().getLongitude(); place = status.getPlace().getCountry() + "," + status.getPlace().getFullName(); date = status.getCreatedAt().toString(); id = Integer.toString(count); name = status.getUser().getScreenName(); message = status.getText(); System.out.println("---------------------------"); System.out.println("ID:" + count); System.out.println("latitude:" + latitude); System.out.println("longtitude:" + longtitude); System.out.println("place:" + place); System.out.println("name:" + name); System.out.println("message:" + message); System.out.println("data:" + date); System.out.println("-------------8-------------"); insertDB(id, count, name, longtitude, latitude, place, message, date); if (++count > 100) { twitterStream.shutdown(); System.out.println("Information Collection Completed"); } // count = (count+1) % 101; } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { // System.out.println("Got a status deletion notice id:" // + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // System.out.println("Got track limitation notice:" // + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { // System.out.println("Got scrub_geo event userId:" + userId // + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { // System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; twitterStream.addListener(listener); twitterStream.sample(); }
From source file:org.apache.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser, File temporaryDirectory) { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override//from w w w . ja v a2 s. c o m public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { log.error(ex, "Got exception"); } @Override public void onStallWarning(StallWarning warning) { log.warn("Got stall warning: %s", warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { @Override public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Nullable @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); Thread.sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(StringUtils.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? StringUtils.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:org.apache.s4.example.twitter.TwitterInputAdapter.java
License:Apache License
public void connectAndRead() throws Exception { ConfigurationBuilder cb = new ConfigurationBuilder(); Properties twitterProperties = new Properties(); File twitter4jPropsFile = new File(System.getProperty("user.home") + "/twitter4j.properties"); if (!twitter4jPropsFile.exists()) { logger.error(/*ww w . j av a 2 s . c om*/ "Cannot find twitter4j.properties file in this location :[{}]. Make sure it is available at this place and includes user/password credentials", twitter4jPropsFile.getAbsolutePath()); return; } twitterProperties.load(new FileInputStream(twitter4jPropsFile)); cb.setDebugEnabled(Boolean.valueOf(twitterProperties.getProperty("debug"))) .setUser(twitterProperties.getProperty("user")) .setPassword(twitterProperties.getProperty("password")); TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener statusListener = new StatusListener() { @Override public void onException(Exception ex) { logger.error("error", ex); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { logger.error("error"); } @Override public void onStatus(Status status) { messageQueue.add(status); } @Override public void onScrubGeo(long userId, long upToStatusId) { logger.error("error"); } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { logger.error("error"); } }; twitterStream.addListener(statusListener); twitterStream.sample(); }
From source file:org.apache.storm.starter.spout.Q2TwitterSampleSpout.java
License:Apache License
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { queue = new LinkedBlockingQueue<Status>(1000); _collector = collector;/*from w w w . j a v a 2 s .c o m*/ StatusListener listener = new StatusListener() { @Override public void onStatus(Status status) { queue.offer(status); } @Override public void onDeletionNotice(StatusDeletionNotice sdn) { } @Override public void onTrackLimitationNotice(int i) { } @Override public void onScrubGeo(long l, long l1) { } @Override public void onException(Exception ex) { } @Override public void onStallWarning(StallWarning arg0) { // TODO Auto-generated method stub } }; _twitterStream = new TwitterStreamFactory(new ConfigurationBuilder().setJSONStoreEnabled(true).build()) .getInstance(); _twitterStream.addListener(listener); _twitterStream.setOAuthConsumer(consumerKey, consumerSecret); AccessToken token = new AccessToken(accessToken, accessTokenSecret); _twitterStream.setOAuthAccessToken(token); if (keyWords.length == 0) { _twitterStream.sample(); } else { FilterQuery query = new FilterQuery().track(keyWords); query.language(new String[] { "en" }); query.track(new String[] { "job", "vote", "local", "Clinton", "Trump", "debate", "tech", "TheWalkingDead", "android", "Election", "USA", "GOT7", "Empire", "Apple", "ISIS", "election", "Football", "iPhone", "Amazon", "uber" }); _twitterStream.filter(query); } }