Example usage for twitter4j StatusListener StatusListener

List of usage examples for twitter4j StatusListener StatusListener

Introduction

In this page you can find the example usage for twitter4j StatusListener StatusListener.

Prototype

StatusListener

Source Link

Usage

From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java

License:Apache License

@Override
public Firehose connect(InputRowParser parser) throws IOException {
    final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() {
        @Override//from   w w  w .j av a 2s .  co  m
        public void onConnect() {
            log.info("Connected_to_Twitter");
        }

        @Override
        public void onDisconnect() {
            log.info("Disconnect_from_Twitter");
        }

        /**
         * called before thread gets cleaned up
         */
        @Override
        public void onCleanUp() {
            log.info("Cleanup_twitter_stream");
        }
    }; // ConnectionLifeCycleListener

    final TwitterStream twitterStream;
    final StatusListener statusListener;
    final int QUEUE_SIZE = 2000;
    /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread.   */
    final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE);
    final long startMsec = System.currentTimeMillis();

    //
    //   set up Twitter Spritzer
    //
    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener);
    statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j
        @Override
        public void onStatus(Status status) {
            // time to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }
            try {
                boolean success = queue.offer(status, 15L, TimeUnit.SECONDS);
                if (!success) {
                    log.warn("queue too slow!");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // This notice will be sent each time a limited stream becomes unlimited.
            // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity.
            log.warn("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }
    };

    twitterStream.addListener(statusListener);
    twitterStream.sample(); // creates a generic StatusStream
    log.info("returned from sample()");

    return new Firehose() {

        private final Runnable doNothingRunnable = new Runnable() {
            public void run() {
            }
        };

        private long rowCount = 0L;
        private boolean waitIfmax = (getMaxEventCount() < 0L);
        private final Map<String, Object> theMap = new TreeMap<>();
        // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper();

        private boolean maxTimeReached() {
            if (getMaxRunMinutes() <= 0) {
                return false;
            } else {
                return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes();
            }
        }

        private boolean maxCountReached() {
            return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount();
        }

        @Override
        public boolean hasMore() {
            if (maxCountReached() || maxTimeReached()) {
                return waitIfmax;
            } else {
                return true;
            }
        }

        @Override
        public InputRow nextRow() {
            // Interrupted to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }

            // all done?
            if (maxCountReached() || maxTimeReached()) {
                if (waitIfmax) {
                    // sleep a long time instead of terminating
                    try {
                        log.info("reached limit, sleeping a long time...");
                        sleep(2000000000L);
                    } catch (InterruptedException e) {
                        throw new RuntimeException("InterruptedException", e);
                    }
                } else {
                    // allow this event through, and the next hasMore() call will be false
                }
            }
            if (++rowCount % 1000 == 0) {
                log.info("nextRow() has returned %,d InputRows", rowCount);
            }

            Status status;
            try {
                status = queue.take();
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }

            theMap.clear();

            HashtagEntity[] hts = status.getHashtagEntities();
            String text = status.getText();
            theMap.put("text", (null == text) ? "" : text);
            theMap.put("htags", (hts.length > 0)
                    ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() {
                        @Nullable
                        @Override
                        public String apply(HashtagEntity input) {
                            return input.getText();
                        }
                    })
                    : ImmutableList.<String>of());

            long[] lcontrobutors = status.getContributors();
            List<String> contributors = new ArrayList<>();
            for (long contrib : lcontrobutors) {
                contributors.add(String.format("%d", contrib));
            }
            theMap.put("contributors", contributors);

            GeoLocation geoLocation = status.getGeoLocation();
            if (null != geoLocation) {
                double lat = status.getGeoLocation().getLatitude();
                double lon = status.getGeoLocation().getLongitude();
                theMap.put("lat", lat);
                theMap.put("lon", lon);
            } else {
                theMap.put("lat", null);
                theMap.put("lon", null);
            }

            if (status.getSource() != null) {
                Matcher m = sourcePattern.matcher(status.getSource());
                theMap.put("source", m.find() ? m.group(1) : status.getSource());
            }

            theMap.put("retweet", status.isRetweet());

            if (status.isRetweet()) {
                Status original = status.getRetweetedStatus();
                theMap.put("retweet_count", original.getRetweetCount());

                User originator = original.getUser();
                theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : "");
                theMap.put("originator_follower_count",
                        originator != null ? originator.getFollowersCount() : "");
                theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : "");
                theMap.put("originator_verified", originator != null ? originator.isVerified() : "");
            }

            User user = status.getUser();
            final boolean hasUser = (null != user);
            theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0);
            theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0);
            theMap.put("lang", hasUser ? user.getLang() : "");
            theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available?
            theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0);
            theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : "");
            theMap.put("screen_name", hasUser ? user.getScreenName() : "");
            theMap.put("location", hasUser ? user.getLocation() : "");
            theMap.put("verified", hasUser ? user.isVerified() : "");

            theMap.put("ts", status.getCreatedAt().getTime());

            List<String> dimensions = Lists.newArrayList(theMap.keySet());

            return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap);
        }

        @Override
        public Runnable commit() {
            // ephemera in, ephemera out.
            return doNothingRunnable; // reuse the same object each time
        }

        @Override
        public void close() throws IOException {
            log.info("CLOSE twitterstream");
            twitterStream.shutdown(); // invokes twitterStream.cleanUp()
        }
    };
}

From source file:kafka.producer.PrintGeoStream.java

License:Apache License

/**
 * Main entry of this application.//from  w ww . j  av  a  2 s.  c o  m
 *
 * @param args <kafka host> <kafka topic>
 * @throws twitter4j.TwitterException
 */
public static void main(String[] args) throws TwitterException {

    //        if (args.length != 2) {
    //            System.out.println("Usage: java twitter4j.examples.PrintFilterStream  <kafka host>  <kafka topic>");
    //            System.exit(-1);
    //        }

    //        final String kafkaZKHost = args[0];
    //        final String kafkaTopic = args[1];

    final String kafkaZKHost = "localhost";
    final String kafkaTopic = "flightDelay";

    BasicConfigurator.configure();

    StatusListener listener = new StatusListener() {

        Gson gson = new Gson();

        Properties kafkaProps = new Properties();
        //kafkaProps.put("serializer.class", "kafka.serializer.StringEncoder");
        //kafkaProps.put("zk.connect", "localhost:2181");
        Producer<Integer, String> kafkaProducer = null;
        int tweetCount = 0;
        int interimCount = 0;

        @Override
        public void onStatus(Status status) {
            if (kafkaProducer == null) {
                kafkaProps.put("serializer.class", "kafka.serializer.StringEncoder");
                //kafkaProps.put("zk.connect", "localhost:2181");
                kafkaProps.put("zk.connect", kafkaZKHost.concat(":2181"));
                kafkaProducer = new Producer<Integer, String>(new ProducerConfig(kafkaProps));

            }
            tweetCount++;
            interimCount++;
            if (interimCount == 100) {
                LOG.info(" total tweets received: " + tweetCount);
                interimCount = 0;
            }

            //System.out.println("@" + status.getUser().getScreenName() + " - "); // + status.getText());
            //if ( status.getPlace() != null ) {
            //    System.out.println("coordinates = " + status.getPlace().getGeometryCoordinates());
            //}

            TweetInfo tweetInfo = new TweetInfo();
            tweetInfo.populate(status);
            String message = gson.toJson(tweetInfo, TweetInfo.class);

            kafkaProducer.send(new ProducerData<Integer, String>(kafkaTopic, message));
            //kafkaProducer.send(new ProducerData<Integer, String>("live_tweets", message));
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            //System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };
    /*
     * - use Redis to pass filter messages in - or ZK?
     * - Read in filter JSON with parms for:
     *     - lat/lon boxj
     *     - screen names to follow
     *     - hashtags to track
     * - Convert screen names to userIds
     *     - Twitter twitter = new TwitterFactory().getInstance();
     *       ResponseList<User> users = twitter.lookupUsers(args[0].split(","));
     * - Kick off filter with new FilterQuery
     *     - periodically check, and if filter updated, restart the TwitterStream:
     *         .shutdown()
     *         new
     *         .addListener
     * 
     * 
     */

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    //        twitterStream.setOAuthAccessToken(new AccessToken("14098069-IgutahzmbaCubdoMU0MxsfsJT2iNVpRnoohCbV67k", "d1kFKqRSnrlA4bP1XWWLiHNsHfYDZQPWDHRVpGP4hM"));
    //        twitterStream.setOAuthConsumer("G0SKTmMXjfMsunBh3Furg", "iVTYNLqVaVdUZz4xOn7H8ywylTaTuW3kH1dT21NUy4");
    twitterStream.addListener(listener);
    /*
     * For CONUS only: 
     *  "lowerLat": "22.0",
     *  "lowerLon": "-127.0",
     *  "upperLat": "47.5",
     *  "upperLon": "-67.0"
     */
    FilterQuery fQuery = new FilterQuery();
    String[] topics = new String[2];
    topics[0] = "delayed flight";
    topics[1] = "flight delay";

    //        double[][] us = {{-180, 24}, {-66, 50}};
    double[][] lax = { { -118.5, 33.8 }, { -118.3, 34.01 } };
    fQuery.locations(lax);
    fQuery.track(topics);
    twitterStream.filter(fQuery);

}

From source file:my.twittergui.TwitterUI.java

private void jButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton2ActionPerformed
    String text;/*from   w  ww.j  a va  2  s . c  om*/
    text = jTextField1.getText();
    //jTextArea1.append(text+"\n");
    jTextArea1.append("Searching for: ");
    String[] strarray = text.split(" ");
    for (int i = 0; i < strarray.length; i++) {
        jTextArea1.append(strarray[i] + "\n");
    }

    File filed = null;
    filed = new File("C:\\Results");
    if (!filed.exists()) {
        filed.mkdir();
    }

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey("").setOAuthConsumerSecret("").setOAuthAccessToken("")
            .setOAuthAccessTokenSecret("");

    StatusListener listener = new StatusListener() {

        public void onStatus(Status status) {

            try {

                int fileno = 0;
                String strI = Integer.toString(fileno);
                String fname = "C:\\results\\DataCol" + strI + ".csv";
                File file = new File(fname);
                FileWriter bw = new FileWriter(file, true);

                if (file.length() == 0) {
                    bw.write("\"Screen Name\", text, \"created at\", geolocation, Retweet? \r\n");
                    bw.flush();
                }
                while (file.length() > 10485760) {
                    fileno += 1;
                    strI = Integer.toString(fileno);
                    fname = "DataCol" + strI + ".txt";
                    file = new File(fname);
                    bw = new FileWriter(file, true);
                    bw.write("\"Screen Name\", \"text\", \"created_at\", \"geolocation\" \r\n");
                    bw.flush();

                }

                // if(!status.isRetweet()){
                bw.write("\r\n");
                bw.write("\"" + status.getUser().getScreenName() + "\",");
                String tweettxt = status.getText();
                tweettxt = tweettxt.replace("\n", "");
                tweettxt = tweettxt.replace(",", "");
                tweettxt = tweettxt.replace("\"", "");
                bw.write("\"" + tweettxt + "\",");

                bw.write("\"" + status.getCreatedAt() + "\",");
                if (status.getGeoLocation() != null) {
                    bw.write("\"" + status.getGeoLocation() + "\"");
                } else
                    bw.write("N/A,");
                if (status.isRetweet())
                    bw.write("Yes");
                else
                    bw.write("No");

                bw.flush();

                bw.close();
                //System.out.print("\n");
                String str = "@" + status.getUser().getScreenName() + "  " + status.getText() + " "
                        + status.getCreatedAt() + " ";
                if (status.getGeoLocation() != null)
                    str += status.getGeoLocation();
                //System.out.print(str);
                jTextArea1.append(str + "\n");
                // }

                bw.close();
            } catch (IOException e) {
                //System.out.print("EXCEPTION");
                e.printStackTrace();
            }

        }

        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
        }

        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
        }

        public void onStallWarning(StallWarning stall) {
        }

        public void onScrubGeo(long a, long b) {
        }

        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };
    TwitterStreamFactory tf = new TwitterStreamFactory(cb.build());
    TwitterStream twitterStream = tf.getInstance();
    twitterStream.addListener(listener);
    // Filter
    FilterQuery filtre = new FilterQuery();
    Scanner in = new Scanner(System.in);

    filtre.track(strarray);
    twitterStream.filter(filtre);

}

From source file:nlptexthatespeechdetection.dataCollection.GetTwitterDoc2VecTrainingData.java

public static void main(String[] args) throws FileNotFoundException, IOException {
    File dir = new File(folderName);
    if (!dir.exists())
        dir.mkdir();//w w  w  .ja  v a 2s.  c  om
    if (!dir.isDirectory()) {
        System.out.println(folderName + " is not a directory");
        return;
    }

    System.out.println("number of tweets required: ");
    int numTweetsRequired = (new Scanner(System.in)).nextInt();

    String path = folderName + "/" + fileName;
    File file = new File(path);
    if (!file.exists())
        file.createNewFile();
    FileWriter writer = new FileWriter(path, true);

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener = new StatusListener() {
        int numTweets = 0;

        @Override
        public void onStatus(Status status) {
            if (status.getLang().equals("in")) {
                try {
                    String statusText = status.getText();
                    writer.write("\n");
                    writer.write(statusText);
                    numTweets++;
                    System.out.println("numTweets: " + numTweets);

                    if (numTweets >= numTweetsRequired) {
                        writer.close();
                        System.exit(0);
                    }
                } catch (IOException ex) {
                    Logger.getLogger(GetTwitterDoc2VecTrainingData.class.getName()).log(Level.SEVERE, null, ex);
                }

            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };

    twitterStream.addListener(listener);

    FilterQuery filterQuery = new FilterQuery();
    filterQuery.track(new String[] { "a", "i", "u", "e", "o" });
    filterQuery.language("in");
    twitterStream.filter(filterQuery);

}

From source file:nlptexthatespeechdetection.dataCollection.TwitterStreamingAnnotator.java

public static void main(String[] args) throws NotDirectoryException {
    Scanner sc = new Scanner(System.in);
    System.out.println("Nama Anda (sebagai anotator): ");
    String namaAnotator = sc.nextLine();
    AnnotatedDataFolder annotatedDataFolder = new AnnotatedDataFolder(dataFolderName);

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener = new StatusListener() {
        @Override/*from   ww w  .  jav  a  2s . c  o m*/
        public void onStatus(Status status) {
            if (status.getLang().equals("in")) {
                System.out.println();
                System.out.println();
                System.out.println("=======ANOTASI=======");
                System.out.println("status: " + status.getText());
                System.out.println();
                System.out.println("is this a hate speech?(y/n. any other if you do not know)");
                String annotatorResponse = sc.nextLine().trim().toLowerCase();

                Date date = new Date();
                String dateString = dateFormat.format(date);

                try {
                    if (annotatorResponse.equals("y")) {
                        String filePath = annotatedDataFolder.saveHateSpeechString(namaAnotator, dateString,
                                status.getText());
                        System.out.println("Saved data to: " + filePath);
                    } else if (annotatorResponse.equals("n")) {
                        String filePath = annotatedDataFolder.saveNotHateSpeechString(namaAnotator, dateString,
                                status.getText());
                        System.out.println("Saved data to: " + filePath);
                    }
                    System.out.println("thank you!");
                } catch (FileNotFoundException ex) {
                    ex.printStackTrace();
                } catch (IOException ex) {
                    Logger.getLogger(TwitterStreamingAnnotator.class.getName()).log(Level.SEVERE, null, ex);
                }

            } else {
                System.out.println("ignoring non-indonesian tweet");
            }
            //                if (status.getGeoLocation() != null) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " $ " + status.getGeoLocation().toString());
            //                }
            //                if (status.getLang().equals("id")) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString()));
            //                }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };

    twitterStream.addListener(listener);

    FilterQuery filterQuery = new FilterQuery();
    filterQuery.track(new String[] { "a", "i", "u", "e", "o" });
    filterQuery.language("in");
    twitterStream.filter(filterQuery);
}

From source file:nlptexthatespeechdetection.NLPTextHateSpeechDetection.java

/**
 * @param args the command line arguments
 *///  w  w w .j  a  va2  s  . c  om
public static void main(String[] args) throws TwitterException, NotDirectoryException, IOException {
    HateSpeechClassifier1 classifier = new HateSpeechClassifier1();
    AnnotatedDataFolder data = new AnnotatedDataFolder("data");
    boolean overSampling = false;
    classifier.train(data.getDateSortedLabeledData(overSampling));

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener = new StatusListener() {
        int numHateSpeech = 0;
        int numTweets = 0;

        @Override
        public void onStatus(Status status) {
            if (status.getLang().equals("in")) {
                numTweets++;
                if (classifier.isHateSpeech(status.getText(), 0.5)) {
                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * "
                            + status.getId() + " # " + status.getLang() + " $ "
                            + (status.getGeoLocation() == null ? "NULLGEO"
                                    : status.getGeoLocation().toString()));
                    System.out.println();
                    System.out.println("lang: " + status.getLang());
                    System.out.println("number of detected hate speech: " + numHateSpeech);
                    System.out.println("total number of streamed tweets: " + numTweets);
                    System.out.println();
                    System.out.println();
                    numHateSpeech++;
                }
            } else {
                System.out.println("ignoring non-Indonesian tweet");
            }
            //                if (status.getGeoLocation() != null) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " $ " + status.getGeoLocation().toString());
            //                }
            //                if (status.getLang().equals("id")) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString()));
            //                }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };

    twitterStream.addListener(listener);

    FilterQuery filterQuery = new FilterQuery();
    filterQuery.track(new String[] { "a", "i", "u", "e", "o" });
    filterQuery.language("in");
    twitterStream.filter(filterQuery);

    twitterStream.sample();
}

From source file:nyu.twitter.lg.FentchTwitter.java

License:Open Source License

public static void invoke() throws Exception {
    init();/*from  w w  w  .  j  a  v a 2  s .c o m*/
    // Create table if it does not exist yet
    if (Tables.doesTableExist(dynamoDB, tableName)) {
        System.out.println("Table " + tableName + " is already ACTIVE");
    } else {
        // Create a table with a primary hash key named 'name', which holds
        // a string
        CreateTableRequest createTableRequest = new CreateTableRequest().withTableName(tableName)
                .withKeySchema(new KeySchemaElement().withAttributeName("id").withKeyType(KeyType.HASH))
                .withAttributeDefinitions(new AttributeDefinition().withAttributeName("id")
                        .withAttributeType(ScalarAttributeType.S))
                .withProvisionedThroughput(
                        new ProvisionedThroughput().withReadCapacityUnits(1L).withWriteCapacityUnits(1L));
        TableDescription createdTableDescription = dynamoDB.createTable(createTableRequest)
                .getTableDescription();
        System.out.println("Created Table: " + createdTableDescription);
        // Wait for it to become active
        System.out.println("Waiting for " + tableName + " to become ACTIVE...");
        Tables.waitForTableToBecomeActive(dynamoDB, tableName);
    }

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey("Emwo2pG").setOAuthConsumerSecret("RM9B7fske5T")
            .setOAuthAccessToken("19ubQOirq").setOAuthAccessTokenSecret("Lbg3C");

    final TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();

    StatusListener listener = new StatusListener() {

        @Override
        public void onStatus(Status status) {

            if (status.getGeoLocation() != null && status.getPlace() != null) {

                //               if (count == 0) {
                //                  count++;
                //               }
                //               
                latitude = status.getGeoLocation().getLatitude();
                longtitude = status.getGeoLocation().getLongitude();
                place = status.getPlace().getCountry() + "," + status.getPlace().getFullName();
                date = status.getCreatedAt().toString();
                id = Integer.toString(count);
                name = status.getUser().getScreenName();
                message = status.getText();
                System.out.println("---------------------------");
                System.out.println("ID:" + count);
                System.out.println("latitude:" + latitude);
                System.out.println("longtitude:" + longtitude);
                System.out.println("place:" + place);
                System.out.println("name:" + name);
                System.out.println("message:" + message);
                System.out.println("data:" + date);
                System.out.println("-------------8-------------");

                insertDB(id, count, name, longtitude, latitude, place, message, date);

                if (++count > 100) {
                    twitterStream.shutdown();
                    System.out.println("Information Collection Completed");
                }
                //      count = (count+1) % 101;
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //            System.out.println("Got a status deletion notice id:"
            //                  + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            //            System.out.println("Got track limitation notice:"
            //                  + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            //            System.out.println("Got scrub_geo event userId:" + userId
            //                  + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            //            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };
    twitterStream.addListener(listener);
    twitterStream.sample();
}

From source file:org.apache.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java

License:Apache License

@Override
public Firehose connect(InputRowParser parser, File temporaryDirectory) {
    final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() {
        @Override//from w w w .  ja v a2  s. c  o m
        public void onConnect() {
            log.info("Connected_to_Twitter");
        }

        @Override
        public void onDisconnect() {
            log.info("Disconnect_from_Twitter");
        }

        /**
         * called before thread gets cleaned up
         */
        @Override
        public void onCleanUp() {
            log.info("Cleanup_twitter_stream");
        }
    }; // ConnectionLifeCycleListener

    final TwitterStream twitterStream;
    final StatusListener statusListener;
    final int QUEUE_SIZE = 2000;
    /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread.   */
    final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE);
    final long startMsec = System.currentTimeMillis();

    //
    //   set up Twitter Spritzer
    //
    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener);
    statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j
        @Override
        public void onStatus(Status status) {
            // time to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }
            try {
                boolean success = queue.offer(status, 15L, TimeUnit.SECONDS);
                if (!success) {
                    log.warn("queue too slow!");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // This notice will be sent each time a limited stream becomes unlimited.
            // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity.
            log.warn("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onException(Exception ex) {
            log.error(ex, "Got exception");
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            log.warn("Got stall warning: %s", warning);
        }
    };

    twitterStream.addListener(statusListener);
    twitterStream.sample(); // creates a generic StatusStream
    log.info("returned from sample()");

    return new Firehose() {

        private final Runnable doNothingRunnable = new Runnable() {
            @Override
            public void run() {
            }
        };

        private long rowCount = 0L;
        private boolean waitIfmax = (getMaxEventCount() < 0L);
        private final Map<String, Object> theMap = new TreeMap<>();
        // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper();

        private boolean maxTimeReached() {
            if (getMaxRunMinutes() <= 0) {
                return false;
            } else {
                return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes();
            }
        }

        private boolean maxCountReached() {
            return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount();
        }

        @Override
        public boolean hasMore() {
            if (maxCountReached() || maxTimeReached()) {
                return waitIfmax;
            } else {
                return true;
            }
        }

        @Nullable
        @Override
        public InputRow nextRow() {
            // Interrupted to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }

            // all done?
            if (maxCountReached() || maxTimeReached()) {
                if (waitIfmax) {
                    // sleep a long time instead of terminating
                    try {
                        log.info("reached limit, sleeping a long time...");
                        Thread.sleep(2000000000L);
                    } catch (InterruptedException e) {
                        throw new RuntimeException("InterruptedException", e);
                    }
                } else {
                    // allow this event through, and the next hasMore() call will be false
                }
            }
            if (++rowCount % 1000 == 0) {
                log.info("nextRow() has returned %,d InputRows", rowCount);
            }

            Status status;
            try {
                status = queue.take();
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }

            theMap.clear();

            HashtagEntity[] hts = status.getHashtagEntities();
            String text = status.getText();
            theMap.put("text", (null == text) ? "" : text);
            theMap.put("htags", (hts.length > 0)
                    ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() {
                        @Nullable
                        @Override
                        public String apply(HashtagEntity input) {
                            return input.getText();
                        }
                    })
                    : ImmutableList.<String>of());

            long[] lcontrobutors = status.getContributors();
            List<String> contributors = new ArrayList<>();
            for (long contrib : lcontrobutors) {
                contributors.add(StringUtils.format("%d", contrib));
            }
            theMap.put("contributors", contributors);

            GeoLocation geoLocation = status.getGeoLocation();
            if (null != geoLocation) {
                double lat = status.getGeoLocation().getLatitude();
                double lon = status.getGeoLocation().getLongitude();
                theMap.put("lat", lat);
                theMap.put("lon", lon);
            } else {
                theMap.put("lat", null);
                theMap.put("lon", null);
            }

            if (status.getSource() != null) {
                Matcher m = sourcePattern.matcher(status.getSource());
                theMap.put("source", m.find() ? m.group(1) : status.getSource());
            }

            theMap.put("retweet", status.isRetweet());

            if (status.isRetweet()) {
                Status original = status.getRetweetedStatus();
                theMap.put("retweet_count", original.getRetweetCount());

                User originator = original.getUser();
                theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : "");
                theMap.put("originator_follower_count",
                        originator != null ? originator.getFollowersCount() : "");
                theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : "");
                theMap.put("originator_verified", originator != null ? originator.isVerified() : "");
            }

            User user = status.getUser();
            final boolean hasUser = (null != user);
            theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0);
            theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0);
            theMap.put("lang", hasUser ? user.getLang() : "");
            theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available?
            theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0);
            theMap.put("user_id", hasUser ? StringUtils.format("%d", user.getId()) : "");
            theMap.put("screen_name", hasUser ? user.getScreenName() : "");
            theMap.put("location", hasUser ? user.getLocation() : "");
            theMap.put("verified", hasUser ? user.isVerified() : "");

            theMap.put("ts", status.getCreatedAt().getTime());

            List<String> dimensions = Lists.newArrayList(theMap.keySet());

            return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap);
        }

        @Override
        public Runnable commit() {
            // ephemera in, ephemera out.
            return doNothingRunnable; // reuse the same object each time
        }

        @Override
        public void close() {
            log.info("CLOSE twitterstream");
            twitterStream.shutdown(); // invokes twitterStream.cleanUp()
        }
    };
}

From source file:org.apache.s4.example.twitter.TwitterInputAdapter.java

License:Apache License

public void connectAndRead() throws Exception {

    ConfigurationBuilder cb = new ConfigurationBuilder();
    Properties twitterProperties = new Properties();
    File twitter4jPropsFile = new File(System.getProperty("user.home") + "/twitter4j.properties");
    if (!twitter4jPropsFile.exists()) {
        logger.error(/*ww w .  j  av  a  2 s . c  om*/
                "Cannot find twitter4j.properties file in this location :[{}]. Make sure it is available at this place and includes user/password credentials",
                twitter4jPropsFile.getAbsolutePath());
        return;
    }
    twitterProperties.load(new FileInputStream(twitter4jPropsFile));

    cb.setDebugEnabled(Boolean.valueOf(twitterProperties.getProperty("debug")))
            .setUser(twitterProperties.getProperty("user"))
            .setPassword(twitterProperties.getProperty("password"));
    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    StatusListener statusListener = new StatusListener() {

        @Override
        public void onException(Exception ex) {
            logger.error("error", ex);
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            logger.error("error");
        }

        @Override
        public void onStatus(Status status) {
            messageQueue.add(status);

        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            logger.error("error");
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            logger.error("error");
        }
    };
    twitterStream.addListener(statusListener);
    twitterStream.sample();

}

From source file:org.apache.storm.starter.spout.Q2TwitterSampleSpout.java

License:Apache License

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    queue = new LinkedBlockingQueue<Status>(1000);
    _collector = collector;/*from  w w  w  .  j  a v a 2 s  .c  o m*/

    StatusListener listener = new StatusListener() {

        @Override
        public void onStatus(Status status) {

            queue.offer(status);
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice sdn) {
        }

        @Override
        public void onTrackLimitationNotice(int i) {
        }

        @Override
        public void onScrubGeo(long l, long l1) {
        }

        @Override
        public void onException(Exception ex) {
        }

        @Override
        public void onStallWarning(StallWarning arg0) {
            // TODO Auto-generated method stub

        }

    };

    _twitterStream = new TwitterStreamFactory(new ConfigurationBuilder().setJSONStoreEnabled(true).build())
            .getInstance();

    _twitterStream.addListener(listener);
    _twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    _twitterStream.setOAuthAccessToken(token);

    if (keyWords.length == 0) {

        _twitterStream.sample();
    }

    else {

        FilterQuery query = new FilterQuery().track(keyWords);
        query.language(new String[] { "en" });
        query.track(new String[] { "job", "vote", "local", "Clinton", "Trump", "debate", "tech",
                "TheWalkingDead", "android", "Election", "USA", "GOT7", "Empire", "Apple", "ISIS", "election",
                "Football", "iPhone", "Amazon", "uber" });
        _twitterStream.filter(query);
    }

}