Example usage for twitter4j TwitterStreamFactory TwitterStreamFactory

List of usage examples for twitter4j TwitterStreamFactory TwitterStreamFactory

  1. HOME
  2. Java
  3. twitter4j
  4. twitter4j.*
  5. TwitterStreamFactory
  6. TwitterStreamFactory

Introduction

In this page you can find the example usage for twitter4j TwitterStreamFactory TwitterStreamFactory.

Prototype

public TwitterStreamFactory()

Source Link

Document

Creates a TwitterStreamFactory with the root configuration.

Usage

From source file:net.lacolaco.smileessence.twitter.TwitterApi.java

License:Open Source License

public TwitterStream getTwitterStream() {
    TwitterStream stream = new TwitterStreamFactory().getInstance();
    stream.setOAuthAccessToken(new AccessToken(token, tokenSecret));
    return stream;
}

From source file:nlptexthatespeechdetection.dataCollection.GetTwitterDoc2VecTrainingData.java

public static void main(String[] args) throws FileNotFoundException, IOException {
    File dir = new File(folderName);
    if (!dir.exists())
        dir.mkdir();//from  w  w  w.  j  a  v  a2  s .c  om
    if (!dir.isDirectory()) {
        System.out.println(folderName + " is not a directory");
        return;
    }

    System.out.println("number of tweets required: ");
    int numTweetsRequired = (new Scanner(System.in)).nextInt();

    String path = folderName + "/" + fileName;
    File file = new File(path);
    if (!file.exists())
        file.createNewFile();
    FileWriter writer = new FileWriter(path, true);

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener = new StatusListener() {
        int numTweets = 0;

        @Override
        public void onStatus(Status status) {
            if (status.getLang().equals("in")) {
                try {
                    String statusText = status.getText();
                    writer.write("\n");
                    writer.write(statusText);
                    numTweets++;
                    System.out.println("numTweets: " + numTweets);

                    if (numTweets >= numTweetsRequired) {
                        writer.close();
                        System.exit(0);
                    }
                } catch (IOException ex) {
                    Logger.getLogger(GetTwitterDoc2VecTrainingData.class.getName()).log(Level.SEVERE, null, ex);
                }

            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };

    twitterStream.addListener(listener);

    FilterQuery filterQuery = new FilterQuery();
    filterQuery.track(new String[] { "a", "i", "u", "e", "o" });
    filterQuery.language("in");
    twitterStream.filter(filterQuery);

}

From source file:nlptexthatespeechdetection.dataCollection.TwitterStreamingAnnotator.java

public static void main(String[] args) throws NotDirectoryException {
    Scanner sc = new Scanner(System.in);
    System.out.println("Nama Anda (sebagai anotator): ");
    String namaAnotator = sc.nextLine();
    AnnotatedDataFolder annotatedDataFolder = new AnnotatedDataFolder(dataFolderName);

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener = new StatusListener() {
        @Override/*www .j a  v a 2s.c o  m*/
        public void onStatus(Status status) {
            if (status.getLang().equals("in")) {
                System.out.println();
                System.out.println();
                System.out.println("=======ANOTASI=======");
                System.out.println("status: " + status.getText());
                System.out.println();
                System.out.println("is this a hate speech?(y/n. any other if you do not know)");
                String annotatorResponse = sc.nextLine().trim().toLowerCase();

                Date date = new Date();
                String dateString = dateFormat.format(date);

                try {
                    if (annotatorResponse.equals("y")) {
                        String filePath = annotatedDataFolder.saveHateSpeechString(namaAnotator, dateString,
                                status.getText());
                        System.out.println("Saved data to: " + filePath);
                    } else if (annotatorResponse.equals("n")) {
                        String filePath = annotatedDataFolder.saveNotHateSpeechString(namaAnotator, dateString,
                                status.getText());
                        System.out.println("Saved data to: " + filePath);
                    }
                    System.out.println("thank you!");
                } catch (FileNotFoundException ex) {
                    ex.printStackTrace();
                } catch (IOException ex) {
                    Logger.getLogger(TwitterStreamingAnnotator.class.getName()).log(Level.SEVERE, null, ex);
                }

            } else {
                System.out.println("ignoring non-indonesian tweet");
            }
            //                if (status.getGeoLocation() != null) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " $ " + status.getGeoLocation().toString());
            //                }
            //                if (status.getLang().equals("id")) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString()));
            //                }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };

    twitterStream.addListener(listener);

    FilterQuery filterQuery = new FilterQuery();
    filterQuery.track(new String[] { "a", "i", "u", "e", "o" });
    filterQuery.language("in");
    twitterStream.filter(filterQuery);
}

From source file:nlptexthatespeechdetection.NLPTextHateSpeechDetection.java

/**
 * @param args the command line arguments
 *//*from   w  w w  .  j a v  a 2  s .  co  m*/
public static void main(String[] args) throws TwitterException, NotDirectoryException, IOException {
    HateSpeechClassifier1 classifier = new HateSpeechClassifier1();
    AnnotatedDataFolder data = new AnnotatedDataFolder("data");
    boolean overSampling = false;
    classifier.train(data.getDateSortedLabeledData(overSampling));

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener = new StatusListener() {
        int numHateSpeech = 0;
        int numTweets = 0;

        @Override
        public void onStatus(Status status) {
            if (status.getLang().equals("in")) {
                numTweets++;
                if (classifier.isHateSpeech(status.getText(), 0.5)) {
                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * "
                            + status.getId() + " # " + status.getLang() + " $ "
                            + (status.getGeoLocation() == null ? "NULLGEO"
                                    : status.getGeoLocation().toString()));
                    System.out.println();
                    System.out.println("lang: " + status.getLang());
                    System.out.println("number of detected hate speech: " + numHateSpeech);
                    System.out.println("total number of streamed tweets: " + numTweets);
                    System.out.println();
                    System.out.println();
                    numHateSpeech++;
                }
            } else {
                System.out.println("ignoring non-Indonesian tweet");
            }
            //                if (status.getGeoLocation() != null) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " $ " + status.getGeoLocation().toString());
            //                }
            //                if (status.getLang().equals("id")) {
            //                    System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText() + " * " + status.getId() + " # " + status.getLang() + " $ " + (status.getGeoLocation() == null ? "NULLGEO" : status.getGeoLocation().toString()));
            //                }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }
    };

    twitterStream.addListener(listener);

    FilterQuery filterQuery = new FilterQuery();
    filterQuery.track(new String[] { "a", "i", "u", "e", "o" });
    filterQuery.language("in");
    twitterStream.filter(filterQuery);

    twitterStream.sample();
}

From source file:org.apache.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java

License:Apache License

@Override
public Firehose connect(InputRowParser parser, File temporaryDirectory) {
    final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() {
        @Override/*from w  w  w  . j av a 2 s . c o  m*/
        public void onConnect() {
            log.info("Connected_to_Twitter");
        }

        @Override
        public void onDisconnect() {
            log.info("Disconnect_from_Twitter");
        }

        /**
         * called before thread gets cleaned up
         */
        @Override
        public void onCleanUp() {
            log.info("Cleanup_twitter_stream");
        }
    }; // ConnectionLifeCycleListener

    final TwitterStream twitterStream;
    final StatusListener statusListener;
    final int QUEUE_SIZE = 2000;
    /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread.   */
    final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE);
    final long startMsec = System.currentTimeMillis();

    //
    //   set up Twitter Spritzer
    //
    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener);
    statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j
        @Override
        public void onStatus(Status status) {
            // time to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }
            try {
                boolean success = queue.offer(status, 15L, TimeUnit.SECONDS);
                if (!success) {
                    log.warn("queue too slow!");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // This notice will be sent each time a limited stream becomes unlimited.
            // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity.
            log.warn("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onException(Exception ex) {
            log.error(ex, "Got exception");
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            log.warn("Got stall warning: %s", warning);
        }
    };

    twitterStream.addListener(statusListener);
    twitterStream.sample(); // creates a generic StatusStream
    log.info("returned from sample()");

    return new Firehose() {

        private final Runnable doNothingRunnable = new Runnable() {
            @Override
            public void run() {
            }
        };

        private long rowCount = 0L;
        private boolean waitIfmax = (getMaxEventCount() < 0L);
        private final Map<String, Object> theMap = new TreeMap<>();
        // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper();

        private boolean maxTimeReached() {
            if (getMaxRunMinutes() <= 0) {
                return false;
            } else {
                return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes();
            }
        }

        private boolean maxCountReached() {
            return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount();
        }

        @Override
        public boolean hasMore() {
            if (maxCountReached() || maxTimeReached()) {
                return waitIfmax;
            } else {
                return true;
            }
        }

        @Nullable
        @Override
        public InputRow nextRow() {
            // Interrupted to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }

            // all done?
            if (maxCountReached() || maxTimeReached()) {
                if (waitIfmax) {
                    // sleep a long time instead of terminating
                    try {
                        log.info("reached limit, sleeping a long time...");
                        Thread.sleep(2000000000L);
                    } catch (InterruptedException e) {
                        throw new RuntimeException("InterruptedException", e);
                    }
                } else {
                    // allow this event through, and the next hasMore() call will be false
                }
            }
            if (++rowCount % 1000 == 0) {
                log.info("nextRow() has returned %,d InputRows", rowCount);
            }

            Status status;
            try {
                status = queue.take();
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }

            theMap.clear();

            HashtagEntity[] hts = status.getHashtagEntities();
            String text = status.getText();
            theMap.put("text", (null == text) ? "" : text);
            theMap.put("htags", (hts.length > 0)
                    ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() {
                        @Nullable
                        @Override
                        public String apply(HashtagEntity input) {
                            return input.getText();
                        }
                    })
                    : ImmutableList.<String>of());

            long[] lcontrobutors = status.getContributors();
            List<String> contributors = new ArrayList<>();
            for (long contrib : lcontrobutors) {
                contributors.add(StringUtils.format("%d", contrib));
            }
            theMap.put("contributors", contributors);

            GeoLocation geoLocation = status.getGeoLocation();
            if (null != geoLocation) {
                double lat = status.getGeoLocation().getLatitude();
                double lon = status.getGeoLocation().getLongitude();
                theMap.put("lat", lat);
                theMap.put("lon", lon);
            } else {
                theMap.put("lat", null);
                theMap.put("lon", null);
            }

            if (status.getSource() != null) {
                Matcher m = sourcePattern.matcher(status.getSource());
                theMap.put("source", m.find() ? m.group(1) : status.getSource());
            }

            theMap.put("retweet", status.isRetweet());

            if (status.isRetweet()) {
                Status original = status.getRetweetedStatus();
                theMap.put("retweet_count", original.getRetweetCount());

                User originator = original.getUser();
                theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : "");
                theMap.put("originator_follower_count",
                        originator != null ? originator.getFollowersCount() : "");
                theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : "");
                theMap.put("originator_verified", originator != null ? originator.isVerified() : "");
            }

            User user = status.getUser();
            final boolean hasUser = (null != user);
            theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0);
            theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0);
            theMap.put("lang", hasUser ? user.getLang() : "");
            theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available?
            theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0);
            theMap.put("user_id", hasUser ? StringUtils.format("%d", user.getId()) : "");
            theMap.put("screen_name", hasUser ? user.getScreenName() : "");
            theMap.put("location", hasUser ? user.getLocation() : "");
            theMap.put("verified", hasUser ? user.isVerified() : "");

            theMap.put("ts", status.getCreatedAt().getTime());

            List<String> dimensions = Lists.newArrayList(theMap.keySet());

            return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap);
        }

        @Override
        public Runnable commit() {
            // ephemera in, ephemera out.
            return doNothingRunnable; // reuse the same object each time
        }

        @Override
        public void close() {
            log.info("CLOSE twitterstream");
            twitterStream.shutdown(); // invokes twitterStream.cleanUp()
        }
    };
}

From source file:org.apache.flume.sink.solr.morphline.TwitterSource.java

License:Apache License

@Override
public void configure(Context context) {
    String consumerKey = context.getString("consumerKey");
    String consumerSecret = context.getString("consumerSecret");
    String accessToken = context.getString("accessToken");
    String accessTokenSecret = context.getString("accessTokenSecret");

    LOGGER.info("Consumer Key:        '" + consumerKey + "'");
    LOGGER.info("Consumer Secret:     '" + consumerSecret + "'");
    LOGGER.info("Access Token:        '" + accessToken + "'");
    LOGGER.info("Access Token Secret: '" + accessTokenSecret + "'");

    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    twitterStream.setOAuthAccessToken(new AccessToken(accessToken, accessTokenSecret));
    twitterStream.addListener(this);
    avroSchema = createAvroSchema();//from  ww  w  .j  a v a 2  s.com

    maxBatchSize = context.getInteger("maxBatchSize", maxBatchSize);
    maxBatchDurationMillis = context.getInteger("maxBatchDurationMillis", maxBatchDurationMillis);
}

From source file:org.apache.flume.source.twitter.TwitterSource.java

License:Apache License

@Override
public void configure(Context context) {
    String consumerKey = context.getString("consumerKey");
    String consumerSecret = context.getString("consumerSecret");
    String accessToken = context.getString("accessToken");
    String accessTokenSecret = context.getString("accessTokenSecret");

    LOGGER.info("Consumer Key:        '" + consumerKey + "'");
    LOGGER.info("Consumer Secret:     '" + consumerSecret + "'");
    LOGGER.info("Access Token:        '" + accessToken + "'");
    LOGGER.info("Access Token Secret: '" + accessTokenSecret + "'");

    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    twitterStream.setOAuthAccessToken(new AccessToken(accessToken, accessTokenSecret));
    twitterStream.addListener(this);
    avroSchema = createAvroSchema();//from   ww w. j  a v  a2 s.c o  m
    dataFileWriter = new DataFileWriter<GenericRecord>(new GenericDatumWriter<GenericRecord>(avroSchema));

    maxBatchSize = context.getInteger("maxBatchSize", maxBatchSize);
    maxBatchDurationMillis = context.getInteger("maxBatchDurationMillis", maxBatchDurationMillis);
}

From source file:org.drools.examples.twittercbr.offline.TwitterDumper.java

License:Apache License

public static void main(String[] args) throws TwitterException, IOException, InterruptedException {
    final ObjectOutputStream oos = new ObjectOutputStream(
            new FileOutputStream("src/main/resources/twitterstream.dump"));

    final TwitterStream twitterStream = new TwitterStreamFactory().getInstance();

    final StatusListener listener = new StatusListener() {
        private int counter = 0;

        @Override/* w w w . j a v  a 2  s . co m*/
        public void onException(Exception arg0) {
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice arg0) {
        }

        @Override
        public void onScrubGeo(long arg0, long arg1) {
        }

        @Override
        public void onTrackLimitationNotice(int arg0) {
        }

        @Override
        public void onStatus(Status arg0) {
            counter++;
            if (counter % 100 == 0) {
                System.out.println("Message = " + counter);
            }
            try {
                oos.writeObject(arg0);
            } catch (IOException e) {
                e.printStackTrace();
                System.exit(0);
            }
        }

        @Override
        public void onStallWarning(StallWarning arg0) {
        }
    };
    StatusStream stream = twitterStream.getSampleStream();
    long start = System.currentTimeMillis();
    while (System.currentTimeMillis() - start < 300000) {
        stream.next(listener);
    }
    twitterStream.shutdown();
    Thread.currentThread().sleep(10000);
    oos.close();
}

From source file:org.drools.examples.twittercbr.TwitterCBR.java

License:Apache License

/**
 * Main method/*from w  w w  .j  a  va  2s .c  om*/
 */
public static void main(String[] args) throws TwitterException, IOException {
    if (args.length == 0) {
        System.out.println("Please provide the rules file name to load.");
        System.exit(0);
    }

    // Creates a knowledge base
    final KnowledgeBase kbase = createKnowledgeBase(args[0]);

    // Creates a knowledge session
    final StatefulKnowledgeSession ksession = createKnowledgeSession(kbase);

    // Gets the stream entry point 
    final WorkingMemoryEntryPoint ep = ksession.getWorkingMemoryEntryPoint("twitter");

    // Connects to the twitter stream and register the listener 
    new Thread(new Runnable() {
        @Override
        public void run() {
            StatusListener listener = new TwitterStatusListener(ep);
            TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
            twitterStream.addListener(listener);
            twitterStream.sample();
        }
    }).start();

    // Starts to fire rules in Drools Fusion
    ksession.fireUntilHalt();
}

From source file:org.example.TwitterDumper.java

License:Apache License

public static void main(String[] args) throws TwitterException, IOException, InterruptedException {
    final ObjectOutputStream oos = new ObjectOutputStream(
            new FileOutputStream("src/main/resources/twitterstream.dump"));

    final TwitterStream twitterStream = new TwitterStreamFactory().getInstance();

    final StatusListener listener = new StatusListener() {
        private int counter = 0;

        public void onException(Exception arg0) {
        }/*from  w  ww.  j a va2 s.c o m*/

        public void onDeletionNotice(StatusDeletionNotice arg0) {
        }

        public void onScrubGeo(long arg0, long arg1) {
        }

        public void onTrackLimitationNotice(int arg0) {
        }

        public void onStatus(Status arg0) {
            counter++;
            if (counter % 100 == 0) {
                System.out.println("Message = " + counter);
            }
            try {
                oos.writeObject(arg0);
            } catch (IOException e) {
                e.printStackTrace();
                System.exit(0);
            }
        }

        public void onStallWarning(StallWarning arg0) {
        }
    };
    StatusStream stream = twitterStream.getSampleStream();
    long start = System.currentTimeMillis();
    while (System.currentTimeMillis() - start < 300000) {
        stream.next(listener);
    }
    twitterStream.shutdown();
    Thread.currentThread().sleep(10000);
    oos.close();
}