List of usage examples for twitter4j TwitterStream shutdown
TwitterStream shutdown();
From source file:nyu.twitter.lg.FentchTwitter.java
License:Open Source License
public static void invoke() throws Exception { init();/* w w w.j ava 2s . co m*/ // Create table if it does not exist yet if (Tables.doesTableExist(dynamoDB, tableName)) { System.out.println("Table " + tableName + " is already ACTIVE"); } else { // Create a table with a primary hash key named 'name', which holds // a string CreateTableRequest createTableRequest = new CreateTableRequest().withTableName(tableName) .withKeySchema(new KeySchemaElement().withAttributeName("id").withKeyType(KeyType.HASH)) .withAttributeDefinitions(new AttributeDefinition().withAttributeName("id") .withAttributeType(ScalarAttributeType.S)) .withProvisionedThroughput( new ProvisionedThroughput().withReadCapacityUnits(1L).withWriteCapacityUnits(1L)); TableDescription createdTableDescription = dynamoDB.createTable(createTableRequest) .getTableDescription(); System.out.println("Created Table: " + createdTableDescription); // Wait for it to become active System.out.println("Waiting for " + tableName + " to become ACTIVE..."); Tables.waitForTableToBecomeActive(dynamoDB, tableName); } ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("Emwo2pG").setOAuthConsumerSecret("RM9B7fske5T") .setOAuthAccessToken("19ubQOirq").setOAuthAccessTokenSecret("Lbg3C"); final TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener listener = new StatusListener() { @Override public void onStatus(Status status) { if (status.getGeoLocation() != null && status.getPlace() != null) { // if (count == 0) { // count++; // } // latitude = status.getGeoLocation().getLatitude(); longtitude = status.getGeoLocation().getLongitude(); place = status.getPlace().getCountry() + "," + status.getPlace().getFullName(); date = status.getCreatedAt().toString(); id = Integer.toString(count); name = status.getUser().getScreenName(); message = status.getText(); System.out.println("---------------------------"); System.out.println("ID:" + count); System.out.println("latitude:" + latitude); System.out.println("longtitude:" + longtitude); System.out.println("place:" + place); System.out.println("name:" + name); System.out.println("message:" + message); System.out.println("data:" + date); System.out.println("-------------8-------------"); insertDB(id, count, name, longtitude, latitude, place, message, date); if (++count > 100) { twitterStream.shutdown(); System.out.println("Information Collection Completed"); } // count = (count+1) % 101; } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { // System.out.println("Got a status deletion notice id:" // + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // System.out.println("Got track limitation notice:" // + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { // System.out.println("Got scrub_geo event userId:" + userId // + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { // System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; twitterStream.addListener(listener); twitterStream.sample(); }
From source file:org.apache.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java
License:Apache License
@Override public Firehose connect(InputRowParser parser, File temporaryDirectory) { final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() { @Override//from w ww .j a v a2 s . c o m public void onConnect() { log.info("Connected_to_Twitter"); } @Override public void onDisconnect() { log.info("Disconnect_from_Twitter"); } /** * called before thread gets cleaned up */ @Override public void onCleanUp() { log.info("Cleanup_twitter_stream"); } }; // ConnectionLifeCycleListener final TwitterStream twitterStream; final StatusListener statusListener; final int QUEUE_SIZE = 2000; /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread. */ final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE); final long startMsec = System.currentTimeMillis(); // // set up Twitter Spritzer // twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener); statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j @Override public void onStatus(Status status) { // time to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } try { boolean success = queue.offer(status, 15L, TimeUnit.SECONDS); if (!success) { log.warn("queue too slow!"); } } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { // This notice will be sent each time a limited stream becomes unlimited. // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity. log.warn("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onException(Exception ex) { log.error(ex, "Got exception"); } @Override public void onStallWarning(StallWarning warning) { log.warn("Got stall warning: %s", warning); } }; twitterStream.addListener(statusListener); twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()"); return new Firehose() { private final Runnable doNothingRunnable = new Runnable() { @Override public void run() { } }; private long rowCount = 0L; private boolean waitIfmax = (getMaxEventCount() < 0L); private final Map<String, Object> theMap = new TreeMap<>(); // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper(); private boolean maxTimeReached() { if (getMaxRunMinutes() <= 0) { return false; } else { return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes(); } } private boolean maxCountReached() { return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount(); } @Override public boolean hasMore() { if (maxCountReached() || maxTimeReached()) { return waitIfmax; } else { return true; } } @Nullable @Override public InputRow nextRow() { // Interrupted to stop? if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Interrupted, time to stop"); } // all done? if (maxCountReached() || maxTimeReached()) { if (waitIfmax) { // sleep a long time instead of terminating try { log.info("reached limit, sleeping a long time..."); Thread.sleep(2000000000L); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } } else { // allow this event through, and the next hasMore() call will be false } } if (++rowCount % 1000 == 0) { log.info("nextRow() has returned %,d InputRows", rowCount); } Status status; try { status = queue.take(); } catch (InterruptedException e) { throw new RuntimeException("InterruptedException", e); } theMap.clear(); HashtagEntity[] hts = status.getHashtagEntities(); String text = status.getText(); theMap.put("text", (null == text) ? "" : text); theMap.put("htags", (hts.length > 0) ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() { @Nullable @Override public String apply(HashtagEntity input) { return input.getText(); } }) : ImmutableList.<String>of()); long[] lcontrobutors = status.getContributors(); List<String> contributors = new ArrayList<>(); for (long contrib : lcontrobutors) { contributors.add(StringUtils.format("%d", contrib)); } theMap.put("contributors", contributors); GeoLocation geoLocation = status.getGeoLocation(); if (null != geoLocation) { double lat = status.getGeoLocation().getLatitude(); double lon = status.getGeoLocation().getLongitude(); theMap.put("lat", lat); theMap.put("lon", lon); } else { theMap.put("lat", null); theMap.put("lon", null); } if (status.getSource() != null) { Matcher m = sourcePattern.matcher(status.getSource()); theMap.put("source", m.find() ? m.group(1) : status.getSource()); } theMap.put("retweet", status.isRetweet()); if (status.isRetweet()) { Status original = status.getRetweetedStatus(); theMap.put("retweet_count", original.getRetweetCount()); User originator = original.getUser(); theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : ""); theMap.put("originator_follower_count", originator != null ? originator.getFollowersCount() : ""); theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : ""); theMap.put("originator_verified", originator != null ? originator.isVerified() : ""); } User user = status.getUser(); final boolean hasUser = (null != user); theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0); theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0); theMap.put("lang", hasUser ? user.getLang() : ""); theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available? theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0); theMap.put("user_id", hasUser ? StringUtils.format("%d", user.getId()) : ""); theMap.put("screen_name", hasUser ? user.getScreenName() : ""); theMap.put("location", hasUser ? user.getLocation() : ""); theMap.put("verified", hasUser ? user.isVerified() : ""); theMap.put("ts", status.getCreatedAt().getTime()); List<String> dimensions = Lists.newArrayList(theMap.keySet()); return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap); } @Override public Runnable commit() { // ephemera in, ephemera out. return doNothingRunnable; // reuse the same object each time } @Override public void close() { log.info("CLOSE twitterstream"); twitterStream.shutdown(); // invokes twitterStream.cleanUp() } }; }
From source file:org.drools.examples.twittercbr.offline.TwitterDumper.java
License:Apache License
public static void main(String[] args) throws TwitterException, IOException, InterruptedException { final ObjectOutputStream oos = new ObjectOutputStream( new FileOutputStream("src/main/resources/twitterstream.dump")); final TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); final StatusListener listener = new StatusListener() { private int counter = 0; @Override/*w w w .j av a 2 s.com*/ public void onException(Exception arg0) { } @Override public void onDeletionNotice(StatusDeletionNotice arg0) { } @Override public void onScrubGeo(long arg0, long arg1) { } @Override public void onTrackLimitationNotice(int arg0) { } @Override public void onStatus(Status arg0) { counter++; if (counter % 100 == 0) { System.out.println("Message = " + counter); } try { oos.writeObject(arg0); } catch (IOException e) { e.printStackTrace(); System.exit(0); } } @Override public void onStallWarning(StallWarning arg0) { } }; StatusStream stream = twitterStream.getSampleStream(); long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < 300000) { stream.next(listener); } twitterStream.shutdown(); Thread.currentThread().sleep(10000); oos.close(); }
From source file:org.example.TwitterDumper.java
License:Apache License
public static void main(String[] args) throws TwitterException, IOException, InterruptedException { final ObjectOutputStream oos = new ObjectOutputStream( new FileOutputStream("src/main/resources/twitterstream.dump")); final TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); final StatusListener listener = new StatusListener() { private int counter = 0; public void onException(Exception arg0) { }/*from w w w . j a v a2 s. c om*/ public void onDeletionNotice(StatusDeletionNotice arg0) { } public void onScrubGeo(long arg0, long arg1) { } public void onTrackLimitationNotice(int arg0) { } public void onStatus(Status arg0) { counter++; if (counter % 100 == 0) { System.out.println("Message = " + counter); } try { oos.writeObject(arg0); } catch (IOException e) { e.printStackTrace(); System.exit(0); } } public void onStallWarning(StallWarning arg0) { } }; StatusStream stream = twitterStream.getSampleStream(); long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < 300000) { stream.next(listener); } twitterStream.shutdown(); Thread.currentThread().sleep(10000); oos.close(); }
From source file:org.primeoservices.cfgateway.twitter.utils.TwitterUtils.java
License:Apache License
/** * Unconditionally close a <code>TwitterStream</code> * //from w w w . j a v a 2 s .c om * @param stream the stream to be closed */ public static void closeQuietly(final TwitterStream stream) { if (stream == null) return; try { stream.cleanUp(); stream.shutdown(); } catch (Throwable t) { // ignore } }
From source file:TweetCollector.TweetCollector.java
/** * @param args//from w w w . ja va 2 s .co m * the command line arguments * @throws java.lang.InterruptedException */ public static void main(String[] args) throws InterruptedException { // The first arg is the thread name, // the second arg is a daemon flag. // Setting it to false so that the process won't terminate unless // the timer is canceled Timer fetchTimer = new Timer("TrendFetcher", false); // Schedule now and every X milliseconds afterwards fetchTimer.scheduleAtFixedRate(new ScheduledTrendFetcher(), new Date(), REPEAT_INTERVAL_IN_SECS * 1000); // Initialize twitter with the custom conf TwitterStream twitterStream = new TwitterStreamFactory(Utils.TwitterConfBuilder.buildConf()).getInstance(); twitterStream.addListener(new StreamingTweetListener()); // Wait till the list has elements, check every 5 seconds while (TrendList.getInstance().isEmpty()) { TimeUnit.SECONDS.sleep(1); } // Initialize the filter query and start tracking the trending topics // The trend tracker updates the filter and // the tweet fetcher automatically makes use of the new filter FilterQuery fq = new FilterQuery(); Timer updateTimer = new Timer("TrendTrackerUpdater", false); // Schedule now and every X milliseconds afterwards updateTimer.scheduleAtFixedRate(new TrendTrackerUpdater(twitterStream, fq), new Date(), REPEAT_INTERVAL_IN_SECS * 1000); // Wait for console input System.out.println("type 'exit' or 'q' to exit."); System.out.println("type 'count' or 'c' to check the current tweet count."); Scanner reader = new Scanner(System.in); String s = reader.nextLine().trim().toLowerCase(); while (!s.equals("exit") && !s.equals("q")) { s = reader.nextLine().trim().toLowerCase(); if (s.equals("count") || s.equals("c")) { System.out.println("Count:\t" + DBManager.tweetCount()); } } reader.close(); fetchTimer.cancel(); updateTimer.cancel(); twitterStream.shutdown(); }
From source file:twittertweetscollection.TwitterTweetsCollection.java
public List<Status> execute() throws TwitterException { final List<Status> statuses = new ArrayList(); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("I7WYWBE6eSNKdV4FYSyiuP0uk") .setOAuthConsumerSecret("oft7Y6WcUCvxovPU5modCZ3BOlHH2z9Px73UkFMILN4ltBhOni") .setOAuthAccessToken("3524247253-o7zXFn3r8PwPb4IUTYqHRQzVabTzfoFn9Ck7FzW") .setOAuthAccessTokenSecret("V7zkLiJmF0vsdX8rUdkXo3q1ha452vKqVtB5OoCuQWAgR"); TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener listener = new StatusListener() { public void onStatus(Status status) { statuses.add(status);//from w w w .j a v a 2s . c o m System.out.println(statuses.size() + ":" + status.getText()); if (statuses.size() > 100) { synchronized (lock) { lock.notify(); } System.out.println("unlocked"); } } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onStallWarning(StallWarning sw) { System.out.println(sw.getMessage()); } }; //FilterQuery fq = new FilterQuery(); //String keywords[] = { "SuperBowlSunday"}; //fq.track(keywords); twitterStream.addListener(listener); twitterStream.sample(); //twitterStream.filter(fq); try { synchronized (lock) { lock.wait(); } } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("returning statuses"); twitterStream.shutdown(); return statuses; }