List of usage examples for twitter4j FilterQuery FilterQuery
public FilterQuery()
From source file:storm.starter.spout.TwitterRandomEnglishSpout.java
License:Apache License
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { queue = new LinkedBlockingQueue<Status>(1000); _collector = collector;/*from w w w . jav a 2 s. c om*/ StatusListener listener = new StatusListener() { @Override public void onStatus(Status status) { queue.offer(status); } @Override public void onDeletionNotice(StatusDeletionNotice sdn) { } @Override public void onTrackLimitationNotice(int i) { } @Override public void onScrubGeo(long l, long l1) { } @Override public void onException(Exception ex) { } @Override public void onStallWarning(StallWarning arg0) { // TODO Auto-generated method stub } }; TwitterStream twitterStream = new TwitterStreamFactory( new ConfigurationBuilder().setJSONStoreEnabled(true).build()).getInstance(); twitterStream.addListener(listener); twitterStream.setOAuthConsumer(consumerKey, consumerSecret); AccessToken token = new AccessToken(accessToken, accessTokenSecret); twitterStream.setOAuthAccessToken(token); _twitterStream = twitterStream; FilterQuery query = new FilterQuery(); //query.track(new String[] {"#politics"}); //query.language(new String[]{"en"}); twitterStream.sample(); /* if (keyWords.length == 0) { twitterStream.sample(); } else { // TODO: Adjust the query below to also track locations and languages. FilterQuery query = new FilterQuery(); query.track(keyWords); query.language(new String[]{"en"}); //query.locations(wholeWorldBoundingBox); twitterStream.filter(query); } */ }
From source file:storm.starter.trident.homework.spouts.TwitterSampleSpout.java
License:Apache License
@Override public void open(Map conf, TopologyContext context) { queue = new LinkedBlockingQueue<Status>(1000); StatusListener listener = new StatusListener() { @Override/*from w w w.j a v a2 s. c o m*/ public void onStatus(Status status) { queue.offer(status); } @Override public void onDeletionNotice(StatusDeletionNotice sdn) { } @Override public void onTrackLimitationNotice(int i) { } @Override public void onScrubGeo(long l, long l1) { } @Override public void onException(Exception ex) { } @Override public void onStallWarning(StallWarning arg0) { // TODO Auto-generated method stub } }; TwitterStream twitterStream = new TwitterStreamFactory( new ConfigurationBuilder().setJSONStoreEnabled(true).build()).getInstance(); twitterStream.addListener(listener); twitterStream.setOAuthConsumer(consumerKey, consumerSecret); AccessToken token = new AccessToken(accessToken, accessTokenSecret); twitterStream.setOAuthAccessToken(token); if (keyWords.length == 0) { twitterStream.sample(); } else { FilterQuery query = new FilterQuery().track(keyWords); twitterStream.filter(query); } }
From source file:storm.twitter.spout.TwitterSampleSpout.java
License:Apache License
/** * {@inheritDoc}//w ww .j ava2 s .com * * The method receives tweets from Twitter Streaming API and puts them into queue variable. * * <p>The method is called when a task for this component is initialized within a worker on the cluster. * It provides the spout with the environment in which the spout executes. * * @param conf The configuration of Apache Storm for the spout. * @param context The context contains information about the place of a task in the topology. It contains * information about the task id, component id, I/O information, etc. * @param collector The collector is used to emit tuples to the output stream of the spout. */ @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { // Create queue of Strings for tweets queue = new LinkedBlockingQueue<String>(1000); _collector = collector; StatusListener listener = new StatusListener() { // Get json from Twitter API and put it to queue @Override public void onStatus(Status status) { String json = TwitterObjectFactory.getRawJSON(status); queue.offer(json); } @Override public void onDeletionNotice(StatusDeletionNotice sdn) { } @Override public void onTrackLimitationNotice(int i) { } @Override public void onScrubGeo(long l, long l1) { } @Override public void onException(Exception ex) { } @Override public void onStallWarning(StallWarning arg0) { } }; // Create twitterStream TwitterStream twitterStream = new TwitterStreamFactory( new ConfigurationBuilder().setJSONStoreEnabled(true).build()).getInstance(); // Twitter API configuration twitterStream.addListener(listener); twitterStream.setOAuthConsumer(consumerKey, consumerSecret); AccessToken token = new AccessToken(accessToken, accessTokenSecret); twitterStream.setOAuthAccessToken(token); // if there is no keywork filter - get all 1% of tweets from API if (keyWords.length == 0) { twitterStream.sample(); } // filter tweets from the stream else { FilterQuery query = new FilterQuery().track(keyWords); twitterStream.filter(query); } }
From source file:sumblr.TwitterStreamData.java
public TwitterStreamData() { this.oathAccessToken = new AccessToken(ACCESS_TOKEN, ACCESS_TOKEN_SECRET); this.twitterStream = new TwitterStreamFactory().getInstance(); this.twitterStream.setOAuthConsumer(CONSUMER_KEY, CONSUMER_KEY_SECRET); this.twitterStream.setOAuthAccessToken(oathAccessToken); this.fq = new FilterQuery(); preprocessor = new Preprocessor(); }
From source file:toninbot.ToninBot.java
/** * @param args the command line arguments *//* w w w .j a va 2 s .co m*/ public static void main(String[] args) { AccessToken accessToken = new AccessToken(Credenciales.token, Credenciales.tokenSecret); ConfigurationBuilder builder = new ConfigurationBuilder(); builder.setOAuthConsumerKey(Credenciales.consumerKey); builder.setOAuthConsumerSecret(Credenciales.consumerSecret); Configuration configuration = builder.build(); TwitterStreamFactory twStreamFactory = new TwitterStreamFactory(configuration); TwitterStream twitterStream = twStreamFactory.getInstance(); twitterStream.setOAuthAccessToken(accessToken); ToninStatusListener listener = new ToninStatusListener(); twitterStream.addListener(listener); FilterQuery filtre = new FilterQuery(); filtre.follow(184742273L, 2841338087L);//Allegue y proyectoPSIa1 //filtre.follow(2841338087L);//proyectoPSIa1 twitterStream.filter(filtre); }
From source file:traffickarmasent.TweetCollection.java
public static void main(String[] args) throws FileNotFoundException, IOException, Exception { // loading slang dictionary with key as slang and value as its full form slangMap = new HashMap<String, String>(); BufferedReader slangRead = new BufferedReader(new FileReader("extras/out.txt")); String line = ""; while ((line = slangRead.readLine()) != null) { String parts[] = line.split("\t"); slangMap.put(parts[0], parts[1]); }/*from w w w . j a v a 2s.c o m*/ slangRead.close(); //loading entity list BufferedReader htm_in = new BufferedReader(new FileReader("extras/html_ent.txt")); entityList = new ArrayList<String>(); while ((line = htm_in.readLine()) != null) { entityList.add(line); } FileInputStream fos1 = new FileInputStream(new File("extras/hash1.dat")); // loading emoticon dictionary, with key as emoticon and value as its sentiment score ObjectInputStream out1 = new ObjectInputStream(fos1); emohash1 = (HashMap<String, Double>) out1.readObject(); //System.out.println(hm1); FileInputStream fos2 = new FileInputStream(new File("extras/hash2.dat")); // loading emoticon dictionary, with key as emoticon and value as its sentiment score ObjectInputStream out2 = new ObjectInputStream(fos2); emohash2 = (HashMap<String, Double>) out2.readObject(); //System.out.println(hm2); //loading senti-wordnet FileReader fr2 = new FileReader("extras/SentiWordNet_scores_final.txt"); BufferedReader br2 = new BufferedReader(fr2); String str2; senti_map = new HashMap<String, Double>(); while ((str2 = br2.readLine()) != null) { StringTokenizer st = new StringTokenizer(str2, "^"); senti_map.put(st.nextToken(), Double.parseDouble(st.nextToken())); } String serializedClassifier = "english.all.3class.distsim.crf.ser.gz"; //NER configuration AbstractSequenceClassifier classifier = CRFClassifier.getClassifierNoExceptions(serializedClassifier); DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder(); InputSource is = new InputSource(); MaxentTagger tagger = new MaxentTagger("taggers/english-left3words-distsim.tagger"); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true); cb.setOAuthConsumerKey("ufulV3imKoYNzdh58LotTC1YD"); cb.setOAuthConsumerSecret("2A781ma736HTenAXXYn9tRIelQYJkbCqY0GLi7W71ZwwDmNU59"); cb.setOAuthAccessToken("2564905075-MY9osfHabaRnonQVHHhHeA1vCLSOhuHWjBNBiIY"); cb.setOAuthAccessTokenSecret("JsD8Woc7iiFiDSwoCwjNAb6KNEurz7tBqSj9pJV8WXabr"); twitter4j.TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); StatusListener listener = new StatusListener() { double score = 0.0; double count = 0; ArrayList<String> locArray = new ArrayList<String>(); @Override public void onStatus(Status status) { String text = status.getText(); double geoLat = 0.0; double geoLng = 0.0; String tweetId = status.getId() + ""; String userName = status.getUser().getName(); String userId = status.getUser().getId() + ""; if (status.getGeoLocation() != null) { geoLat = status.getGeoLocation().getLatitude(); geoLng = status.getGeoLocation().getLongitude(); } tweetClean(text, status.getGeoLocation()); System.out.println(text + "\n" + tweetId + " " + userName + " " + userId); } @Override public void onDeletionNotice(StatusDeletionNotice sdn) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override public void onTrackLimitationNotice(int i) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override public void onScrubGeo(long l, long l1) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override public void onStallWarning(StallWarning sw) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override public void onException(Exception excptn) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } public void tweetClean(String message, GeoLocation loc) { try { // URL removal message = removeUrl(message); System.out.println("lalala" + message); //removing user mentions message = userMentions(message); //slang removal String[] acro = slangRemoval(message); //entity removal String[] finaldata = entityRemoval(acro); message = ""; for (String word : finaldata) { message += word + " "; } //System.out.println(message); //Ner Taggging String XmlData = classifier.classifyWithInlineXML(message); message = XmlData; //handling words to the spell_checked String[] data = message.split("<"); String val = ""; for (String word : data) { if (word.startsWith("PER") || word.startsWith("LOC")) { word = word.replaceAll("PERSON>", ""); word = word.replaceAll("/PERSON>", ""); word = word.replaceAll("LOCATION>", ""); word = word.replaceAll("/LOCATION>", ""); //insert word into database here locArray.add(word); } else { word = word.replaceAll("/PERSON>", ""); word = word.replaceAll("/LOCATION>", ""); //System.out.println(word); val += word; } } //System.out.println("see" + val); if (loc != null || locArray.size() > 0) { val = val.replaceAll("\\s+", " "); String[] temp = val.split(" "); String match = ""; //spell_check for (String word : temp) { //System.out.println(word); if (emohash2.containsKey(word)) { score += emohash2.get(word); count++; message = message.replace(word, ""); } Process p = Runtime.getRuntime().exec("python extras/text_blob.py " + word); BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); //System.out.println(in.readLine()); match = in.readLine(); //match = spell_check(word); //System.out.println(match); if (!match.equals(word)) { message = message.replaceAll(word, match); } } System.out.println(message); //UTF-8 emoji's emojiDetection(message); //handle NerTags message = af_spellcheck(message); //System.out.println(a); //removing irrelevant chars message = removeChars(message); //System.out.println(a); //POS- TAGGING message = posTagging(message); // removing prepositions and nouns message = removePrepn(message); //System.out.println(a); sentiScores(message); System.out.println(score); System.out.println(score / count); } } catch (Exception ex) { System.out.println(ex.getMessage()); } } public String removeUrl(String message) { String urlPattern = "((https?|ftp|gopher|telnet|file|Unsure|http):((//)|(\\\\))+[\\w\\d:#@%/;$()~_?\\+-=\\\\\\.&]*)"; Pattern p = Pattern.compile(urlPattern, Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(message); int i = 0; while (m.find()) { message = message.replaceAll(m.group(i), "").trim(); i++; } return message; } public String af_spellcheck(String message) { message = message.replaceAll("<PERSON>", ""); message = message.replaceAll("</PERSON>", ""); message = message.replaceAll("<LOCATION>", ""); message = message.replaceAll("</LOCATION>", ""); return message; } public String removeChars(String message) { message = message.replaceAll("\\.", ""); message = message.replaceAll("\\!", ""); message = message.replaceAll("\\$", ""); message = message.replaceAll("\\%", ""); message = message.replaceAll("\\^", ""); message = message.replaceAll("\\|", ""); message = message.replaceAll("\\+", ""); message = message.replaceAll("\\:", ""); message = message.replaceAll("\\(", ""); message = message.replaceAll("\\)", ""); message = message.replaceAll("\\*", ""); message = message.replaceAll("\\{", ""); return message; } public void sentiScores(String message) { message = message.replaceAll("_NNS", "_n"); message = message.replaceAll("_NN", "_n"); message = message.replaceAll("_RBR", "_r"); message = message.replaceAll("_RBS", "_r"); message = message.replaceAll("_RB", "_r"); message = message.replaceAll("_JJR", "_a"); message = message.replaceAll("_JJS", "_a"); message = message.replaceAll("_JJ", "_a"); message = message.replaceAll("_VBD", "_v"); message = message.replaceAll("_VBG", "_v"); message = message.replaceAll("_VBN", "_v"); message = message.replaceAll("_VBP", "_v"); message = message.replaceAll("_VBZ", "_vs"); message = message.replaceAll("_VB", "_v"); message = message.replaceAll("\\s+", " "); // System.out.println(message); String[] senti_token = message.split(" "); for (String word : senti_token) { word = word.toLowerCase(); System.out.println(word); if (senti_map.containsKey(word)) { score += senti_map.get(word); //System.out.println(score); count++; } } } public void emojiDetection(String message) { Pattern emo = Pattern.compile("[\\uD83D\\uDE01-\\uD83D\\uDE4F]"); Matcher m_emo = emo.matcher(message); while (m_emo.find()) { if (emohash1.containsKey(m_emo.group())) //System.out.println("llalala"); { score += emohash1.get(m_emo.group()); } count++; } } public String userMentions(String message) { Pattern p = Pattern.compile("\\@\\w+"); Matcher m = p.matcher(message); while (m.find()) { //System.out.println(m.group()); message = message.replaceAll(m.group(), ""); } return message; } public String[] slangRemoval(String message) { ArrayList<String> slangRemovalList = new ArrayList<String>(); String[] words = message.split(" "); for (String single : words) { if (slangMap.containsKey(single.toUpperCase())) { slangRemovalList.add(slangMap.get(single.toUpperCase())); } else { slangRemovalList.add(single); } } String[] myArray = new String[slangRemovalList.size()]; slangRemovalList.toArray(myArray); return myArray; } public String posTagging(String message) throws Exception { String tagged = tagger.tagString(message); return tagged; } public String removePrepn(String message) { String delims = " "; String[] tokens = message.split(delims); for (String word : tokens) { if (word.endsWith("_IN") || word.endsWith("_NNP") || word.endsWith("_NNPS")) { message = message.replace(word, ""); } } return message; } public String[] entityRemoval(String[] message) { List<String> finalList = new ArrayList<String>(); for (String word : message) { if (!entityList.contains(word.trim())) { finalList.add(word); } } String[] myArray = new String[finalList.size()]; finalList.toArray(myArray); return myArray; } }; FilterQuery fq = new FilterQuery(); String keywords[] = { "Mumbai traffic", "@TrafflineMUM", "TrafficMum", "MumbaiTrafficPol", "avoid traffic Mumbai", "Breakdown Mumbai traffic", "@smart_mumbaikar", "@TrafficBOM", "#StreetSmartWithTraffline mumbai", "#mumbai #TRAFFICALERT ", "#mumbai #TRAFFIC" }; fq.track(keywords); twitterStream.addListener(listener); twitterStream.filter(fq); }
From source file:trendulo.ingest.twitter.TwitterStreamingStringSequenceSource.java
License:Apache License
public TwitterStreamingStringSequenceSource(String username, String password, String archiveTwitterFilePath, StatusFilter statusFilter) {/*from www . jav a 2 s . c om*/ this.statusFilter = statusFilter; statusQueue = new ArrayBlockingQueue<Status>(DEFAULT_BUFFER_SIZE); // If the user wants status messages archived, create an OutputStream if (archiveTwitterFilePath != null) { try { // Verify that the file doesn't already exist so we don't clobber the archive if (new File(archiveTwitterFilePath).exists()) { throw new RuntimeException( "Archive File Already Exists. Please move/rename before starting Ingest: " + archiveTwitterFilePath); } FileOutputStream fos = new FileOutputStream(archiveTwitterFilePath); if (archiveTwitterFilePath.endsWith("gz")) { archiveOutputStream = new GZIPOutputStream(new BufferedOutputStream(fos)); } else { archiveOutputStream = fos; } } catch (IOException e) { log.error("Error creating archive file", e); } } // Configure the Twitter Streaming API client // this system property is required to access raw json System.setProperty("twitter4j.jsonStoreEnabled", "true"); auth = new BasicAuthorization(username, password); twitterStream = new TwitterStreamFactory().getInstance(auth); twitterStream.addListener(this); // We only want status messages with a geo. This needs to be pulled // out so it is configurable // Right now just the lower 48 of the US FilterQuery filterQuery = new FilterQuery(); double[][] locations = { { -126.0, 24.0 }, { -67.0, 49.0 } }; filterQuery.locations(locations); twitterStream.filter(filterQuery); }
From source file:TweetCollector.TweetCollector.java
/** * @param args//from w w w . j av a 2s .c om * the command line arguments * @throws java.lang.InterruptedException */ public static void main(String[] args) throws InterruptedException { // The first arg is the thread name, // the second arg is a daemon flag. // Setting it to false so that the process won't terminate unless // the timer is canceled Timer fetchTimer = new Timer("TrendFetcher", false); // Schedule now and every X milliseconds afterwards fetchTimer.scheduleAtFixedRate(new ScheduledTrendFetcher(), new Date(), REPEAT_INTERVAL_IN_SECS * 1000); // Initialize twitter with the custom conf TwitterStream twitterStream = new TwitterStreamFactory(Utils.TwitterConfBuilder.buildConf()).getInstance(); twitterStream.addListener(new StreamingTweetListener()); // Wait till the list has elements, check every 5 seconds while (TrendList.getInstance().isEmpty()) { TimeUnit.SECONDS.sleep(1); } // Initialize the filter query and start tracking the trending topics // The trend tracker updates the filter and // the tweet fetcher automatically makes use of the new filter FilterQuery fq = new FilterQuery(); Timer updateTimer = new Timer("TrendTrackerUpdater", false); // Schedule now and every X milliseconds afterwards updateTimer.scheduleAtFixedRate(new TrendTrackerUpdater(twitterStream, fq), new Date(), REPEAT_INTERVAL_IN_SECS * 1000); // Wait for console input System.out.println("type 'exit' or 'q' to exit."); System.out.println("type 'count' or 'c' to check the current tweet count."); Scanner reader = new Scanner(System.in); String s = reader.nextLine().trim().toLowerCase(); while (!s.equals("exit") && !s.equals("q")) { s = reader.nextLine().trim().toLowerCase(); if (s.equals("count") || s.equals("c")) { System.out.println("Count:\t" + DBManager.tweetCount()); } } reader.close(); fetchTimer.cancel(); updateTimer.cancel(); twitterStream.shutdown(); }
From source file:tweetmining.MiningFunctions.java
/** * Mines data from stream based on the Listener created in the constructor. */// w w w . j ava2 s. c o m public void MineFromStream() { FilterQuery fq = new FilterQuery(); twitterStream.addListener(listener); twitterStream.sample(); }
From source file:twitter.stream.PrintFilterStream.java
License:Apache License
public static void main(String[] args) throws TwitterException, IOException { if (args.length < 1) { System.out.println(/*from ww w . j a v a 2s . co m*/ "Usage: java twitter4j.examples.PrintFilterStream [follow(comma separated numerical user ids)] [track(comma separated filter terms)]"); System.exit(-1); } final ConnectionManager c = new ConnectionManager(); try { c.createConnection(); c.createDB(); c.closeConnection(); } catch (SQLException e) { // TODO Auto-generated catch block System.out.println(e); } System.out.println("conexion creada"); StatusListener listener = new StatusListener() { @Override public void onStatus(Status status) { // pw.println("@" + status.getUser().getScreenName() + " - " + status.getText()); // if (status.getGeoLocation() != null) // pw.println("Lat:" + status.getGeoLocation().getLatitude() + "Long" // + status.getGeoLocation().getLongitude()); // User u = status.getUser(); // try { // System.out.println("User: id: " + u.getId() + " name: " + u.getName()); // c.insertTwitterUser(u); // } catch (SQLException e1) { // // TODO Auto-generated catch block // System.out.println(e1); // } // // if (status.isRetweet()) { // // System.out.println("RT Found!, getting the original tweet"); // Status retweetedStat = status.getRetweetedStatus(); // u = retweetedStat.getUser(); // System.out.println("RT User: id: " + u.getId() + " name: " + u.getName()); // // try { // // c.insertTwitterUser(u); // // c.insertTweet(retweetedStat); // // // // // } catch (SQLException e) { // // TODO Auto-generated catch block // System.out.println(e); // } // // } // if(status.getQuotedStatus() != null){ // // System.out.println("Quoted Found!, getting the original tweet"); // Status quotedStatus = status.getQuotedStatus(); // u = quotedStatus.getUser(); // System.out.println("Quoted User: id: " + u.getId() + " name: " + u.getName()); // // try { // c.insertTwitterUser(u); // c.insertTweet(quotedStatus); // } catch (SQLException e) { // // TODO Auto-generated catch block // System.out.println(e); // } // } // // // // try { // c.insertTweet(status); // } catch (SQLException e) { // // TODO Auto-generated catch block // System.out.println(e); // } try { c.recursiveInsert(status, 0); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } @Override public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId()); } @Override public void onTrackLimitationNotice(int numberOfLimitedStatuses) { System.out.println("Got track limitation notice:" + numberOfLimitedStatuses); } @Override public void onScrubGeo(long userId, long upToStatusId) { System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId); } @Override public void onStallWarning(StallWarning warning) { System.out.println("Got stall warning:" + warning); } @Override public void onException(Exception ex) { ex.printStackTrace(); } }; TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); FilterQuery fq = new FilterQuery(); String keywords[] = { "election2016", "NeverTrump", "Hillary2016", "Trump", "HillarysEmail", "2016election" }; fq.track(keywords); twitterStream.addListener(listener); twitterStream.filter(fq); }