Example usage for twitter4j Status getText

List of usage examples for twitter4j Status getText

  1. HOME
  2. Java
  3. twitter4j
  4. twitter4j.*
  5. Status
  6. getText

Introduction

In this page you can find the example usage for twitter4j Status getText.

Prototype

String getText();

Source Link

Document

Returns the text of the status

Usage

From source file:gui.project2.v1.FXMLDocumentController.java

@FXML
public void searchloaction() throws InterruptedException {
    int maxFollowerCount = 0;
    userslocations = new ArrayList<>();
    nodes = new ArrayList<>();

    GraphicsContext gc = graph.getGraphicsContext2D();
    gc.setFill(Color.GAINSBORO);/*from w ww  . j  a v a 2  s .  c  om*/
    gc.fillRect(0, 0, 308, 308);
    Twitter twitter;
    twitter = tf.getInstance();
    ArrayList<User> users = new ArrayList<>();
    try {
        Query query = new Query("");

        GeoLocation location;
        location = new GeoLocation(parseDouble(latitude.getText()), parseDouble(longitude.getText()));
        Query.Unit unit = Query.KILOMETERS;
        query.setGeoCode(location, parseDouble(radius.getText()), unit);
        QueryResult result;

        do {
            result = twitter.search(query);
            List<Status> tweets = result.getTweets();

            for (Status tweet : tweets) {
                System.out.println("@" + tweet.getUser().getScreenName() + " - " + tweet.getText());
                boolean q = false;
                if (userslocations != null && !userslocations.isEmpty()) {
                    for (int i = 0; i < userslocations.size(); i++) {
                        if (userslocations.get(i).getName().equals(tweet.getUser().getScreenName())) {
                            q = true;
                            break;
                        }
                    }
                }

                if (!q && tweet.getGeoLocation() != null) {
                    pair n;
                    String latString = "";
                    String lonString = "";
                    int la = 0;
                    int lo = 0;
                    String geoString = tweet.getGeoLocation().toString();
                    for (int i = 0; i < geoString.length(); i++) {
                        if (geoString.charAt(i) == '=') {
                            if (la == 0) {
                                la = 1;
                            } else if (la == -1) {
                                lo = 1;
                            }
                        } else if (geoString.charAt(i) == ',') {
                            la = -1;
                        } else if (geoString.charAt(i) == '}') {
                            lo = -1;
                        } else if (la == 1) {
                            latString = latString + geoString.charAt(i);
                        } else if (lo == 1) {
                            lonString = lonString + geoString.charAt(i);
                        }
                    }
                    User thisUser;
                    thisUser = tweet.getUser();
                    double lat = parseDouble(latString);
                    double lon = parseDouble(lonString);
                    System.out.println(tweet.getGeoLocation().toString());
                    n = new pair(tweet.getUser().getScreenName(), lat, lon);
                    userslocations.add(n);
                    users.add(thisUser);
                    if (thisUser.getFollowersCount() > maxFollowerCount) {
                        maxFollowerCount = thisUser.getFollowersCount();
                    }
                }

            }
        } while ((query = result.nextQuery()) != null);
        for (int i = 0; i < users.size(); i++) {
            if (i % 14 == 0 && i != 0) {
                Thread.sleep(1000 * 60 * 15 + 30);
            }
            IDs friends;
            friends = twitter.getFriendsIDs(users.get(i).getId(), -1);
            for (long j : friends.getIDs()) {
                for (int k = i + 1; k < users.size(); k++) {
                    if (users.get(k).getId() == j) {
                        nodes.add(users.get(i).getScreenName() + ":" + users.get(k).getScreenName());
                    }
                }
            }
        }

    } catch (TwitterException te) {
        System.out.println("Failed to search tweets: " + te.getMessage());
        System.exit(-1);
    }
    double xmin;
    double xmax;
    double ymin;
    double ymax;
    xmin = userslocations.get(0).getA();
    xmax = userslocations.get(0).getA();
    ymin = userslocations.get(0).getB();
    ymax = userslocations.get(0).getB();
    for (int i = 1; i < userslocations.size(); i++) {
        if (xmin > userslocations.get(i).getA()) {
            xmin = userslocations.get(i).getA();
        }
        if (xmax < userslocations.get(i).getA()) {
            xmax = userslocations.get(i).getA();
        }
        if (ymin > userslocations.get(i).getB()) {
            ymin = userslocations.get(i).getB();
        }
        if (ymax < userslocations.get(i).getB()) {
            ymax = userslocations.get(i).getB();
        }
    }
    for (int i = 0; i < userslocations.size(); i++) {
        if (userslocations.get(i).getA() - xmin >= 0 && userslocations.get(i).getB() - ymin >= 0) {
            gc.setLineWidth(users.get(i).getFollowersCount() / maxFollowerCount * 3 + 1);
            gc.strokeOval((userslocations.get(i).getA() - xmin) / (xmax - xmin) * 300 + 4,
                    (userslocations.get(i).getB() - ymin) / (ymax - ymin) * 300 + 4, 4, 4);
        }

    }
    ObservableList<String> usersLeftList = FXCollections.observableArrayList();
    for (int i = 0; i < users.size() - 1; i++) {
        User k = null;
        for (int j = i + 1; j < users.size(); j++) {
            if (users.get(j).getFollowersCount() > users.get(i).getFollowersCount()) {
                k = users.get(i);
                users.set(i, users.get(j));
                users.set(j, k);
            }
        }
    }
    for (int i = 0; i < users.size() / 5; i++) {
        usersLeftList.add(users.get(i).getScreenName() + " " + users.get(i).getFollowersCount());
    }
    listView.setItems(usersLeftList);
    gc.setLineWidth(1);
    gc.setFill(Color.BLUE);
    for (int i = 0; i < nodes.size(); i++) {
        String user1 = "";
        String user2 = "";
        int p = 0;
        double x1 = 0;
        double x2 = 0;
        double y1 = 0;
        double y2 = 0;
        for (int j = 0; j < nodes.get(i).length(); j++) {
            if (nodes.get(i).charAt(j) == ':') {
                p = 1;
            } else if (p == 0) {
                user1 = user1 + nodes.get(i).charAt(j);
            } else if (p == 1) {
                user2 = user2 + nodes.get(i).charAt(j);
            }
        }
        for (int j = 0; j < userslocations.size(); j++) {
            if (userslocations.get(j).getName().equals(user1)) {
                x1 = (userslocations.get(j).getA() - xmin) / (xmax - xmin) * 300 + 6;
                y1 = (userslocations.get(j).getB() - ymin) / (ymax - ymin) * 300 + 6;
            } else if (userslocations.get(j).getName().equals(user2)) {
                x2 = (userslocations.get(j).getA() - xmin) / (xmax - xmin) * 300 + 6;
                y2 = (userslocations.get(j).getB() - ymin) / (ymax - ymin) * 300 + 6;
            }
        }
        gc.strokeLine(x1, y1, x2, y2);
        gc.fillOval(x1 - 2, y1 - 2, 4, 4);
        gc.fillOval(x2 - 2, y2 - 2, 4, 4);
    }
}

From source file:h2weibo.Twitter2Weibo.java

License:Open Source License

public void syncTwitter(String userId) {
    T2WUser user = helper.findOneByUser(userId);

    weibo.setToken(user.getToken(), user.getTokenSecret());

    Twitter twitter = new TwitterFactory().getInstance();
    if (user.getTwitterToken() != null) {
        twitter.setOAuthAccessToken(new AccessToken(user.getTwitterToken(), user.getTwitterTokenSecret()));
        log.debug(String.format("Using OAuth for %s", user.getUserId()));
    }//from  ww w .  ja va 2 s .  c  o m

    StatusFilters filters = new StatusFilters();
    filters.use(new NoSyncFilter()); // should be used first
    filters.use(new TcoStatusFilter()).use(new URLStatusFilter()).use(new TagStatusFilter())
            .use(new FlickrImageFilter());

    NoMentionFilter mentionFilter = new NoMentionFilter();
    UserMappingFilter mappingFilter = new UserMappingFilter(helper);

    if (!user.ready()) {
        log.debug(String.format("Skipping @%s ...", user.getUserId()));
        return;
    }

    // gets Twitter instance with default credentials
    String screenName = user.getUserId();
    long latestId = user.getLatestId();

    log.debug(String.format("Checking @%s's timeline, latest ID = %d.", userId, latestId));

    try {
        if (latestId == 0) {
            List<Status> statuses = twitter.getUserTimeline(screenName);
            if (statuses.size() > 0) {
                user.setLatestId(statuses.get(0).getId()); // Record latestId, and sync next time
            }
            log.info(String.format("First time use for @%s. Set latest ID to %d.", userId, latestId));
        } else {
            Paging paging = new Paging(latestId);
            List<Status> statuses = twitter.getUserTimeline(screenName, paging);

            // sync from the oldest one
            for (int i = statuses.size() - 1; i >= 0; i--) {
                Status status = statuses.get(i);

                if (status.getId() < user.getLatestId())
                    continue; // safe keeper

                String name = status.getUser().getScreenName();
                String statusText = status.getText();
                log.info(String.format("%s - %s", name, statusText));
                try {
                    if (status.isRetweet()) {
                        if (user.isDropRetweets()) {
                            user.setLatestId(status.getId());
                            log.debug("Skipped " + statusText + " because status is a retweet.");
                            continue;
                        } else {
                            filters.remove(mentionFilter);
                            filters.use(mappingFilter);
                        }
                    } else {
                        if (user.isDropMentions()) {
                            filters.remove(mappingFilter);
                            filters.use(mentionFilter);
                        } else {
                            filters.remove(mentionFilter);
                            filters.use(mappingFilter);
                        }
                    }

                    statusText = filters.filter(statusText);
                    if (statusText == null) {
                        user.setLatestId(status.getId());
                        log.info(String.format("Skipped %s because of the filter.", statusText));
                        continue;
                    }

                    if (!user.isNoImage()) {
                        // add twitter images to status text
                        MediaEntity[] mediaEntities = status.getMediaEntities();
                        if (mediaEntities != null) {
                            for (MediaEntity entity : mediaEntities) {
                                statusText += " " + entity.getMediaURL();
                            }
                            log.info("with media url: " + statusText);
                        }

                        StatusImageExtractor ex = new StatusImageExtractor();
                        StringBuffer buf = new StringBuffer(statusText);
                        byte[] image = ex.extract(buf);
                        if (image != null) {
                            user.setLatestId(status.getId());
                            try {
                                statusText = buf.toString(); // with image urls removed
                                weibo.uploadStatus(statusText, new ImageItem(image));
                                log.info(String.format("@%s - %s sent with image.", name, statusText));
                            } catch (WeiboException e) {
                                log.error("Faile to update image.", e);
                            }
                            continue;
                        }
                    }

                    GeoLocation location = status.getGeoLocation();
                    if (user.isWithGeo() && location != null) {
                        weibo.updateStatus(statusText, location.getLatitude(), location.getLongitude());
                        log.info(String.format("@%s - %s sent with geo locations.", name, statusText));
                    } else {
                        weibo.updateStatus(statusText);
                        log.info(String.format("@%s - %s sent.", name, statusText));
                    }
                } catch (WeiboException e) {
                    if (e.getStatusCode() != 400) { // resending same tweet
                        log.error("Failed to update Weibo", e);
                        break;
                    } else {
                        log.error("Sending same message", e);
                    }
                }
                log.info(String.format("Sent: by %s - %s", name, statusText));
                user.setLatestId(status.getId());
            }
        }
        helper.saveUser(user);
    } catch (Exception e) {
        if (!(e instanceof TwitterException)) {
            log.error("Failed to update.", e);
        }
    }
}

From source file:hashimotonet.UpdateStatus.java

License:Apache License

/**
 * Usage: java twitter4j.examples.tweets.UpdateStatus [text]
 * /* w w  w  .  j a  v a2  s. com*/
 * @param args
 *            message
 */
public static void main(String[] args) {
    /*
          try {
             new UpdateStatus().connectTwitter();
          } catch(Exception e) {
             e.printStackTrace();
          }
    */

    if (args.length < 1) {
        System.out.println("Usage: java twitter4j.examples.tweets.UpdateStatus [text]");
        System.exit(-1);
    }

    String message = "";
    for (int i = 0; i < args.length; i++) {
        message += args[i] + " ";
    }

    try {
        Twitter twitter = new TwitterFactory().getInstance();
        try {
            // get request token.
            // this will throw IllegalStateException if access token is
            // already available
            RequestToken requestToken = twitter.getOAuthRequestToken();

            //            RequestToken requestToken = new RequestToken(Globals.KEY,Globals.SECRET);
            String url = requestToken.getAuthenticationURL();
            System.out.println("url = " + url);

            System.out.println("Got request token.");
            System.out.println("Request token: " + requestToken.getToken());
            System.out.println("Request token secret: " + requestToken.getTokenSecret());
            AccessToken accessToken = null;

            BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
            while (null == accessToken) {
                System.out.println("Open the following URL and grant access to your account:");
                System.out.println(requestToken.getAuthorizationURL());
                System.out.print("Enter the PIN(if available) and hit enter after you granted access.[PIN]:");
                String pin = br.readLine();
                try {
                    if (pin.length() > 0) {
                        accessToken = twitter.getOAuthAccessToken(requestToken, pin);
                    } else {
                        accessToken = twitter.getOAuthAccessToken(requestToken);
                    }
                } catch (TwitterException te) {
                    if (401 == te.getStatusCode()) {
                        System.out.println("Unable to get the access token.");
                    } else {
                        te.printStackTrace();
                    }
                }
            }
            System.out.println("Got access token.");
            System.out.println("Access token: " + accessToken.getToken());
            System.out.println("Access token secret: " + accessToken.getTokenSecret());
        } catch (IllegalStateException ie) {
            // access token is already available, or consumer key/secret is
            // not set.
            if (!twitter.getAuthorization().isEnabled()) {
                System.out.println("OAuth consumer key/secret is not set.");
                System.exit(-1);
            }
        }
        Status status = twitter.updateStatus(message);
        System.out.println("Successfully updated the status to [" + status.getText() + "].");
        System.exit(0);
    } catch (TwitterException te) {
        te.printStackTrace();
        System.out.println("Failed to get timeline: " + te.getMessage());
        System.exit(-1);
    } catch (IOException ioe) {
        ioe.printStackTrace();
        System.out.println("Failed to read the system input.");
        System.exit(-1);
    }
}

From source file:ikemen.Main.java

License:Apache License

public static void main(String[] args) {
    final Twitter twitter = TwitterFactory.getSingleton();
    final TwitterStream stream = TwitterStreamFactory.getSingleton();
    stream.addListener(new UserStreamAdapter() {
        public void onStatus(Status status) {
            if (yonda(status.getText())) {
                try {
                    twitter.createFavorite(status.getId());
                } catch (TwitterException ignore) {
                }/*  w ww.jav  a 2  s. c  o m*/
                try {
                    twitter.sendDirectMessage("yusuke", "http://twitter.com/yusuke/status/" + status.getId());
                } catch (TwitterException ignore) {
                }
            }
        }
    });
    stream.user();
}

From source file:info.maslowis.twitterripper.util.Util.java

License:Open Source License

/**
 * Returns a representation the status (tweet) as string
 *
 * @return a representation the status as string in format <em>Status{id=, text='', lang='', createdAt=, geoLocation=, isFavorited=, isRetweeted=, retweetCount=, isTruncated=}</em>
 *//*w w  w .  jav  a  2s.  co m*/
public static String toString(final Status status) {
    return "Status{id=" + status.getId() + ", text='" + status.getText() + "', lang='" + status.getLang()
            + "', createdAt=" + status.getCreatedAt() + ", geoLocation=" + status.getGeoLocation()
            + ", isFavorited=" + status.isFavorited() + ", isRetweeted=" + status.isRetweeted()
            + ", retweetCount=" + status.getRetweetCount() + ", isTruncated=" + status.isTruncated() + "}";
}

From source file:info.usbo.skypetwitter.Run.java

License:Apache License

public static void main(String[] args) throws SkypeException, IOException {
    System.out.println("Working Directory = " + System.getProperty("user.dir"));
    Properties props = new Properties();
    loadProperties(props, "twitter4j.properties");
    loadProperties(props, "skype.properties");
    loadProperties(props, "app.properties");
    String chat_group_id = props.getProperty("skype.chat_group_id");
    String twitter_user_id = props.getProperty("twitter.user");
    data_dir = props.getProperty("data.dir");
    Integer twitter_timeout = Integer.parseInt(props.getProperty("twitter.timeout"));
    System.out.println("Twitter user: " + twitter_user_id);
    System.out.println("Twitter timeout: " + twitter_timeout);
    if ("".equals(twitter_user_id)) {
        return;/*from w w w  .  j  av  a2 s  .c om*/
    }
    if (load_file() == 0) {
        System.out.println("File not found");
        return;
    }
    while (true) {
        bChanged = 0;
        SimpleDateFormat sdf = new SimpleDateFormat("dd.MM.yyyy HH:mm");
        System.out.println("Looking at " + sdf.format(Calendar.getInstance().getTime()));
        Chat ch = Skype.chat(chat_group_id);
        Twitter twitter = new TwitterFactory().getInstance();
        try {
            List<Status> statuses;
            statuses = twitter.getUserTimeline(twitter_user_id);
            String sText;
            for (Status status : statuses) {
                Date d = status.getCreatedAt();
                //  ?
                Calendar cal = Calendar.getInstance();
                cal.setTime(d);
                cal.add(Calendar.HOUR_OF_DAY, 7);
                d = cal.getTime();
                sText = "@" + status.getUser().getScreenName() + "  " + sdf.format(d)
                        + " ( https://twitter.com/" + twitter_user_id + "/status/" + status.getId() + " ): \r\n"
                        + status.getText() + "\r\n***";
                for (URLEntity e : status.getURLEntities()) {
                    sText = sText.replaceAll(e.getURL(), e.getExpandedURL());
                }
                for (MediaEntity e : status.getMediaEntities()) {
                    sText = sText.replaceAll(e.getURL(), e.getMediaURL());
                }
                if (twitter_ids.indexOf(status.getId()) == -1) {
                    System.out.println(sText);
                    ch.send(sText);
                    twitter_ids.add(status.getId());
                    bChanged = 1;
                }
            }
        } catch (TwitterException te) {
            te.printStackTrace();
            System.out.println("Failed to get timeline: " + te.getMessage());
            // System.exit(-1);
        } catch (SkypeException ex) {
            ex.printStackTrace();
            System.out.println("Failed to send message: " + ex.getMessage());
        }

        // VK
        try {
            vk();
            for (VK v : vk) {
                if (vk_ids.indexOf(v.getId()) == -1) {
                    Date d = v.getDate();
                    //  ?
                    Calendar cal = Calendar.getInstance();
                    cal.setTime(d);
                    cal.add(Calendar.HOUR_OF_DAY, 7);
                    d = cal.getTime();
                    String sText = "@Depersonilized (VK)  " + sdf.format(d)
                            + " ( http://vk.com/Depersonilized?w=wall-0_" + v.getId() + " ): \r\n"
                            + v.getText();
                    if (!"".equals(v.getAttachment())) {
                        sText += "\r\n" + v.getAttachment();
                    }
                    sText += "\r\n***";
                    System.out.println(sText);
                    ch.send(sText);
                    vk_ids.add(v.getId());
                    bChanged = 1;
                }
            }
        } catch (ParseException e) {
            e.printStackTrace();
            System.out.println("Failed to get vk: " + e.getMessage());
            // System.exit(-1);
        } catch (SkypeException ex) {
            ex.printStackTrace();
            System.out.println("Failed to send message: " + ex.getMessage());
        }
        if (bChanged == 1) {
            save_file();
        }
        try {
            Thread.sleep(1000 * 60 * twitter_timeout);
        } catch (InterruptedException ex) {
            ex.printStackTrace();
            System.out.println("Failed to sleep: " + ex.getMessage());
        }
    }
}

From source file:io.anserini.index.UpdateIndex.java

License:Apache License

@SuppressWarnings("static-access")
public static void main(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(new Option(HELP_OPTION, "show help"));
    options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment"));
    options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors"));

    options.addOption(/*ww w.j a  v  a  2  s.com*/
            OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION));
    options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids")
            .create(DELETES_OPTION));
    options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION));

    CommandLine cmdline = null;
    CommandLineParser parser = new GnuParser();
    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        System.exit(-1);
    }

    if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(INDEX_OPTION)) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(UpdateIndex.class.getName(), options);
        System.exit(-1);
    }

    String indexPath = cmdline.getOptionValue(INDEX_OPTION);

    final FieldType textOptions = new FieldType();
    textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
    textOptions.setStored(true);
    textOptions.setTokenized(true);
    textOptions.setStoreTermVectors(true);

    LOG.info("index: " + indexPath);

    File file = new File("PittsburghUserTimeline");
    if (!file.exists()) {
        System.err.println("Error: " + file + " does not exist!");
        System.exit(-1);
    }

    final StatusStream stream = new JsonStatusCorpusReader(file);

    Status status;
    String s;
    HashMap<Long, String> hm = new HashMap<Long, String>();
    try {
        while ((s = stream.nextRaw()) != null) {
            try {
                status = DataObjectFactory.createStatus(s);

                if (status.getText() == null) {
                    continue;
                }

                hm.put(status.getUser().getId(),
                        hm.get(status.getUser().getId()) + status.getText().replaceAll("[\\r\\n]+", " "));

            } catch (Exception e) {

            }
        }

    } catch (Exception e) {
        e.printStackTrace();
    } finally {

        stream.close();
    }

    ArrayList<String> userIDList = new ArrayList<String>();
    try (BufferedReader br = new BufferedReader(new FileReader(new File("userID")))) {
        String line;
        while ((line = br.readLine()) != null) {
            userIDList.add(line.replaceAll("[\\r\\n]+", ""));

            // process the line.
        }
    }

    try {
        reader = DirectoryReader
                .open(FSDirectory.open(new File(cmdline.getOptionValue(INDEX_OPTION)).toPath()));
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    final Directory dir = new SimpleFSDirectory(Paths.get(cmdline.getOptionValue(INDEX_OPTION)));
    final IndexWriterConfig config = new IndexWriterConfig(ANALYZER);

    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

    final IndexWriter writer = new IndexWriter(dir, config);

    IndexSearcher searcher = new IndexSearcher(reader);
    System.out.println("The total number of docs indexed "
            + searcher.collectionStatistics(TweetStreamReader.StatusField.TEXT.name).docCount());

    for (int city = 0; city < cityName.length; city++) {

        // Pittsburgh's coordinate -79.976389, 40.439722

        Query q_long = NumericRangeQuery.newDoubleRange(TweetStreamReader.StatusField.LONGITUDE.name,
                new Double(longitude[city] - 0.05), new Double(longitude[city] + 0.05), true, true);
        Query q_lat = NumericRangeQuery.newDoubleRange(TweetStreamReader.StatusField.LATITUDE.name,
                new Double(latitude[city] - 0.05), new Double(latitude[city] + 0.05), true, true);

        BooleanQuery bqCityName = new BooleanQuery();

        Term t = new Term("place", cityName[city]);
        TermQuery query = new TermQuery(t);
        bqCityName.add(query, BooleanClause.Occur.SHOULD);
        System.out.println(query.toString());

        for (int i = 0; i < cityNameAlias[city].length; i++) {
            t = new Term("place", cityNameAlias[city][i]);
            query = new TermQuery(t);
            bqCityName.add(query, BooleanClause.Occur.SHOULD);
            System.out.println(query.toString());
        }

        BooleanQuery bq = new BooleanQuery();

        BooleanQuery finalQuery = new BooleanQuery();

        // either a coordinate match
        bq.add(q_long, BooleanClause.Occur.MUST);
        bq.add(q_lat, BooleanClause.Occur.MUST);

        finalQuery.add(bq, BooleanClause.Occur.SHOULD);
        // or a place city name match
        finalQuery.add(bqCityName, BooleanClause.Occur.SHOULD);

        TotalHitCountCollector totalHitCollector = new TotalHitCountCollector();

        // Query hasFieldQuery = new ConstantScoreQuery(new
        // FieldValueFilter("timeline"));
        //
        // searcher.search(hasFieldQuery, totalHitCollector);
        //
        // if (totalHitCollector.getTotalHits() > 0) {
        // TopScoreDocCollector collector =
        // TopScoreDocCollector.create(Math.max(0,
        // totalHitCollector.getTotalHits()));
        // searcher.search(finalQuery, collector);
        // ScoreDoc[] hits = collector.topDocs().scoreDocs;
        //
        //
        // HashMap<String, Integer> hasHit = new HashMap<String, Integer>();
        // int dupcount = 0;
        // for (int i = 0; i < hits.length; ++i) {
        // int docId = hits[i].doc;
        // Document d;
        //
        // d = searcher.doc(docId);
        //
        // System.out.println(d.getFields());
        // }
        // }

        // totalHitCollector = new TotalHitCountCollector();
        searcher.search(finalQuery, totalHitCollector);

        if (totalHitCollector.getTotalHits() > 0) {
            TopScoreDocCollector collector = TopScoreDocCollector
                    .create(Math.max(0, totalHitCollector.getTotalHits()));
            searcher.search(finalQuery, collector);
            ScoreDoc[] hits = collector.topDocs().scoreDocs;

            System.out.println("City " + cityName[city] + " " + collector.getTotalHits() + " hits.");

            HashMap<String, Integer> hasHit = new HashMap<String, Integer>();
            int dupcount = 0;
            for (int i = 0; i < hits.length; ++i) {
                int docId = hits[i].doc;
                Document d;

                d = searcher.doc(docId);

                if (userIDList.contains(d.get(IndexTweets.StatusField.USER_ID.name))
                        && hm.containsKey(Long.parseLong(d.get(IndexTweets.StatusField.USER_ID.name)))) {
                    //            System.out.println("Has timeline field?" + (d.get("timeline") != null));
                    //            System.out.println(reader.getDocCount("timeline"));
                    //            d.add(new Field("timeline", hm.get(Long.parseLong(d.get(IndexTweets.StatusField.USER_ID.name))),
                    //                textOptions));
                    System.out.println("Found a user hit");
                    BytesRefBuilder brb = new BytesRefBuilder();
                    NumericUtils.longToPrefixCodedBytes(Long.parseLong(d.get(IndexTweets.StatusField.ID.name)),
                            0, brb);
                    Term term = new Term(IndexTweets.StatusField.ID.name, brb.get());
                    //            System.out.println(reader.getDocCount("timeline"));

                    Document d_new = new Document();
                    //            for (IndexableField field : d.getFields()) {
                    //              d_new.add(field);
                    //            }
                    // System.out.println(d_new.getFields());
                    d_new.add(new StringField("userBackground", d.get(IndexTweets.StatusField.USER_ID.name),
                            Store.YES));
                    d_new.add(new Field("timeline",
                            hm.get(Long.parseLong(d.get(IndexTweets.StatusField.USER_ID.name))), textOptions));
                    // System.out.println(d_new.get());
                    writer.addDocument(d_new);
                    writer.commit();

                    //            t = new Term("label", "why");
                    //            TermQuery tqnew = new TermQuery(t);
                    //
                    //            totalHitCollector = new TotalHitCountCollector();
                    //
                    //            searcher.search(tqnew, totalHitCollector);
                    //
                    //            if (totalHitCollector.getTotalHits() > 0) {
                    //              collector = TopScoreDocCollector.create(Math.max(0, totalHitCollector.getTotalHits()));
                    //              searcher.search(tqnew, collector);
                    //              hits = collector.topDocs().scoreDocs;
                    //
                    //              System.out.println("City " + cityName[city] + " " + collector.getTotalHits() + " hits.");
                    //
                    //              for (int k = 0; k < hits.length; k++) {
                    //                docId = hits[k].doc;
                    //                d = searcher.doc(docId);
                    //                System.out.println(d.get(IndexTweets.StatusField.ID.name));
                    //                System.out.println(d.get(IndexTweets.StatusField.PLACE.name));
                    //              }
                    //            }

                    // writer.deleteDocuments(term);
                    // writer.commit();
                    // writer.addDocument(d);
                    // writer.commit();

                    //            System.out.println(reader.getDocCount("timeline"));
                    // writer.updateDocument(term, d);
                    // writer.commit();

                }

            }
        }
    }
    reader.close();
    writer.close();

}

From source file:io.druid.examples.twitter.TwitterSpritzerFirehoseFactory.java

License:Apache License

@Override
public Firehose connect(InputRowParser parser) throws IOException {
    final ConnectionLifeCycleListener connectionLifeCycleListener = new ConnectionLifeCycleListener() {
        @Override// w w w.  ja v  a  2s . c o m
        public void onConnect() {
            log.info("Connected_to_Twitter");
        }

        @Override
        public void onDisconnect() {
            log.info("Disconnect_from_Twitter");
        }

        /**
         * called before thread gets cleaned up
         */
        @Override
        public void onCleanUp() {
            log.info("Cleanup_twitter_stream");
        }
    }; // ConnectionLifeCycleListener

    final TwitterStream twitterStream;
    final StatusListener statusListener;
    final int QUEUE_SIZE = 2000;
    /** This queue is used to move twitter events from the twitter4j thread to the druid ingest thread.   */
    final BlockingQueue<Status> queue = new ArrayBlockingQueue<Status>(QUEUE_SIZE);
    final long startMsec = System.currentTimeMillis();

    //
    //   set up Twitter Spritzer
    //
    twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addConnectionLifeCycleListener(connectionLifeCycleListener);
    statusListener = new StatusListener() { // This is what really gets called to deliver stuff from twitter4j
        @Override
        public void onStatus(Status status) {
            // time to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }
            try {
                boolean success = queue.offer(status, 15L, TimeUnit.SECONDS);
                if (!success) {
                    log.warn("queue too slow!");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            //log.info("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
        }

        @Override
        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // This notice will be sent each time a limited stream becomes unlimited.
            // If this number is high and or rapidly increasing, it is an indication that your predicate is too broad, and you should consider a predicate with higher selectivity.
            log.warn("Got track limitation notice:" + numberOfLimitedStatuses);
        }

        @Override
        public void onScrubGeo(long userId, long upToStatusId) {
            //log.info("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
        }

        @Override
        public void onException(Exception ex) {
            ex.printStackTrace();
        }

        @Override
        public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
        }
    };

    twitterStream.addListener(statusListener);
    twitterStream.sample(); // creates a generic StatusStream
    log.info("returned from sample()");

    return new Firehose() {

        private final Runnable doNothingRunnable = new Runnable() {
            public void run() {
            }
        };

        private long rowCount = 0L;
        private boolean waitIfmax = (getMaxEventCount() < 0L);
        private final Map<String, Object> theMap = new TreeMap<>();
        // DIY json parsing // private final ObjectMapper omapper = new ObjectMapper();

        private boolean maxTimeReached() {
            if (getMaxRunMinutes() <= 0) {
                return false;
            } else {
                return (System.currentTimeMillis() - startMsec) / 60000L >= getMaxRunMinutes();
            }
        }

        private boolean maxCountReached() {
            return getMaxEventCount() >= 0 && rowCount >= getMaxEventCount();
        }

        @Override
        public boolean hasMore() {
            if (maxCountReached() || maxTimeReached()) {
                return waitIfmax;
            } else {
                return true;
            }
        }

        @Override
        public InputRow nextRow() {
            // Interrupted to stop?
            if (Thread.currentThread().isInterrupted()) {
                throw new RuntimeException("Interrupted, time to stop");
            }

            // all done?
            if (maxCountReached() || maxTimeReached()) {
                if (waitIfmax) {
                    // sleep a long time instead of terminating
                    try {
                        log.info("reached limit, sleeping a long time...");
                        sleep(2000000000L);
                    } catch (InterruptedException e) {
                        throw new RuntimeException("InterruptedException", e);
                    }
                } else {
                    // allow this event through, and the next hasMore() call will be false
                }
            }
            if (++rowCount % 1000 == 0) {
                log.info("nextRow() has returned %,d InputRows", rowCount);
            }

            Status status;
            try {
                status = queue.take();
            } catch (InterruptedException e) {
                throw new RuntimeException("InterruptedException", e);
            }

            theMap.clear();

            HashtagEntity[] hts = status.getHashtagEntities();
            String text = status.getText();
            theMap.put("text", (null == text) ? "" : text);
            theMap.put("htags", (hts.length > 0)
                    ? Lists.transform(Arrays.asList(hts), new Function<HashtagEntity, String>() {
                        @Nullable
                        @Override
                        public String apply(HashtagEntity input) {
                            return input.getText();
                        }
                    })
                    : ImmutableList.<String>of());

            long[] lcontrobutors = status.getContributors();
            List<String> contributors = new ArrayList<>();
            for (long contrib : lcontrobutors) {
                contributors.add(String.format("%d", contrib));
            }
            theMap.put("contributors", contributors);

            GeoLocation geoLocation = status.getGeoLocation();
            if (null != geoLocation) {
                double lat = status.getGeoLocation().getLatitude();
                double lon = status.getGeoLocation().getLongitude();
                theMap.put("lat", lat);
                theMap.put("lon", lon);
            } else {
                theMap.put("lat", null);
                theMap.put("lon", null);
            }

            if (status.getSource() != null) {
                Matcher m = sourcePattern.matcher(status.getSource());
                theMap.put("source", m.find() ? m.group(1) : status.getSource());
            }

            theMap.put("retweet", status.isRetweet());

            if (status.isRetweet()) {
                Status original = status.getRetweetedStatus();
                theMap.put("retweet_count", original.getRetweetCount());

                User originator = original.getUser();
                theMap.put("originator_screen_name", originator != null ? originator.getScreenName() : "");
                theMap.put("originator_follower_count",
                        originator != null ? originator.getFollowersCount() : "");
                theMap.put("originator_friends_count", originator != null ? originator.getFriendsCount() : "");
                theMap.put("originator_verified", originator != null ? originator.isVerified() : "");
            }

            User user = status.getUser();
            final boolean hasUser = (null != user);
            theMap.put("follower_count", hasUser ? user.getFollowersCount() : 0);
            theMap.put("friends_count", hasUser ? user.getFriendsCount() : 0);
            theMap.put("lang", hasUser ? user.getLang() : "");
            theMap.put("utc_offset", hasUser ? user.getUtcOffset() : -1); // resolution in seconds, -1 if not available?
            theMap.put("statuses_count", hasUser ? user.getStatusesCount() : 0);
            theMap.put("user_id", hasUser ? String.format("%d", user.getId()) : "");
            theMap.put("screen_name", hasUser ? user.getScreenName() : "");
            theMap.put("location", hasUser ? user.getLocation() : "");
            theMap.put("verified", hasUser ? user.isVerified() : "");

            theMap.put("ts", status.getCreatedAt().getTime());

            List<String> dimensions = Lists.newArrayList(theMap.keySet());

            return new MapBasedInputRow(status.getCreatedAt().getTime(), dimensions, theMap);
        }

        @Override
        public Runnable commit() {
            // ephemera in, ephemera out.
            return doNothingRunnable; // reuse the same object each time
        }

        @Override
        public void close() throws IOException {
            log.info("CLOSE twitterstream");
            twitterStream.shutdown(); // invokes twitterStream.cleanUp()
        }
    };
}

From source file:io.rakam.datasource.twitter.TweetProcessor.java

License:Apache License

@Override
public void onStatus(Status status) {
    Map<String, Object> map = new HashMap<>();

    GeoLocation geoLocation = status.getGeoLocation();
    if (geoLocation != null) {
        map.put("latitude", geoLocation.getLatitude());
        map.put("longitude", geoLocation.getLongitude());
    }//from  w  ww.  j  a  v  a2  s  . com

    map.put("_time", status.getCreatedAt().getTime());
    Place place = status.getPlace();
    if (place != null) {
        map.put("country_code", place.getCountryCode());
        map.put("place", place.getName());
        map.put("place_type", place.getPlaceType());
        map.put("place_id", place.getId());
    }

    User user = status.getUser();
    map.put("_user", user.getId());
    map.put("user_lang", user.getLang());
    map.put("user_created", user.getCreatedAt());
    map.put("user_followers", user.getFollowersCount());
    map.put("user_status_count", user.getStatusesCount());
    map.put("user_verified", user.isVerified());

    map.put("id", status.getId());
    map.put("is_reply", status.getInReplyToUserId() > -1);
    map.put("is_retweet", status.isRetweet());
    map.put("has_media", status.getMediaEntities().length > 0);
    map.put("urls",
            Arrays.stream(status.getURLEntities()).map(URLEntity::getText).collect(Collectors.toList()));
    map.put("hashtags", Arrays.stream(status.getHashtagEntities()).map(HashtagEntity::getText)
            .collect(Collectors.toList()));
    map.put("user_mentions", Arrays.stream(status.getUserMentionEntities()).map(UserMentionEntity::getText)
            .collect(Collectors.toList()));
    map.put("language", "und".equals(status.getLang()) ? null : status.getLang());
    map.put("is_positive", classifier.isPositive(status.getText()));

    Event event = new Event().properties(map).collection(collection);
    buffer.add(event);

    commitIfNecessary();
}

From source file:io.spring.batch.spark.Main.java

License:Apache License

public static void main(String[] args) {
    // URL of the Spark cluster
    String sparkUrl = "local[2]";

    SparkConf conf = new SparkConf();

    conf.setMaster(sparkUrl);/*from w w w  .j  av a  2  s .com*/
    conf.setAppName("Twitter");
    conf.validateSettings();

    JavaStreamingContext ssc = new JavaStreamingContext(sparkUrl, "Twitter", new Duration(1000));

    Configuration configuration = new ConfigurationBuilder()
            .setOAuthConsumerKey(System.getProperty("twitter4j.oauth.consumerKey"))
            .setOAuthConsumerSecret(System.getProperty("twitter4j.oauth.consumerSecret"))
            .setOAuthAccessToken(System.getProperty("twitter4j.oauth.accessKey"))
            .setOAuthAccessTokenSecret(System.getProperty("twitter4j.oauth.accessSecret")).setDebugEnabled(true)
            .build();

    JavaDStream<Status> tweets = TwitterUtils.createStream(ssc, new OAuthAuthorization(configuration));

    tweets.map(new Function<Status, String>() {
        public String call(Status status) {
            return status.getText();
        }
    }).dstream().saveAsTextFiles("hdfs://localhost:8020/spark/twitter/", "txt");

    ssc.start();

    // Just run stream for 20 seconds
    ssc.awaitTermination(20000);
}