Example usage for twitter4j HashtagEntity getText

List of usage examples for twitter4j HashtagEntity getText

  1. HOME
  2. Java
  3. twitter4j
  4. twitter4j.*
  5. HashtagEntity
  6. getText

Introduction

In this page you can find the example usage for twitter4j HashtagEntity getText.

Prototype

@Override
String getText();

Source Link

Document

Returns the text of the hashtag without #.

Usage

From source file:aic2013.extractor.TwitterExtractor.java

public static void main(String[] args) throws Exception {
    EntityManagerFactory emf = null;//from  w  w  w.jav a 2 s. c  o m
    MongoClient mongoClient = null;
    Twitter twitter = null;
    Neo4jConnection neo4j = null;

    try {
        // Database configuration
        Properties prop = new Properties();
        prop.load(new FileInputStream("hibernate.properties"));
        // prop.load(TwitterExtractor.class.getClassLoader().getResourceAsStream("hibernate.properties"));

        emf = Persistence.createEntityManagerFactory("twitterdb", prop);
        final String mongoServer = "localhost";
        mongoClient = new MongoClient(mongoServer);
        twitter = TwitterFactory.getSingleton();

        //         final String neo4jJdbc = "jdbc:neo4j:ec2-54-217-131-208.eu-west-1.compute.amazonaws.com:7474";
        final String neo4jJdbc = "jdbc:neo4j://localhost:7474";
        neo4j = new Driver().connect(neo4jJdbc, new Properties());
        neo4j.setAutoCommit(true);//false);

        final TwitterDataAccess twitterDataAccess = new TwitterDataAccess(twitter);
        final MongoDataAccess mongoDataAccess = new MongoDataAccess(mongoClient);
        final Neo4jService neo4jService = new Neo4jService(neo4j);
        final EntityManager em = emf.createEntityManager();
        final UserDataAccess userDataAcces = new UserDataAccess(em);

        final TopicExtractionCoordinator extractionCoordinator = new TopicExtractionCoordinatorImpl();

        //         final ConcurrentLinkedQueue<E>
        final UserService userService = new UserService(em);
        final Processor<Status> mongoProcessor = new Processor<Status>() {

            @Override
            public void process(final Status status) {
                final TwitterUser user = new TwitterUser(status.getUser());
                userService.persist(user);
                neo4jService.transactional(new Neo4jUnitOfWork() {

                    @Override
                    public void process() throws SQLException {
                        neo4jService.createPersonIfAbsent(user);
                        // HashtagEntity[] topics =
                        // status.getHashtagEntities();
                        Status originalStatus = status;
                        if (status.isRetweet()) {
                            originalStatus = status.getRetweetedStatus();
                        }

                        extractionCoordinator.doExtraction(originalStatus, new TopicExtractionCallback() {

                            @Override
                            public void handleExtractionResult(Set<Topic> extractedTopics) {
                                /* TODO: comment out outputs */
                                //                              System.out.println(status.getText());
                                //                              for (Topic t : extractedTopics) {
                                //                                 System.out.println(t.getName());
                                //                              }
                                try {
                                    /* add hash tags to topics */
                                    for (HashtagEntity tag : status.getHashtagEntities()) {
                                        extractedTopics.add(new Topic(new String[] { tag.getText() }));
                                    }
                                    for (Topic topic : extractedTopics) {
                                        neo4jService.createTopicIfAbsent(topic);
                                        if (status.isRetweet()) {
                                            neo4jService.createRelationIfAbsent("RETWEETS", user, topic);
                                        } else {
                                            neo4jService.createRelationIfAbsent("TWEETS", user, topic);
                                        }
                                    }
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }

                            }

                            @Override
                            public void handleExtractionError(ExtractionException e) {
                                throw new RuntimeException(e);
                            }
                        });
                    }
                });
            }
        };
        final Processor<TwitterUser> userProcessor = new Processor<TwitterUser>() {

            @Override
            public void process(final TwitterUser user) {
                try {
                    twitterDataAccess.forAllFollowers(user, new Processor<User>() {

                        @Override
                        public void process(User u) {
                            final TwitterUser follower = em.find(TwitterUser.class, u.getId());

                            if (follower != null) {
                                neo4jService.transactional(new Neo4jUnitOfWork() {

                                    @Override
                                    public void process() throws SQLException {
                                        neo4jService.createUniqueRelation("FOLLOWS", user, follower);
                                    }
                                });
                            }
                        }
                    });
                } catch (Exception ex) {
                    throw new RuntimeException(ex);
                }
            }
        };

        // invoke the processor for every entry in mongo
        mongoDataAccess.forAll(mongoProcessor);
        extractionCoordinator.awaitTermination(-1, TimeUnit.SECONDS);
        // invoke the processor for every entry in the rdbms
        //         userDataAcces.forAll(userProcessor);
    } finally {
        if (emf != null && emf.isOpen()) {
            emf.close();
        }
        if (mongoClient != null) {
            mongoClient.close();
        }
        if (twitter != null) {
            twitter.shutdown();
        }
        if (neo4j != null) {
            neo4j.close();
        }
    }
}

From source file:co.cask.tigon.analytics.TweetCollector.java

License:Apache License

@Tick(unit = TimeUnit.MILLISECONDS, delay = 100)
public void collect() throws InterruptedException {
    if (this.queue == null) {
        // Sleep and return if public timeline is disabled
        Thread.sleep(1000);//from   w w w . j a  va  2 s.c o  m
        return;
    }
    int batchSize = 100;

    for (int i = 0; i < batchSize; i++) {
        Status tweet = queue.poll();
        if (tweet == null) {
            break;
        }

        // emitting more data to get higher throughput
        for (int k = 0; k < tweetAmplification; k++) {
            List<String> hashtags = Lists.newArrayList();
            for (HashtagEntity hashtag : tweet.getHashtagEntities()) {
                hashtags.add(hashtag.getText());
            }
            output.emit(new SimpleTweet(tweet.getText(), hashtags));
        }
    }
}

From source file:com.cask.twitter.TweetStream.java

License:Apache License

@Override
public void initialize(FlowletContext context) {
    statusListener = new StatusListener() {
        @Override/*from w w w  . j a  va  2  s .com*/
        public void onStatus(Status status) {
            for (HashtagEntity hash : status.getHashtagEntities()) {
                tweetQ.add(hash.getText());
            }
        }

        @Override
        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {

        }

        @Override
        public void onTrackLimitationNotice(int i) {

        }

        @Override
        public void onScrubGeo(long l, long l2) {

        }

        @Override
        public void onStallWarning(StallWarning stallWarning) {

        }

        @Override
        public void onException(Exception e) {
            LOG.error(e.getMessage());
        }
    };

    cb = new ConfigurationBuilder();
    cb.setDebugEnabled(false)
            //Provide Twitter Credentials through Runtime Arguments.
            .setOAuthConsumerKey(context.getRuntimeArguments().get("ConsumerKey"))
            .setOAuthConsumerSecret(context.getRuntimeArguments().get("ConsumerSecret"))
            .setOAuthAccessToken(context.getRuntimeArguments().get("AccessToken"))
            .setOAuthAccessTokenSecret(context.getRuntimeArguments().get("AccessTokenSecret"));

    tStream = new TwitterStreamFactory(cb.build()).getInstance();
    //For testing, we can disable Twitter Stream and use testStream to send sample hashTags.
    if (!context.getRuntimeArguments().containsKey("disableLiveStream")) {
        tStream.addListener(statusListener);
        tStream.sample();
    }
}

From source file:com.daemon.database.Transactor.java

License:Open Source License

/**
 * Adds the hashtag to the SQL Batch, if not already in the DB. Returns its id.
 * This method is called by saveTweetsHashtags.
 * /*from ww w . j a  va 2  s.  c  o m*/
 * @param htEntity   An Hashtag-Entity to be saved.
 * @throws SQLException
 */
private void saveHashtag(HashtagEntity htEntity) throws SQLException {
    prepStatementHashtag.setString(1, htEntity.getText());
    prepStatementHashtag.addBatch();
}

From source file:com.daemon.database.Transactor.java

License:Open Source License

/**
 * Adds the relation between tweet and hashtag to the SQL Batch. This method is called by
 * saveTweetsHashtags().//from  w  w  w.  j a v a 2  s.  c o m
 * 
 * @param tweet   The tweet related to the search term-id.
 * @param htEntity   The hashtag object related to the tweet.
 * @throws SQLException
 */
private void saveTweetHashtagRelation(Status tweet, HashtagEntity htEntity) throws SQLException {
    prepStatementTweetHasHashtag.setLong(1, tweet.getId());
    prepStatementTweetHasHashtag.setString(2, htEntity.getText());
    prepStatementTweetHasHashtag.addBatch();
}

From source file:com.daiv.android.twitter.utils.TweetLinkUtils.java

License:Apache License

public static String[] getLinksInStatus(Status status) {
    UserMentionEntity[] users = status.getUserMentionEntities();
    String mUsers = "";

    for (UserMentionEntity name : users) {
        String n = name.getScreenName();
        if (n.length() > 1) {
            mUsers += n + "  ";
        }/*from  w  ww.jav a 2  s.co  m*/
    }

    HashtagEntity[] hashtags = status.getHashtagEntities();
    String mHashtags = "";

    for (HashtagEntity hashtagEntity : hashtags) {
        String text = hashtagEntity.getText();
        if (text.length() > 1) {
            mHashtags += text + "  ";
        }
    }

    URLEntity[] urls = status.getURLEntities();
    String expandedUrls = "";
    String compressedUrls = "";

    for (URLEntity entity : urls) {
        String url = entity.getExpandedURL();
        if (url.length() > 1) {
            expandedUrls += url + "  ";
            compressedUrls += entity.getURL() + "  ";
        }
    }

    MediaEntity[] medias = status.getMediaEntities();
    String mediaExp = "";
    String mediaComp = "";
    String mediaDisplay = "";

    for (MediaEntity e : medias) {
        String url = e.getURL();
        if (url.length() > 1) {
            mediaComp += url + "  ";
            mediaExp += e.getExpandedURL() + "  ";
            mediaDisplay += e.getDisplayURL() + "  ";
        }
    }

    String[] sExpandedUrls;
    String[] sCompressedUrls;
    String[] sMediaExp;
    String[] sMediaComp;
    String[] sMediaDisplay;

    try {
        sCompressedUrls = compressedUrls.split("  ");
    } catch (Exception e) {
        sCompressedUrls = new String[0];
    }

    try {
        sExpandedUrls = expandedUrls.split("  ");
    } catch (Exception e) {
        sExpandedUrls = new String[0];
    }

    try {
        sMediaComp = mediaComp.split("  ");
    } catch (Exception e) {
        sMediaComp = new String[0];
    }

    try {
        sMediaExp = mediaExp.split("  ");
    } catch (Exception e) {
        sMediaExp = new String[0];
    }

    try {
        sMediaDisplay = mediaDisplay.split("  ");
    } catch (Exception e) {
        sMediaDisplay = new String[0];
    }

    String tweetTexts = status.getText();

    String imageUrl = "";
    String otherUrl = "";

    for (int i = 0; i < sCompressedUrls.length; i++) {
        String comp = sCompressedUrls[i];
        String exp = sExpandedUrls[i];

        if (comp.length() > 1 && exp.length() > 1) {
            String str = exp.toLowerCase();

            try {
                String replacement = exp.replace("http://", "").replace("https://", "").replace("www.", "");

                boolean hasCom = replacement.contains(".com");
                replacement = replacement.substring(0, 30) + "...";

                if (hasCom && !replacement.contains(".com")) { // the link was too long...
                    replacement = exp.replace("http://", "").replace("https://", "").replace("www.", "");
                    replacement = replacement.substring(0, replacement.indexOf(".com") + 6) + "...";
                }

                tweetTexts = tweetTexts.replace(comp, replacement);
            } catch (Exception e) {
                tweetTexts = tweetTexts.replace(comp,
                        exp.replace("http://", "").replace("https://", "").replace("www.", ""));
            }
            if (str.contains("instag") && !str.contains("blog.insta")) {
                imageUrl = exp + "media/?size=l";
                otherUrl += exp + "  ";
            } else if (exp.toLowerCase().contains("youtub")
                    && !(str.contains("channel") || str.contains("user"))) {
                // first get the youtube video code
                int start = exp.indexOf("v=") + 2;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                } catch (Exception e) {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1)
                            + "/hqdefault.jpg";
                }
                otherUrl += exp + "  ";
            } else if (str.contains("youtu.be")) {
                // first get the youtube video code
                int start = exp.indexOf(".be/") + 4;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                } catch (Exception e) {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1)
                            + "/hqdefault.jpg";
                }
                otherUrl += exp + "  ";
            } else if (str.contains("twitpic")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://twitpic.com/show/full/" + exp.substring(start).replace("/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("i.imgur") && !str.contains("/a/")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://i.imgur.com/" + exp.replace("http://i.imgur.com/", "").replace(".jpg", "")
                        + "l.jpg";
                imageUrl = imageUrl.replace("gallery/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("imgur") && !str.contains("/a/")) {
                int start = exp.indexOf(".com/") + 6;
                imageUrl = "http://i.imgur.com/" + exp.replace("http://imgur.com/", "").replace(".jpg", "")
                        + "l.jpg";
                imageUrl = imageUrl.replace("gallery/", "").replace("a/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("pbs.twimg.com")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("ow.ly/i/")) {
                imageUrl = "http://static.ow.ly/photos/original/"
                        + exp.substring(exp.lastIndexOf("/")).replaceAll("/", "") + ".jpg";
                otherUrl += exp + "  ";
            } else if (str.contains("p.twipple.jp")) {
                imageUrl = "http://p.twipple.jp/show/large/" + exp.replace("p.twipple.jp/", "")
                        .replace("http://", "").replace("https://", "").replace("www.", "");
                otherUrl += exp + "  ";
            } else if (str.contains(".jpg") || str.contains(".png")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("img.ly")) {
                imageUrl = exp.replace("https", "http").replace("http://img.ly/", "http://img.ly/show/large/");
                otherUrl += exp + "  ";
            } else {
                otherUrl += exp + "  ";
            }
        }
    }

    for (int i = 0; i < sMediaComp.length; i++) {
        String comp = sMediaComp[i];
        String exp = sMediaExp[i];

        if (comp.length() > 1 && exp.length() > 1) {
            try {
                String replacement = sMediaDisplay[i].replace("http://", "").replace("https://", "")
                        .replace("www.", "");

                boolean hasCom = replacement.contains(".com");
                replacement = replacement.substring(0, 22) + "...";

                if (hasCom && !replacement.contains(".com")) { // the link was too long...
                    replacement = sMediaDisplay[i].replace("http://", "").replace("https://", "")
                            .replace("www.", "");
                    replacement = replacement.substring(0, replacement.indexOf(".com") + 6) + "...";
                }

                tweetTexts = tweetTexts.replace(comp, replacement);
            } catch (Exception e) {
                tweetTexts = tweetTexts.replace(comp,
                        sMediaDisplay[i].replace("http://", "").replace("https://", "").replace("www.", ""));
            }
            imageUrl = status.getMediaEntities()[0].getMediaURL();

            for (MediaEntity m : status.getExtendedMediaEntities()) {
                if (m.getType().equals("photo")) {
                    if (!imageUrl.contains(m.getMediaURL())) {
                        imageUrl += " " + m.getMediaURL();
                    }
                }
            }

            otherUrl += sMediaDisplay[i];
        }
    }

    return new String[] { tweetTexts, imageUrl, otherUrl, mHashtags, mUsers };
}

From source file:com.daiv.android.twitter.utils.TweetLinkUtils.java

License:Apache License

public static String[] getLinksInStatus(DirectMessage status) {
    UserMentionEntity[] users = status.getUserMentionEntities();
    String mUsers = "";

    for (UserMentionEntity name : users) {
        String n = name.getScreenName();
        if (n.length() > 1) {
            mUsers += n + "  ";
        }/*from  w w w .  j  a v  a  2  s .  c  om*/
    }

    HashtagEntity[] hashtags = status.getHashtagEntities();
    String mHashtags = "";

    for (HashtagEntity hashtagEntity : hashtags) {
        String text = hashtagEntity.getText();
        if (text.length() > 1) {
            mHashtags += text + "  ";
        }
    }

    URLEntity[] urls = status.getURLEntities();
    String expandedUrls = "";
    String compressedUrls = "";

    for (URLEntity entity : urls) {
        String url = entity.getExpandedURL();
        if (url.length() > 1) {
            expandedUrls += url + "  ";
            compressedUrls += entity.getURL() + "  ";
        }
    }

    MediaEntity[] medias = status.getMediaEntities();
    String mediaExp = "";
    String mediaComp = "";
    String mediaDisplay = "";

    for (MediaEntity e : medias) {
        String url = e.getURL();
        if (url.length() > 1) {
            mediaComp += url + "  ";
            mediaExp += e.getExpandedURL() + "  ";
            mediaDisplay += e.getDisplayURL() + "  ";
        }
    }

    String[] sExpandedUrls;
    String[] sCompressedUrls;
    String[] sMediaExp;
    String[] sMediaComp;
    String[] sMediaDisply;

    try {
        sCompressedUrls = compressedUrls.split("  ");
    } catch (Exception e) {
        sCompressedUrls = new String[0];
    }

    try {
        sExpandedUrls = expandedUrls.split("  ");
    } catch (Exception e) {
        sExpandedUrls = new String[0];
    }

    try {
        sMediaComp = mediaComp.split("  ");
    } catch (Exception e) {
        sMediaComp = new String[0];
    }

    try {
        sMediaExp = mediaExp.split("  ");
    } catch (Exception e) {
        sMediaExp = new String[0];
    }

    try {
        sMediaDisply = mediaDisplay.split("  ");
    } catch (Exception e) {
        sMediaDisply = new String[0];
    }

    String tweetTexts = status.getText();

    String imageUrl = "";
    String otherUrl = "";

    for (int i = 0; i < sCompressedUrls.length; i++) {
        String comp = sCompressedUrls[i];
        String exp = sExpandedUrls[i];

        if (comp.length() > 1 && exp.length() > 1) {
            String str = exp.toLowerCase();

            tweetTexts = tweetTexts.replace(comp,
                    exp.replace("http://", "").replace("https://", "").replace("www.", ""));

            if (str.contains("instag") && !str.contains("blog.instag")) {
                imageUrl = exp + "media/?size=m";
                otherUrl += exp + "  ";
            } else if (str.contains("youtub") && !(str.contains("channel") || str.contains("user"))) { // normal youtube link
                // first get the youtube video code
                int start = exp.indexOf("v=") + 2;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                otherUrl += exp + "  ";
            } else if (str.contains("youtu.be")) { // shortened youtube link
                // first get the youtube video code
                int start = exp.indexOf(".be/") + 4;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                otherUrl += exp + "  ";
            } else if (str.contains("twitpic")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://twitpic.com/show/full/" + exp.substring(start).replace("/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("imgur") && !str.contains("/a/")) {
                int start = exp.indexOf(".com/") + 6;
                imageUrl = "http://i.imgur.com/" + exp.substring(start) + "l.jpg";
                imageUrl = imageUrl.replace("gallery/", "").replace("a/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("pbs.twimg.com")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("ow.ly/i/")) {
                Log.v("Test_owly", exp);
                imageUrl = "http://static.ow.ly/photos/original/"
                        + exp.substring(exp.lastIndexOf("/")).replaceAll("/", "") + ".jpg";
                otherUrl += exp + "  ";
            } else if (str.contains(".jpg") || str.contains(".png")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("img.ly")) {
                imageUrl = exp.replace("https", "http").replace("http://img.ly/", "http://img.ly/show/large/");
                otherUrl += exp + "  ";
            } else {
                otherUrl += exp + "  ";
            }
        }
    }

    for (int i = 0; i < sMediaComp.length; i++) {
        String comp = sMediaComp[i];
        String exp = sMediaExp[i];

        if (comp.length() > 1 && exp.length() > 1) {
            tweetTexts = tweetTexts.replace(comp, sMediaDisply[i]);
            imageUrl = status.getMediaEntities()[0].getMediaURL();
            otherUrl += sMediaDisply[i];
        }
    }

    return new String[] { tweetTexts, imageUrl, otherUrl, mHashtags, mUsers };
}

From source file:com.datatorrent.contrib.twitter.TwitterSampleInput.java

License:Open Source License

@Override
public void emitTuples() {
    for (int size = statuses.size(); size-- > 0;) {
        Status s = statuses.poll();/*from  w  w  w. j a  v a  2 s  . c o  m*/
        if (status.isConnected()) {
            status.emit(s);
        }

        if (text.isConnected()) {
            text.emit(s.getText());
        }

        if (url.isConnected()) {
            URLEntity[] entities = s.getURLEntities();
            if (entities != null) {
                for (URLEntity ue : entities) {
                    url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString());
                }
            }
        }

        if (hashtag.isConnected()) {
            HashtagEntity[] hashtagEntities = s.getHashtagEntities();
            if (hashtagEntities != null) {
                for (HashtagEntity he : hashtagEntities) {
                    hashtag.emit(he.getText());
                }
            }
        }
    }
}

From source file:com.github.daytron.twaattin.ui.tabledecorator.TweetColumnDecorator.java

License:Open Source License

void createFragmentsWithTag(HashtagEntity[] tags) {

    if (tags != null) {

        for (HashtagEntity tag : tags) {

            int start = tag.getStart();
            int end = tag.getEnd();

            try {

                String encodedUrl = TWITTER_SEARCH_URL + URLEncoder.encode('#' + tag.getText(), "UTF-8");

                String href = "<a href='" + encodedUrl + "' target='search'>";

                TweetFragment fragment = new TweetFragment(start, end, href + '#' + tag.getText() + "</a>");

                fragments.add(fragment);

            } catch (UnsupportedEncodingException e) {

                e.printStackTrace();/*from w ww.j a v a 2 s  . c  o m*/
            }
        }
    }
}

From source file:com.github.jcustenborder.kafka.connect.twitter.StatusConverter.java

License:Apache License

static Struct convertHashtagEntity(HashtagEntity hashtagEntity) {
    return new Struct(SCHEMA_HASHTAG_ENTITY).put("Text", hashtagEntity.getText())
            .put("Start", hashtagEntity.getStart()).put("End", hashtagEntity.getEnd());
}