Example usage for twitter4j Status getURLEntities

List of usage examples for twitter4j Status getURLEntities

  1. HOME
  2. Java
  3. twitter4j
  4. twitter4j.*
  5. Status
  6. getURLEntities

Introduction

In this page you can find the example usage for twitter4j Status getURLEntities.

Prototype

URLEntity[] getURLEntities();

Source Link

Document

Returns an array if URLEntity mentioned in the tweet.

Usage

From source file:com.klinker.android.twitter.utils.TweetLinkUtils.java

License:Apache License

public static String[] getLinksInStatus(Status status) {
    UserMentionEntity[] users = status.getUserMentionEntities();
    String mUsers = "";

    for (UserMentionEntity name : users) {
        String n = name.getScreenName();
        if (n.length() > 1) {
            mUsers += n + "  ";
        }/*from  w  w  w.  jav a2s  . c om*/
    }

    HashtagEntity[] hashtags = status.getHashtagEntities();
    String mHashtags = "";

    for (HashtagEntity hashtagEntity : hashtags) {
        String text = hashtagEntity.getText();
        if (text.length() > 1) {
            mHashtags += text + "  ";
        }
    }

    URLEntity[] urls = status.getURLEntities();
    String expandedUrls = "";
    String compressedUrls = "";

    for (URLEntity entity : urls) {
        String url = entity.getExpandedURL();
        if (url.length() > 1) {
            expandedUrls += url + "  ";
            compressedUrls += entity.getURL() + "  ";
        }
    }

    MediaEntity[] medias = status.getMediaEntities();
    String mediaExp = "";
    String mediaComp = "";
    String mediaDisplay = "";

    for (MediaEntity e : medias) {
        String url = e.getURL();
        if (url.length() > 1) {
            mediaComp += url + "  ";
            mediaExp += e.getExpandedURL() + "  ";
            mediaDisplay += e.getDisplayURL() + "  ";
        }
    }

    String[] sExpandedUrls;
    String[] sCompressedUrls;
    String[] sMediaExp;
    String[] sMediaComp;
    String[] sMediaDisplay;

    try {
        sCompressedUrls = compressedUrls.split("  ");
    } catch (Exception e) {
        sCompressedUrls = new String[0];
    }

    try {
        sExpandedUrls = expandedUrls.split("  ");
    } catch (Exception e) {
        sExpandedUrls = new String[0];
    }

    try {
        sMediaComp = mediaComp.split("  ");
    } catch (Exception e) {
        sMediaComp = new String[0];
    }

    try {
        sMediaExp = mediaExp.split("  ");
    } catch (Exception e) {
        sMediaExp = new String[0];
    }

    try {
        sMediaDisplay = mediaDisplay.split("  ");
    } catch (Exception e) {
        sMediaDisplay = new String[0];
    }

    String tweetTexts = status.getText();

    String imageUrl = "";
    String otherUrl = "";

    for (int i = 0; i < sCompressedUrls.length; i++) {
        String comp = sCompressedUrls[i];
        String exp = sExpandedUrls[i];

        if (comp.length() > 1 && exp.length() > 1) {
            String str = exp.toLowerCase();

            try {
                tweetTexts = tweetTexts.replace(comp,
                        exp.replace("http://", "").replace("https://", "").replace("www.", "").substring(0, 30)
                                + "...");
            } catch (Exception e) {
                tweetTexts = tweetTexts.replace(comp,
                        exp.replace("http://", "").replace("https://", "").replace("www.", ""));
            }
            if (str.contains("instag") && !str.contains("blog.insta")) {
                imageUrl = exp + "media/?size=m";
                otherUrl += exp + "  ";
            } else if (exp.toLowerCase().contains("youtub")
                    && !(str.contains("channel") || str.contains("user"))) {
                // first get the youtube video code
                int start = exp.indexOf("v=") + 2;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                } catch (Exception e) {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1)
                            + "/hqdefault.jpg";
                }
                otherUrl += exp + "  ";
            } else if (str.contains("youtu.be")) {
                // first get the youtube video code
                int start = exp.indexOf(".be/") + 4;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg";
                } catch (Exception e) {
                    imageUrl = "http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1)
                            + "/mqefault.jpg";
                }
                otherUrl += exp + "  ";
            } else if (str.contains("twitpic")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://twitpic.com/show/full/" + exp.substring(start).replace("/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("i.imgur") && !str.contains("/a/")) {
                int start = exp.indexOf(".com/") + 5;
                imageUrl = "http://i.imgur.com/" + exp.replace("http://i.imgur.com/", "").replace(".jpg", "")
                        + "m.jpg";
                imageUrl = imageUrl.replace("gallery/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("imgur") && !str.contains("/a/")) {
                int start = exp.indexOf(".com/") + 6;
                imageUrl = "http://i.imgur.com/" + exp.replace("http://imgur.com/", "").replace(".jpg", "")
                        + "m.jpg";
                imageUrl = imageUrl.replace("gallery/", "").replace("a/", "");
                otherUrl += exp + "  ";
            } else if (str.contains("pbs.twimg.com")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else if (str.contains("ow.ly/i")) {
                imageUrl = "http://static.ow.ly/photos/original/"
                        + exp.substring(exp.lastIndexOf("/")).replaceAll("/", "") + ".jpg";
                otherUrl += exp + "  ";
            } else if (str.contains("p.twipple.jp")) {
                imageUrl = "http://p.twipple.jp/show/large/" + exp.replace("p.twipple.jp/", "")
                        .replace("http://", "").replace("https://", "").replace("www.", "");
                otherUrl += exp + "  ";
            } else if (str.contains(".jpg") || str.contains(".png")) {
                imageUrl = exp;
                otherUrl += exp + "  ";
            } else {
                otherUrl += exp + "  ";
            }
        }
    }

    for (int i = 0; i < sMediaComp.length; i++) {
        String comp = sMediaComp[i];
        String exp = sMediaExp[i];

        if (comp.length() > 1 && exp.length() > 1) {
            try {
                tweetTexts = tweetTexts.replace(comp, sMediaDisplay[i].replace("http://", "")
                        .replace("https://", "").replace("www.", "").substring(0, 22) + "...");
            } catch (Exception e) {
                tweetTexts = tweetTexts.replace(comp,
                        sMediaDisplay[i].replace("http://", "").replace("https://", "").replace("www.", ""));
            }
            imageUrl = status.getMediaEntities()[0].getMediaURL();

            for (MediaEntity m : status.getExtendedMediaEntities()) {
                if (m.getType().equals("photo")) {
                    if (!imageUrl.contains(m.getMediaURL())) {
                        imageUrl += " " + m.getMediaURL();
                    }
                }
            }

            otherUrl += sMediaDisplay[i];
        }
    }

    return new String[] { tweetTexts, imageUrl, otherUrl, mHashtags, mUsers };
}

From source file:com.klinker.android.twitter.utils.TweetLinkUtils.java

License:Apache License

public static ArrayList<String> getAllExternalPictures(Status status) {
    URLEntity[] urls = status.getURLEntities();
    String expandedUrls = "";
    String compressedUrls = "";

    for (URLEntity entity : urls) {
        String url = entity.getExpandedURL();
        if (url.length() > 1) {
            expandedUrls += url + "  ";
            compressedUrls += entity.getURL() + "  ";
        }/*from w  w w . j  a va2 s .  com*/
    }

    MediaEntity[] medias = status.getMediaEntities();
    String mediaExp = "";
    String mediaComp = "";

    for (MediaEntity e : medias) {
        String url = e.getURL();
        if (url.length() > 1) {
            mediaComp += url + "  ";
            mediaExp += e.getExpandedURL() + "  ";
        }
    }

    String[] sExpandedUrls;
    String[] sCompressedUrls;
    String[] sMediaExp;
    String[] sMediaComp;

    try {
        sCompressedUrls = compressedUrls.split("  ");
    } catch (Exception e) {
        sCompressedUrls = new String[0];
    }

    try {
        sExpandedUrls = expandedUrls.split("  ");
    } catch (Exception e) {
        sExpandedUrls = new String[0];
    }

    try {
        sMediaComp = mediaComp.split("  ");
    } catch (Exception e) {
        sMediaComp = new String[0];
    }

    try {
        sMediaExp = mediaExp.split("  ");
    } catch (Exception e) {
        sMediaExp = new String[0];
    }

    ArrayList<String> images = new ArrayList<String>();

    for (int i = 0; i < sCompressedUrls.length; i++) {
        String comp = sCompressedUrls[i];
        String exp = sExpandedUrls[i];

        if (comp.length() > 1 && exp.length() > 1) {
            String str = exp.toLowerCase();

            if (str.contains("instag") && !str.contains("blog.insta")) {
                images.add(exp + "media/?size=m");
            } else if (exp.toLowerCase().contains("youtub")
                    && !(str.contains("channel") || str.contains("user"))) {
                // first get the youtube video code
                int start = exp.indexOf("v=") + 2;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    images.add("http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg");
                } catch (Exception e) {
                    images.add("http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1)
                            + "/hqdefault.jpg");
                }
            } else if (str.contains("youtu.be")) {
                // first get the youtube video code
                int start = exp.indexOf(".be/") + 4;
                int end = exp.length();
                if (exp.substring(start).contains("&")) {
                    end = exp.indexOf("&");
                } else if (exp.substring(start).contains("?")) {
                    end = exp.indexOf("?");
                }
                try {
                    images.add("http://img.youtube.com/vi/" + exp.substring(start, end) + "/hqdefault.jpg");
                } catch (Exception e) {
                    images.add("http://img.youtube.com/vi/" + exp.substring(start, exp.length() - 1)
                            + "/mqefault.jpg");
                }
            } else if (str.contains("twitpic")) {
                int start = exp.indexOf(".com/") + 5;
                images.add("http://twitpic.com/show/full/" + exp.substring(start).replace("/", ""));
            } else if (str.contains("i.imgur") && !str.contains("/a/")) {
                images.add(("http://i.imgur.com/" + exp.replace("http://i.imgur.com/", "").replace(".jpg", "")
                        + "m.jpg").replace("gallery/", ""));
            } else if (str.contains("imgur") && !str.contains("/a/")) {
                images.add(("http://i.imgur.com/" + exp.replace("http://imgur.com/", "").replace(".jpg", "")
                        + "m.jpg").replace("gallery/", "").replace("a/", ""));
            } else if (str.contains("pbs.twimg.com")) {
                images.add(exp);
            } else if (str.contains("ow.ly/i")) {
                images.add("http://static.ow.ly/photos/original/"
                        + exp.substring(exp.lastIndexOf("/")).replaceAll("/", "") + ".jpg");
            } else if (str.contains("p.twipple.jp")) {
                images.add("http://p.twipple.jp/show/large/" + exp.replace("p.twipple.jp/", "")
                        .replace("http://", "").replace("https://", "").replace("www.", ""));
            } else if (str.contains(".jpg") || str.contains(".png")) {
                images.add(exp);
            }
        }
    }

    for (int i = 0; i < sMediaComp.length; i++) {
        String comp = sMediaComp[i];
        String exp = sMediaExp[i];

        if (comp.length() > 1 && exp.length() > 1) {
            images.add(status.getMediaEntities()[0].getMediaURL());
        }
    }

    return images;
}

From source file:com.mothsoft.alexis.engine.retrieval.TwitterRetrievalTaskImpl.java

License:Apache License

private List<TweetLink> readLinks(Status status) {
    final List<TweetLink> links = new ArrayList<TweetLink>();

    if (status.getURLEntities() != null) {
        for (final URLEntity entity : status.getURLEntities()) {
            final String displayUrl = entity.getDisplayURL();
            final String expandedUrl = entity.getExpandedURL();
            final String url = entity.getURL();
            final TweetLink link = new TweetLink((short) entity.getStart(), (short) entity.getEnd(), displayUrl,
                    expandedUrl, url);/*from  w w  w  .j a  v  a  2  s. c  o  m*/
            links.add(link);
        }
    }

    return links;
}

From source file:com.mycompany.omnomtweets.Search.java

public static boolean writeToStdErr(Status tweet) {
    //System.out.println("Writing " + tweets.size() + " tweets");
    boolean success = true;
    if (tweet != null) {
        String tweetText;/*from  w  w w. j  ava 2 s  . c om*/
        String idOfRetweetee = "";
        if (tweet.getRetweetedStatus() != null) {
            tweetText = "RT " + tweet.getRetweetedStatus().getText();
            idOfRetweetee = "" + tweet.getRetweetedStatus().getUser().getScreenName();
            //System.out.println("retweeted" + tweetText);
        } else {
            tweetText = tweet.getText();
        }
        String urlText = "";
        if (tweet.getURLEntities().length > 0) {
            for (URLEntity url : tweet.getURLEntities()) {
                if (url.getExpandedURL() != null) {
                    urlText += url.getExpandedURL() + " ";
                    tweetText = tweetText.replace(url.getURL(), url.getExpandedURL());
                    //System.out.println("Expanded URL " + url.getExpandedURL());
                } else {
                    urlText += url.getURL() + " ";
                    //System.out.println("URL " + url.getURL());
                }
            }
        }
        if (tweet.getMediaEntities().length > 0) {
            for (MediaEntity media : tweet.getMediaEntities()) {
                if (media.getExpandedURL() != null) {
                    urlText += media.getExpandedURL() + " ";
                    tweetText = tweetText.replace(media.getMediaURL(), media.getExpandedURL());
                    //System.out.println("Expanded URL " + media.getExpandedURL());
                } else {
                    urlText += media.getMediaURL() + " ";
                    //System.out.println("URL " + media.getMediaURL());
                }
            }
        }
        String encodedText = tweetText.replaceAll("\"", "\"\"");
        encodedText = encodedText.replaceAll("\\s", " ");
        String writeMe = "\"" + encodedText + "\"," + urlText + "," + tweet.getUser().getId() + ","
                + tweet.getId() + "," + candidate.name + "," + sdf.format(tweet.getCreatedAt()) + ","
                + idOfRetweetee + "\n";
        System.err.print(writeMe);
        //writeTweets.write(writeMe);
    }
    return success;
}

From source file:com.mycompany.omnomtweets.Search.java

/**
 * Method to write the tweets to file, base 64 encoded tweet text.
 * @param tweets the tweets to be written
 * @param filename the file to write the tweets into
 * @return true unless something bad happens
 *//*from   ww w  . ja  va  2 s.  c o m*/
public static boolean writeTweetsToFile(List<Status> tweets, String filename) {
    //System.out.println("Writing " + tweets.size() + " tweets");
    boolean success = true;
    try {
        FileWriter addTweets = new FileWriter(new File(filename), true);
        if (tweets != null && tweets.size() > 0) {
            for (Status tweet : tweets) {
                String tweetText;
                String idOfRetweetee = "";
                if (tweet.getRetweetedStatus() != null) {
                    tweetText = "RT " + tweet.getRetweetedStatus().getText();
                    idOfRetweetee = "" + tweet.getRetweetedStatus().getUser().getScreenName();
                    //System.out.println("retweeted" + tweetText);
                } else {
                    tweetText = tweet.getText();
                }
                String urlText = "";
                if (tweet.getURLEntities().length > 0) {
                    for (URLEntity url : tweet.getURLEntities()) {
                        if (url.getExpandedURL() != null) {
                            urlText += url.getExpandedURL() + " ";
                            tweetText = tweetText.replace(url.getURL(), url.getExpandedURL());
                            //System.out.println("Expanded URL " + url.getExpandedURL());
                        } else {
                            urlText += url.getURL() + " ";
                            //System.out.println("URL " + url.getURL());    
                        }
                    }
                }
                if (tweet.getMediaEntities().length > 0) {
                    for (MediaEntity media : tweet.getMediaEntities()) {
                        if (media.getExpandedURL() != null) {
                            urlText += media.getExpandedURL() + " ";
                            tweetText = tweetText.replace(media.getMediaURL(), media.getExpandedURL());
                            //System.out.println("Expanded URL " + media.getExpandedURL());
                        } else {
                            urlText += media.getMediaURL() + " ";
                            //System.out.println("URL " + media.getMediaURL());    
                        }
                    }
                }
                String encodedText = tweetText.replaceAll("\"", "\"\"");
                encodedText = encodedText.replaceAll("\n", " ");
                String writeMe = "\"" + encodedText + "\"," + urlText + "," + tweet.getUser().getId() + ","
                        + tweet.getId() + "," + candidate.name + "," + tweet.getCreatedAt() + ","
                        + idOfRetweetee + "\n";
                //System.out.println(writeMe);
                addTweets.write(writeMe);
            }
        }
        addTweets.close();
    } catch (IOException ex) {
        //System.out.println("Something broke lol");
        success = false;
    }
    return success;
}

From source file:com.mycompany.omnomtweets.TweetsAboutCandidates.java

/**
 * Method to write the tweets to file, base 64 encoded tweet text.
 * @param tweets the tweets to be written
 * @param filename the file to write the tweets into
 * @return true unless something bad happens
 *//*from   w ww.ja  v a2 s .  c o m*/
public boolean writeTweetsToFile(List<Status> tweets, String filename) {
    //System.out.println("Writing " + tweets.size() + " tweets");
    boolean success = true;
    try {
        FileWriter addTweets = new FileWriter(new File(filename), true);
        if (tweets != null && tweets.size() > 0) {
            for (Status tweet : tweets) {
                String tweetText;
                String idOfRetweetee = "";
                if (tweet.getRetweetedStatus() != null) {
                    tweetText = "RT " + tweet.getRetweetedStatus().getText();
                    idOfRetweetee = "" + tweet.getRetweetedStatus().getUser().getScreenName();
                    //System.out.println("retweeted" + tweetText);
                } else {
                    tweetText = tweet.getText();
                }
                String urlText = "";
                if (tweet.getURLEntities().length > 0) {
                    for (URLEntity url : tweet.getURLEntities()) {
                        if (url.getExpandedURL() != null) {
                            urlText += url.getExpandedURL() + " ";
                            //System.out.println("Expanded URL " + url.getExpandedURL());
                        } else {
                            urlText += url.getURL() + " ";
                            //System.out.println("URL " + url.getURL());    
                        }
                    }
                }
                if (tweet.getMediaEntities().length > 0) {
                    for (MediaEntity media : tweet.getMediaEntities()) {
                        if (media.getExpandedURL() != null) {
                            urlText += media.getExpandedURL() + " ";
                            //System.out.println("Expanded URL " + media.getExpandedURL());
                        } else {
                            urlText += media.getMediaURL() + " ";
                            //System.out.println("URL " + media.getMediaURL());    
                        }
                    }
                }
                String encodedText = tweet.getText().replaceAll("\"", "\"\"");
                String writeMe = "\"" + encodedText + "\"," + urlText + "," + tweet.getUser().getId() + ","
                        + tweet.getId() + "," + candidate.name + "," + tweet.getCreatedAt() + ","
                        + idOfRetweetee + "\n";
                //System.out.println(writeMe);
                addTweets.write(writeMe);
            }
        }
        addTweets.close();
    } catch (IOException ex) {
        System.out.println("Something broke lol");
        success = false;
    }
    return success;
}

From source file:com.ocpsoft.hatchling.twitter.PersistentStatusListener.java

License:Open Source License

@Override
public void onStatus(final Status status) {
    Tweet t = new Tweet();
    t.setReceived(status.getCreatedAt());
    t.setText(status.getText());//from  www.  j  a  va 2  s  .  c om
    t.setScreenName(status.getUser().getScreenName());
    t.setUserName(status.getUser().getName());
    t.setUserProfileImageURL(status.getUser().getProfileImageURL());
    t.setProfileURL(null);
    t.setTweetId(status.getId());

    URLEntity[] urlEntities = status.getURLEntities();
    if (urlEntities != null) {
        for (URLEntity url : urlEntities) {
            TweetURL tweetURL = new TweetURL();
            if (url.getExpandedURL() != null) {
                tweetURL.setURL(url.getExpandedURL());
            } else if (url.getURL() != null) {
                tweetURL.setURL(url.getURL());
            }
            t.getURLs().add(tweetURL);
        }
    }

    buffer.add(t);
}

From source file:com.raythos.sentilexo.twitter.utils.StatusArraysHelper.java

public static List<String> getUrlsList(Status status) {
    @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")

    List<String> result = new ArrayList<>();
    for (URLEntity url : status.getURLEntities()) {
        result.add(url.getURL());//from w ww  .  j  av  a  2  s.  co m
    }
    return result;
}

From source file:crawling.GetUserTimelineByFile.java

License:Apache License

public static void main(String[] args) {

    if (args.length < 1) {
        System.out.println("Usage: java twitter4j.examples.GetUserTimelineByFile UserFile");
        System.exit(-1);//from   w  w w.j av  a2 s.com
    }
    userFile = args[0];

    try {
        FileWriter outFileWhole = new FileWriter("CrawlTweets" + ".txt", true);
        out = new PrintWriter(outFileWhole);

        // out.close();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    File inFile = new File(userFile);

    if (!inFile.isFile()) {
        System.out.println("Parameter is not an existing file");
        return;
    }

    BufferedReader br = null;
    try {
        br = new BufferedReader(new FileReader(userFile));
    } catch (FileNotFoundException e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    // gets Twitter instance with default credentials
    Twitter twitter = new TwitterFactory().getInstance();
    int count = 0;
    int totalCount = 0;

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy, MM, dd");
    Date start = null;
    try {
        start = sdf.parse("2000, 1, 1");
    } catch (ParseException e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    List<Status> statuses = null;
    // String user = "Porter_Anderson";
    // user = "pqtad";
    // user = "paradunaa6";
    // user = "palifarous";

    ArrayList<Long> users = new ArrayList<Long>();

    String line = null;

    //Read from the original file and write to the new
    //unless content matches data to be removed.
    try {
        while ((line = br.readLine()) != null)
            users.add(Long.parseLong(line));
        br.close();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    //users.add(584928891L);
    //users.add(700425265L);

    DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    Date date = new Date();
    System.out.println();
    System.out.println("----------------------------------------------");

    System.out.println("Start at: " + dateFormat.format(date));

    Long sinceID =
            //218903304682471424L; // 2012/06/29
            238893053518151680L; // 2012/08/24
    Long maxID = 250409135600975873L; // 2012/09/24
    //238893053518151680L;   // 2012/08/24
    // 227659323180974080L;   // 2012/07/24

    boolean overflow = false;
    for (Long usr : users) {
        System.out.println("%" + usr);
        out.println("%" + usr);
        out.println("%" + usr);
        Paging paging = null;
        count = 0;
        for (int i = 1; i < 21; i++) {
            //paging = new Paging(i, 200);
            paging = new Paging(i, 200, sinceID, maxID);

            overflow = false;
            try {
                // statuses = twitter.getUserTimeline(user, paging);
                statuses = twitter.getUserTimeline(usr, paging);
                Thread.currentThread();
                try {
                    Thread.sleep(10500);
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            } catch (TwitterException te) {
                te.printStackTrace();
                System.out.println("Failed to get timeline: " + te.getMessage());
                //System.exit(-1);
            }
            if (statuses.isEmpty()) {
                if (i > 16)
                    overflow = true;
                break;
            }
            for (Status status : statuses) {
                String record = "";
                record += status.getId();
                record += "::" + status.getInReplyToStatusId();
                record += "::" + status.getInReplyToUserId();
                record += "::" + status.getRetweetCount();
                if (status.getRetweetedStatus() != null)
                    record += "::" + status.getRetweetedStatus().getId();
                else
                    record += "::" + "-1";
                //record += "::" + status.isRetweet();
                int len = status.getUserMentionEntities().length;
                if (len > 0) {
                    record += "::";
                    for (int l = 0; l < len; l++) {
                        UserMentionEntity ent = status.getUserMentionEntities()[l];
                        record += "," + ent.getId();
                    }
                } else
                    record += "::" + "-1";
                len = status.getURLEntities().length;
                if (len > 0) {
                    record += "::";
                    for (int l = 0; l < len; l++) {
                        URLEntity ent = status.getURLEntities()[l];
                        record += "," + ent.getURL() + "|"
                        //+ ent.getDisplayURL() + "|"
                                + ent.getExpandedURL();
                    }
                } else
                    record += "::" + "-1";
                record += "::" + cleanText(status.getText());
                record += "::" +
                // status.getCreatedAt();
                        (status.getCreatedAt().getTime() - start.getTime()) / 1000;

                record += "::" + getSource(status.getSource());
                //System.out.println(record);
                out.println(record);
            }

            count += statuses.size();
            out.flush();
        }
        totalCount += count;
        out.println("%" + usr + ", " + count + ", " + overflow);
        System.out.println("%" + usr + ", " + count + ", " + overflow);
        out.println("------------------------------------------");
    }
    System.out.println("Total status count is " + totalCount);
    out.println("#" + totalCount);
    out.close();
    date = new Date();
    System.out.println();
    System.out.println("----------------------------------------------");

    System.out.println("End at: " + dateFormat.format(date));
}

From source file:crawling.GetUserTimelineByFileDiff.java

License:Apache License

public static void main(String[] args) {

    if (args.length < 1) {
        System.out.println("Usage: java twitter4j.examples.GetUserTimelineByFile UserFile");
        System.exit(-1);//from   ww w  .  j  a va  2 s .co  m
    }
    userFile = args[0];

    try {
        FileWriter outFileWhole = new FileWriter("CrawlTweets" + ".txt", true);
        outWhole = new PrintWriter(outFileWhole);

        FileWriter outFileInter = new FileWriter("CrawlInter" + ".txt", true);
        outInter = new PrintWriter(outFileInter);
        // out.close();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    File inFile = new File(userFile);

    if (!inFile.isFile()) {
        System.out.println("Parameter is not an existing file");
        return;
    }

    BufferedReader br = null;
    try {
        br = new BufferedReader(new FileReader(userFile));
    } catch (FileNotFoundException e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    // gets Twitter instance with default credentials
    Twitter twitter = new TwitterFactory().getInstance();
    int count = 0;
    int countInter = 0;
    int totalCount = 0;
    int totalCountInter = 0;

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy, MM, dd");
    Date start = null;
    try {
        start = sdf.parse("2000, 1, 1");
    } catch (ParseException e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    List<Status> statuses = null;
    // String user = "Porter_Anderson";
    // user = "pqtad";
    // user = "paradunaa6";
    // user = "palifarous";
    // statuses = twitter.getUserTimeline(user);
    //String usersS = "490495670,210462629,731570426,174646182,381163846,308812526,595154838,47356666,590795627,466854690,560386951,406747522,767819150,193105498,725874775,381257304,486727765,39469575,68239634,187739283,27048882,172979072,16075589,618047540,841958365,707631710,76274554,334451278,573978436,782711790,498017142,764236789,88185343,293950091,726606902,221410697,566098451,32987449,73497385,223182159,571182929,769441794,16225570,404748449,249953519,713693096,727692930,178473099,366957548,255171527,411892240,370333418,753509292,765928471,49389074,422867487,185298515,112841821,524731824,132207254,431289858,35602356,747638244,76398937,276388822,221106768,608597994,532423674,509070257,235773945,263995811,61575477,48606725,518136068,450781132,631504150,842236698,840479664,306291201,59829276,541212815,264436593,262466303,302157661,135587748,399229428,265344815,225280446,540254332,218019401,260453139,187424025,177051847,351065900,406684867,293234971,219356963,280763734,59290083,214750688";

    //String[] splits = usersS.split(",");
    ArrayList<Long> users = new ArrayList<Long>();

    String line = null;

    //Read from the original file and write to the new
    //unless content matches data to be removed.
    try {
        while ((line = br.readLine()) != null) {

            users.add(Long.parseLong(line));
        }
        br.close();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    //users.add(584928891L);
    //users.add(700425265L);

    DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    Date date = new Date();
    System.out.println();
    System.out.println("----------------------------------------------");

    System.out.println("Start at: " + dateFormat.format(date));

    Long sinceID =
            //218903304682471424L; // 2012/06/29
            238893053518151680L; // 2012/08/24
    Long maxID = 250409135600975873L; // 2012/09/24
    //238893053518151680L;   // 2012/08/24
    // 227659323180974080L;   // 2012/07/24

    boolean overflow = false;
    for (Long usr : users) {
        System.out.println("%" + usr);
        outWhole.println("%" + usr);
        outInter.println("%" + usr);
        Paging paging = null;
        count = 0;
        countInter = 0;
        for (int i = 1; i < 21; i++) {
            //paging = new Paging(i, 200);
            paging = new Paging(i, 200, sinceID, maxID);

            overflow = false;
            try {
                // statuses = twitter.getUserTimeline(user, paging);
                statuses = twitter.getUserTimeline(usr, paging);
                Thread.currentThread();
                try {
                    Thread.sleep(10500);
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            } catch (TwitterException te) {
                te.printStackTrace();
                System.out.println("Failed to get timeline: " + te.getMessage());
                //System.exit(-1);
            }
            if (statuses.isEmpty()) {
                if (i > 16)
                    overflow = true;
                break;
            }

            for (Status status : statuses) {
                boolean inter = false;
                //countRT += status.getRetweetCount();
                String record = "";
                String recordInter = "";
                record += status.getId();

                // For reply
                Long replyId = status.getInReplyToStatusId();
                if (replyId == -1)
                    record += "::-1::-1";
                else {
                    record += "::" + replyId;
                    record += "::" + status.getInReplyToUserId();
                    inter = true;
                }

                // For retweeting
                Long retweets = (long) status.getRetweetCount();
                record += "::" + retweets;
                if (retweets > 0) {
                    inter = true;

                    if (status.getRetweetedStatus() != null)
                        record += "::" + status.getRetweetedStatus().getId();
                    else
                        record += "::" + "-1"; // The source of retweeting
                }
                //record += "::" + status.isRetweet();
                // For mentions
                int len = status.getUserMentionEntities().length;
                if (len > 0) {
                    record += "::";
                    for (int l = 0; l < len; l++) {
                        UserMentionEntity ent = status.getUserMentionEntities()[l];
                        record += "," + ent.getId();
                    }
                    inter = true;
                } else
                    record += "::" + "-1";

                recordInter += record;

                // For URL
                len = status.getURLEntities().length;
                if (len > 0) {
                    record += "::";
                    for (int l = 0; l < len; l++) {
                        URLEntity ent = status.getURLEntities()[l];
                        record += "," + ent.getURL() + "|"
                        //+ ent.getDisplayURL() + "|"
                                + ent.getExpandedURL();
                    }
                } else
                    record += "::" + "-1";

                // For text
                record += "::" + cleanText(status.getText());

                // For creating time
                // record += "::" + status.getCreatedAt().toString();
                Long seconds =
                        // status.getCreatedAt();
                        (status.getCreatedAt().getTime() - start.getTime()) / 1000;
                record += "::" + seconds;
                recordInter += "::" + seconds;

                // For publishing source
                record += "::" + getSource(status.getSource());
                //System.out.println(record);
                outWhole.println(record);

                if (inter) {
                    outInter.println(recordInter);
                    countInter++;
                }
            }

            count += statuses.size();
            outWhole.flush();
            outInter.flush();
        }
        totalCount += count;
        totalCountInter += countInter;
        outWhole.println("%" + usr + "," + count + "," + countInter + "," + overflow);
        outInter.println("%" + usr + "," + count + "," + countInter + "," + overflow);
        System.out.println("%" + usr + "," + count + "," + countInter + "," + overflow);
        outWhole.println("------------------------------------------");
        outInter.println("------------------------------------------");
    }
    System.out.println("Total status count is " + totalCount + "," + totalCountInter);
    outWhole.println("#" + totalCount + "," + totalCountInter);
    outInter.println("#" + totalCount + "," + totalCountInter);
    outWhole.close();
    outInter.close();
    date = new Date();
    System.out.println();
    System.out.println("----------------------------------------------");

    System.out.println("End at: " + dateFormat.format(date));
}