Example usage for twitter4j HttpResponseCode NOT_FOUND

List of usage examples for twitter4j HttpResponseCode NOT_FOUND

Introduction

In this page you can find the example usage for twitter4j HttpResponseCode NOT_FOUND.

Prototype

int NOT_FOUND

To view the source code for twitter4j HttpResponseCode NOT_FOUND.

Click Source Link

Usage

From source file:com.avishkar.NewGetFollowersIDs.java

License:Apache License

private static List<Long> getUserFollwers(long twitterUserId)
        throws UnknownHostException, TwitterException, InterruptedException {
    Twitter twitter = new TwitterFactory(AccessTokenUtil.getConfig()).getInstance();
    List<Long> filteredUserListOnFollowerCount = new ArrayList<Long>();
    final Gson gson = new Gson();
    try {//from w ww.ja  v a 2 s .  c  o m

        checkRateLimit("/followers/ids", twitter);
        System.out.println("Listing followers's Follower for ID:" + twitterUserId + System.lineSeparator());
        IDs followerIDs = twitter.getFollowersIDs(twitterUserId, -1);
        JsonObject followerJSON = new JsonObject();

        followerJSON.addProperty("id", twitterUserId);
        followerJSON.addProperty("followers", gson.toJson(followerIDs.getIDs()));
        DBAccess.insert(gson.toJson(followerJSON));

        System.out.println("Current Followers Fetched Size:" + followerIDs.getIDs().length);
        // Filtering for influential user
        if (followerIDs.getIDs().length > 1000) {
            System.out.println("User assumed as Influential");
            return null;
        }
        int from = 0;
        int to = 99;
        int limit = checkRateLimit("/users/lookup", twitter);
        while (from < followerIDs.getIDs().length) {
            if (to > followerIDs.getIDs().length)
                to = followerIDs.getIDs().length - 1;
            if (limit == 0)
                checkRateLimit("/users/lookup", twitter);
            ResponseList<User> followers = twitter
                    .lookupUsers(Arrays.copyOfRange(followerIDs.getIDs(), from, to));
            System.out.println("Recieved User count:" + followers.size());
            for (User user : followers) {
                DBAccess.insertUser(gson.toJson(user));
                if (user.getFollowersCount() < 1000) {
                    // if(user.getStatusesCount()>0 &&
                    // user.getStatus()!=null &&
                    // sevenDaysAgo.after(user.getStatus().getCreatedAt()))
                    filteredUserListOnFollowerCount.add(user.getId());
                    getStatuses(user.getScreenName(), twitter);
                } else
                    System.out.println("User " + user.getScreenName()
                            + " is pruned for Influential or over subscription." + " Follower count:"
                            + user.getFollowersCount() + " Friends Count:" + user.getFriendsCount());
            }

            from += 100;
            to += 100;
            limit--;
        }
    } catch (TwitterException te) {
        if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {
            System.out.println("Encountered locked profile. Skipping " + twitterUserId);
            return null;

            // log something here
        }
        te.printStackTrace();
        System.out.println("Failed to get followers' Follower: " + te.getMessage());
        // System.exit(-1);
    }

    return filteredUserListOnFollowerCount;
}

From source file:friendsandfollowers.DBFollowersIDs.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException,
        FileNotFoundException, UnsupportedEncodingException {

    // Check arguments that passed in
    if ((args == null) || (args.length == 0)) {
        System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job.");
        System.err.println("First: String 'OUTPUT: /output/path/'");
        System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 "
                + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids.");
        System.err.println("Third (optional): 'screen_name / user_id_str'");
        System.err.println("If 3rd argument not provided then provide" + " Twitter users through database.");
        System.exit(-1);//  w w w.j  a v  a2  s  .c om
    }

    MysqlDB DB = new MysqlDB();
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/followers/ids";

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    int IDS_TO_FETCH_INT = -1;
    try {
        IDS_TO_FETCH_INT = Integer.parseInt(args[1]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[1] + " must be an integer.");
        System.exit(-1);
    }

    int IDS_TO_FETCH = 0;
    if (IDS_TO_FETCH_INT > 5000) {

        float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000;
        IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F);
    } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) {
        IDS_TO_FETCH = 1;
    }

    String targetedUser = "";
    if (args.length == 3) {
        try {
            targetedUser = StringEscapeUtils.escapeJava(args[2]);
        } catch (Exception e) {
            System.err.println("Argument" + args[2] + " must be an String.");
            System.exit(-1);
        }
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        IDs ids;
        System.out.println("Listing followers ids.");

        // if targetedUser not provided by argument, then look into database.
        if (StringUtils.isEmpty(targetedUser)) {

            String selectQuery = "SELECT * FROM `followers_parent` WHERE " + "`targeteduser` != '' AND "
                    + "`nextcursor` != '0' AND " + "`nextcursor` != '2'";

            ResultSet results = DB.selectQ(selectQuery);

            int numRows = DB.numRows(results);
            if (numRows < 1) {
                System.err.println("No User in database to get followersIDS");
                System.exit(-1);
            }

            OUTERMOST: while (results.next()) {

                int followers_parent_id = results.getInt("id");
                targetedUser = results.getString("targeteduser");
                long cursor = results.getLong("nextcursor");
                System.out.println("Targeted User: " + targetedUser);

                int idsLoopCounter = 0;
                int totalIDs = 0;

                // put idsJSON in a file
                PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8");

                // call different functions for screen_name and id_str
                Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                do {
                    ids = null;
                    try {

                        if (chckedNumaric) {

                            long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                            ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor);
                        } else {
                            ids = twitter.getFollowersIDs(targetedUser, cursor);
                        }

                    } catch (TwitterException te) {

                        // do not throw if user has protected tweets, 
                        // or if they deleted their account
                        if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                            System.out.println(targetedUser + " is protected or account is deleted");
                        } else {
                            System.out.println("Followers Get Exception: " + te.getMessage());
                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // load auth user
                            tf = AppOAuths.loadOAuthUser(endpoint);
                            twitter = tf.getInstance();

                            System.out.println(
                                    "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls;
                            RemainingCallsCounter = 0;

                            System.out.println("New Remianing Calls: " + RemainingCalls);
                        }

                        // update cursor in "followers_parent"
                        String fieldValues = "`nextcursor` = 2";
                        String where = "id = " + followers_parent_id;
                        DB.Update("`followers_parent`", fieldValues, where);

                        // If error then switch to next user
                        continue OUTERMOST;
                    }

                    if (ids.getIDs().length > 0) {

                        idsLoopCounter++;
                        totalIDs += ids.getIDs().length;
                        System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                        JSONObject responseDetailsJson = new JSONObject();
                        JSONArray jsonArray = new JSONArray();
                        for (long id : ids.getIDs()) {
                            jsonArray.put(id);
                        }
                        Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                        writer.println(idsJSON);
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // load auth user
                        tf = AppOAuths.loadOAuthUser(endpoint);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                    if (IDS_TO_FETCH_INT != -1) {
                        if (idsLoopCounter == IDS_TO_FETCH) {
                            break;
                        }
                    }

                } while ((cursor = ids.getNextCursor()) != 0);
                writer.close();
                System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);
                System.out.println();

                // update cursor in "followers_parent"
                String fieldValues = "`nextcursor` = " + cursor;
                String where = "id = " + followers_parent_id;
                DB.Update("`followers_parent`", fieldValues, where);

            } // loop through every result found in db
        } else {

            // Second Argument Set, so we are here.
            System.out.println("screen_name / user_id_str passed by argument");

            int idsLoopCounter = 0;
            int totalIDs = 0;

            // put idsJSON in a file
            PrintWriter writer = new PrintWriter(
                    OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8");

            // call different functions for screen_name and id_str
            Boolean chckedNumaric = helpers.isNumeric(targetedUser);
            long cursor = -1;

            do {
                ids = null;
                try {

                    if (chckedNumaric) {

                        long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                        ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor);
                    } else {
                        ids = twitter.getFollowersIDs(targetedUser, cursor);
                    }

                } catch (TwitterException te) {

                    // do not throw if user has protected tweets, or if they deleted their account
                    if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                            || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                        System.out.println(targetedUser + " is protected or account is deleted");
                    } else {
                        System.out.println("Followers Get Exception: " + te.getMessage());
                    }
                    System.exit(-1);
                }

                if (ids.getIDs().length > 0) {

                    idsLoopCounter++;
                    totalIDs += ids.getIDs().length;
                    System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                    JSONObject responseDetailsJson = new JSONObject();
                    JSONArray jsonArray = new JSONArray();

                    for (long id : ids.getIDs()) {
                        jsonArray.put(id);
                    }

                    Object idsJSON = responseDetailsJson.put("ids", jsonArray);
                    writer.println(idsJSON);
                }

                // If rate limit reach then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // load auth user
                    tf = AppOAuths.loadOAuthUser(endpoint);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

                if (IDS_TO_FETCH_INT != -1) {
                    if (idsLoopCounter == IDS_TO_FETCH) {
                        break;
                    }
                }

            } while ((cursor = ids.getNextCursor()) != 0);
            writer.close();

            System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);
            System.out.println();
        }

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.err.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:friendsandfollowers.DBFriendsIDs.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException,
        FileNotFoundException, UnsupportedEncodingException {

    // Check arguments that passed in
    if ((args == null) || (args.length == 0)) {
        System.err.println("2 Parameters are required plus one optional " + "parameter to launch a Job.");
        System.err.println("First: String 'OUTPUT: /output/path/'");
        System.err.println("Second: (int) Number of ids to fetch. " + "Provide number which increment by 5000 "
                + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids.");
        System.err.println("Third (optional): 'screen_name / user_id_str'");
        System.err.println("If 3rd argument not provided then provide" + " Twitter users through database.");
        System.exit(-1);/*from w w w . java  2s. c om*/
    }

    MysqlDB DB = new MysqlDB();
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/friends/ids";

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    int IDS_TO_FETCH_INT = -1;
    try {
        IDS_TO_FETCH_INT = Integer.parseInt(args[1]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[1] + " must be an integer.");
        System.exit(-1);
    }

    int IDS_TO_FETCH = 0;
    if (IDS_TO_FETCH_INT > 5000) {

        float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000;
        IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F);
    } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) {
        IDS_TO_FETCH = 1;
    }

    String targetedUser = "";
    if (args.length == 3) {
        try {
            targetedUser = StringEscapeUtils.escapeJava(args[2]);
        } catch (Exception e) {
            System.err.println("Argument" + args[2] + " must be an String.");
            System.exit(-1);
        }
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        IDs ids;
        System.out.println("Listing friends ids.");

        // if targetedUser not provided by argument, then look into database.
        if (StringUtils.isEmpty(targetedUser)) {

            String selectQuery = "SELECT * FROM `followings_parent` WHERE " + "`targeteduser` != '' AND "
                    + "`nextcursor` != '0' AND " + "`nextcursor` != '2'";

            ResultSet results = DB.selectQ(selectQuery);

            int numRows = DB.numRows(results);
            if (numRows < 1) {
                System.err.println("No User in database to get friendsIDS");
                System.exit(-1);
            }

            OUTERMOST: while (results.next()) {

                int following_parent_id = results.getInt("id");
                targetedUser = results.getString("targeteduser");
                long cursor = results.getLong("nextcursor");
                System.out.println("Targeted User: " + targetedUser);

                int idsLoopCounter = 0;
                int totalIDs = 0;

                // put idsJSON in a file
                PrintWriter writer = new PrintWriter(OutputDirPath + "/" + targetedUser, "UTF-8");

                // call different functions for screen_name and id_str
                Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                do {
                    ids = null;
                    try {

                        if (chckedNumaric) {

                            long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                            ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor);
                        } else {
                            ids = twitter.getFriendsIDs(targetedUser, cursor);
                        }

                    } catch (TwitterException te) {

                        // do not throw if user has protected tweets, 
                        // or if they deleted their account
                        if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                            System.out.println(targetedUser + " is protected or account is deleted");
                        } else {
                            System.out.println("Friends Get Exception: " + te.getMessage());
                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // load auth user
                            tf = AppOAuths.loadOAuthUser(endpoint);
                            twitter = tf.getInstance();

                            System.out.println(
                                    "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls;
                            RemainingCallsCounter = 0;

                            System.out.println("New Remianing Calls: " + RemainingCalls);
                        }

                        // update cursor in "followings_parent"
                        String fieldValues = "`nextcursor` = 2";
                        String where = "id = " + following_parent_id;
                        DB.Update("`followings_parent`", fieldValues, where);

                        // If error then switch to next user
                        continue OUTERMOST;
                    }

                    if (ids.getIDs().length > 0) {

                        idsLoopCounter++;
                        totalIDs += ids.getIDs().length;
                        System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                        JSONObject responseDetailsJson = new JSONObject();
                        JSONArray jsonArray = new JSONArray();
                        for (long id : ids.getIDs()) {
                            jsonArray.put(id);
                        }
                        Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                        writer.println(idsJSON);
                    }

                    // If rate limit reached then switch Auth user.
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // load auth user
                        tf = AppOAuths.loadOAuthUser(endpoint);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth " + "Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                    if (IDS_TO_FETCH_INT != -1) {
                        if (idsLoopCounter == IDS_TO_FETCH) {
                            break;
                        }
                    }

                } while ((cursor = ids.getNextCursor()) != 0);
                writer.close();
                System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);
                System.out.println();

                // update cursor in "followings_parent"
                String fieldValues = "`nextcursor` = " + cursor;
                String where = "id = " + following_parent_id;
                DB.Update("`followings_parent`", fieldValues, where);

            } // loop through every result found in db
        } else {

            // Second Argument Sets, so we are here.
            System.out.println("screen_name / user_id_str " + "passed by argument");

            int idsLoopCounter = 0;
            int totalIDs = 0;

            // put idsJSON in a file
            PrintWriter writer = new PrintWriter(
                    OutputDirPath + "/" + targetedUser + "_ids_" + helpers.getUnixTimeStamp(), "UTF-8");

            // call different functions for screen_name and id_str
            Boolean chckedNumaric = helpers.isNumeric(targetedUser);
            long cursor = -1;

            do {
                ids = null;
                try {

                    if (chckedNumaric) {

                        long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                        ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor);
                    } else {
                        ids = twitter.getFriendsIDs(targetedUser, cursor);
                    }

                } catch (TwitterException te) {

                    // do not throw if user has protected tweets, 
                    // or if they deleted their account
                    if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                            || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                        System.out.println(targetedUser + " is protected or account is deleted");
                    } else {
                        System.out.println("Friends Get Exception: " + te.getMessage());
                    }
                    System.exit(-1);
                }

                if (ids.getIDs().length > 0) {

                    idsLoopCounter++;
                    totalIDs += ids.getIDs().length;
                    System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                    JSONObject responseDetailsJson = new JSONObject();
                    JSONArray jsonArray = new JSONArray();
                    for (long id : ids.getIDs()) {
                        jsonArray.put(id);
                    }
                    Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                    writer.println(idsJSON);

                }

                // If rate limit reach then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // load auth user
                    tf = AppOAuths.loadOAuthUser(endpoint);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

                if (IDS_TO_FETCH_INT != -1) {
                    if (idsLoopCounter == IDS_TO_FETCH) {
                        break;
                    }
                }

            } while ((cursor = ids.getNextCursor()) != 0);
            writer.close();
            System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);
            System.out.println();

        }

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.err.println("Failed to get friends' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:friendsandfollowers.FilesThreaderFollowersIDsParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException,
        FileNotFoundException, UnsupportedEncodingException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 5)) {
        System.err.println("5 Parameters are required  plus one optional " + "parameter to launch a Job.");
        System.err.println("First: String 'INPUT: DB or /input/path/'");
        System.err.println("Second: String 'OUTPUT: /output/path/'");
        System.err.println("Third: (int) Total Number Of Jobs");
        System.err.println("Fourth: (int) This Job Number");
        System.err.println("Fifth: (int) Number of seconds to pause");
        System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 "
                + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids.");
        System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000");
        System.exit(-1);//ww  w .  j a v  a2  s  .co  m
    }

    // TODO documentation for command line
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/followers/ids";

    String inputPath = null;
    try {
        inputPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument " + args[0] + " must be an String.");
        System.exit(-1);
    }

    String outputPath = null;
    try {
        outputPath = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument " + args[1] + " must be an String.");
        System.exit(-1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[2] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[3] + " must be an integer.");
        System.exit(1);
    }

    int secondsToPause = 0;
    try {
        secondsToPause = Integer.parseInt(args[4]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[4] + " must be an integer.");
        System.exit(-1);
    }

    int IDS_TO_FETCH_INT = -1;
    if (args.length == 6) {
        try {
            IDS_TO_FETCH_INT = Integer.parseInt(args[5]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[5] + " must be an integer.");
            System.exit(-1);
        }
    }

    int IDS_TO_FETCH = 0;
    if (IDS_TO_FETCH_INT > 5000) {

        float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000;
        IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F);
    } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) {
        IDS_TO_FETCH = 1;
    }

    secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
    System.out.println("secondsToPause: " + secondsToPause);
    helpers.pause(secondsToPause);

    try {

        int TotalWorkLoad = 0;
        ArrayList<String> allFiles = null;
        try {
            final File folder = new File(inputPath);
            allFiles = helpers.listFilesForSingleFolder(folder);
            TotalWorkLoad = allFiles.size();
        } catch (Exception e) {

            System.err.println("Input folder is not exists: " + e.getMessage());
            System.exit(-1);
        }

        System.out.println("Total Workload is: " + TotalWorkLoad);

        if (TotalWorkLoad < 1) {
            System.err.println("No screen names file exists in: " + inputPath);
            System.exit(-1);
        }

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load. Please reduce 'Number of jobs' to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;
        int chunkSizeToGet = (JOB_NO + 1) * chunkSize;

        System.out.println("My Share is " + chunkSize);
        System.out.println();

        // Load OAuh User
        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls;
        int RemainingCallsCounter = 0;
        System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);
        System.out.println();

        IDs ids;
        System.out.println("Going to get followers ids.");

        // to write output in a file
        System.out.flush();

        if (JOB_NO + 1 == TOTAL_JOBS) {
            chunkSizeToGet = TotalWorkLoad;
        }

        List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet);

        for (String myFile : myFilesShare) {
            System.out.println("Going to parse file: " + myFile);

            try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) {
                String line;
                OUTERMOST: while ((line = br.readLine()) != null) {
                    // process the line.

                    System.out.println("Going to get followers ids of Screen-name / user_id: " + line);
                    System.out.println();

                    String targetedUser = line.trim(); // tmp
                    long cursor = -1;
                    int idsLoopCounter = 0;
                    int totalIDs = 0;

                    PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8");

                    // call different functions for screen_name and id_str
                    Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                    do {
                        ids = null;
                        try {

                            if (chckedNumaric) {

                                long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                                ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor);
                            } else {
                                ids = twitter.getFollowersIDs(targetedUser, cursor);
                            }

                        } catch (TwitterException te) {

                            // do not throw if user has protected tweets, or
                            // if they deleted their account
                            if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                    || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                                System.out.println(targetedUser + " is protected or account is deleted");
                            } else {
                                System.out.println("Followers Get Exception: " + te.getMessage());
                            }

                            // If rate limit reached then switch Auth user
                            RemainingCallsCounter++;
                            if (RemainingCallsCounter >= RemainingCalls) {

                                // load auth user
                                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                                twitter = tf.getInstance();

                                System.out.println(
                                        "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name);

                                RemainingCalls = AppOAuths.RemainingCalls;
                                RemainingCallsCounter = 0;

                                System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                            }

                            // Remove file if ids not found
                            if (totalIDs == 0) {

                                System.out.println("No ids fetched so removing " + "file " + targetedUser);

                                File fileToDelete = new File(outputPath + "/" + targetedUser);
                                fileToDelete.delete();
                            }
                            System.out.println();

                            // If error then switch to next user
                            continue OUTERMOST;
                        }

                        if (ids.getIDs().length > 0) {

                            idsLoopCounter++;
                            totalIDs += ids.getIDs().length;
                            System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                            JSONObject responseDetailsJson = new JSONObject();
                            JSONArray jsonArray = new JSONArray();
                            for (long id : ids.getIDs()) {
                                jsonArray.put(id);
                            }
                            Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                            writer.println(idsJSON);

                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // load auth user
                            tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                            twitter = tf.getInstance();

                            System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls;
                            RemainingCallsCounter = 0;

                            System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                        }

                        if (IDS_TO_FETCH_INT != -1) {
                            if (idsLoopCounter == IDS_TO_FETCH) {
                                break;
                            }
                        }

                    } while ((cursor = ids.getNextCursor()) != 0);

                    writer.close();
                    System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);

                    // Remove file if ids not found
                    if (totalIDs == 0) {

                        System.out.println("No ids fetched so removing " + "file " + targetedUser);

                        File fileToDelete = new File(outputPath + "/" + targetedUser);
                        fileToDelete.delete();
                    }
                    System.out.println();

                } // while get records from single file
            } // read my single file
            catch (IOException e) {
                System.err.println("Failed to read lines from " + myFile);
            }

            // to write output in a file
            System.out.flush();
        } // all my files share

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.err.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}

From source file:friendsandfollowers.FilesThreaderFriendsIDsParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException,
        FileNotFoundException, UnsupportedEncodingException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 5)) {
        System.err.println("5 Parameters are required  plus one optional " + "parameter to launch a Job.");
        System.err.println("First: String 'INPUT: DB or /input/path/'");
        System.err.println("Second: String 'OUTPUT: /output/path/'");
        System.err.println("Third: (int) Total Number Of Jobs");
        System.err.println("Fourth: (int) This Job Number");
        System.err.println("Fifth: (int) Number of seconds to pause");
        System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 "
                + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids.");
        System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000");
        System.exit(-1);//  www  . j a v a2  s. com
    }

    // TODO documentation for command line
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/friends/ids";

    String inputPath = null;
    try {
        inputPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument " + args[0] + " must be an String.");
        System.exit(-1);
    }

    String outputPath = null;
    try {
        outputPath = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument " + args[1] + " must be an String.");
        System.exit(-1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[2] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[3] + " must be an integer.");
        System.exit(1);
    }

    int secondsToPause = 0;
    try {
        secondsToPause = Integer.parseInt(args[4]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[4] + " must be an integer.");
        System.exit(-1);
    }

    int IDS_TO_FETCH_INT = -1;
    if (args.length == 6) {
        try {
            IDS_TO_FETCH_INT = Integer.parseInt(args[5]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[5] + " must be an integer.");
            System.exit(-1);
        }
    }

    int IDS_TO_FETCH = 0;
    if (IDS_TO_FETCH_INT > 5000) {

        float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000;
        IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F);
    } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) {
        IDS_TO_FETCH = 1;
    }

    secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
    System.out.println("secondsToPause: " + secondsToPause);
    helpers.pause(secondsToPause);

    try {

        int TotalWorkLoad = 0;
        ArrayList<String> allFiles = null;
        try {
            final File folder = new File(inputPath);
            allFiles = helpers.listFilesForSingleFolder(folder);
            TotalWorkLoad = allFiles.size();
        } catch (Exception e) {

            System.err.println("Input folder is not exists: " + e.getMessage());
            System.exit(-1);
        }

        System.out.println("Total Workload is: " + TotalWorkLoad);

        if (TotalWorkLoad < 1) {
            System.err.println("No screen names file exists in: " + inputPath);
            System.exit(-1);
        }

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load. Please reduce 'Number of jobs' to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;
        int chunkSizeToGet = (JOB_NO + 1) * chunkSize;

        System.out.println("My Share is " + chunkSize);
        System.out.println();

        // Load OAuh User
        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls;
        int RemainingCallsCounter = 0;
        System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);
        System.out.println();

        IDs ids;
        System.out.println("Going to get friends ids.");

        // to write output in a file
        System.out.flush();

        if (JOB_NO + 1 == TOTAL_JOBS) {
            chunkSizeToGet = TotalWorkLoad;
        }

        List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet);

        for (String myFile : myFilesShare) {
            System.out.println("Going to parse file: " + myFile);

            try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) {
                String line;
                OUTERMOST: while ((line = br.readLine()) != null) {
                    // process the line.

                    System.out.println("Going to get friends ids of Screen-name / user_id: " + line);
                    System.out.println();

                    String targetedUser = line.trim(); // tmp
                    long cursor = -1;
                    int idsLoopCounter = 0;
                    int totalIDs = 0;

                    PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8");

                    // call different functions for screen_name and id_str
                    Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                    do {
                        ids = null;
                        try {

                            if (chckedNumaric) {

                                long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                                ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor);
                            } else {
                                ids = twitter.getFriendsIDs(targetedUser, cursor);
                            }

                        } catch (TwitterException te) {

                            // do not throw if user has protected tweets, or
                            // if they deleted their account
                            if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                    || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                                System.out.println(targetedUser + " is protected or account is deleted");
                            } else {
                                System.out.println("Friends Get Exception: " + te.getMessage());
                            }

                            // If rate limit reached then switch Auth user
                            RemainingCallsCounter++;
                            if (RemainingCallsCounter >= RemainingCalls) {

                                // load auth user
                                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                                twitter = tf.getInstance();

                                System.out.println(
                                        "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name);

                                RemainingCalls = AppOAuths.RemainingCalls;
                                RemainingCallsCounter = 0;

                                System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                            }

                            // Remove file if ids not found
                            if (totalIDs == 0) {

                                System.out.println("No ids fetched so removing " + "file " + targetedUser);

                                File fileToDelete = new File(outputPath + "/" + targetedUser);
                                fileToDelete.delete();
                            }
                            System.out.println();

                            // If error then switch to next user
                            continue OUTERMOST;
                        }

                        if (ids.getIDs().length > 0) {

                            idsLoopCounter++;
                            totalIDs += ids.getIDs().length;
                            System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                            JSONObject responseDetailsJson = new JSONObject();
                            JSONArray jsonArray = new JSONArray();
                            for (long id : ids.getIDs()) {
                                jsonArray.put(id);
                            }
                            Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                            writer.println(idsJSON);

                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // load auth user
                            tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                            twitter = tf.getInstance();

                            System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls;
                            RemainingCallsCounter = 0;

                            System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                        }

                        if (IDS_TO_FETCH_INT != -1) {
                            if (idsLoopCounter == IDS_TO_FETCH) {
                                break;
                            }
                        }

                    } while ((cursor = ids.getNextCursor()) != 0);

                    writer.close();
                    System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);

                    // Remove file if ids not found
                    if (totalIDs == 0) {

                        System.out.println("No ids fetched so removing " + "file " + targetedUser);

                        File fileToDelete = new File(outputPath + "/" + targetedUser);
                        fileToDelete.delete();
                    }
                    System.out.println();

                } // while get records from single file
            } // read my single file
            catch (IOException e) {
                System.err.println("Failed to read lines from " + myFile);
            }

            // to write output in a file
            System.out.flush();
        } // all my files share

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.err.println("Failed to get friends' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}

From source file:timeline.CmdUserTimeline.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length != 7)) {

        System.err.println("Please provide command as following.");
        System.err.println("java -cp twitter4j-multi-oauth-0.3.jar "
                + "timeline.CmdUserTimeline consumer_key consumer_secret"
                + " user_token user_secret output_path " + "screen_name_or_userid "
                + "number_of_tweets_to_get_max_is_3200");
        System.exit(-1);/*from  ww  w . j  av  a2 s . c  o  m*/
    }

    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    String consumer_key = null;
    try {
        consumer_key = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    String consumer_secret = null;
    try {
        consumer_secret = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(-1);
    }
    String user_token = null;
    try {
        user_token = StringEscapeUtils.escapeJava(args[2]);
    } catch (Exception e) {
        System.err.println("Argument" + args[2] + " must be an String.");
        System.exit(-1);
    }
    String user_secret = null;
    try {
        user_secret = StringEscapeUtils.escapeJava(args[3]);
    } catch (Exception e) {
        System.err.println("Argument" + args[3] + " must be an String.");
        System.exit(-1);
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[4]);
    } catch (Exception e) {
        System.err.println("Argument" + args[4] + " must be an String.");
        System.exit(-1);
    }

    String targetedUser = "";
    try {
        targetedUser = StringEscapeUtils.escapeJava(args[5]);
    } catch (Exception e) {
        System.err.println("Argument" + args[5] + " must be an String.");
        System.exit(-1);
    }

    int NUMBER_OF_TWEETS = 0;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[6]);
    } catch (Exception e) {
        System.err.println("Argument" + args[6] + " must be an integer.");
        System.exit(-1);
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, consumer_key, consumer_secret, user_token,
                user_secret);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("Remianing Calls: " + RemainingCalls);

        // screen_name / user_id provided by arguments
        System.out.println("Trying to create output directory");
        String filesPath = OutputDirPath + "/";
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {

            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        String fileName = filesPath + targetedUser;
        PrintWriter writer = new PrintWriter(fileName, "UTF-8");

        // Call different functions for screen_name and id_str
        Boolean chckedNumaric = helpers.isNumeric(targetedUser);

        List<Status> statuses = new ArrayList<>();
        int size = statuses.size();
        int pageno = 1;
        int totalTweets = 0;
        boolean tweetCounterReached = false;
        System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
        while (true) {

            try {

                Paging page = new Paging(pageno++, 200);

                if (chckedNumaric) {

                    long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                    statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                    if (statuses.size() > 0) {
                        for (Status status : statuses) {
                            String rawJSON = TwitterObjectFactory.getRawJSON(status);
                            writer.println(rawJSON);

                            totalTweets += 1;
                            if (totalTweets >= NUMBER_OF_TWEETS) {
                                tweetCounterReached = true;
                                break;
                            }
                        }
                        if (tweetCounterReached) {
                            break;
                        }
                    }
                } else {

                    statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                    if (statuses.size() > 0) {
                        for (Status status : statuses) {
                            String rawJSON = TwitterObjectFactory.getRawJSON(status);
                            writer.println(rawJSON);

                            totalTweets += 1;
                            if (totalTweets >= NUMBER_OF_TWEETS) {
                                tweetCounterReached = true;
                                break;
                            }
                        }
                        if (tweetCounterReached) {
                            break;
                        }
                    }
                }

                // If user's total tweet are less than 195 then no next call
                if (size == 0) {
                    if (totalTweets < 195) {
                        break;
                    }
                }

                // If user's all tweets parsed then exit
                if (totalTweets == size) {
                    break;
                }

                size = totalTweets;

                statuses.clear();

            } catch (TwitterException e) {

                // e.printStackTrace();
                // do not throw if user has protected tweets, or if they
                // deleted their account
                if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                        || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                    System.out.println(targetedUser + " is protected or account is deleted");
                } else {
                    System.out.println("Tweets Get Exception: " + e.getMessage());
                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    System.out.println("No more remianing calls");
                }

                if (totalTweets < 1) {
                    writer.close();
                    // Remove file if tweets not found
                    File fileToDelete = new File(fileName);
                    fileToDelete.delete();
                }
                break;
            }

            // If rate limit reached then switch Auth user
            RemainingCallsCounter++;
            if (RemainingCallsCounter >= RemainingCalls) {

                System.out.println("No more remianing calls");
                break;
            }

        } // while get tweets
        writer.close();

        if (totalTweets > 0) {
            System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);
        } else {

            // Remove file if tweets not found
            File fileToDelete = new File(fileName);
            fileToDelete.delete();
        }

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:timeline.DBNonThreader.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length == 0)) {
        System.err.println("First: 'OUTPUT_PATH' is mendatory.");
        System.err.println("Second: (int) Number of Tweets to get. Max 3200");
        System.err.println("Third: 'screen_name / id_str'" + " is optional.");
        System.err.println("If 3rd argument not provided then provide" + " Twitter users through database.");

        System.exit(-1);/* w  ww .ja  va2 s  .c o  m*/
    }

    MysqlDB DB = new MysqlDB();
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[0] + " must be an String.");
        System.exit(-1);
    }

    int NUMBER_OF_TWEETS = 3200;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an integer.");
        System.exit(-1);
    }

    String targetedUser = "";
    if (args.length == 3) {
        try {
            targetedUser = StringEscapeUtils.escapeJava(args[2]);
        } catch (Exception e) {
            System.err.println("Argument" + args[2] + " must be an String.");
            System.exit(-1);
        }
    }

    try {

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        System.out.println("User's Tweets");

        // If targetedUser not provided by argument, then look into
        // database.
        if (StringUtils.isEmpty(targetedUser)) {

            String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE "
                    + "`tweets_dumped_all` = 0";

            ResultSet results = DB.selectQ(selectQuery);

            int numRows = DB.numRows(results);
            if (numRows < 1) {
                System.err.println("No User in database to get Tweets");
                System.exit(-1);
            }

            System.out.println("Trying to create output directory");
            String filesPath = OutputDirPath + "/";
            File theDir = new File(filesPath);

            // If the directory does not exist, create it
            if (!theDir.exists()) {

                try {
                    theDir.mkdirs();

                } catch (SecurityException se) {

                    System.err.println("Could not create output " + "directory: " + OutputDirPath);
                    System.err.println(se.getMessage());
                    System.exit(-1);
                }
            }

            OUTERMOST: while (results.next()) {

                int targetedUserID = results.getInt("id");
                targetedUser = results.getString("targeteduser");
                System.out.println("Targeted User: " + targetedUser);

                String fileName = filesPath + "/" + targetedUser;
                PrintWriter writer = new PrintWriter(fileName, "UTF-8");

                // Call different functions for screen_name and id_str
                Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                List<Status> statuses = new ArrayList<>();
                int size = statuses.size();
                int pageno = 1;
                int totalTweets = 0;
                boolean tweetCounterReached = false;
                System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
                while (true) {

                    try {

                        Paging page = new Paging(pageno++, 200);

                        if (chckedNumaric) {

                            long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                            statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                            if (statuses.size() > 0) {
                                for (Status status : statuses) {
                                    String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                    writer.println(rawJSON);

                                    totalTweets += 1;
                                    if (totalTweets >= NUMBER_OF_TWEETS) {
                                        tweetCounterReached = true;
                                        break;
                                    }
                                }
                                if (tweetCounterReached) {
                                    break;
                                }
                            }
                        } else {

                            statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                            if (statuses.size() > 0) {
                                for (Status status : statuses) {
                                    String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                    writer.println(rawJSON);

                                    totalTweets += 1;
                                    if (totalTweets >= NUMBER_OF_TWEETS) {
                                        tweetCounterReached = true;
                                        break;
                                    }
                                }
                                if (tweetCounterReached) {
                                    break;
                                }
                            }
                        }

                        // If user's total tweet are less than 195 
                        // then no next call
                        if (size == 0) {
                            if (totalTweets < 195) {
                                break;
                            }
                        }

                        // If user's all tweets parsed 
                        // then proceed to next user 
                        if (totalTweets == size) {
                            break;
                        }

                        size = totalTweets;

                        statuses.clear();

                    } catch (TwitterException e) {

                        // e.printStackTrace();
                        // do not throw if user has protected tweets, 
                        // or if they deleted their account
                        if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                            System.out.println(targetedUser + " is protected or account is deleted");
                        } else {
                            System.out.println("Tweets Get Exception: " + e.getMessage());
                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // Load OAuth user
                            tf = AppOAuths.loadOAuthUser(endpoint);
                            twitter = tf.getInstance();

                            System.out.println(
                                    "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls - 2;
                            RemainingCallsCounter = 0;

                            System.out.println("New Remianing Calls: " + RemainingCalls);
                        }

                        if (totalTweets < 1) {
                            writer.close();
                            // Remove file if tweets not found
                            File fileToDelete = new File(fileName);
                            fileToDelete.delete();
                        }

                        // Update stats
                        String fieldValues = "`tweets_dumped_all` = 2";
                        String where = "id = " + targetedUserID;
                        DB.Update("`twitter_users`", fieldValues, where);
                        continue OUTERMOST;

                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }
                } // while get tweets
                writer.close();

                if (totalTweets > 0) {

                    System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);

                    // Update stats
                    String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets;
                    String where = "id = " + targetedUserID;
                    DB.Update("`twitter_users`", fieldValues, where);
                } else {

                    // Update stats
                    String fieldValues = "`tweets_dumped_all` = 2";
                    String where = "id = " + targetedUserID;
                    DB.Update("`twitter_users`", fieldValues, where);

                    // Remove file if tweets not found
                    File fileToDelete = new File(fileName);
                    fileToDelete.delete();
                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // Load OAuth user
                    tf = AppOAuths.loadOAuthUser(endpoint);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls - 2;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

            } // while get users from database

        } else {
            // screen_name / user_id provided by arguments
            System.out.println("screen_name / user_id provided by arguments");

            System.out.println("Trying to create output directory");
            String filesPath = OutputDirPath + "/";
            File theDir = new File(filesPath);

            // If the directory does not exist, create it
            if (!theDir.exists()) {

                try {
                    theDir.mkdirs();

                } catch (SecurityException se) {

                    System.err.println("Could not create output " + "directory: " + OutputDirPath);
                    System.err.println(se.getMessage());
                    System.exit(-1);
                }
            }

            System.out.println("Targeted User: " + targetedUser);

            String fileName = filesPath + "/" + targetedUser;
            PrintWriter writer = new PrintWriter(fileName, "UTF-8");

            // Call different functions for screen_name and id_str
            Boolean chckedNumaric = helpers.isNumeric(targetedUser);

            List<Status> statuses = new ArrayList<>();
            int size = statuses.size();
            int pageno = 1;
            int totalTweets = 0;
            boolean tweetCounterReached = false;
            System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
            while (true) {

                try {

                    Paging page = new Paging(pageno++, 200);

                    if (chckedNumaric) {

                        long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                        statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    } else {

                        statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    }

                    // If user's total tweet are less than 195 then no next call
                    if (size == 0) {
                        if (totalTweets < 195) {
                            break;
                        }
                    }

                    // If user's all tweets parsed then proceed to next user 
                    if (totalTweets == size) {
                        break;
                    }

                    size = totalTweets;

                    statuses.clear();

                } catch (TwitterException e) {

                    // e.printStackTrace();
                    // do not throw if user has protected tweets, or if they deleted their account
                    if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                            || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                        System.out.println(targetedUser + " is protected or account is deleted");
                    } else {
                        System.out.println("Tweets Get Exception: " + e.getMessage());
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                    if (totalTweets < 1) {
                        writer.close();
                        // Remove file if tweets not found
                        File fileToDelete = new File(fileName);
                        fileToDelete.delete();
                    }
                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // Load OAuth user
                    tf = AppOAuths.loadOAuthUser(endpoint);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls - 2;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

            } // while get tweets
            writer.close();

            if (totalTweets > 0) {
                System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);
            } else {

                // Remove file if tweets not found
                File fileToDelete = new File(fileName);
                fileToDelete.delete();
            }

        } // screen_name / user_id provided by arguments

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");
}

From source file:timeline.DBThreaderParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 5)) {
        System.err.println("5 Parameters are required to launch a Job.");
        System.err.println("First: String 'OUTPUT_PATH'");
        System.err.println("Second: (int) Total Number Of Jobs");
        System.err.println("Third: (int) This Job Number");
        System.err.println("Fourth: (int) Seconds to pause between next launch");
        System.err.println("Fifth: (int) Number of Tweets to get. Max 3200");
        System.err.println("Example: fileName.class /output/path 10 1 2 3200");
        System.exit(-1);/*from w ww. jav a 2  s  .  co  m*/
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[1]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[1] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[2] + " must be an integer.");
        System.exit(1);
    }

    int NUMBER_OF_TWEETS = 0;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[4]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[4] + " must be an integer.");
        System.exit(1);
    }

    MysqlDB DB = new MysqlDB();
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    try {

        String totalRowsQuery = "SELECT count( * ) as wLoad FROM `twitter_users`"
                + " WHERE `tweets_dumped_all` = 0";
        ResultSet totalRowsResults = DB.selectQ(totalRowsQuery);

        int TotalWorkLoad = 0;
        while (totalRowsResults.next()) {
            TotalWorkLoad = totalRowsResults.getInt("wLoad");
        }

        if (TotalWorkLoad < 1) {
            System.err.println("No User in database to get Tweets");
            System.exit(-1);
        }

        // Free memmory
        totalRowsResults = null;

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load Please reduce Number of jobs to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;

        String selectQuery = "SELECT `id`,`targeteduser` FROM " + "`twitter_users` WHERE "
                + "`tweets_dumped_all` = 0 LIMIT " + offSet + "," + chunkSize;

        ResultSet results = DB.selectQ(selectQuery);

        int numRows = DB.numRows(results);
        if (numRows < 1) {
            System.err.println(
                    "No User in database to get Tweets" + " with offset " + offSet + " and limit " + chunkSize);
            System.exit(-1);
        }

        /**
         * wait before launching actual job
         */
        int secondsToPause = 0;
        try {
            secondsToPause = Integer.parseInt(args[3]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[3] + " must be an integer.");
            System.exit(-1);
        }

        secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
        System.out.println("secondsToPause: " + secondsToPause);
        helpers.pause(secondsToPause);
        /**
         * wait before launching actual job
         */

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        System.out.println("User's Tweets");

        System.out.println("Trying to create output directory");
        String filesPath = OutputDirPath + "/";
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {
            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        System.out.flush();

        OUTERMOST: while (results.next()) {

            int targetedUserID = results.getInt("id");
            String targetedUser = results.getString("targeteduser");
            System.out.println("Targeted User: " + targetedUser);

            // Create User file to push tweets in it
            String fileName = filesPath + "/" + targetedUser;
            PrintWriter writer = new PrintWriter(fileName, "UTF-8");

            // Call different functions for screen_name and id_str
            Boolean chckedNumaric = helpers.isNumeric(targetedUser);

            List<Status> statuses = new ArrayList<>();
            int size = statuses.size();
            int pageno = 1;
            int totalTweets = 0;
            boolean tweetCounterReached = false;
            System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
            while (true) {
                try {

                    Paging page = new Paging(pageno++, 200);

                    if (chckedNumaric) {

                        long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                        statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    } else {

                        statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                        if (statuses.size() > 0) {
                            for (Status status : statuses) {
                                String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                writer.println(rawJSON);

                                totalTweets += 1;
                                if (totalTweets >= NUMBER_OF_TWEETS) {
                                    tweetCounterReached = true;
                                    break;
                                }
                            }
                            if (tweetCounterReached) {
                                break;
                            }
                        }
                    }

                    // If user's total tweet are less 
                    // than 195 then no next call
                    if (size == 0) {
                        if (totalTweets < 195) {
                            break;
                        }
                    }

                    // If user's all tweets parsed 
                    // then proceed to next user 
                    if (totalTweets == size) {
                        break;
                    }

                    size = totalTweets;

                    statuses.clear();

                } catch (TwitterException e) {

                    // e.printStackTrace();
                    // do not throw if user has protected tweets, 
                    // or if they deleted their account
                    if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                            || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                        System.out.println(targetedUser + " is protected or account is deleted");
                    } else {
                        System.out.println("Tweets Get Exception: " + e.getMessage());
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                    if (totalTweets < 1) {
                        writer.close();
                        // Remove file if tweets not found
                        File fileToDelete = new File(fileName);
                        fileToDelete.delete();
                    }

                    // Update stats
                    String fieldValues = "`tweets_dumped_all` = 2";
                    String where = "id = " + targetedUserID;
                    DB.Update("`twitter_users`", fieldValues, where);
                    continue OUTERMOST;

                }

                // If rate limit reached then switch Auth user
                RemainingCallsCounter++;
                if (RemainingCallsCounter >= RemainingCalls) {

                    // Load OAuth user
                    tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                    twitter = tf.getInstance();

                    System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                    RemainingCalls = AppOAuths.RemainingCalls - 2;
                    RemainingCallsCounter = 0;

                    System.out.println("New Remianing Calls: " + RemainingCalls);
                }

            } // while get tweets
            writer.close();

            if (totalTweets > 0) {

                System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);
                // Update stats
                String fieldValues = "`tweets_dumped_all` = 1, " + " `tweets_all_count` = " + totalTweets;
                String where = "id = " + targetedUserID;
                DB.Update("`twitter_users`", fieldValues, where);
            } else {

                // Update stats
                String fieldValues = "`tweets_dumped_all` = 2, " + " `tweets_all_count` = " + totalTweets;
                String where = "id = " + targetedUserID;
                DB.Update("`twitter_users`", fieldValues, where);

                // Remove file if tweets not found
                File fileToDelete = new File(fileName);
                fileToDelete.delete();
            }

            // If rate limit reached then switch Auth user
            RemainingCallsCounter++;
            if (RemainingCallsCounter >= RemainingCalls) {

                // Load OAuth user
                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                twitter = tf.getInstance();

                System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                RemainingCalls = AppOAuths.RemainingCalls - 2;
                RemainingCallsCounter = 0;

                System.out.println("New Remianing Calls: " + RemainingCalls);
            }

            System.out.flush();
        } // while get users from database
    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get tweets: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}

From source file:timeline.FilesThreaderParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException, IOException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 7)) {
        System.err.println("7 Parameters are required to launch a Job.");
        System.err.println("First: String 'INPUT: " + "/path/to/perline/screen_names/files/'");
        System.err.println("Second: String 'OUTPUT_PATH'");
        System.err.println("Third: (int) Total Number Of Jobs");
        System.err.println("Fourth: (int) This Job Number");
        System.err.println("Fifth: (int) Seconds to pause between " + "next launch");
        System.err.println("Sixth: (int) Number of Tweets to get. Max 3200");
        System.err.println("Name of current threader.");
        System.err.println("Example: fileName.class /output/path " + "10 2 3200 pool-1-thread-1");
        System.exit(-1);//w w w  . ja va2 s.com
    }

    String inputPath = null;
    try {
        inputPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument " + args[0] + " must be an String.");
        System.exit(-1);
    }

    String OutputDirPath = null;
    try {
        OutputDirPath = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument" + args[1] + " must be an String.");
        System.exit(1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[2] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[3] + " must be an integer.");
        System.exit(1);
    }

    int NUMBER_OF_TWEETS = 0;
    try {
        NUMBER_OF_TWEETS = Integer.parseInt(args[5]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[5] + " must be an integer.");
        System.exit(1);
    }

    String ThreadName = null;
    try {
        ThreadName = StringEscapeUtils.escapeJava(args[6]);
    } catch (Exception e) {
        System.err.println("Argument" + args[6] + " must be an String.");
        System.exit(1);
    }

    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/statuses/user_timeline";

    try {

        int TotalWorkLoad = 0;
        ArrayList<String> allFiles = null;
        try {
            final File folder = new File(inputPath);
            allFiles = helpers.listFilesForSingleFolder(folder);
            TotalWorkLoad = allFiles.size();
        } catch (Exception e) {

            System.err.println("Input folder is not exists: " + e.getMessage());
            System.exit(-1);
        }

        System.out.println("Total Workload is: " + TotalWorkLoad);

        if (TotalWorkLoad < 1) {
            System.err.println("No input file exists in: " + inputPath);
            System.exit(-1);
        }

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load. Please reduce 'Number of jobs' to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;
        int chunkSizeToGet = (JOB_NO + 1) * chunkSize;

        System.out.println("My Share is " + chunkSize);
        System.out.println("My offSet is " + offSet);
        System.out.println("My chunkSizeToGet is " + chunkSizeToGet);
        System.out.println();

        /**
         * wait before launching actual job
         */
        int secondsToPause = 0;
        try {
            secondsToPause = Integer.parseInt(args[4]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[4] + " must be an integer.");
            System.exit(-1);
        }

        secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
        System.out.println("secondsToPause: " + secondsToPause);
        helpers.pause(secondsToPause);
        /**
         * wait before launching actual job
         */

        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls - 2;
        int RemainingCallsCounter = 0;
        System.out.println("First Time Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);

        System.out.println("User's Tweets");

        System.out.println("Trying to create output directory");

        int dirCounter = 1;
        int fileCounter = 0;

        String filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter;
        File theDir = new File(filesPath);

        // If the directory does not exist, create it
        if (!theDir.exists()) {
            try {
                theDir.mkdirs();

            } catch (SecurityException se) {

                System.err.println("Could not create output " + "directory: " + OutputDirPath + "/" + ThreadName
                        + "-tweets-" + dirCounter);
                System.err.println(se.getMessage());
                System.exit(-1);
            }
        }

        if (JOB_NO + 1 == TOTAL_JOBS) {
            chunkSizeToGet = TotalWorkLoad;
        }

        // to write output in a file
        System.out.flush();

        List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet);

        for (String myFile : myFilesShare) {
            System.out.println("Going to parse file: " + myFile);
            System.out.println();

            try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) {
                String line;

                OUTERMOST: while ((line = br.readLine()) != null) {

                    if (fileCounter >= 30000) {

                        dirCounter++;
                        fileCounter = 0;
                        filesPath = OutputDirPath + "/" + ThreadName + "-tweets-" + dirCounter;
                        theDir = new File(filesPath);

                        // If the directory does not exist, create it
                        if (!theDir.exists()) {
                            try {
                                theDir.mkdirs();

                            } catch (SecurityException se) {

                                System.err.println("Could not create output " + "directory: " + OutputDirPath
                                        + "/" + ThreadName + "-tweets-" + dirCounter);
                                System.err.println(se.getMessage());
                                System.exit(-1);
                            }
                        }
                    }

                    System.out.println("Going to get tweets of " + "Screen-name / user_id: " + line);

                    String targetedUser = line.trim();
                    System.out.println("Targeted User: " + targetedUser);

                    // Create User file to push tweets in it
                    String fileName = filesPath + "/" + targetedUser;
                    PrintWriter writer = new PrintWriter(fileName, "UTF-8");

                    // Call different functions for screen_name and id_str
                    Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                    List<Status> statuses = new ArrayList<>();
                    int size = statuses.size();
                    int pageno = 1;
                    int totalTweets = 0;
                    boolean tweetCounterReached = false;
                    System.out.println("NUMBER_OF_TWEETS to get:" + NUMBER_OF_TWEETS);
                    System.out.println();

                    while (true) {
                        try {

                            Paging page = new Paging(pageno++, 200);

                            if (chckedNumaric) {

                                long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                                statuses.addAll(twitter.getUserTimeline(LongValueTargetedUser, page));

                                if (statuses.size() > 0) {
                                    for (Status status : statuses) {
                                        String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                        writer.println(rawJSON);

                                        totalTweets += 1;
                                        if (totalTweets >= NUMBER_OF_TWEETS) {
                                            tweetCounterReached = true;
                                            break;
                                        }
                                    }
                                    if (tweetCounterReached) {
                                        break;
                                    }
                                }
                            } else {

                                statuses.addAll(twitter.getUserTimeline(targetedUser, page));

                                if (statuses.size() > 0) {
                                    for (Status status : statuses) {
                                        String rawJSON = TwitterObjectFactory.getRawJSON(status);
                                        writer.println(rawJSON);

                                        totalTweets += 1;
                                        if (totalTweets >= NUMBER_OF_TWEETS) {
                                            tweetCounterReached = true;
                                            break;
                                        }
                                    }
                                    if (tweetCounterReached) {
                                        break;
                                    }
                                }
                            }

                            // If user's total tweet are less 
                            // than 195 then no next call
                            if (size == 0) {
                                if (totalTweets < 195) {
                                    break;
                                }
                            }

                            // If user's all tweets parsed 
                            // then proceed to next user 
                            if (totalTweets == size) {
                                break;
                            }

                            size = totalTweets;

                            statuses.clear();

                        } catch (TwitterException e) {

                            // e.printStackTrace();
                            // do not throw if user has protected tweets, 
                            // or if they deleted their account
                            if (e.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                    || e.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                                System.out.println(targetedUser + " is protected or account is deleted");
                            } else {
                                System.out.println("Tweets Get Exception: " + e.getMessage());
                            }

                            // If rate limit reached then switch Auth user
                            RemainingCallsCounter++;
                            if (RemainingCallsCounter >= RemainingCalls) {

                                // Load OAuth user
                                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                                twitter = tf.getInstance();

                                System.out.println(
                                        "New User Loaded OAuth" + " Screen_name: " + AppOAuths.screen_name);

                                RemainingCalls = AppOAuths.RemainingCalls - 2;
                                RemainingCallsCounter = 0;

                                System.out.println("New Remianing Calls: " + RemainingCalls);
                            }

                            if (totalTweets < 1) {
                                writer.close();
                                // Remove file if tweets not found
                                File fileToDelete = new File(fileName);
                                fileToDelete.delete();
                            }

                            continue OUTERMOST;

                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // Load OAuth user
                            tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                            twitter = tf.getInstance();

                            System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls - 2;
                            RemainingCallsCounter = 0;

                            System.out.println("New Remianing Calls: " + RemainingCalls);
                        }

                    } // while get tweets
                    writer.close();

                    if (totalTweets > 0) {
                        fileCounter++;
                        System.out.println("Total dumped tweets of " + targetedUser + " are: " + totalTweets);

                    } else {

                        // Remove file if tweets not found
                        File fileToDelete = new File(fileName);
                        fileToDelete.delete();
                    }

                    // If rate limit reached then switch Auth user
                    RemainingCallsCounter++;
                    if (RemainingCallsCounter >= RemainingCalls) {

                        // Load OAuth user
                        tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                        twitter = tf.getInstance();

                        System.out.println("New User Loaded OAuth Screen_name: " + AppOAuths.screen_name);

                        RemainingCalls = AppOAuths.RemainingCalls - 2;
                        RemainingCallsCounter = 0;

                        System.out.println("New Remianing Calls: " + RemainingCalls);
                    }

                } // while get users

            } // read my single file
            catch (IOException e) {
                System.err.println("Failed to read lines from " + myFile);
            }

            File currentFile = new File(inputPath + "/" + myFile);
            currentFile.delete();

            // to write output in a file
            System.out.flush();
        } // all my files share
    } catch (TwitterException te) {
        // te.printStackTrace();
        System.out.println("Failed to get tweets: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}