Example usage for twitter4j IDs getNextCursor

List of usage examples for twitter4j IDs getNextCursor

Introduction

In this page you can find the example usage for twitter4j IDs getNextCursor.

Prototype

@Override
    long getNextCursor();

Source Link

Usage

From source file:friendsandfollowers.FilesThreaderFollowersIDsParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException,
        FileNotFoundException, UnsupportedEncodingException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 5)) {
        System.err.println("5 Parameters are required  plus one optional " + "parameter to launch a Job.");
        System.err.println("First: String 'INPUT: DB or /input/path/'");
        System.err.println("Second: String 'OUTPUT: /output/path/'");
        System.err.println("Third: (int) Total Number Of Jobs");
        System.err.println("Fourth: (int) This Job Number");
        System.err.println("Fifth: (int) Number of seconds to pause");
        System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 "
                + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids.");
        System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000");
        System.exit(-1);//  w ww  .  j a  va 2s.c  om
    }

    // TODO documentation for command line
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/followers/ids";

    String inputPath = null;
    try {
        inputPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument " + args[0] + " must be an String.");
        System.exit(-1);
    }

    String outputPath = null;
    try {
        outputPath = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument " + args[1] + " must be an String.");
        System.exit(-1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[2] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[3] + " must be an integer.");
        System.exit(1);
    }

    int secondsToPause = 0;
    try {
        secondsToPause = Integer.parseInt(args[4]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[4] + " must be an integer.");
        System.exit(-1);
    }

    int IDS_TO_FETCH_INT = -1;
    if (args.length == 6) {
        try {
            IDS_TO_FETCH_INT = Integer.parseInt(args[5]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[5] + " must be an integer.");
            System.exit(-1);
        }
    }

    int IDS_TO_FETCH = 0;
    if (IDS_TO_FETCH_INT > 5000) {

        float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000;
        IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F);
    } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) {
        IDS_TO_FETCH = 1;
    }

    secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
    System.out.println("secondsToPause: " + secondsToPause);
    helpers.pause(secondsToPause);

    try {

        int TotalWorkLoad = 0;
        ArrayList<String> allFiles = null;
        try {
            final File folder = new File(inputPath);
            allFiles = helpers.listFilesForSingleFolder(folder);
            TotalWorkLoad = allFiles.size();
        } catch (Exception e) {

            System.err.println("Input folder is not exists: " + e.getMessage());
            System.exit(-1);
        }

        System.out.println("Total Workload is: " + TotalWorkLoad);

        if (TotalWorkLoad < 1) {
            System.err.println("No screen names file exists in: " + inputPath);
            System.exit(-1);
        }

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load. Please reduce 'Number of jobs' to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;
        int chunkSizeToGet = (JOB_NO + 1) * chunkSize;

        System.out.println("My Share is " + chunkSize);
        System.out.println();

        // Load OAuh User
        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls;
        int RemainingCallsCounter = 0;
        System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);
        System.out.println();

        IDs ids;
        System.out.println("Going to get followers ids.");

        // to write output in a file
        System.out.flush();

        if (JOB_NO + 1 == TOTAL_JOBS) {
            chunkSizeToGet = TotalWorkLoad;
        }

        List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet);

        for (String myFile : myFilesShare) {
            System.out.println("Going to parse file: " + myFile);

            try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) {
                String line;
                OUTERMOST: while ((line = br.readLine()) != null) {
                    // process the line.

                    System.out.println("Going to get followers ids of Screen-name / user_id: " + line);
                    System.out.println();

                    String targetedUser = line.trim(); // tmp
                    long cursor = -1;
                    int idsLoopCounter = 0;
                    int totalIDs = 0;

                    PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8");

                    // call different functions for screen_name and id_str
                    Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                    do {
                        ids = null;
                        try {

                            if (chckedNumaric) {

                                long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                                ids = twitter.getFollowersIDs(LongValueTargetedUser, cursor);
                            } else {
                                ids = twitter.getFollowersIDs(targetedUser, cursor);
                            }

                        } catch (TwitterException te) {

                            // do not throw if user has protected tweets, or
                            // if they deleted their account
                            if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                    || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                                System.out.println(targetedUser + " is protected or account is deleted");
                            } else {
                                System.out.println("Followers Get Exception: " + te.getMessage());
                            }

                            // If rate limit reached then switch Auth user
                            RemainingCallsCounter++;
                            if (RemainingCallsCounter >= RemainingCalls) {

                                // load auth user
                                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                                twitter = tf.getInstance();

                                System.out.println(
                                        "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name);

                                RemainingCalls = AppOAuths.RemainingCalls;
                                RemainingCallsCounter = 0;

                                System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                            }

                            // Remove file if ids not found
                            if (totalIDs == 0) {

                                System.out.println("No ids fetched so removing " + "file " + targetedUser);

                                File fileToDelete = new File(outputPath + "/" + targetedUser);
                                fileToDelete.delete();
                            }
                            System.out.println();

                            // If error then switch to next user
                            continue OUTERMOST;
                        }

                        if (ids.getIDs().length > 0) {

                            idsLoopCounter++;
                            totalIDs += ids.getIDs().length;
                            System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                            JSONObject responseDetailsJson = new JSONObject();
                            JSONArray jsonArray = new JSONArray();
                            for (long id : ids.getIDs()) {
                                jsonArray.put(id);
                            }
                            Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                            writer.println(idsJSON);

                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // load auth user
                            tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                            twitter = tf.getInstance();

                            System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls;
                            RemainingCallsCounter = 0;

                            System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                        }

                        if (IDS_TO_FETCH_INT != -1) {
                            if (idsLoopCounter == IDS_TO_FETCH) {
                                break;
                            }
                        }

                    } while ((cursor = ids.getNextCursor()) != 0);

                    writer.close();
                    System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);

                    // Remove file if ids not found
                    if (totalIDs == 0) {

                        System.out.println("No ids fetched so removing " + "file " + targetedUser);

                        File fileToDelete = new File(outputPath + "/" + targetedUser);
                        fileToDelete.delete();
                    }
                    System.out.println();

                } // while get records from single file
            } // read my single file
            catch (IOException e) {
                System.err.println("Failed to read lines from " + myFile);
            }

            // to write output in a file
            System.out.flush();
        } // all my files share

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.err.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}

From source file:friendsandfollowers.FilesThreaderFriendsIDsParser.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, SQLException, JSONException,
        FileNotFoundException, UnsupportedEncodingException {

    // Check how many arguments were passed in
    if ((args == null) || (args.length < 5)) {
        System.err.println("5 Parameters are required  plus one optional " + "parameter to launch a Job.");
        System.err.println("First: String 'INPUT: DB or /input/path/'");
        System.err.println("Second: String 'OUTPUT: /output/path/'");
        System.err.println("Third: (int) Total Number Of Jobs");
        System.err.println("Fourth: (int) This Job Number");
        System.err.println("Fifth: (int) Number of seconds to pause");
        System.err.println("Sixth: (int) Number of ids to fetch" + "Provide number which increment by 5000 "
                + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids.");
        System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 1 3 75000");
        System.exit(-1);/*from   w ww . ja  v a  2s  . c om*/
    }

    // TODO documentation for command line
    AppOAuth AppOAuths = new AppOAuth();
    Misc helpers = new Misc();
    String endpoint = "/friends/ids";

    String inputPath = null;
    try {
        inputPath = StringEscapeUtils.escapeJava(args[0]);
    } catch (Exception e) {
        System.err.println("Argument " + args[0] + " must be an String.");
        System.exit(-1);
    }

    String outputPath = null;
    try {
        outputPath = StringEscapeUtils.escapeJava(args[1]);
    } catch (Exception e) {
        System.err.println("Argument " + args[1] + " must be an String.");
        System.exit(-1);
    }

    int TOTAL_JOBS = 0;
    try {
        TOTAL_JOBS = Integer.parseInt(args[2]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[2] + " must be an integer.");
        System.exit(1);
    }

    int JOB_NO = 0;
    try {
        JOB_NO = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
        System.err.println("Argument " + args[3] + " must be an integer.");
        System.exit(1);
    }

    int secondsToPause = 0;
    try {
        secondsToPause = Integer.parseInt(args[4]);
    } catch (NumberFormatException e) {
        System.err.println("Argument" + args[4] + " must be an integer.");
        System.exit(-1);
    }

    int IDS_TO_FETCH_INT = -1;
    if (args.length == 6) {
        try {
            IDS_TO_FETCH_INT = Integer.parseInt(args[5]);
        } catch (NumberFormatException e) {
            System.err.println("Argument" + args[5] + " must be an integer.");
            System.exit(-1);
        }
    }

    int IDS_TO_FETCH = 0;
    if (IDS_TO_FETCH_INT > 5000) {

        float IDS_TO_FETCH_F = (float) IDS_TO_FETCH_INT / 5000;
        IDS_TO_FETCH = (int) Math.ceil(IDS_TO_FETCH_F);
    } else if ((IDS_TO_FETCH_INT <= 5000) && (IDS_TO_FETCH_INT > 0)) {
        IDS_TO_FETCH = 1;
    }

    secondsToPause = (TOTAL_JOBS * secondsToPause) - (JOB_NO * secondsToPause);
    System.out.println("secondsToPause: " + secondsToPause);
    helpers.pause(secondsToPause);

    try {

        int TotalWorkLoad = 0;
        ArrayList<String> allFiles = null;
        try {
            final File folder = new File(inputPath);
            allFiles = helpers.listFilesForSingleFolder(folder);
            TotalWorkLoad = allFiles.size();
        } catch (Exception e) {

            System.err.println("Input folder is not exists: " + e.getMessage());
            System.exit(-1);
        }

        System.out.println("Total Workload is: " + TotalWorkLoad);

        if (TotalWorkLoad < 1) {
            System.err.println("No screen names file exists in: " + inputPath);
            System.exit(-1);
        }

        if (TOTAL_JOBS > TotalWorkLoad) {
            System.err.println("Number of jobs are more than total work"
                    + " load. Please reduce 'Number of jobs' to launch.");
            System.exit(-1);
        }

        float TotalWorkLoadf = TotalWorkLoad;
        float TOTAL_JOBSf = TOTAL_JOBS;
        float res = (TotalWorkLoadf / TOTAL_JOBSf);

        int chunkSize = (int) Math.ceil(res);
        int offSet = JOB_NO * chunkSize;
        int chunkSizeToGet = (JOB_NO + 1) * chunkSize;

        System.out.println("My Share is " + chunkSize);
        System.out.println();

        // Load OAuh User
        TwitterFactory tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
        Twitter twitter = tf.getInstance();

        int RemainingCalls = AppOAuths.RemainingCalls;
        int RemainingCallsCounter = 0;
        System.out.println("First Time OAuth Remianing Calls: " + RemainingCalls);

        String Screen_name = AppOAuths.screen_name;
        System.out.println("First Time Loaded OAuth Screen_name: " + Screen_name);
        System.out.println();

        IDs ids;
        System.out.println("Going to get friends ids.");

        // to write output in a file
        System.out.flush();

        if (JOB_NO + 1 == TOTAL_JOBS) {
            chunkSizeToGet = TotalWorkLoad;
        }

        List<String> myFilesShare = allFiles.subList(offSet, chunkSizeToGet);

        for (String myFile : myFilesShare) {
            System.out.println("Going to parse file: " + myFile);

            try (BufferedReader br = new BufferedReader(new FileReader(inputPath + "/" + myFile))) {
                String line;
                OUTERMOST: while ((line = br.readLine()) != null) {
                    // process the line.

                    System.out.println("Going to get friends ids of Screen-name / user_id: " + line);
                    System.out.println();

                    String targetedUser = line.trim(); // tmp
                    long cursor = -1;
                    int idsLoopCounter = 0;
                    int totalIDs = 0;

                    PrintWriter writer = new PrintWriter(outputPath + "/" + targetedUser, "UTF-8");

                    // call different functions for screen_name and id_str
                    Boolean chckedNumaric = helpers.isNumeric(targetedUser);

                    do {
                        ids = null;
                        try {

                            if (chckedNumaric) {

                                long LongValueTargetedUser = Long.valueOf(targetedUser).longValue();

                                ids = twitter.getFriendsIDs(LongValueTargetedUser, cursor);
                            } else {
                                ids = twitter.getFriendsIDs(targetedUser, cursor);
                            }

                        } catch (TwitterException te) {

                            // do not throw if user has protected tweets, or
                            // if they deleted their account
                            if (te.getStatusCode() == HttpResponseCode.UNAUTHORIZED
                                    || te.getStatusCode() == HttpResponseCode.NOT_FOUND) {

                                System.out.println(targetedUser + " is protected or account is deleted");
                            } else {
                                System.out.println("Friends Get Exception: " + te.getMessage());
                            }

                            // If rate limit reached then switch Auth user
                            RemainingCallsCounter++;
                            if (RemainingCallsCounter >= RemainingCalls) {

                                // load auth user
                                tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                                twitter = tf.getInstance();

                                System.out.println(
                                        "New Loaded OAuth User " + " Screen_name: " + AppOAuths.screen_name);

                                RemainingCalls = AppOAuths.RemainingCalls;
                                RemainingCallsCounter = 0;

                                System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                            }

                            // Remove file if ids not found
                            if (totalIDs == 0) {

                                System.out.println("No ids fetched so removing " + "file " + targetedUser);

                                File fileToDelete = new File(outputPath + "/" + targetedUser);
                                fileToDelete.delete();
                            }
                            System.out.println();

                            // If error then switch to next user
                            continue OUTERMOST;
                        }

                        if (ids.getIDs().length > 0) {

                            idsLoopCounter++;
                            totalIDs += ids.getIDs().length;
                            System.out.println(idsLoopCounter + ": IDS length: " + ids.getIDs().length);

                            JSONObject responseDetailsJson = new JSONObject();
                            JSONArray jsonArray = new JSONArray();
                            for (long id : ids.getIDs()) {
                                jsonArray.put(id);
                            }
                            Object idsJSON = responseDetailsJson.put("ids", jsonArray);

                            writer.println(idsJSON);

                        }

                        // If rate limit reached then switch Auth user
                        RemainingCallsCounter++;
                        if (RemainingCallsCounter >= RemainingCalls) {

                            // load auth user
                            tf = AppOAuths.loadOAuthUser(endpoint, TOTAL_JOBS, JOB_NO);
                            twitter = tf.getInstance();

                            System.out.println("New Loaded OAuth User Screen_name: " + AppOAuths.screen_name);

                            RemainingCalls = AppOAuths.RemainingCalls;
                            RemainingCallsCounter = 0;

                            System.out.println("New OAuth Remianing Calls: " + RemainingCalls);
                        }

                        if (IDS_TO_FETCH_INT != -1) {
                            if (idsLoopCounter == IDS_TO_FETCH) {
                                break;
                            }
                        }

                    } while ((cursor = ids.getNextCursor()) != 0);

                    writer.close();
                    System.out.println("Total ids dumped of " + targetedUser + " are: " + totalIDs);

                    // Remove file if ids not found
                    if (totalIDs == 0) {

                        System.out.println("No ids fetched so removing " + "file " + targetedUser);

                        File fileToDelete = new File(outputPath + "/" + targetedUser);
                        fileToDelete.delete();
                    }
                    System.out.println();

                } // while get records from single file
            } // read my single file
            catch (IOException e) {
                System.err.println("Failed to read lines from " + myFile);
            }

            // to write output in a file
            System.out.flush();
        } // all my files share

    } catch (TwitterException te) {
        // te.printStackTrace();
        System.err.println("Failed to get friends' ids: " + te.getMessage());
        System.exit(-1);
    }
    System.out.println("!!!! DONE !!!!");

    // Close System.out for this thread which will
    // flush and close this thread.
    System.out.close();
}

From source file:net.lacolaco.smileessence.twitter.task.BlockIDsTask.java

License:Open Source License

@Override
protected Long[] doInBackground(Void... params) {
    try {//from w  w w . ja  v  a2 s  .c o  m
        List<Long> idList = new ArrayList<>();
        long cursor = -1;
        do {
            IDs blocksIDs = twitter.getBlocksIDs(cursor);
            cursor = blocksIDs.getNextCursor();
            for (long id : blocksIDs.getIDs()) {
                idList.add(id);
            }
        } while (cursor != 0);

        return idList.toArray(new Long[idList.size()]);
    } catch (TwitterException e) {
        Logger.error(e);
        return new Long[0];
    }
}

From source file:net.lacolaco.smileessence.twitter.task.MutesIDsTask.java

License:Open Source License

@Override
protected Long[] doInBackground(Void... params) {
    try {/*from   w  w  w.j  a va 2 s .co m*/
        List<Long> idList = new ArrayList<>();
        long cursor = -1;
        do {
            IDs mutesIDs = twitter.getMutesIDs(cursor);
            cursor = mutesIDs.getNextCursor();
            for (long id : mutesIDs.getIDs()) {
                idList.add(id);
            }
        } while (cursor != 0);

        return idList.toArray(new Long[idList.size()]);
    } catch (TwitterException e) {
        Logger.error(e);
        return new Long[0];
    }
}

From source file:org.apache.streams.twitter.provider.TwitterFollowingProviderTask.java

License:Apache License

private void collectIds(Long id) {
    int keepTrying = 0;

    long curser = -1;

    twitter4j.User user;//  w  w  w  . ja  v a2 s .  co m
    String userJson;
    try {
        user = client.users().showUser(id);
        userJson = TwitterObjectFactory.getRawJSON(user);
    } catch (TwitterException ex) {
        LOGGER.error("Failure looking up " + id);
        return;
    }

    do {
        try {
            twitter4j.IDs ids = null;
            if (provider.getConfig().getEndpoint().equals("followers")) {
                ids = client.friendsFollowers().getFollowersIDs(id, curser,
                        provider.getConfig().getMaxItems().intValue());
            } else if (provider.getConfig().getEndpoint().equals("friends")) {
                ids = client.friendsFollowers().getFriendsIDs(id, curser,
                        provider.getConfig().getMaxItems().intValue());
            }

            Objects.requireNonNull(ids);
            Preconditions.checkArgument(ids.getIDs().length > 0);

            for (long otherId : ids.getIDs()) {

                try {
                    Follow follow = null;
                    if (provider.getConfig().getEndpoint().equals("followers")) {
                        follow = new Follow().withFollowee(new User().withId(id))
                                .withFollower(new User().withId(otherId));
                    } else if (provider.getConfig().getEndpoint().equals("friends")) {
                        follow = new Follow().withFollowee(new User().withId(otherId))
                                .withFollower(new User().withId(id));
                    }

                    Objects.requireNonNull(follow);

                    if (item_count < provider.getConfig().getMaxItems()) {
                        ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue);
                        item_count++;
                    }
                } catch (Exception ex) {
                    LOGGER.warn("Exception: {}", ex);
                }
            }
            if (!ids.hasNext()) {
                break;
            }
            if (ids.getNextCursor() == 0) {
                break;
            }
            curser = ids.getNextCursor();
            page_count++;
        } catch (TwitterException twitterException) {
            keepTrying += TwitterErrorHandler.handleTwitterError(client, id, twitterException);
        } catch (Exception ex) {
            keepTrying += TwitterErrorHandler.handleTwitterError(client, null, ex);
        }
    } while (shouldContinuePulling() && curser != 0 && keepTrying < provider.getConfig().getRetryMax());
}

From source file:org.loklak.harvester.TwitterAPI.java

License:Open Source License

public static JSONObject getNetworkerNames(final String screen_name, final int max_count,
        final Networker networkRelation) throws IOException, TwitterException {
    if (max_count == 0)
        return new JSONObject();
    boolean complete = true;
    Set<Number> networkingIDs = new LinkedHashSet<>();
    Set<Number> unnetworkingIDs = new LinkedHashSet<>();
    JsonFactory mapcapsule = (networkRelation == Networker.FOLLOWERS ? DAO.followers_dump : DAO.following_dump)
            .get("screen_name", screen_name);
    if (mapcapsule == null) {
        JsonDataset ds = networkRelation == Networker.FOLLOWERS ? DAO.followers_dump : DAO.following_dump;
        mapcapsule = ds.get("screen_name", screen_name);
    }/* ww w.  j a  v  a2  s.  c o m*/

    if (mapcapsule != null) {
        JSONObject json = mapcapsule.getJSON();

        // check date and completeness
        complete = json.has("complete") ? (Boolean) json.get("complete") : Boolean.FALSE;
        String retrieval_date_string = json.has("retrieval_date") ? (String) json.get("retrieval_date") : null;
        DateTime retrieval_date = retrieval_date_string == null ? null
                : AbstractObjectEntry.utcFormatter.parseDateTime(retrieval_date_string);
        if (complete && System.currentTimeMillis() - retrieval_date.getMillis() < DateParser.DAY_MILLIS)
            return json;

        // load networking ids for incomplete retrievals (untested)
        String nr = networkRelation == Networker.FOLLOWERS ? "follower" : "following";
        if (json.has(nr)) {
            JSONArray fro = json.getJSONArray(nr);
            for (Object f : fro)
                networkingIDs.add((Number) f);
        }
    }
    TwitterFactory tf = getUserTwitterFactory(screen_name);
    if (tf == null)
        tf = getAppTwitterFactory();
    if (tf == null)
        return new JSONObject();
    Twitter twitter = tf.getInstance();
    long cursor = -1;
    collect: while (cursor != 0) {
        try {
            IDs ids = networkRelation == Networker.FOLLOWERS ? twitter.getFollowersIDs(screen_name, cursor)
                    : twitter.getFriendsIDs(screen_name, cursor);
            RateLimitStatus rateStatus = ids.getRateLimitStatus();
            if (networkRelation == Networker.FOLLOWERS) {
                getFollowerIdRemaining = rateStatus.getRemaining();
                getFollowerIdResetTime = System.currentTimeMillis() + rateStatus.getSecondsUntilReset() * 1000;
            } else {
                getFollowingIdRemaining = rateStatus.getRemaining();
                getFollowingIdResetTime = System.currentTimeMillis() + rateStatus.getSecondsUntilReset() * 1000;
            }
            //System.out.println("got: " + ids.getIDs().length + " ids");
            //System.out.println("Rate Status: " + rateStatus.toString() + "; time=" + System.currentTimeMillis());
            boolean dd = false;
            for (long id : ids.getIDs()) {
                if (networkingIDs.contains(id))
                    dd = true; // don't break loop here
                networkingIDs.add(id);
            }
            if (dd)
                break collect; // this is complete!
            if (rateStatus.getRemaining() == 0) {
                complete = false;
                break collect;
            }
            if (networkingIDs.size() >= Math.min(10000, max_count >= 0 ? max_count : 10000)) {
                complete = false;
                break collect;
            }
            cursor = ids.getNextCursor();
        } catch (TwitterException e) {
            complete = false;
            break collect;
        }
    }
    // create result
    JSONObject json = new JSONObject(true);
    json.put("screen_name", screen_name);
    json.put("retrieval_date", AbstractObjectEntry.utcFormatter.print(System.currentTimeMillis()));
    json.put("complete", complete);
    Map<String, Number> networking = getScreenName(networkingIDs, max_count, true);
    Map<String, Number> unnetworking = getScreenName(unnetworkingIDs, max_count, true);
    if (networkRelation == Networker.FOLLOWERS) {
        json.put("followers_count", networking.size());
        json.put("unfollowers_count", unnetworking.size());
        json.put("followers_names", networking);
        json.put("unfollowers_names", unnetworking);
        if (complete)
            DAO.followers_dump.putUnique(json); // currently we write only complete data sets. In the future the update of datasets shall be supported
    } else {
        json.put("following_count", networking.size());
        json.put("unfollowing_count", unnetworking.size());
        json.put("following_names", networking);
        json.put("unfollowing_names", unnetworking);
        if (complete)
            DAO.following_dump.putUnique(json);
    }
    return json;
}

From source file:org.sociotech.communitymashup.source.twitter.TwitterSourceService.java

License:Open Source License

private void addFollowing(User twitterUser, boolean createNewUsers) {

    if (twitterUser == null) {
        return;//from w w  w. j  a  v  a  2s.  com
    }

    IDs following;
    long cursor = -1;

    // look up person for twitter user and connect
    Person connectToPerson = getPersonForTwitterUser(twitterUser);

    do {
        try {
            following = twitterAPI.getFriendsIDs(twitterUser.getId(), cursor);
            log("Retrieving following with cursor: " + cursor, LogService.LOG_INFO);
        } catch (TwitterException e) {
            log("Could not get more following Users from twitter. (Cursor: " + cursor + "): " + e.getMessage(),
                    LogService.LOG_DEBUG);
            break;
        }

        if (following == null) {
            return;
        }

        if (connectToPerson != null) {
            connectPersonsForTwitterUserIds(following.getIDs(), connectToPerson, true, createNewUsers);
        }

        // next cursor
        cursor = following.getNextCursor();
    } while (following.hasNext());
}

From source file:org.sociotech.communitymashup.source.twitter.TwitterSourceService.java

License:Open Source License

private void addFollower(User twitterUser, boolean createNewUsers) {

    if (twitterUser == null) {
        return;//from w  w  w.java 2s  .  c  o  m
    }

    int followersMax = 1000;

    int followersCount = twitterUser.getFollowersCount();
    if (followersCount > followersMax) {
        // ignore users with to much followers
        log("Ignoring the " + followersCount + " Followers of " + twitterUser.getName());
        return;
    }

    IDs followers;
    long cursor = -1;

    do {
        try {
            followers = twitterAPI.getFollowersIDs(twitterUser.getId(), cursor);
            log("Retrieving followers with cursor: " + cursor, LogService.LOG_INFO);
        } catch (TwitterException e) {
            log("Could not get more followers from twitter. (Cursor: " + cursor + "): " + e.getMessage(),
                    LogService.LOG_DEBUG);
            break;
        }

        if (followers == null) {
            return;
        }

        // look up person for twitter user and connect
        Person connectToPerson = getPersonForTwitterUser(twitterUser);
        if (connectToPerson != null) {
            connectPersonsForTwitterUserIds(followers.getIDs(), connectToPerson, false, createNewUsers);
        }
        // next cursor
        cursor = followers.getNextCursor();
    } while (followers.hasNext());
}

From source file:org.yukung.following2ldr.command.impl.FindFeedUrlCommand.java

License:Apache License

@Override
public void run() throws Throwable {
    // Twitter??/*from  ww w.  j  a v  a2s  .c  o  m*/
    long start = System.currentTimeMillis();
    String consumerKey = config.getProperty(Constants.CONSUMER_KEY);
    String consumerSecret = config.getProperty(Constants.CONSUMER_SECRET);
    String userName = params.get(0);
    Twitter twitter = new TwitterFactory().getInstance();
    twitter.setOAuthConsumer(consumerKey, consumerSecret);
    RequestToken requestToken = twitter.getOAuthRequestToken();
    String authorizationURL = requestToken.getAuthorizationURL();
    System.out.println(":" + authorizationURL);
    BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
    System.out.print("Please enter the PIN:");
    String pin = br.readLine();
    AccessToken accessToken = twitter.getOAuthAccessToken(requestToken, pin);
    twitter.setOAuthAccessToken(accessToken);
    long cursor = -1L;
    IDs friendIDs;
    List<Long> iDsList = new ArrayList<Long>(5000);
    do {
        friendIDs = twitter.getFriendsIDs(userName, cursor);
        long[] iDs = friendIDs.getIDs();
        for (long iD : iDs) {
            iDsList.add(iD);
        }
        cursor = friendIDs.getNextCursor();
    } while (friendIDs.hasNext());
    List<long[]> list = new ArrayList<long[]>();
    int offset = 0;
    long[] tmp = new long[100];
    for (Long id : iDsList) {
        if (offset < 100) {
            tmp[offset] = id;
            offset++;
        } else {
            list.add(tmp);
            offset = 0;
            tmp = new long[100];
        }
    }
    list.add(tmp);
    List<URL> urlList = new ArrayList<URL>();
    for (long[] array : list) {
        ResponseList<User> lookupUsers = twitter.lookupUsers(array);
        for (User user : lookupUsers) {
            log.info("URL:" + user.getURL());
            urlList.add(user.getURL());
        }
    }
    String path = "C:\\Users\\ikeda_yusuke\\Documents\\sandbox\\java\\data\\" + userName + ".txt";
    FileWriter writer = new FileWriter(path);
    BufferedWriter out = new BufferedWriter(writer);
    //      PrintWriter pw = new PrintWriter(writer);
    for (URL url : urlList) {
        if (url != null) {
            out.write(url.toString() + "\n");
        }
    }
    out.flush();
    out.close();
    long end = System.currentTimeMillis();
    log.info("?:" + (end - start) + " ms");

    // ??IDor??
    // Twitter API??????ID?
    // ?????ID?URL100???
    // ??URL?

}

From source file:peoplesearch.FindFriendsAndFollowers.java

public void GetFollowersIDs() {
    try {//  w  w  w  .  j  av  a 2s.  co m
        // I need to pass the Person name and the TwitterID.
        //String targetname="Philip Bergkvist";
        Twitter twitter = new TwitterFactory().getInstance();
        long cursor = -1;
        IDs ids;

        ResponseList<User> users1 = null;
        ResponseList<User> users2 = null;
        System.out.println("Listing followers's ids.");

        GraphManager mgr = EmbeddedGraphManager.getInstance();

        mgr.init(new File("/usr/local/Cellar/neo4j/2.1.7/libexec/data/forlang1.db"));
        //mgr.addTwitterAccount(new TwitterAccountImpl(new Date(), "I am Studying", 13, 82, true,"Aalborg", "Philiptwoshoes", 2730631792L));
        List<TwitterAccount> twitteraccountslist;
        twitteraccountslist = null;
        twitteraccountslist = mgr.listTwitterAccounts();
        System.out.println("the number of twitter account in the neo4J DB is" + twitteraccountslist.size());
        for (TwitterAccount Twit : twitteraccountslist) {

            do {
                TwitterLimitWait tlw = new TwitterLimitWait();
                tlw.CheckLimit();

                if (0 < twitteraccountslist.size()) {
                    ids = twitter.getFollowersIDs(Twit.getScreenName(), cursor); //.getFollowersIDs(pep[0], cursor);
                    //ids = twitter.getFollowersIDs("Philiptwoshoes", cursor); //.getFollowersIDs(pep[0], cursor);
                    tlw.CheckLimit();
                    users1 = twitter.getFollowersList(Twit.getScreenName(), cursor);
                    tlw.CheckLimit();
                    users2 = twitter.getFriendsList(Twit.getScreenName(), cursor);

                } else {
                    tlw.CheckLimit();
                    ids = twitter.getFollowersIDs(cursor);

                }

                for (User user : users1) {
                    tlw.CheckLimit();
                    System.out.println("the follower called " + user.getName() + " with twitter handler "
                            + user.getScreenName());
                    String username = user.getName();
                    //mgr.addPerson(new PersonImpl(username));
                    Date Creation = user.getCreatedAt();
                    tlw.CheckLimit();
                    String descript = user.getDescription();
                    boolean empty1 = user.getDescription().isEmpty();
                    if (empty1 == true) {
                        descript = " ";
                    }
                    tlw.CheckLimit();
                    int followers = user.getFollowersCount();
                    tlw.CheckLimit();
                    int following = user.getFriendsCount();
                    boolean geo = user.isGeoEnabled();
                    String loc = user.getLocation();
                    boolean empty2 = user.getLocation().isEmpty();
                    if (empty2 == true) {
                        loc = " ";
                    }
                    String screenname = user.getScreenName();
                    boolean empty3 = user.getScreenName().isEmpty();
                    if (empty3 == true) {
                        screenname = " ";
                    }
                    tlw.CheckLimit();
                    long twitID = user.getId();

                    mgr.linkPersonToTwitterAccount(new PersonImpl(username), new TwitterAccountImpl(Creation,
                            descript, followers, following, geo, loc, screenname, twitID));
                    mgr.linkTwitterAccounts(new TwitterAccountImpl(Creation, descript, followers, following,
                            geo, loc, screenname, twitID), Twit);
                }

                System.out.println("The total number of followers is: " + users1.size());
                // the same procedure for the Following
                for (User user : users2) {
                    tlw.CheckLimit();
                    System.out.println("the following called " + user.getName() + " with twitter handler "
                            + user.getScreenName());
                    String username1 = user.getName();
                    //mgr.addPerson(new PersonImpl(username1));
                    Date Creation = user.getCreatedAt();
                    String descript = user.getDescription();
                    boolean empty1 = user.getDescription().isEmpty();
                    if (empty1 == true) {
                        descript = " ";
                    }
                    int followers = user.getFollowersCount();
                    int following = user.getFriendsCount();
                    boolean geo = user.isGeoEnabled();
                    String loc = user.getLocation();
                    boolean empty2 = user.getLocation().isEmpty();
                    if (empty2 == true) {
                        loc = " ";
                    }
                    String screenname = user.getScreenName();
                    tlw.CheckLimit();
                    boolean empty3 = user.getScreenName().isEmpty();
                    if (empty3 == true) {
                        screenname = " ";
                    }
                    tlw.CheckLimit();
                    long twitID = user.getId();
                    //mgr.addTwitterAccount(new TwitterAccountImpl(Creation,descript,followers,following,geo,loc,screenname,twitID));
                    mgr.linkPersonToTwitterAccount(new PersonImpl(username1), new TwitterAccountImpl(Creation,
                            descript, followers, following, geo, loc, screenname, twitID));
                    mgr.linkTwitterAccounts(Twit, new TwitterAccountImpl(Creation, descript, followers,
                            following, geo, loc, screenname, twitID));
                }
                System.out.println("The total number of friend is: " + users2.size());

                //}
            } while ((cursor = ids.getNextCursor()) != 0);

        }
        mgr.destroy(); // I have to check that the second iteration works fine, because i could not test that.
        System.exit(0);
    } catch (TwitterException te) {
        te.printStackTrace();
        System.out.println("Failed to get followers' ids: " + te.getMessage());
        System.exit(-1);
    }
}