Java tutorial
/* * Copyright 2012 Nodeable Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.streamreduce.util; import com.google.common.collect.ImmutableSet; import com.streamreduce.ProviderIdConstants; import com.streamreduce.connections.AuthType; import com.streamreduce.core.model.Connection; import com.streamreduce.core.model.InventoryItem; import com.streamreduce.core.service.exception.InvalidCredentialsException; import net.sf.json.JSONArray; import net.sf.json.JSONException; import net.sf.json.JSONNull; import net.sf.json.JSONObject; import org.apache.http.Header; import org.scribe.oauth.OAuthService; import org.springframework.util.Assert; import javax.ws.rs.HttpMethod; import javax.ws.rs.core.MediaType; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; /** * GitHubClient provides necessary methods for interacting with GitHub. */ public class GitHubClient extends AbstractProjectHostingClient { public static final Set<String> SUPPORTED_EVENT_TYPES = ImmutableSet.of("CommitCommentEvent", // This tests a commit comment "CreateEvent", // This tests create events (branch, repository and tag) "DeleteEvent", // This tests delete events (branch and tag) "DownloadEvent", // This tests file upload events "ForkEvent", // This tests for events "ForkApplyEvent", // This tests the fork apply (apply patch from fork) event "GollumEvent", // This tests the wiki page changes "IssueCommentEvent", // This tests issue comment events "IssuesEvent", // This tests issue events "MemberEvent", // This tests membership events "PublicEvent", // This tests when a private repository goes public "PullRequestEvent", // This tests pull request events "PushEvent", // This tests push events "WatchEvent" // This tests watch events ); public static final String GITHUB_API_BASE = "https://api.github.com/"; final OAuthService oAuthService; /** * Constructs a client for GitHub using the credentials in the connection provided. * * @param connection the connection to use for interacting with GitHub */ public GitHubClient(Connection connection, OAuthService oAuthService) { super(connection); this.oAuthService = oAuthService; Assert.isTrue(connection.getProviderId().equals(ProviderIdConstants.GITHUB_PROVIDER_ID)); debugLog(LOGGER, "Client created for " + getConnectionCredentials().getIdentity()); } /** * Returns a JSON array representing the all GitHub repositories connection user has access to. * * Note: This includes all repositories the user owns (public/private), watches and can see as part * of an organization. * * @return the JSON representation of the available GitHub projects * * @throws InvalidCredentialsException if the connection's credentials are invalid * @throws IOException if anything else goes wrong */ public List<JSONObject> getRepositories() throws InvalidCredentialsException, IOException { // To gather the full list of repositories we're interested in, we have to make multiple calls to the GitHub API: // // * user/repos: Repositories the user owns // * user/watched: Repositories the user watches // * orgs/<org_name>/repos: Repositories the user can see in the organization the user is a member of (if any) debugLog(LOGGER, "Getting repositories visible to " + getConnectionCredentials().getIdentity()); List<JSONObject> repositories = new ArrayList<>(); Set<String> reposUrls = new HashSet<>(); reposUrls.add(GITHUB_API_BASE + "user/repos"); reposUrls.add(GITHUB_API_BASE + "user/watched"); for (String repoUrl : reposUrls) { List<JSONObject> reposFromUrl = makeRequest(repoUrl); for (JSONObject repo : reposFromUrl) { repositories.add(repo); } } debugLog(LOGGER, " Repositories found: " + repositories.size()); return repositories; } /** * Returns a list of organizations the user is a member of. * * @return list of JSONObjects representing the organizations the user is a member of * * @throws InvalidCredentialsException if the connection associated with this client has invalid credentials * @throws IOException if anything goes wrong making the actual request */ @SuppressWarnings("unused") //Default Oauth scope causes 404 when reading from orgs public List<JSONObject> getOrganizations() throws InvalidCredentialsException, IOException { debugLog(LOGGER, "Getting organzation " + getConnectionCredentials().getIdentity() + " is a member of"); List<JSONObject> organizations = makeRequest(GITHUB_API_BASE + "user/orgs"); debugLog(LOGGER, " Organizations: " + organizations.size()); return organizations; } /** * {@inheritDoc} */ @Override public void validateConnection() throws InvalidCredentialsException, IOException { debugLog(LOGGER, "Validating connection"); getUser(); } public JSONObject getUser() throws InvalidCredentialsException, IOException { debugLog(LOGGER, "Validating connection"); String userUrl = GITHUB_API_BASE + "user"; try { List<JSONObject> response = makeRequest(userUrl); return (response.size() == 1 ? response.get(0) : null); } catch (InvalidCredentialsException e) { throw new InvalidCredentialsException("The GitHub connection credentials for " + getConnectionCredentials().getIdentity() + " are invalid."); } } /** * Returns a JSONObject representing the comparison of two commits on a repository. * * @param repoName the repository whose revisions we're comparing * @param before the first revision * @param after the second revision * * @return a JSONObject describing the commits comparison * * @throws InvalidCredentialsException if the connection credentials are invalid * @throws IOException if any other communication error happens */ public JSONObject compareCommits(String repoName, String before, String after) throws InvalidCredentialsException, IOException { debugLog(LOGGER, "Comparing commits between " + before + " and " + after + " on " + repoName); String commitsUrl = GITHUB_API_BASE + "repos/" + repoName + "/compare/" + before + "..." + after; List<JSONObject> rawResponse = makeRequest(commitsUrl); // Should never happen where we're returning null but just in case return (rawResponse.size() == 1 ? rawResponse.get(0) : null); } /** * Returns a JSONObject representing a GitHub project/repository. * * @param repoName the repository name * * @return a JSONObject representing the GitHub project/repository * * @throws InvalidCredentialsException if the connection credentials are invalid * @throws IOException if any other communication error happens */ public JSONObject getRepositoryDetails(String repoName) throws InvalidCredentialsException, IOException { debugLog(LOGGER, "Getting repository details for " + repoName); String projectUrl = GITHUB_API_BASE + "repos/" + repoName; List<JSONObject> rawResponse = makeRequest(projectUrl); // Should never happen where we're returning null but just in case return (rawResponse.size() == 1 ? rawResponse.get(0) : null); } /** * Retrieves all activity for the projects passed in with no limit on the maximum number of activity * entries returned. * * @see #getActivity(java.util.Set, int) */ public List<JSONObject> getActivity(Set<String> projectKeys) throws InvalidCredentialsException, IOException { return getActivity(projectKeys, Integer.MAX_VALUE); } /** * Retrieves the activity for the given connection based on the last poll date stored in the connection. * * Note: This list is already sorted in the proper order, contains no duplicates and contains only entries that * are pertinent: * * * Entries will correspond with a project in the projectKeys set unless that set is empty/null and then * entries can be for any project * * Entries will after the last activity date in the connection * * @param projectKeys the project keys we're interested in or null for all * @param maxActivities the maximum number of results to return * * @return list of JSONObjects representing activity entries * * @throws InvalidCredentialsException if the connection associated with this client has invalid credentials * @throws IOException if anything goes wrong making the actual request */ public List<JSONObject> getActivity(Set<String> projectKeys, int maxActivities) throws InvalidCredentialsException, IOException { // The way we gather activity for a GitHub connection is by making a few events feeds calls, merging them // together and then returning the results. The end result should be a list of pertinent events that have no // duplicates and includes all necessary events after the last poll period. // // The GitHub API request logic looks like this: // // * /users/<user_id>/received_events: This is the list of events that the user has "received" by watching // repositories/users. // * /users/<user_id>/events: This is a list of events that the user itself has created. // * /users/<user_id>/events/orgs/<org_id>: This is a list of events that have been performed within the // the organization. (This will also require a call prior to this // to get the user's organizations, if any.) debugLog(LOGGER, "Getting activity"); // Establish some defaults for fields that can be null projectKeys = (projectKeys != null ? projectKeys : new HashSet<String>()); maxActivities = (maxActivities >= 1 ? maxActivities : 100); List<JSONObject> allActivityItems = new ArrayList<>(); Set<Integer> processedActivityHashes = new HashSet<>(); Date lastActivity = getLastActivityPollDate(); Set<String> eventsUrls = new HashSet<>(); String username = getConnectionCredentials().getIdentity(); // Generate the list of events URLs to process eventsUrls.add(GITHUB_API_BASE + "users/" + username + "/received_events"); // User's received events eventsUrls.add(GITHUB_API_BASE + "users/" + username + "/events"); // User's events // // To generate the list of organization URLs to process, we need to walk through the user's organizations list // List<JSONObject> organizations = getOrganizations(); // // for (JSONObject organization : organizations) { // eventsUrls.add(GITHUB_API_BASE + "users/" + username + "/events/orgs/" + organization.getString("login")); // } for (String eventUrl : eventsUrls) { List<JSONObject> rawActivity = makeRequest(eventUrl, maxActivities, false); for (JSONObject activity : rawActivity) { String eventType = activity.getString("type"); String repoName = activity.getJSONObject("repo").getString("name"); Date activityDate = getCreatedDate(activity); // If we do not support the event type or its for a repository we don't monitor, move on if (!SUPPORTED_EVENT_TYPES.contains(eventType) || !projectKeys.contains(repoName)) { continue; } if (activityDate.before(lastActivity)) { break; } int activityHash = activity.hashCode(); if (!processedActivityHashes.contains(activityHash) && allActivityItems.size() < maxActivities) { allActivityItems.add(activity); processedActivityHashes.add(activityHash); } } } // Sort the activities Collections.sort(allActivityItems, new Comparator<JSONObject>() { /** * {@inheritDoc} */ @Override public int compare(JSONObject jo0, JSONObject jo1) { Date jod0 = getCreatedDate(jo0); Date jod1 = getCreatedDate(jo1); return jod0.compareTo(jod1); } }); // Return only the maximum number of results if the list of activities is greater than the maximum requested if (allActivityItems.size() > maxActivities) { allActivityItems = allActivityItems.subList(0, maxActivities); } debugLog(LOGGER, " Activities found: " + allActivityItems.size()); return allActivityItems; } /** * Returns a map with the following keys in it or null if the entry is unhandleable: * * * title: This is the title of the activity * * content: This is the content of the activity (Summarizing changes when necessary) * * hashtags: A set of hashtags * * The event processing only handles repository events at this time. That being said, these events are known events * that are not handled at this time: * * * FollowEvent: This is a user-specific event * * GistEvent: Gists are outside of the repository/project boundary * * TeamAddEvent: This is an organization-specific event * * @param inventoryItem the inventory item the activity entry corresponds to * @param entry the JSONObject to parse * * @return the map described above * * @throws InvalidCredentialsException if the connection credentials are invalid (in the event they are used) * @throws IOException if there is an issue making requests (if necessary) */ public Map<String, Object> getPartsForActivity(InventoryItem inventoryItem, JSONObject entry) throws InvalidCredentialsException, IOException { Assert.isTrue(getConnectionId().equals(inventoryItem.getConnection().getId())); // GitHub activity information gathering is pretty straight forward thanks to GitHub's events // API (http://developer.github.com/v3/events/). Basically, there are a finite set of GitHub // event types (http://developer.github.com/v3/events/types/) and knowing this we can deduce // the activity/event title, content and hashtags in a pretty simple fashion. For extensive // details, look at the large if/else statement below and each branch will outline the format // for the title, content and the generated hashtags for each event type. // // Note: We do generate the title and content of each activity to be just like what the stream // text would look like. So unlike Jira, where the title/content are given to us, we // do in fact generate the title/content. String activityType = entry.getString("type"); JSONObject payload = entry.getJSONObject("payload"); String repoName = entry.getJSONObject("repo").getString("name"); Map<String, Object> activityParts = new HashMap<>(); StringBuilder title = new StringBuilder(); StringBuilder content = new StringBuilder(); Set<String> hashtags = new HashSet<>(); // Bring in the inventory item hashtags for (String hashtag : inventoryItem.getHashtags()) { hashtags.add(hashtag); } // Always add the project hashtags.add("#" + repoName.toLowerCase()); // Generate the title based on common event properties. The titles will all have the same // common structure: [actor.login] [action] at [repo.name] title.append(entry.getJSONObject("actor").getString("login")).append(" "); // Generate the action based on the event type if (activityType.equals("CommitCommentEvent")) { // [actor.login] commented on [repo.name] title.append("commented on ").append(repoName); // Comment in [payload.comment.commit_id]: [payload.comment.body] JSONObject comment = payload.getJSONObject("comment"); content.append("Comment in ").append(comment.getString("commit_id").substring(0, 10)).append(": ") .append(comment.getString("body")); // Commit comment hashtags: // // #comment // #source // #changeset hashtags.add("#comment"); hashtags.add("#source"); hashtags.add("#changeset"); } else if (activityType.equals("CreateEvent")) { // Branch : [actor.login] created branch [payload.ref] at [repo.name] // Tag : [actor.login] created tag [payload.ref] at [repo.name] // Repository: [actor.login] created repository [repo.name (without owner)] String refType = payload.getString("ref_type"); String ref = payload.getString("ref"); title.append("created ").append(refType).append(" "); if (refType.equals("repository")) { title.append(repoName.split("/")[1]); } else { title.append(ref).append(" at ").append(repoName); } // Branch : New branch is at /[repo.name]/tree/[payload.ref] // Tag : New tag is at /[repo.name]/tree/[payload.ref] // Repository: [payload.description] if (refType.equals("branch") || refType.equals("tag")) { content.append("New ").append(refType).append(" is at /").append(repoName).append("/tree/") .append(ref); } else if (refType.equals("repository")) { content.append(payload.getString("description")); } // Create event hashtags: // // #source // #create // #[payload.ref_type] hashtags.add("#source"); hashtags.add("#create"); hashtags.add("#" + refType); } else if (activityType.equals("DeleteEvent")) { // Branch: [actor.login] deleted branch [payload.ref] at [repo.name] // Tag : [actor.login] deleted tag [payload.ref] at [repo.name] String refType = payload.getString("ref_type"); String ref = payload.getString("ref"); title.append("deleted ").append(refType).append(" ").append(ref).append(" at ").append(repoName); // Branch : Deleted branch was at /[repo.name]/tree/[payload.ref] // Tag : Deleted tag was at /[repo.name]/tree/[payload.ref] content.append("Deleted ").append(refType).append(" was at /").append(repoName).append("/tree/") .append(ref); // Delete event hashtags: // // #source // #delete // #[payload.ref_type] hashtags.add("#source"); hashtags.add("#delete"); hashtags.add("#" + refType); } else if (activityType.equals("DownloadEvent")) { // [actor.login] uploaded a file to [repo.name] title.append("uploaded a file to ").append(repoName); // "[download.name]" is at /[repo.name]/downloads // [download.description] JSONObject download = payload.getJSONObject("download"); content.append("\"").append(download.getString("name")).append("\" is at /").append(repoName) .append("/downloads").append("\n").append(download.getString("description")); // Download event hashtags: // // #download hashtags.add("#download"); } else if (activityType.equals("ForkEvent")) { // [actor.login] forked [repo.name] title.append("forked ").append(repoName); // Forked repository is at [actor.login/payload.forkee/name] JSONObject forkee = payload.getJSONObject("forkee"); content.append("Forked repository is at ").append(entry.getJSONObject("actor").getString("login")) .append("/").append(forkee.getString("name")); // Fork event hashtags: // // #repository // #fork // #create hashtags.add("#repository"); hashtags.add("#fork"); hashtags.add("#create"); } else if (activityType.equals("ForkApplyEvent")) { // [actor.login] applied fork commits to [repo.name] title.append("applied fork commits to ").append(repoName); // We have to retrieve the commits between the [payload.before] and [payload.after] and then render // each commit as follows: // [commit.sha] [commit.message] JSONObject commitsComparison = compareCommits(repoName, payload.getString("before"), payload.getString("after")); JSONArray commits = commitsComparison.getJSONArray("commits"); for (int i = 0; i < commits.size(); i++) { JSONObject commit = commits.getJSONObject(i); if (i != 0) { content.append("\n"); } content.append(commit.getString("sha").substring(0, 7)).append(" ") .append(commit.getJSONObject("commit").getString("message").split("\n")[0]); } // Fork apply event hashtags: // // #fork // #apply // #source // #changeset hashtags.add("#fork"); hashtags.add("#apply"); hashtags.add("#source"); hashtags.add("#changeset"); } else if (activityType.equals("GollumEvent")) { // Single change : [actor.login] [payload.pages[0].action] the [repo.name] wiki // Multiple changes: [actor.login] made multiple changes to the [repo.name] wiki JSONArray pages = payload.getJSONArray("pages"); if (pages.size() == 1) { JSONObject page = pages.getJSONObject(0); title.append(page.getString("action")).append(" the"); } else { title.append("made multiple changes to the"); } title.append(" ").append(repoName).append(" wiki"); // For each page change: // [page[i].action] [page[i].title] for (int i = 0; i < pages.size(); i++) { JSONObject page = pages.getJSONObject(i); String action = page.getString("action"); if (i != 0) { content.append("\n"); } content.append(action.substring(0, 1).toUpperCase()).append(action.substring(1)).append(" ") .append(page.getString("title")).append("."); } // Gollum event hashtags // // #wiki hashtags.add("#wiki"); } else if (activityType.equals("IssueCommentEvent")) { // [actor.login] commented on issue [payload.issue.number] on [repo.name] JSONObject issue = payload.getJSONObject("issue"); title.append("commented on ") .append(issue.getJSONObject("pull_request").get("diff_url") instanceof JSONNull ? "issue " : "pull request ") .append(issue.getInt("number")).append(" on ").append(repoName); // [payload.comment.body] content.append(payload.getJSONObject("comment").getString("body")); // Issue comment event hashtags // // #comment // #issue // #[issue.state] // #[labels] hashtags.add("#comment"); hashtags.add("#issue"); hashtags.add("#" + issue.getString("state")); JSONArray labels = issue.getJSONArray("labels"); for (int i = 0; i < labels.size(); i++) { hashtags.add("#" + labels.getJSONObject(i).getString("name")); } } else if (activityType.equals("IssuesEvent")) { // [actor.login] [payload.issue.action] issue [payload.issue.number] on [repo.name] JSONObject issue = payload.getJSONObject("issue"); title.append(payload.getString("action")).append(" issue ").append(issue.getInt("number")) .append(" on ").append(repoName); // [payload.issue.title] content.append(issue.getString("title")); // Issue event hashtags // // #issue // #[issue.state] // #[labels] hashtags.add("#issue"); hashtags.add("#" + issue.getString("state")); JSONArray labels = issue.getJSONArray("labels"); for (int i = 0; i < labels.size(); i++) { hashtags.add("#" + labels.getJSONObject(i).getString("name")); } } else if (activityType.equals("MemberEvent")) { // [actor.login] added [payload.member.login] to [repo.name] title.append("added ").append(payload.getJSONObject("member")).append(" to ").append(repoName); // [repo.name] is at [repo.name] content.append(repoName.split("/")[1]).append(" is at ").append(repoName); // Member event hashtags // // #repository // #membership hashtags.add("#repository"); hashtags.add("#membership"); } else if (activityType.equals("PublicEvent")) { // [actor.login] open sourced [repo.name] title.append("open sourced ").append(repoName); // We have to retrieve the project information and then this is the format: // [project.description] content.append(getRepositoryDetails(repoName).getString("description")); // Public event hashtags // // #repository // #opensourced hashtags.add("#repository"); hashtags.add("#opensourced"); } else if (activityType.equals("PullRequestEvent")) { // [actor.login] [action] pull request [payload.issue.number] on [repo.name] String action = payload.getString("action"); boolean merged = payload.getJSONObject("pull_request").getBoolean("merged"); title.append(action.equals("closed") && merged ? "merged" : action).append(" pull request ") .append(payload.getJSONObject("pull_request").getInt("number")).append(" on ").append(repoName); // [payload.pull_request.title] // [payload.pull_request.commits] commits with [payload.pull_request.additions] additions and [payload.pull_request.deletions] deletions JSONObject pullRequest = payload.getJSONObject("pull_request"); content.append(pullRequest.getString("title")).append("\n").append(pullRequest.getInt("commits")) .append(" commits with ").append(pullRequest.getInt("additions")).append(" additions and ") .append(pullRequest.getInt("deletions")).append(" deletions"); // Pull request hashtags // // #source // #pullrequest // #[action] hashtags.add("#source"); hashtags.add("#pullrequest"); hashtags.add("#" + (action.equals("closed") && merged ? "merged" : action)); } else if (activityType.equals("PushEvent")) { // [actor.login] pushed to [branch] at [repo.name] String ref = payload.getString("ref"); // Shorten the ref ref = ref.substring(ref.lastIndexOf("/") + 1); title.append("pushed to ").append(ref).append(" at ").append(repoName); // We have to list each commit in the push with this format: // [commit.sha] [commit.message] // Sometimes a PushEvent can omit the commits property if (payload.has("commits")) { JSONArray commits = payload.getJSONArray("commits"); for (int i = 0; i < commits.size(); i++) { JSONObject commit = commits.getJSONObject(i); if (i != 0) { content.append("\n"); } content.append(commit.getString("sha").substring(0, 7)).append(" ") .append(commit.getString("message").split("\n")[0]); } } // Push event hashtags // // #source // #changeset hashtags.add("#source"); hashtags.add("#changeset"); } else if (activityType.equals("WatchEvent")) { // [actor.login] [payload.action] watching [repo.name] title.append(payload.getString("action")).append(" watching ").append(repoName); // [repo.name] description: // [project.description] content.append(repoName.split("/")[1]).append(" description: \n") .append(getRepositoryDetails(repoName).getString("description")); // Watch event hashtags // // #repository // #watch hashtags.add("#repository"); hashtags.add("#watch"); } else { LOGGER.error("Unsupported GitHub event type: " + activityType); return null; } activityParts.put("title", title.toString()); activityParts.put("content", content.toString()); activityParts.put("hashtags", hashtags); return activityParts; } /** * Returns a Date object for the value represented in the JSON object's 'created_at' property. * * @param jsonObject the JSON object to parse * * @return the date value of the 'created_at' property of the JSON object or null otherwise */ private Date getCreatedDate(JSONObject jsonObject) { String rawActivityDate = (jsonObject.has("created_at") ? jsonObject.getString("created_at") : null); Date activityDate = null; if (rawActivityDate != null) { // Example date: 2011-09-06T17:26:27Z try { activityDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(rawActivityDate); } catch (ParseException e) { LOGGER.error("Unable to parse the date (" + rawActivityDate + "): " + e.getMessage()); e.printStackTrace(); } } return activityDate; } /** * Make a request using cache with no maximum number of items returned. * * @see #makeRequest(String, int, boolean) */ private List<JSONObject> makeRequest(String url) throws InvalidCredentialsException, IOException { return makeRequest(url, Integer.MAX_VALUE, true); } /** * Make a request using cache and limited by maxResults. * * @see #makeRequest(String, int, boolean) */ private List<JSONObject> makeRequest(String url, int maxResults) throws InvalidCredentialsException, IOException { return makeRequest(url, maxResults, true); } /** * Makes a call to GitHub and handles pagination of the results. * * @param url the GitHub URL to make a call to * @param maxResults the maximum number of results to return * @param cache use caching * * @return a list of JSONObjects or an empty list if the response had no content * * @throws InvalidCredentialsException if the connection associated with this client has invalid credentials * @throws IOException if anything goes wrong making the actual request */ @SuppressWarnings("unchecked") public List<JSONObject> makeRequest(String url, int maxResults, boolean cache) throws InvalidCredentialsException, IOException { // Caching in GitHub is very, very simple. Since all requests are made from the same user and each request // URL is unique, caching is as simple as mapping the cached result (when necessary) to the request URL. // // Note: We do not use cache for the actual activity requests Object objectFromCache = (cache ? requestCache.getIfPresent(url) : null); List<JSONObject> response = (objectFromCache != null ? (List<JSONObject>) objectFromCache : null); int pageSize = 100; // Quick return if there was an entry in the cache if (response != null) { debugLog(LOGGER, " (From cache)"); return response; } else { response = new ArrayList<>(); } List<Header> responseHeaders = new ArrayList<>(); JSONArray rawResponse = new JSONArray(); String payload = ""; if (getAuthType().equals(AuthType.USERNAME_PASSWORD)) { payload = HTTPUtils.openUrl(url + "?per_page=" + pageSize, HttpMethod.GET, null, MediaType.APPLICATION_JSON, getConnectionCredentials().getIdentity(), getConnectionCredentials().getCredential(), null, responseHeaders); } else if (getAuthType().equals(AuthType.OAUTH)) { payload = HTTPUtils.openOAuthUrl(url + "?per_page=" + pageSize, HttpMethod.GET, null, MediaType.APPLICATION_JSON, oAuthService, getConnectionCredentials(), null, responseHeaders); } try { // Try to parse as a JSONArray knowing that it might be a JSONObject rawResponse = JSONArray.fromObject(payload); } catch (JSONException e) { try { // Try to parse as a JSONObject JSONObject rawObject = JSONObject.fromObject(payload); rawResponse.add(rawObject); } catch (JSONException e2) { // Fail return null; } } int sizeOfArray = rawResponse.size(); List<Integer> processedObjectHashes = new ArrayList<>(); int page = 1; boolean complete = false; while (!complete && response.size() < maxResults) { for (Object anArrayResponse : rawResponse) { JSONObject arrayEntry = JSONObject.fromObject(anArrayResponse); int entryHash = arrayEntry.hashCode(); if (sizeOfArray < maxResults && !processedObjectHashes.contains(entryHash)) { response.add(arrayEntry); processedObjectHashes.add(entryHash); } if (sizeOfArray == maxResults) { complete = true; break; } } // If we returned fewer entries than the max, we know pagination isn't necessary if (sizeOfArray < pageSize) { complete = true; } // GitHub pagination is pretty simple. Basically if there is a Link header, we can expect that pagination // is necessary: http://developer.github.com/v3/#pagination boolean linkHeaderFound = false; for (Header header : responseHeaders) { if (header.getName() != null && header.getName().equals("Link")) { linkHeaderFound = true; break; } } if (!linkHeaderFound) { complete = true; } // If we've decided that pagination is necessary, move on to the next page and continue processing if (!complete) { page++; if (getAuthType().equals(AuthType.USERNAME_PASSWORD)) { rawResponse = JSONArray .fromObject(HTTPUtils.openUrl(url + "?per_page=" + pageSize + "&page=" + page, "GET", null, MediaType.APPLICATION_JSON, getConnectionCredentials().getIdentity(), getConnectionCredentials().getCredential(), null, responseHeaders)); } else if (getAuthType().equals(AuthType.OAUTH)) { rawResponse = JSONArray .fromObject(HTTPUtils.openOAuthUrl(url + "?per_page=" + pageSize + "&page=" + page, "GET", null, MediaType.APPLICATION_JSON, oAuthService, getConnectionCredentials(), null, responseHeaders)); } sizeOfArray = rawResponse.size(); } } // Cache the entry if necessary if (cache) { requestCache.put(url, response); } return response; } }