Java tutorial
/* * Copyright (c) 2017, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.wso2.security.tools.reposcanner.repository; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.eclipse.egit.github.core.Repository; import org.eclipse.egit.github.core.RepositoryTag; import org.eclipse.egit.github.core.client.GitHubClient; import org.eclipse.egit.github.core.service.RepositoryService; import org.wso2.security.tools.reposcanner.AppConfig; import org.wso2.security.tools.reposcanner.downloader.GitHubMasterDownloader; import org.wso2.security.tools.reposcanner.downloader.GitHubTagDownloader; import org.wso2.security.tools.reposcanner.downloader.RepoDownloader; import org.wso2.security.tools.reposcanner.entiry.Repo; import org.wso2.security.tools.reposcanner.entiry.RepoType; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; /** * Responsible of calling Github API and get repository information for each user */ public class GitHubRepoInfoGenerator implements RepoInfoGenerator { private static Logger log = Logger.getLogger(GitHubRepoInfoGenerator.class.getName()); private GitHubClient client; private RepoDownloader gitMasterDownloader; private File masterDownloadFolder; private RepoDownloader gitTagDownloader; private File tagsDownloadFolder; public GitHubRepoInfoGenerator(char[] oAuth2Token) { client = new GitHubClient(); client.setOAuth2Token(new String(oAuth2Token)); if (AppConfig.isDownloadMaster()) { gitMasterDownloader = new GitHubMasterDownloader(); masterDownloadFolder = new File("source-master"); if (masterDownloadFolder.exists()) { try { FileUtils.deleteDirectory(masterDownloadFolder); } catch (IOException e) { log.error("Error in removing master download folder: " + masterDownloadFolder.getAbsolutePath(), e); } } masterDownloadFolder.mkdir(); } if (AppConfig.isDownloadTags()) { gitTagDownloader = new GitHubTagDownloader(); tagsDownloadFolder = new File("source-tags"); if (tagsDownloadFolder.exists()) { try { FileUtils.deleteDirectory(tagsDownloadFolder); } catch (IOException e) { log.error("Error in removing tags download folder: " + tagsDownloadFolder.getAbsolutePath(), e); } } tagsDownloadFolder.mkdir(); } } @Override public List<Repo> getRepoList(String consoleTag, List<String> users) { RepositoryService repositoryService = new RepositoryService(client); List<Repo> repoList = Collections.synchronizedList(new ArrayList()); //Get the list of git repositories for each GitHub user account users.parallelStream().forEach(user -> { log.info(consoleTag + "Fetching repositories for GitHub user account: " + user); try { List<Repository> userRepositoryList = repositoryService.getRepositories(user); log.info(consoleTag + userRepositoryList.size() + " repositories found for user account: " + user); //Download master branches only if download-master flag is enabled if (AppConfig.isDownloadMaster()) { userRepositoryList.parallelStream().forEach(repository -> { if (AppConfig.getGithubRepos() == null || AppConfig.getGithubRepos().contains(repository.getName())) { try { log.info(consoleTag + "[DownloadMaster] Started downloading master branch of: " + repository.getName()); Repo tempRepo = new Repo(RepoType.GIT, repository.getOwner().getLogin(), repository.getName(), repository.getCloneUrl(), null, null, null); gitMasterDownloader.downloadRepo(tempRepo, masterDownloadFolder, false); log.info(consoleTag + "[DownloadMaster] Completed downloading master branch of: " + repository.getName()); } catch (Exception e) { log.error(consoleTag + "Error in downloading master branch ZIP for GitHub user account: " + user + " repository: " + repository.getName(), e); } } else { log.info(consoleTag + "[DownloadMaster][Skipping] Skipping since the repo is not in include list : " + repository.getName()); } }); } //Do this only if scan should be done (skip skip-flag is not set) or skip-flag is set but tag download should happen if (!AppConfig.isSkipScan() || AppConfig.isDownloadTags()) { //Get the list of tags for each repository userRepositoryList.parallelStream().forEach(repository -> { if (AppConfig.getGithubRepos() == null || AppConfig.getGithubRepos().contains(repository.getName())) { log.info(consoleTag + "Fetching tags for GitHub user account: " + user + " repository: " + repository.getName()); try { List<RepositoryTag> repositoryTagLists = repositoryService.getTags(repository); log.info(consoleTag + repositoryTagLists.size() + " tags found for user account: " + user + " repository:" + repository.getName()); //Create persistable Repo object with repository and tag information repositoryTagLists.parallelStream().forEach(repositoryTag -> { Repo repo = new Repo(RepoType.GIT, repository.getOwner().getLogin(), repository.getName(), repository.getCloneUrl(), repositoryTag.getName(), repositoryTag.getZipballUrl(), new Date()); repoList.add(repo); //Download tags only if download-tag flag is enabled if (AppConfig.isDownloadTags()) { try { log.info(consoleTag + "[DownloadTags] Started downloading tag: " + repo.getTagName() + " of: " + repository.getName()); gitTagDownloader.downloadRepo(repo, tagsDownloadFolder, false); log.info(consoleTag + "[DownloadTags] Completed downloading tag: " + repo.getTagName() + " of: " + repository.getName()); } catch (Exception e) { log.error(consoleTag + "Error in downloading master branch ZIP for GitHub user account: " + user + " repository: " + repository.getName(), e); } } }); } catch (Exception e) { log.error(consoleTag + "Error in fetching tags for GitHub user account: " + user + " repository: " + repository.getName(), e); } } else { log.info(consoleTag + "[Skipping] Skipping since the repo is not in include list : " + repository.getName()); } }); } else { log.warn(consoleTag + "[Skipping] SkipScan parameter is set and tag download is not enabled. Skipping tag information retrieval."); } } catch (Exception e) { log.error("Error in fetching repositories for GitHub user account: " + user, e); } }); return repoList; } }