Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.addthis.hydra.job.store; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; import java.util.List; import com.addthis.basis.util.LessFiles; import com.addthis.basis.util.Parameter; import com.addthis.maljson.JSONArray; import com.addthis.maljson.JSONException; import com.addthis.maljson.JSONObject; import com.google.common.collect.Iterables; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.DiffCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.AbstractTreeIterator; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A class for storing/retrieving files from a git repository. */ public class JobStoreGit { private final File baseDir; private final File gitDir; private final File jobDir; private final FileRepository repository; private final Git git; private final boolean remote = Parameter.boolValue("job.store.remote", false); private static final String branchName = Parameter.value("cluster.name", "localhost"); private static final String gitUser = Parameter.value("git.user", "anonymous"); private static final String gitPassword = Parameter.value("git.password", ""); private static final String gitUrl = Parameter.value("git.url", ""); private static final UsernamePasswordCredentialsProvider provider = new UsernamePasswordCredentialsProvider( gitUser, gitPassword); @SuppressWarnings("unused") private static final Logger log = LoggerFactory.getLogger(JobStoreGit.class); boolean haveRemoteBranch = false; /** * Set up a git repository in a directory. * * @param baseDir The directory that will store the files. * @throws IOException If there is a problem writing to the directory */ public JobStoreGit(File baseDir) throws Exception { this.baseDir = baseDir; this.gitDir = new File(baseDir, ".git"); this.jobDir = new File(baseDir, "jobs"); repository = new FileRepository(gitDir); git = new Git(repository); initialize(); } /** * Fetch the version of a config after a particular commit in its history * * @param jobId The job id to check * @param commitId The commit id to fetch * @return The contents of the file after the specified time * @throws GitAPIException If there is a problem retrieving the file after the specified commit * @throws IOException If there is a problem writing to the file */ public String fetchJobConfigFromHistory(String jobId, String commitId) throws GitAPIException, IOException { synchronized (gitDir) { File file = getFileForJobId(jobId); git.checkout().addPath(getPathForJobId(jobId)).setStartPoint(commitId).setName("master").call(); String rv = new String(LessFiles.read(file)); git.reset().setMode(ResetCommand.ResetType.HARD).call(); return rv; } } /** * Returns the hash of the last commit before a job was deleted. * * This method uses the following git command equivalent: * <pre> * git log --all --skip=1 -n 1 -- jobs/[jobId] * </pre> * which only works as intended with deleted jobs. * * @param jobId The id of a deleted job. * @throws GitAPIException * @throws IOException */ public String getCommitHashBeforeJobDeletion(String jobId) throws GitAPIException, IOException { // skip=1 skips the deletion, and -n 1 returns only the commit immediately before that Iterable<RevCommit> iter = git.log().all().addPath(getPathForJobId(jobId)).setSkip(1).setMaxCount(1).call(); RevCommit commit = Iterables.getFirst(iter, null); return commit == null ? null : commit.getName(); } /** * Get a summary of the diff, mainly used to automatically generate useful commit messages * * @param jobId The job to summarize the diff for * @return A string description of how the file has changed, e.g. "(added 3 lines, removed 1)" * @throws IOException If there is a problem reading from the file * @throws GitAPIException If there is a problem getting the diff from git */ public String getDiffSummary(String jobId) throws IOException, GitAPIException { // Have to start each count at -1 to account for the line that looks like ---a/filename int added = -1; int removed = -1; String diff = getDiff(jobId, null); for (String line : diff.split("\n")) { if (line.startsWith("-")) { removed++; } else if (line.startsWith("+")) { added++; } } if (added <= 0 && removed <= 0) { return "(no changes)"; } else { return "(added " + added + " line" + (added == 1 ? "" : "s") + ", removed " + removed + ")"; } } /** * Get the diff of a file against a particular commit * * @param jobId The job config to diff * @param commitId If specified, the commit to compare against; otherwise, compare against the latest revision * @return A string description of the diff, comparable to the output of "git diff filename" * @throws GitAPIException If there is a problem fetching the diff from git * @throws IOException If there is a problem reading from the file */ public String getDiff(String jobId, String commitId) throws GitAPIException, IOException { OutputStream out = new ByteArrayOutputStream(); DiffCommand diff = git.diff().setPathFilter(PathFilter.create(getPathForJobId(jobId))).setOutputStream(out) .setSourcePrefix("old:").setDestinationPrefix("new:"); if (commitId != null) { diff.setOldTree(getTreeIterator(commitId)); diff.setNewTree(getTreeIterator("HEAD")); } diff.call(); return out.toString(); } /* * Internal function for generating an automatic commit message based on who changed the file and how it changed */ private String generateFinalCommitMessage(String jobId, String author, String commitMessage) throws GitAPIException, IOException { return (author != null ? author : "unknown") + ": " + (commitMessage != null && !commitMessage.isEmpty() ? commitMessage : "[AUTO]") + " " + getDiffSummary(jobId); } /** * Get a JSON representation of the log of changes to a file * * @param jobId The jobId to fetch the log for * @return A log of the form [{commit:commitid, time:time, msg:commitmessage}, ...] * @throws GitAPIException If there is a problem fetching the log * @throws JSONException If there is a problem generating the JSON */ public JSONArray getGitLog(String jobId) throws Exception { JSONArray rv = new JSONArray(); for (RevCommit commit : git.log().addPath(getPathForJobId(jobId)).call()) { JSONObject commitJson = new JSONObject().put("commit", commit.getName()) .put("time", 1000L * (commit.getCommitTime())).put("msg", commit.getFullMessage()); rv.put(commitJson); } return rv; } /** * Write a job config to a file and commit it * * @param jobId The jobId being updated * @param author The author who made the most recent change * @param config The latest version of the config * @param commitMessage The base commit message to use, or null to use an automatic one * @throws GitAPIException If there is a problem talking to the git repo * @throws IOException If there is a problem writing to the file system */ public void commit(String jobId, String author, String config, String commitMessage) throws GitAPIException, IOException { synchronized (gitDir) { File file = getFileForJobId(jobId); assert (file.exists() || file.createNewFile()); LessFiles.write(file, config.getBytes(), false); commitMessage = generateFinalCommitMessage(jobId, author, commitMessage); git.add().addFilepattern(getPathForJobId(jobId)).call(); git.commit().setMessage(commitMessage).call(); push(); } } /** * Pull-rebase and push to the remote repo if git.remote is enabled * * @throws GitAPIException If there is a failure talking to the remote repository */ public void push() throws GitAPIException { if (remote) { synchronized (gitDir) { if (haveRemoteBranch()) { git.pull().setCredentialsProvider(provider).setRebase(true).call(); } git.push().setCredentialsProvider(provider).setRemote("origin").call(); } } } /** * Initialize the git repo in the specified directory * * @throws Exception If there is a problem during initialization */ private void initialize() throws Exception { synchronized (gitDir) { FileBasedConfig config = repository.getConfig(); for (String branch : Arrays.asList(branchName, "master")) { config.setString("branch", branch, "merge", "refs/heads/" + branch); config.setString("branch", branch, "remote", "origin"); } config.setString("remote", "origin", "fetch", "+refs/*:refs/*"); config.setString("remote", "origin", "url", gitUrl); if (!gitDir.exists()) { if (remote) { new CloneCommand().setCredentialsProvider(provider).setRemote("origin").setBranch("master") .setURI(gitUrl).setDirectory(baseDir).call(); git.branchCreate().setName(branchName).call(); git.checkout().setName(branchName).call(); } else { repository.create(); } } if (!jobDir.exists()) { LessFiles.initDirectory(jobDir); } } } /* * Internal function for fetching the TreeIterator object corresponding to the git state after a particular commit */ private AbstractTreeIterator getTreeIterator(String name) throws IOException { final ObjectId id = repository.resolve(name); if (id == null) { throw new IllegalArgumentException(name); } final CanonicalTreeParser p = new CanonicalTreeParser(); try (ObjectReader or = repository.newObjectReader(); RevWalk walk = new RevWalk(repository)) { p.reset(or, walk.parseTree(id)); return p; } } /* * Internal function for generating an automatic message for deleting a config */ private String getDeleteCommitMessage(String jobId) { return "Deleted " + jobId; } /** * Delete a config and remove it from git tracking * * @param jobId The job config to remove */ public void remove(String jobId) { File file = getFileForJobId(jobId); synchronized (gitDir) { try { git.rm().addFilepattern(getPathForJobId(jobId)).call(); git.commit().setMessage(getDeleteCommitMessage(jobId)).call(); push(); } catch (Exception e) { // Maybe the file hadn't been committed yet, or was not tracked in the first place assert (file.delete()); } } } /** * Internal function to fetch the file corresponding to a job id * * @param jobId The job id to look for * @return The corresponding file */ private File getFileForJobId(String jobId) { return new File(jobDir, jobId); } /** * Internal function to fetch the particular relative file path for a jobId that jgit needs to work right * * @param jobId The jobId to look for * @return The corresponding path */ private String getPathForJobId(String jobId) { return jobDir.getName() + "/" + jobId; } private boolean haveRemoteBranch() throws GitAPIException { if (haveRemoteBranch) { return true; // If found once, the branch should stick around under any reasonable circumstances } List<Ref> refs = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE).call(); for (Ref ref : refs) { String name = ref.getName(); if (("/refs/remotes/origin/" + branchName).equals(name)) { haveRemoteBranch = true; return true; } } return false; } }