Java tutorial
/* * Copyright 2011 Oliver B. Fischer * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package de.fischer.thotti.core.distbuilder; import de.fischer.thotti.awscommon.AWSAccessCredentials; import de.fischer.thotti.awscommon.AWSIllegalConfigurationException; import de.fischer.thotti.awscommon.AWSRessourceNotFoundException; import de.fischer.thotti.core.runner.NDRunner; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; import org.apache.commons.compress.archivers.ArchiveException; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.ArchiveStreamFactory; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.utils.IOUtils; import org.sonatype.aether.RepositorySystem; import org.sonatype.aether.RepositorySystemSession; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.StringWriter; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; /** * The * <h2>Local and Remote Maven Repositories</h2> * <p/> * dd * <p/> * <h2>Limitations of the Distribution Builder</h2> * <p/> * <ul> * <li>Exclusions are note supported for Maven * artefacts at the moment. But they can be added * quite easily.</li> * </ul> * * @author Oliver Fischer */ public class DistributionBuilder { // See http://aether.sonatype.org/using-aether-in-maven-plugins.html public final static String MAHOUT_GROUP_ID = "org.apache.mahout"; public final static String BASE_DIRECTORY = "thottindtests"; /** * The directory used in the distribution file to hold all * needed external artefacts. */ public final static String EXTERNAL_LIB_DIRECTORY = "thirdparty"; /** * The directory used in the distribution file to hold all * needed artefacts of Mahout. */ public final static String MAHOUT_LIB_DIRECTORY = "mahout"; /** * The directory where all data files and directories * will go. It denotes is a constant for * {@value #DATA_DIRECTORY}. */ public final static String DATA_DIRECTORY = "testdata"; /** * The directory used in the distribution file to store * the artifact with the testcases itself. This is * a constant for {@value #TEST_CASE_LIB_DIRECTORY}. */ public final static String TEST_CASE_LIB_DIRECTORY = "tests"; /** * The directory where all shell scripts and executables * will go. It denotes is a constant for * {@value #BIN_DIRECTORY}. */ public final static String BIN_DIRECTORY = "bin"; MavenDependencyFetcher depFetcher = new MavenDependencyFetcher(); private File targetFile; private BuilderListener listener; private ClasspathBuilder cpBuilder = new ClasspathBuilder(); protected List<File> dataFiles = new LinkedList<File>(); private File testArtifact; /** * Sets the base directory of the local Maven repository. * <p/> * See the {@link DistributionBuilder} class documentation * for additional information. * * @param directory Directory pointing to the base directory of the to * be used local Maven repository. * Must not be {@code null}. */ public DistributionBuilder withLocalRepository(File directory) { if (null == directory) throw new NullPointerException(); depFetcher.withLocalRepository(directory); return this; } public AWSAccessCredentials getRemoteUsersCredentials() throws AWSRessourceNotFoundException, AWSIllegalConfigurationException, IOException { String home = System.getProperty("user.home"); // @todo Put this file into a constant String credentialsName = "amazon.remote.credentials"; File credentialFile = new File(home, credentialsName); return AWSAccessCredentials.fromResource(credentialFile); } public DistributionBuilder addMavenDependency(String artifactCoords) { // @todo No Unit tests, write them! if (artifactCoords == null) throw new NullPointerException(); depFetcher.addMavenDependency(artifactCoords); return this; } /** * @param id The identifier of the remote repository. * @param type The type of the remote repository. * @param url The URL of the remote repository. */ public DistributionBuilder withRemoteRepository(String id, String type, String url) { if (null == id) throw new NullPointerException(); if (null == type) throw new NullPointerException(); if (null == url) throw new NullPointerException(); depFetcher.addRemoteRepository(id, type, url); return this; } /** * Adds a specific file to the distribution to build. The * file will be added directo * * @param fileToAdd The file to add to the distribtion, * must not be {@code null}. * * @see #DATA_DIRECTORY */ public DistributionBuilder addDataFile(File fileToAdd) { // @todo Write Unit tests for this // @todo assert a file dataFiles.add(fileToAdd); return this; } public DistributionBuilder addDataDirectory(File directoryToAdd) { if (null == directoryToAdd) { throw new NullPointerException(); } // @todo Write Unit tests return this; } public DistributionBuilder setTestArtifact(File artifact) { if (null == artifact) { throw new NullPointerException(); } // @todo Write Unit tests for this testArtifact = artifact; return this; } public File buildDistribution() throws Exception, ArchiveException { String mahoutVersion = org.apache.mahout.Version.version(); // @todo Throw an exception if the distfile cannot created File output = targetFile; Set<String> addedFiles = new HashSet<String>(); List<FetchResult> fetchedArtifacts = depFetcher.fetchDependencies(); final OutputStream out = new FileOutputStream(output); ArchiveOutputStream os = new ArchiveStreamFactory().createArchiveOutputStream("zip", out); String subDirectory; for (FetchResult fetch : fetchedArtifacts) { String coord = fetch.getArtifactCoordinates(); // @todo This is not safe! It is a hack! if (coord.split(":")[0].equals(MAHOUT_GROUP_ID)) { subDirectory = MAHOUT_LIB_DIRECTORY; } else { subDirectory = EXTERNAL_LIB_DIRECTORY; } for (File file : fetch.getFiles()) { String targetCP = subDirectory + "/" + file.getName(); String targetArchive = BASE_DIRECTORY + "/" + targetCP; if (!addedFiles.contains(targetCP)) { cpBuilder.addPath(targetCP); if (getListener() != null) { StringBuilder sb = new StringBuilder(); sb.append("Add: ").append(file.toURI()).append(" as ").append(targetCP); getListener().onAddingMavenArtifact(sb.toString()); } os.putArchiveEntry(new ZipArchiveEntry(targetArchive)); IOUtils.copy(new FileInputStream(file), os); os.closeArchiveEntry(); addedFiles.add(targetCP); } } } cpBuilder.addPath(DATA_DIRECTORY); for (File file : dataFiles) { String targetCP = DATA_DIRECTORY + "/" + file.getName(); String targetArchive = BASE_DIRECTORY + "/" + targetCP; if (getListener() != null) { StringBuilder sb = new StringBuilder(); sb.append("Add: ").append(file.toURI()).append(" as ").append(targetCP); getListener().onAddingTestData(sb.toString()); } os.putArchiveEntry(new ZipArchiveEntry(targetArchive)); IOUtils.copy(new FileInputStream(file), os); os.closeArchiveEntry(); } String testCaseTargetCP = TEST_CASE_LIB_DIRECTORY + "/" + testArtifact.getName(); String testCaseTargetArchive = BASE_DIRECTORY + "/" + testCaseTargetCP; if (getListener() != null) { StringBuilder sb = new StringBuilder(); sb.append("Add: ").append(testArtifact.toURI()).append(" as ").append(testCaseTargetCP); listener.onAddingTestCases(sb.toString()); } os.putArchiveEntry(new ZipArchiveEntry(testCaseTargetArchive)); IOUtils.copy(new FileInputStream(testArtifact), os); os.closeArchiveEntry(); cpBuilder.addPath(testCaseTargetCP); generateRunSkript(os); out.flush(); os.close(); return output; } public DistributionBuilder withTargetArchive(File file) { targetFile = file; return this; } public void setRepositorySystem(RepositorySystem system) { depFetcher.setRepoSystem(system); } public void setSession(RepositorySystemSession session) { depFetcher.setSession(session); } public void setListener(BuilderListener l) { listener = l; } public BuilderListener getListener() { return listener; } protected void generateRunSkript(ArchiveOutputStream os) throws IOException, TemplateException, AWSRessourceNotFoundException, AWSIllegalConfigurationException { // @todo Improve Exception handling AWSAccessCredentials remoteCredentials = getRemoteUsersCredentials(); Configuration cfg = new Configuration(); cfg.setClassForTemplateLoading(this.getClass(), "/freemarker/distribution"); cfg.setLocalizedLookup(false); cfg.setObjectWrapper(new DefaultObjectWrapper()); Map varMap = new HashMap(); varMap.put("var_classpath", cpBuilder.toString()); varMap.put("var_ak", remoteCredentials.getAccessKeyId()); varMap.put("var_sk", remoteCredentials.getSecretKey()); varMap.put("var_resultfile", NDRunner.THOTTI_RESULT_FILE); Template temp = null; temp = cfg.getTemplate("run.sh.ftl"); StringWriter out = new StringWriter(); temp.process(varMap, out); out.flush(); String content = out.getBuffer().toString().replace("\r\n", "\n"); String target = BASE_DIRECTORY + "/" + BIN_DIRECTORY + "/ndrunner"; os.putArchiveEntry(new ZipArchiveEntry(target)); IOUtils.copy(new ByteArrayInputStream(content.getBytes()), os); os.closeArchiveEntry(); } }