de.fischer.thotti.core.distbuilder.DistributionBuilder.java Source code

Java tutorial

Introduction

Here is the source code for de.fischer.thotti.core.distbuilder.DistributionBuilder.java

Source

/*
 * Copyright 2011 Oliver B. Fischer
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.fischer.thotti.core.distbuilder;

import de.fischer.thotti.awscommon.AWSAccessCredentials;
import de.fischer.thotti.awscommon.AWSIllegalConfigurationException;
import de.fischer.thotti.awscommon.AWSRessourceNotFoundException;
import de.fischer.thotti.core.runner.NDRunner;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.utils.IOUtils;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * The
 * <h2>Local and Remote Maven Repositories</h2>
 * <p/>
 * dd
 * <p/>
 * <h2>Limitations of the Distribution Builder</h2>
 * <p/>
 * <ul>
 * <li>Exclusions are note supported for Maven
 * artefacts at the moment. But they can be added
 * quite easily.</li>
 * </ul>
 *
 * @author Oliver Fischer
 */
public class DistributionBuilder {
    // See http://aether.sonatype.org/using-aether-in-maven-plugins.html

    public final static String MAHOUT_GROUP_ID = "org.apache.mahout";

    public final static String BASE_DIRECTORY = "thottindtests";

    /**
     * The directory used in the distribution file to hold all
     * needed external artefacts.
     */
    public final static String EXTERNAL_LIB_DIRECTORY = "thirdparty";

    /**
     * The directory used in the distribution file to hold all
     * needed artefacts of Mahout.
     */
    public final static String MAHOUT_LIB_DIRECTORY = "mahout";

    /**
     * The directory where all data files and directories
     * will go. It denotes is a constant for
     * {@value #DATA_DIRECTORY}.
     */
    public final static String DATA_DIRECTORY = "testdata";

    /**
     * The directory used in the distribution file to store
     * the artifact with the testcases itself. This is
     * a constant for {@value #TEST_CASE_LIB_DIRECTORY}.
     */
    public final static String TEST_CASE_LIB_DIRECTORY = "tests";

    /**
     * The directory where all shell scripts and executables
     * will go. It denotes is a constant for
     * {@value #BIN_DIRECTORY}.
     */
    public final static String BIN_DIRECTORY = "bin";

    MavenDependencyFetcher depFetcher = new MavenDependencyFetcher();

    private File targetFile;
    private BuilderListener listener;
    private ClasspathBuilder cpBuilder = new ClasspathBuilder();

    protected List<File> dataFiles = new LinkedList<File>();
    private File testArtifact;

    /**
     * Sets the base directory of the local Maven repository.
     * <p/>
     * See the {@link DistributionBuilder} class documentation
     * for additional information.
     *
     * @param directory Directory pointing to the base directory of the to
     *                  be used local Maven repository.
     *                  Must not be {@code null}.
     */
    public DistributionBuilder withLocalRepository(File directory) {
        if (null == directory)
            throw new NullPointerException();

        depFetcher.withLocalRepository(directory);

        return this;
    }

    public AWSAccessCredentials getRemoteUsersCredentials()
            throws AWSRessourceNotFoundException, AWSIllegalConfigurationException, IOException {
        String home = System.getProperty("user.home");

        // @todo Put this file into a constant
        String credentialsName = "amazon.remote.credentials";

        File credentialFile = new File(home, credentialsName);

        return AWSAccessCredentials.fromResource(credentialFile);
    }

    public DistributionBuilder addMavenDependency(String artifactCoords) {
        // @todo No Unit tests, write them!
        if (artifactCoords == null)
            throw new NullPointerException();

        depFetcher.addMavenDependency(artifactCoords);

        return this;
    }

    /**
     * @param id   The identifier of the remote repository.
     * @param type The type of the remote repository.
     * @param url  The URL of the remote repository.
     */
    public DistributionBuilder withRemoteRepository(String id, String type, String url) {
        if (null == id)
            throw new NullPointerException();
        if (null == type)
            throw new NullPointerException();
        if (null == url)
            throw new NullPointerException();

        depFetcher.addRemoteRepository(id, type, url);

        return this;
    }

    /**
     * Adds a specific file to the distribution to build. The
     * file will be added directo
     *
     * @param fileToAdd The file to add to the distribtion,
     *                  must not be {@code null}.
     *
     * @see #DATA_DIRECTORY
     */
    public DistributionBuilder addDataFile(File fileToAdd) {
        // @todo Write Unit tests for this

        // @todo assert a file
        dataFiles.add(fileToAdd);

        return this;
    }

    public DistributionBuilder addDataDirectory(File directoryToAdd) {
        if (null == directoryToAdd) {
            throw new NullPointerException();
        }

        // @todo Write Unit tests

        return this;
    }

    public DistributionBuilder setTestArtifact(File artifact) {
        if (null == artifact) {
            throw new NullPointerException();
        }

        // @todo Write Unit tests for this

        testArtifact = artifact;

        return this;
    }

    public File buildDistribution() throws Exception, ArchiveException {
        String mahoutVersion = org.apache.mahout.Version.version();
        // @todo Throw an exception if the distfile cannot created
        File output = targetFile;

        Set<String> addedFiles = new HashSet<String>();

        List<FetchResult> fetchedArtifacts = depFetcher.fetchDependencies();

        final OutputStream out = new FileOutputStream(output);
        ArchiveOutputStream os = new ArchiveStreamFactory().createArchiveOutputStream("zip", out);

        String subDirectory;

        for (FetchResult fetch : fetchedArtifacts) {
            String coord = fetch.getArtifactCoordinates();

            // @todo This is not safe! It is a hack!
            if (coord.split(":")[0].equals(MAHOUT_GROUP_ID)) {
                subDirectory = MAHOUT_LIB_DIRECTORY;
            } else {
                subDirectory = EXTERNAL_LIB_DIRECTORY;
            }

            for (File file : fetch.getFiles()) {
                String targetCP = subDirectory + "/" + file.getName();
                String targetArchive = BASE_DIRECTORY + "/" + targetCP;

                if (!addedFiles.contains(targetCP)) {
                    cpBuilder.addPath(targetCP);

                    if (getListener() != null) {
                        StringBuilder sb = new StringBuilder();

                        sb.append("Add: ").append(file.toURI()).append(" as ").append(targetCP);

                        getListener().onAddingMavenArtifact(sb.toString());
                    }

                    os.putArchiveEntry(new ZipArchiveEntry(targetArchive));
                    IOUtils.copy(new FileInputStream(file), os);
                    os.closeArchiveEntry();

                    addedFiles.add(targetCP);
                }
            }
        }

        cpBuilder.addPath(DATA_DIRECTORY);

        for (File file : dataFiles) {
            String targetCP = DATA_DIRECTORY + "/" + file.getName();
            String targetArchive = BASE_DIRECTORY + "/" + targetCP;

            if (getListener() != null) {
                StringBuilder sb = new StringBuilder();

                sb.append("Add: ").append(file.toURI()).append(" as ").append(targetCP);

                getListener().onAddingTestData(sb.toString());
            }

            os.putArchiveEntry(new ZipArchiveEntry(targetArchive));
            IOUtils.copy(new FileInputStream(file), os);
            os.closeArchiveEntry();
        }

        String testCaseTargetCP = TEST_CASE_LIB_DIRECTORY + "/" + testArtifact.getName();
        String testCaseTargetArchive = BASE_DIRECTORY + "/" + testCaseTargetCP;

        if (getListener() != null) {
            StringBuilder sb = new StringBuilder();

            sb.append("Add: ").append(testArtifact.toURI()).append(" as ").append(testCaseTargetCP);

            listener.onAddingTestCases(sb.toString());
        }

        os.putArchiveEntry(new ZipArchiveEntry(testCaseTargetArchive));
        IOUtils.copy(new FileInputStream(testArtifact), os);
        os.closeArchiveEntry();

        cpBuilder.addPath(testCaseTargetCP);

        generateRunSkript(os);

        out.flush();
        os.close();

        return output;
    }

    public DistributionBuilder withTargetArchive(File file) {
        targetFile = file;

        return this;
    }

    public void setRepositorySystem(RepositorySystem system) {
        depFetcher.setRepoSystem(system);
    }

    public void setSession(RepositorySystemSession session) {
        depFetcher.setSession(session);
    }

    public void setListener(BuilderListener l) {
        listener = l;
    }

    public BuilderListener getListener() {
        return listener;
    }

    protected void generateRunSkript(ArchiveOutputStream os)
            throws IOException, TemplateException, AWSRessourceNotFoundException, AWSIllegalConfigurationException {
        // @todo Improve Exception handling

        AWSAccessCredentials remoteCredentials = getRemoteUsersCredentials();

        Configuration cfg = new Configuration();
        cfg.setClassForTemplateLoading(this.getClass(), "/freemarker/distribution");
        cfg.setLocalizedLookup(false);
        cfg.setObjectWrapper(new DefaultObjectWrapper());

        Map varMap = new HashMap();
        varMap.put("var_classpath", cpBuilder.toString());
        varMap.put("var_ak", remoteCredentials.getAccessKeyId());
        varMap.put("var_sk", remoteCredentials.getSecretKey());
        varMap.put("var_resultfile", NDRunner.THOTTI_RESULT_FILE);

        Template temp = null;
        temp = cfg.getTemplate("run.sh.ftl");
        StringWriter out = new StringWriter();
        temp.process(varMap, out);
        out.flush();

        String content = out.getBuffer().toString().replace("\r\n", "\n");

        String target = BASE_DIRECTORY + "/" + BIN_DIRECTORY + "/ndrunner";
        os.putArchiveEntry(new ZipArchiveEntry(target));
        IOUtils.copy(new ByteArrayInputStream(content.getBytes()), os);
        os.closeArchiveEntry();
    }
}