com.shmsoft.dmass.main.SHMcloudMain.java Source code

Java tutorial

Introduction

Here is the source code for com.shmsoft.dmass.main.SHMcloudMain.java

Source

/*
 *
 * Copyright SHMsoft, Inc. 
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/
package com.shmsoft.dmass.main;

import com.google.common.io.Files;
import java.io.File;
import java.text.DecimalFormat;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import com.shmsoft.dmass.services.Project;
import com.shmsoft.dmass.services.Stats;
import com.shmsoft.dmass.services.Util;
import com.shmsoft.dmass.ui.StagingProgressUI;

/**
 * Main application instance
 *
 * @author mark
 */
public class SHMcloudMain {

    private static SHMcloudMain instance = new SHMcloudMain();
    private CommandLine commandLine;
    //private Configuration processingParameters;    

    public static SHMcloudMain getInstance() {
        return instance;
    }

    private Options options = formOptions();

    private Options formOptions() {
        Options buildOptions = new Options();
        for (CommandLineOption o : CommandLineOption.values()) {
            buildOptions.addOption(o.getName(), o.isHasArg(), o.getHelp());
        }
        return buildOptions;
    }

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        instance.processOptions(args);
    }

    private SHMcloudMain() {
    }

    /**
     * Process the command line arguments
     *
     * @param args command line arguments
     */
    private void processOptions(String[] args) {
        String customParameterFile;
        Project project = null;
        try {
            BasicParser parser = new BasicParser();
            commandLine = parser.parse(options, args);

            // one-time actions
            if (commandLine.hasOption(CommandLineOption.HELP.getName()) || commandLine.getOptions().length == 0) {
                HelpFormatter f = new HelpFormatter();
                f.printHelp("java -jar FreeEed.jar [options]\n\n" + "where options include:", options);
            } else if (commandLine.hasOption(CommandLineOption.VERSION.getName())) {
                System.out.println(Version.getVersionAndBuild());
            } else if (commandLine.hasOption(CommandLineOption.GUI.getName())) {
                openGUI();
            } else if (commandLine.hasOption(CommandLineOption.ENRON.getName())) {
                processEnronDataSet();
            } else {
                if (commandLine.hasOption(CommandLineOption.PARAM_FILE.getName())) {
                    // independent actions
                    customParameterFile = commandLine.getOptionValue(CommandLineOption.PARAM_FILE.getName());
                    project = new Project().loadFromFile(new File(customParameterFile));
                    Project.setProject(project);
                    SHMcloudLogging.init(false);
                }
                if (commandLine.hasOption(CommandLineOption.DRY.getName())) {
                    System.out.println("Dry run - exiting now.");
                } else {
                    if (project.isStage()) {
                        stagePackageInput();
                    }
                    String runWhere = project.getProcessWhere();
                    if (runWhere != null) {
                        process(runWhere);
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace(System.out);
        }
    }

    /**
     * Process staged files
     *
     * @param runWhere determines whether processing occurs on local, private, or EC2 Hadoop instance
     * @throws FreeEedException
     */
    public void process(String runWhere) {
        String projectName = Project.getProject().getProjectName();
        Stats.getInstance().setJobStarted(projectName);
        try {
            new ActionProcessing(runWhere).process();
        } catch (Exception e) {
            // TODO - what to do in case of exception
            e.printStackTrace(System.out);
        }
    }

    /**
     * Process from GUI in a thread
     *
     * @param runWhere determines whether processing occurs on local, private, or EC2 Hadoop instance
     * @throws FreeEedException
     */
    public void runProcessing(String runWhere) throws SHMcloudException {
        String projectName = Project.getProject().getProjectName();
        Stats.getInstance().setJobStarted(projectName);
        new Thread(new ActionProcessing(runWhere)).start();
    }

    /**
     * Stage (package) the input files
     * 
     * @throws Exception
     */
    public void stagePackageInput() throws Exception {
        new ActionStaging().stagePackageInput();
    }

    /**
     * Stage from GUI in a thread
     * 
     * @throws Exception
     */
    public void runStagePackageInput() throws Exception {
        StagingProgressUI ui = new StagingProgressUI(null, true);
        ui.setVisible(true);
    }

    // TODO main engine should not mention gui
    private void openGUI() {
        //SHMcloudUI.main(null);
    }

    private void processEnronDataSet() {
        int ENRON_SET_SIZE = 154;
        String localDir = "freeeed-output/";
        String output = "output/";
        String dir = "/mnt/tmp/";
        new File(dir + "results").mkdirs();
        for (int i = 1; i <= ENRON_SET_SIZE; ++i) {
            try {
                DecimalFormat decimalFormat = new DecimalFormat("enron000");
                String projectName = decimalFormat.format(i);
                if (new File(dir + projectName + ".project").exists() == false) {
                    continue;
                }
                String outputPath = dir + "results/" + projectName + "/";
                if (new File(outputPath).exists()) {
                    continue;
                }
                File localDirFile = new File(localDir);
                if (localDirFile.exists()) {
                    Util.deleteDirectory(localDirFile);
                }
                String[] argv = new String[2];
                argv[0] = "-param_file";
                argv[1] = dir + projectName + ".project";
                processOptions(argv);
                // copy to local output dir
                new File(outputPath).mkdirs();
                String command = "cp " + localDir + output + "native.zip " + outputPath + projectName + ".zip";
                PlatformUtil.runUnixCommand(command);
                command = "cp " + localDir + output + "part-r-00000 " + outputPath + projectName
                        + ParameterProcessing.METADATA_FILE_EXT;
                PlatformUtil.runUnixCommand(command);
                command = "mv logs/stats.txt " + outputPath + projectName + ".txt";
                PlatformUtil.runUnixCommand(command);
                // place on amazon s3
                // like this, aws put freeeed.org/enron/results/enron001/enron001.zip enron001.zip
                command = "aws put freeeed.org/enron/results/" + projectName + ".zip " + outputPath + projectName
                        + ".zip";
                PlatformUtil.runUnixCommand(command);
                command = "aws put freeeed.org/enron/results/" + projectName + ".csv " + outputPath + projectName
                        + ParameterProcessing.METADATA_FILE_EXT;
                PlatformUtil.runUnixCommand(command);
                command = "aws put freeeed.org/enron/results/" + projectName + ".txt " + outputPath + projectName
                        + ".report.txt";
                PlatformUtil.runUnixCommand(command);
            } catch (Exception e) {
                e.printStackTrace(System.out);
            }
        }
    }
}