com.simple.sftpfetch.App.java Source code

Java tutorial

Introduction

Here is the source code for com.simple.sftpfetch.App.java

Source

/*
 * Copyright (c) 2012 Simple Finance Technology Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.simple.sftpfetch;

import com.amazonaws.services.s3.AmazonS3Client;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.SftpException;
import com.rabbitmq.client.ConnectionFactory;
import com.simple.sftpfetch.decrypt.FileDecrypter;
import com.simple.sftpfetch.decrypt.NoopDecrypter;
import com.simple.sftpfetch.decrypt.PGPFileDecrypter;
import com.simple.sftpfetch.publish.RabbitClient;
import com.simple.sftpfetch.publish.RabbitConnectionInfo;
import com.simple.sftpfetch.publish.S3;
import com.simple.sftpfetch.publish.SuppliedAWSCredentials;
import com.simple.sftpfetch.sftp.SftpClient;
import com.simple.sftpfetch.sftp.SftpConnectionInfo;
import org.apache.commons.cli.*;

import java.io.*;
import java.security.NoSuchProviderException;
import java.security.Security;
import java.util.List;
import java.util.Properties;
import java.util.regex.Pattern;

import static java.util.Arrays.asList;

/**
 * Main application entry point
 */
public class App {
    public static final String FETCH_DAYS = "fetch.days";
    public static final Pattern MATCH_EVERYTHING = Pattern.compile(".*");
    private SftpClient sftp;
    private S3 s3;
    private RabbitClient rabbit;
    private FileDecrypter decrypter;
    private PrintStream out;

    public App(SftpClient sftpClient, S3 s3, RabbitClient rabbitClient, FileDecrypter decrypter, PrintStream out) {
        this.sftp = sftpClient;
        this.s3 = s3;
        this.rabbit = rabbitClient;
        this.decrypter = decrypter;
        this.out = out;
    }

    private static S3 s3FromProperties(Properties properties) {
        String s3Bucket = properties.getProperty("s3.bucket");
        final String awsAccessKey = properties.getProperty("s3.access.key", "");
        final String awsSecretKey = properties.getProperty("s3.secret.key", "");

        AmazonS3Client client;
        if (awsAccessKey.isEmpty() || awsSecretKey.isEmpty()) {
            client = new AmazonS3Client();
        } else {
            client = new AmazonS3Client(new SuppliedAWSCredentials(awsAccessKey, awsSecretKey));
        }

        return new S3(client, s3Bucket);
    }

    /**
     * A simpler entry point with sane defaults, used by the tests
     *
     * @param routingKey the routing key to use for publishing rabbit messages
     * @param daysToFetch the number of days back to fetch from SFTP
     *
     * @throws IOException
     * @throws NoSuchProviderException
     * @throws SftpException
     */
    void run(String routingKey, int daysToFetch) throws IOException, NoSuchProviderException, SftpException {
        run(routingKey, daysToFetch, App.MATCH_EVERYTHING, false, false);
    }

    /**
     * Run the application
     *
     * @param routingKey the routing key to use for publishing rabbit messages
     * @param daysToFetch the number of days back to fetch from SFTP
     * @param pattern a regular expression pattern that files must match to be processed
     * @param noop if true do not actually modify anything
     * @param overwrite re-publish previously seen files
     *
     * @throws SftpException
     * @throws IOException
     * @throws NoSuchProviderException
     */
    public void run(String routingKey, int daysToFetch, Pattern pattern, boolean noop, boolean overwrite)
            throws SftpException, IOException, NoSuchProviderException {
        for (String filename : sftp.getFilesNewerThan(daysToFetch, pattern)) {
            if (s3.keyExists(filename)) {
                out.println("Previously seen: " + filename);
                if (!overwrite) {
                    continue;
                }
            }
            if (noop) {
                out.println("Would process: " + filename);
            } else {
                File downloaded = sftp.downloadFile(filename);
                File toUpload = decrypter.decryptFile(downloaded);
                s3.upload(filename, toUpload);
                rabbit.publishURL(routingKey, s3.getURLFor(filename));
                out.println("Processed: " + filename);
            }
        }
    }

    public static void main(String[] args) throws Exception {
        Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());

        Options options = getOptions();

        List<String> requiredProperties = asList("c");

        CommandLineParser parser = new PosixParser();
        try {
            CommandLine commandLine = parser.parse(options, args);
            if (commandLine.hasOption("h")) {
                printUsage(options);
                System.exit(0);
            }

            for (String opt : requiredProperties) {
                if (!commandLine.hasOption(opt)) {
                    System.err.println("The option: " + opt + " is required.");
                    printUsage(options);
                    System.exit(1);
                }
            }

            Pattern pattern;
            if (commandLine.hasOption("p")) {
                pattern = Pattern.compile(commandLine.getOptionValue("p"));
            } else {
                pattern = MATCH_EVERYTHING;
            }

            String filename = commandLine.getOptionValue("c");
            Properties properties = new Properties();
            try {
                InputStream stream = new FileInputStream(new File(filename));
                properties.load(stream);
            } catch (IOException ioe) {
                System.err.println("Unable to read properties from: " + filename);
                System.exit(2);
            }

            String routingKey = "";
            if (commandLine.hasOption("r")) {
                routingKey = commandLine.getOptionValue("r");
            } else if (properties.containsKey("rabbit.routingkey")) {
                routingKey = properties.getProperty("rabbit.routingkey");
            }

            int daysToFetch;
            if (commandLine.hasOption("d")) {
                daysToFetch = Integer.valueOf(commandLine.getOptionValue("d"));
            } else {
                daysToFetch = Integer.valueOf(properties.getProperty(FETCH_DAYS));
            }

            FileDecrypter decrypter = null;
            if (properties.containsKey("decryption.key.path")) {
                decrypter = new PGPFileDecrypter(new File(properties.getProperty("decryption.key.path")));
            } else {
                decrypter = new NoopDecrypter();
            }

            SftpClient sftpClient = new SftpClient(new JSch(), new SftpConnectionInfo(properties));
            try {
                App app = new App(sftpClient, s3FromProperties(properties),
                        new RabbitClient(new ConnectionFactory(), new RabbitConnectionInfo(properties)), decrypter,
                        System.out);
                app.run(routingKey, daysToFetch, pattern, commandLine.hasOption("n"), commandLine.hasOption("o"));
            } finally {
                sftpClient.close();
            }
            System.exit(0);
        } catch (UnrecognizedOptionException uoe) {
            System.err.println(uoe.getMessage());
            printUsage(options);
            System.exit(10);
        }
    }

    public static Options getOptions() {
        Options options = new Options();
        options.addOption("n", "noop", false,
                "Don't download or publish anything, simply mention what would be done");
        options.addOption("o", "overwrite", false, "Re-publish previously seen files");
        options.addOption("p", "pattern", true, "Only download files that match this pattern.");
        options.addOption("r", "routing-key", true, "Routing key for posting messages");
        options.addOption("c", "config", true, "Properties file containing configuration options");
        options.addOption("d", "days", true, "Download files newer than this many days ago");
        options.addOption("h", "help", false, "Show this screen");
        return options;
    }

    public static void printUsage(Options options) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("java -jar sftp-fetch.jar [options]", options);
    }
}