maebackup.MaeBackup.java Source code

Java tutorial

Introduction

Here is the source code for maebackup.MaeBackup.java

Source

/*
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
    
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
    
You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
    
Before people complain, yes, I know this isn't the "proper" Java coding style.
It was originally written in C, but I switched it using Glacier. 
Amazon  doesn't offer a C library, so I adapted it to Java, and kept the C coding style.
But it works. And it's one file instead of a gazillion. So bite me. :p
 - Mae
*/

package maebackup;

import java.io.*;
import java.security.*;
import java.util.*;

import com.ice.tar.*;
import com.amazonaws.*;
import com.amazonaws.auth.*;
import com.amazonaws.services.glacier.*;
import com.amazonaws.services.glacier.model.*;
import com.amazonaws.services.glacier.transfer.*;

public class MaeBackup {
    //public static final String compresscmd = "lrzip -U -L9 -z -v ";
    public static String[] compresscmd = { "lrzip", "-U", "-L9", "-z", "-v", "%tar%" };
    public static String[] tarcmd = { "tar", "-cf", "%tar%", "--files-from", "%listfile%" };
    public static boolean useExternalTar = true;
    public static String vaultname;
    public static AWSCredentials credentials;
    public static String endpoint;
    public static int chunksize;
    public static File cachedir;

    public static void usage() {
        System.out.println("Commands:");
        System.out.println("[f]ull <name> <path>          Creates a full backup");
        System.out.println("[p]artial <name> <path>       Backup of changes since last full");
        System.out.println("[i]ncremental <name> <path>   Backup of changes since last full/partial");
        System.out.println("[u]pload <filename>           Uploads file to Glacier");
        System.out.println("[d]ownload <filename> [<job>] Starts or completes a download job");
        System.out.println("[l]ist [<job ID>]             Starts or completes an inventory fetch job");
        System.out.println("delete <archive key>          Deletes a file from Glacier");
        System.exit(1);
    }

    public static void main(String[] args) {
        if (args.length < 1 || args.length > 3)
            usage();

        String cachename = System.getenv("HOME") + "/.maebackup";
        cachedir = new File(cachename);
        if (!cachedir.exists())
            cachedir.mkdir();

        try {
            Properties props = new Properties();
            props.load(new FileReader(new File(cachedir, "maebackup.properties")));
            vaultname = props.getProperty("vaultname").trim();
            credentials = new BasicAWSCredentials(props.getProperty("awspublic").trim(),
                    props.getProperty("awssecret").trim());
            chunksize = Integer.parseInt(props.getProperty("chunksize", "1048576").trim());
            endpoint = props.getProperty("endpoint", "https://glacier.us-east-1.amazonaws.com/").trim();
        } catch (Exception e) {
            System.err.println(cachename + "/maebackup.properties not found or could not be read.");
            System.exit(1);
        }

        switch (args[0]) {
        case "f":
        case "full": {
            if (args.length != 3)
                usage();
            String name = args[1];
            String tarname = name + "-" + String.format("%tF", new Date());

            LinkedList<File> files = findFiles(args[2]);

            File tar = new File(tarname + ".f.tar");
            LinkedHashMap<File, String> hashes;
            if (useExternalTar) {
                hashes = hashFiles(files);
                archiveExternal(files, tar);
            } else {
                hashes = archiveAndHash(files, tar);
            }

            String lrz = compress(tar);
            upload(lrz);

            writeHashes(hashes, new File(cachedir, name + ".f.sha256"));
            File partFile = new File(cachedir, name + ".p.sha256");
            if (partFile.exists())
                partFile.delete();
        }
            break;

        case "p":
        case "partial": {
            if (args.length != 3)
                usage();
            String name = args[1];
            String tarname = name + "-" + String.format("%tF", new Date());

            LinkedHashMap<File, String> fullHashes = loadHashes(new File(cachedir, name + ".f.sha256"));

            LinkedHashMap<File, String> hashes = hashFiles(findFiles(args[2]));

            LinkedList<File> newFiles = hashDiff(fullHashes, hashes);

            File tar = new File(tarname + ".p.tar");
            if (useExternalTar) {
                archiveExternal(newFiles, tar);
            } else {
                archiveFiles(newFiles, tar);
            }
            String lrz = compress(tar);
            upload(lrz);

            writeHashes(hashes, new File(cachedir, name + ".p.sha256"));
        }
            break;

        case "i":
        case "incremental": {
            if (args.length != 3)
                usage();
            String name = args[1];
            String tarname = name + "-" + String.format("%tF", new Date());

            LinkedHashMap<File, String> fullHashes = loadHashes(new File(cachedir, name + ".f.sha256"));
            LinkedHashMap<File, String> partHashes = loadHashes(new File(cachedir, name + ".p.sha256"));
            fullHashes.putAll(partHashes);

            LinkedHashMap<File, String> hashes = hashFiles(findFiles(args[2]));

            LinkedList<File> newFiles = hashDiff(fullHashes, hashes);

            File tar = new File(tarname + ".i.tar");
            if (useExternalTar) {
                archiveExternal(newFiles, tar);
            } else {
                archiveFiles(newFiles, tar);
            }
            String lrz = compress(tar);
            upload(lrz);
        }
            break;

        case "u":
        case "upload": {
            if (args.length != 2)
                usage();
            upload(args[1]);
        }
            break;

        case "d":
        case "download": {
            if (args.length == 2)
                download(args[1], null);
            else if (args.length == 3)
                download(args[1], args[2]);
            else
                usage();
        }

        case "delete": {
            if (args.length != 2)
                usage();
            delete(args[1]);
        }
            break;

        case "l":
        case "list": {
            if (args.length == 1)
                list(null);
            else if (args.length == 2)
                list(args[1]);
            else
                usage();
        }
            break;

        default:
            usage();
        }
    }

    public static LinkedList<File> findFiles(String pathName) {
        File path = new File(pathName);
        LinkedList<File> files = new LinkedList<File>();
        return findFiles(path, files);
    }

    public static LinkedList<File> findFiles(File path, LinkedList<File> fileList) {
        System.out.println("Searching: " + path);
        File[] files = path.listFiles();
        for (File file : files) {
            if (file.isDirectory()) {
                findFiles(file, fileList);
            } else if (file.isFile()) {
                fileList.add(file);
            }
        }
        return fileList;
    }

    public static LinkedHashMap<File, String> hashFiles(Collection<File> files) {
        LinkedHashMap<File, String> hashes = new LinkedHashMap<File, String>();

        int count = files.size();
        System.out.println("Hashing " + count + " files");

        int num = 0;
        for (File file : files) {
            if (num++ % (count / 10) == 0) {
                System.out.printf("  %d/%d\n", num, count);
            }
            try {
                MessageDigest md = MessageDigest.getInstance("SHA-256");

                FileInputStream fis = new FileInputStream(file);
                byte[] buffer = new byte[65536];
                int bytes;

                while ((bytes = fis.read(buffer)) > 0) {
                    md.update(buffer, 0, bytes);
                }

                fis.close();

                byte[] hash = md.digest();

                StringBuffer sb = new StringBuffer();
                for (byte b : hash) {
                    sb.append(String.format("%02x", b));
                }

                hashes.put(file, sb.toString());
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        return hashes;
    }

    public static LinkedHashMap<File, String> loadHashes(File file) {
        LinkedHashMap<File, String> hashes = new LinkedHashMap<File, String>();

        System.out.println("Loading data from " + file);
        try {
            BufferedReader br = new BufferedReader(new FileReader(file));
            String line;
            while ((line = br.readLine()) != null) {
                String[] parts = line.split(" ", 2);
                if (parts.length != 2)
                    continue;
                hashes.put(new File(parts[1]), parts[0]);
            }
            br.close();
        } catch (Exception e) {
            e.printStackTrace();
        }

        return hashes;
    }

    public static void writeHashes(LinkedHashMap<File, String> hashes, File file) {
        writeHashes(file, hashes);
    }

    public static void writeHashes(File file, LinkedHashMap<File, String> hashes) {
        System.out.println("Writing data to " + file);
        try {
            PrintWriter pw = new PrintWriter(file);
            for (Map.Entry<File, String> entry : hashes.entrySet()) {
                pw.println(entry.getValue() + " " + entry.getKey());
            }
            pw.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static LinkedList<File> hashDiff(LinkedHashMap<File, String> old, LinkedHashMap<File, String> cur) {
        LinkedList<File> diff = new LinkedList<File>();

        for (File f : cur.keySet()) {
            if (!cur.get(f).equalsIgnoreCase(old.get(f))) {
                diff.add(f);
            }
        }

        return diff;
    }

    public static void archiveExternal(Collection<File> files, File archive) {
        try {
            System.out.println("Creating tar file " + archive);

            BufferedWriter bw = new BufferedWriter(new FileWriter("files.lst"));
            for (File file : files) {
                bw.write(file.toString());
                bw.newLine();
            }
            bw.close();

            tarcmd[2] = archive.toString();
            tarcmd[4] = "files.lst";
            Process tar = new ProcessBuilder(tarcmd).redirectErrorStream(true).start();

            InputStream is = tar.getInputStream();
            int len;
            byte[] data = new byte[1024];
            while ((len = is.read(data)) != -1) {
                System.out.write(data, 0, len);
                System.out.flush();
            }
            tar.waitFor();

        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static void archiveFiles(Collection<File> files, File archive) {
        try {
            TarOutputStream tar = new TarOutputStream(new FileOutputStream(archive));
            byte[] buffer = new byte[65536];
            int bytes;
            int count = files.size();
            int num = 0;

            System.out.println("Creating tar file " + archive);
            for (File f : files) {
                if (num++ % (count / 10) == 0) {
                    System.out.printf("  %d/%d\n", num, count);
                }
                FileInputStream fis = new FileInputStream(f);
                TarEntry entry = new TarEntry(f);
                tar.putNextEntry(entry);
                while ((bytes = fis.read(buffer)) > 0) {
                    tar.write(buffer, 0, bytes);
                }
                tar.closeEntry();
                fis.close();
            }

            tar.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static LinkedHashMap<File, String> archiveAndHash(Collection<File> files, File archive) {
        LinkedHashMap<File, String> hashes = new LinkedHashMap<File, String>();
        try {
            TarOutputStream tar = new TarOutputStream(new FileOutputStream(archive));
            byte[] buffer = new byte[65536];
            int bytes;
            int count = files.size();
            int num = 0;

            System.out.println("Hashing and creating tar file " + archive);
            for (File f : files) {
                if (num++ % (count / 10) == 0) {
                    System.out.printf("  %d/%d\n", num, count);
                }
                MessageDigest md = MessageDigest.getInstance("SHA-256");

                FileInputStream fis = new FileInputStream(f);
                TarEntry entry = new TarEntry(f);
                tar.putNextEntry(entry);
                while ((bytes = fis.read(buffer)) > 0) {
                    md.update(buffer, 0, bytes);
                    tar.write(buffer, 0, bytes);
                }
                tar.closeEntry();
                fis.close();

                byte[] hash = md.digest();

                StringBuffer sb = new StringBuffer();
                for (byte b : hash) {
                    sb.append(String.format("%02x", b));
                }
                hashes.put(f, sb.toString());
            }

            tar.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return hashes;
    }

    public static String compress(File archive) {
        try {
            String lrzname = archive.toString() + ".lrz";
            compresscmd[compresscmd.length - 1] = archive.toString();
            System.out.println("Compressing " + archive + " to " + lrzname);
            Process lrz = new ProcessBuilder(compresscmd).redirectErrorStream(true).start();

            InputStream is = lrz.getInputStream();
            int len;
            byte[] data = new byte[1024];
            while ((len = is.read(data)) != -1) {
                System.out.write(data, 0, len);
                System.out.flush();
            }
            lrz.waitFor();
            return lrzname;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static void upload(String lrzname) {
        try {
            System.out.println("Uploading to Glacier...");
            ClientConfiguration config = new ClientConfiguration();
            config.setProtocol(Protocol.HTTPS);
            AmazonGlacierClient client = new AmazonGlacierClient(credentials, config);
            client.setEndpoint(endpoint);

            File file = new File(lrzname);
            String archiveid = "";
            if (file.length() < 5 * 1024 * 1024) {
                System.out.println("File is small, uploading as single chunk");
                String treehash = TreeHashGenerator.calculateTreeHash(file);

                InputStream is = new FileInputStream(file);
                byte[] buffer = new byte[(int) file.length()];
                int bytes = is.read(buffer);
                if (bytes != file.length())
                    throw new RuntimeException("Only read " + bytes + " of " + file.length()
                            + " byte file when preparing for upload.");
                InputStream bais = new ByteArrayInputStream(buffer);

                UploadArchiveRequest request = new UploadArchiveRequest(vaultname, lrzname, treehash, bais);
                UploadArchiveResult result = client.uploadArchive(request);
                archiveid = result.getArchiveId();
            } else {
                long chunks = file.length() / chunksize;
                while (chunks > 10000) {
                    chunksize <<= 1;
                    chunks = file.length() / chunksize;
                }
                String chunksizestr = new Integer(chunksize).toString();
                System.out.println(
                        "Starting multipart upload: " + chunks + " full chunks of " + chunksizestr + " bytes");

                InitiateMultipartUploadResult imures = client.initiateMultipartUpload(
                        new InitiateMultipartUploadRequest(vaultname, lrzname, chunksizestr));

                String uploadid = imures.getUploadId();
                RandomAccessFile raf = new RandomAccessFile(file, "r");

                byte[] buffer = new byte[chunksize];

                for (long x = 0; x < chunks; x++) {
                    try {
                        System.out.println("Uploading chunk " + x + "/" + chunks);

                        raf.seek(x * chunksize);
                        raf.read(buffer);

                        String parthash = TreeHashGenerator.calculateTreeHash(new ByteArrayInputStream(buffer));
                        String range = "bytes " + (x * chunksize) + "-" + ((x + 1) * chunksize - 1) + "/*";

                        client.uploadMultipartPart(new UploadMultipartPartRequest(vaultname, uploadid, parthash,
                                range, new ByteArrayInputStream(buffer)));
                    } catch (Exception e) {
                        e.printStackTrace();
                        System.err.println("Error uploading chunk " + x + ", retrying...");
                        x--;
                    }
                }

                if (file.length() > chunks * chunksize) {
                    do {
                        try {
                            System.out.println("Uploading final partial chunk");
                            raf.seek(chunks * chunksize);
                            int bytes = raf.read(buffer);

                            String parthash = TreeHashGenerator
                                    .calculateTreeHash(new ByteArrayInputStream(buffer, 0, bytes));
                            String range = "bytes " + (chunks * chunksize) + "-" + (file.length() - 1) + "/*";

                            client.uploadMultipartPart(new UploadMultipartPartRequest(vaultname, uploadid, parthash,
                                    range, new ByteArrayInputStream(buffer, 0, bytes)));
                        } catch (Exception e) {
                            e.printStackTrace();
                            System.err.println("Error uploading final chunk, retrying...");
                            continue;
                        }
                    } while (false);
                }

                System.out.println("Completing upload");
                String treehash = TreeHashGenerator.calculateTreeHash(file);
                CompleteMultipartUploadResult result = client
                        .completeMultipartUpload(new CompleteMultipartUploadRequest(vaultname, uploadid,
                                new Long(file.length()).toString(), treehash));
                archiveid = result.getArchiveId();
            }

            System.out.println("Uploaded " + lrzname + " to Glacier as ID " + archiveid);

            File listfile = new File(cachedir, "archives.lst");
            FileWriter fw = new FileWriter(listfile, true);
            fw.write(archiveid + " " + lrzname + "\n");
            fw.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static void download(String filename, String jobid) {
        try {
            System.out.println("Starting download...");
            ClientConfiguration config = new ClientConfiguration();
            config.setProtocol(Protocol.HTTPS);
            AmazonGlacierClient client = new AmazonGlacierClient(credentials, config);
            client.setEndpoint(endpoint);

            if (jobid == null || jobid == "") {
                String archiveid;
                // Yes, this will screw up on actual 138-character file names, but... yeah.
                if (filename.length() == 138) {
                    archiveid = filename;
                } else {
                    File listfile = new File(cachedir, "archives.lst");
                    Map<File, String> filemap = loadHashes(listfile);
                    archiveid = filemap.get(filename);
                    if (archiveid == null) {
                        System.err.println("Error: Could not find archive ID for file " + filename);
                        System.exit(1);
                        return;
                    }
                }

                InitiateJobResult result = client.initiateJob(new InitiateJobRequest(vaultname,
                        new JobParameters().withType("archive-retrieval").withArchiveId(archiveid)));
                jobid = result.getJobId();
                System.out.println("Started download job as ID " + jobid);
            } else {
                DescribeJobResult djres = client.describeJob(new DescribeJobRequest(vaultname, jobid));
                if (!djres.getStatusCode().equals("Succeeded")) {
                    System.out.println("Job is not listed as Succeeded. It is: " + djres.getStatusCode());
                    System.out.println(djres.getStatusMessage());
                    System.exit(2);
                }
                long size = djres.getArchiveSizeInBytes();
                long chunks = size / chunksize;
                while (chunks > 10000) {
                    chunksize <<= 1;
                    chunks = size / chunksize;
                }
                RandomAccessFile raf = new RandomAccessFile(filename, "w");
                raf.setLength(size);
                byte[] buffer = new byte[chunksize];

                for (int x = 0; x < chunks; x++) {
                    try {
                        System.out.println("Downloading chunk " + x + " of " + chunks);
                        String range = "bytes " + (x * chunksize) + "-" + ((x + 1) * chunksize - 1) + "/*";

                        GetJobOutputResult gjores = client
                                .getJobOutput(new GetJobOutputRequest(vaultname, jobid, range));

                        gjores.getBody().read(buffer);

                        MessageDigest md = MessageDigest.getInstance("SHA-256");
                        md.update(buffer, 0, chunksize);

                        byte[] hash = md.digest();

                        StringBuffer sb = new StringBuffer();
                        for (byte b : hash) {
                            sb.append(String.format("%02x", b));
                        }
                        if (!sb.toString().equalsIgnoreCase(gjores.getChecksum())) {
                            System.err.println("Error: Chunk " + x + " does not match SHA-256. Retrying.");
                            x--;
                            continue;
                        }

                        raf.seek(x * chunksize);
                        raf.write(buffer);
                    } catch (Exception e) {
                        System.err.println("Error: Exception while downloading chunk " + x + ". Retrying.");
                        x--;
                    }
                }

                if (size > chunks * chunksize) {
                    do {
                        try {
                            System.out.println("Downloading final partial chunk");
                            String range = "bytes " + (chunks * chunksize) + "-" + (size - 1) + "/*";

                            GetJobOutputResult gjores = client
                                    .getJobOutput(new GetJobOutputRequest(vaultname, jobid, range));

                            int bytes = gjores.getBody().read(buffer);

                            MessageDigest md = MessageDigest.getInstance("SHA-256");
                            md.update(buffer, 0, bytes);

                            byte[] hash = md.digest();

                            StringBuffer sb = new StringBuffer();
                            for (byte b : hash) {
                                sb.append(String.format("%02x", b));
                            }
                            if (!sb.toString().equalsIgnoreCase(gjores.getChecksum())) {
                                System.err.println("Error: Final chunk does not match SHA-256. Retrying.");
                                continue;
                            }

                            raf.seek(chunks * chunksize);
                            raf.write(buffer, 0, bytes);
                        } catch (Exception e) {
                            System.err.println("Error: Exception while downloading final chunk. Retrying.");
                            continue;
                        }
                    } while (false);
                }
                raf.close();

                String treehash = TreeHashGenerator.calculateTreeHash(new File(filename));
                if (!treehash.equalsIgnoreCase(djres.getSHA256TreeHash())) {
                    System.err.println("Error: File failed final tree hash check.");
                    System.exit(3);
                }

                System.out.println("Download complete.");
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static void delete(String archive) {
        try {
            System.out.println("Deleting from Glacier...");
            ClientConfiguration config = new ClientConfiguration();
            config.setProtocol(Protocol.HTTPS);
            AmazonGlacierClient client = new AmazonGlacierClient(credentials, config);
            client.setEndpoint(endpoint);
            client.deleteArchive(new DeleteArchiveRequest(vaultname, archive));
            System.out.println("Archive deleted.");
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static void list(String arg) {
        try {
            System.out.println("Listing Glacier vault...");
            ClientConfiguration config = new ClientConfiguration();
            config.setProtocol(Protocol.HTTPS);
            AmazonGlacierClient client = new AmazonGlacierClient(credentials, config);
            client.setEndpoint(endpoint);

            if (arg == null || arg == "") {
                InitiateJobResult result = client.initiateJob(
                        new InitiateJobRequest(vaultname, new JobParameters().withType("inventory-retrieval")));
                String jobid = result.getJobId();
                System.out.println("Started inventory retrival job as ID " + jobid);
            } else {
                DescribeJobResult djres = client.describeJob(new DescribeJobRequest(vaultname, arg));
                if (!djres.getStatusCode().equals("Succeeded")) {
                    System.out.println("Job is not listed as Succeeded. It is: " + djres.getStatusCode());
                    System.out.println(djres.getStatusMessage());
                    System.exit(2);
                }

                GetJobOutputResult gjores = client
                        .getJobOutput(new GetJobOutputRequest().withVaultName(vaultname).withJobId(arg));
                byte[] buffer = new byte[1024];
                int bytes;
                while ((bytes = gjores.getBody().read(buffer)) > 0) {
                    System.out.write(buffer, 0, bytes);
                }
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

// EOF