alluxio.examples.MultiMount.java Source code

Introduction

Here is the source code for alluxio.examples.MultiMount.java

Source

/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.examples;

import alluxio.AlluxioURI;
import alluxio.client.WriteType;
import alluxio.client.file.FileInStream;
import alluxio.client.file.FileOutStream;
import alluxio.client.file.FileSystem;
import alluxio.client.file.options.CreateFileOptions;

import org.apache.commons.io.IOUtils;

/**
 * Example program that demonstrates Alluxio's ability to read and write data across different
 * types of storage systems. In particular, this example reads data from S3 and writes data to HDFS.
 *
 * NOTE: This example assumes that the existence of an Alluxio cluster, the existence of an HDFS
 * cluster, and that the properties for accessing S3 (fs.s3n.awsAccessKeyId and fs.s3n
 * .awsSecretAccessKey) are set in your Alluxio configuration.
 *
 * If the above assumptions are met, you can run the example using:
 *
 * ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>
 */
public final class MultiMount {

    private MultiMount() {
    } // prevent instantiation

    /**
     * Entry point for the {@link MultiMount} program.
     *
     * @param args command-line arguments
     */
    public static void main(String[] args) {
        if (args.length != 1) {
            System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
            System.exit(-1);
        }

        AlluxioURI mntPath = new AlluxioURI("/mnt");
        AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
        AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
        AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
        AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
        AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
        AlluxioURI hdfsPath = new AlluxioURI(args[0]);
        FileSystem fileSystem = FileSystem.Factory.get();

        try {
            // Make sure mount directory exists.
            if (!fileSystem.exists(mntPath)) {
                System.out.print("creating " + mntPath + " ... ");
                fileSystem.createDirectory(mntPath);
                System.out.println("done");
            }

            // Make sure the S3 mount point does not exist.
            if (fileSystem.exists(s3Mount)) {
                System.out.print("unmounting " + s3Mount + " ... ");
                fileSystem.unmount(s3Mount);
                System.out.println("done");
            }

            // Make sure the HDFS mount point does not exist.
            if (fileSystem.exists(hdfsMount)) {
                System.out.print("unmounting " + hdfsMount + " ... ");
                fileSystem.unmount(hdfsMount);
                System.out.println("done");
            }

            // Mount S3.
            System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
            fileSystem.mount(s3Mount, s3Path);
            System.out.println("done");

            // Mount HDFS.
            System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
            fileSystem.mount(hdfsMount, hdfsPath);
            System.out.println("done");

            // Make sure output file does not exist.
            if (fileSystem.exists(outputPath)) {
                System.out.print("deleting " + outputPath + " ... ");
                fileSystem.delete(outputPath);
                System.out.println("done");
            }

            // Open the input stream.
            System.out.print("opening " + inputPath + " ... ");
            FileInStream is = fileSystem.openFile(inputPath);
            System.out.println("done");

            // Open the output stream, setting the write type to make sure result is persisted.
            System.out.print("opening " + outputPath + " ... ");
            CreateFileOptions options = CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
            FileOutStream os = fileSystem.createFile(outputPath, options);
            System.out.println("done");

            // Copy the data
            System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
            IOUtils.copy(is, os);
            System.out.println("done");

            // Close the input stream.
            System.out.print("closing " + inputPath + " ... ");
            is.close();
            System.out.println("done");

            // Close the output stream.
            System.out.print("closing " + outputPath + " ... ");
            os.close();
            System.out.println("done");
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        } finally {
            // Make sure the S3 mount point is removed.
            try {
                if (fileSystem.exists(s3Mount)) {
                    System.out.print("unmounting " + s3Mount + " ... ");
                    fileSystem.unmount(s3Mount);
                    System.out.println("done");
                }
            } catch (Exception e) {
                System.out.println("fail");
                e.printStackTrace();
            }

            // Make sure the HDFS mount point is removed.
            try {
                if (fileSystem.exists(hdfsMount)) {
                    System.out.print("unmounting " + hdfsMount + " ... ");
                    fileSystem.unmount(hdfsMount);
                    System.out.println("done");
                }
            } catch (Exception e) {
                System.out.println("fail");
                e.printStackTrace();
            }
        }
    }
}