com.google.cloud.hadoop.gcsio.GoogleCloudStorageIntegrationTest.java Source code

Java tutorial

Introduction

Here is the source code for com.google.cloud.hadoop.gcsio.GoogleCloudStorageIntegrationTest.java

Source

/**
 * Copyright 2014 Google Inc. All Rights Reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.cloud.hadoop.gcsio;

import com.google.api.client.auth.oauth2.Credential;
import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemBase;
import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper;
import com.google.cloud.hadoop.util.CredentialFactory;
import com.google.cloud.hadoop.util.LogUtil;
import com.google.common.base.Strings;

import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.joda.time.Instant;
import org.junit.Assert;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

/**
 * Integration tests for GoogleCloudStorage class.
 */
public abstract class GoogleCloudStorageIntegrationTest {
    // Logger.
    protected static LogUtil log = new LogUtil(GoogleCloudStorageIntegrationTest.class);

    // GCS access instance.
    protected static GoogleCloudStorage gcs;

    // GCS test instance. Allows calling helper methods polymorphically.
    protected static GoogleCloudStorageIntegrationTest gcsit;

    // Application name for OAuth.
    static final String APP_NAME = "GCS-test";

    // Name of environment variables from which to get GCS access info.
    public static final String GCS_TEST_PROJECT_ID = "GCS_TEST_PROJECT_ID";
    public static final String GCS_TEST_CLIENT_ID = "GCS_TEST_CLIENT_ID";
    public static final String GCS_TEST_CLIENT_SECRET = "GCS_TEST_CLIENT_SECRET";

    // Prefix used for naming test buckets.
    public static final String TEST_BUCKET_NAME_PREFIX = "gcsio-test-";

    // Regex pattern to match UUID string.
    public static final String UUID_PATTERN = "\\p{XDigit}{8}+-\\p{XDigit}{4}+-\\p{XDigit}{4}+-\\p{XDigit}{4}+-\\p{XDigit}{12}+";

    // Regex that matches a test bucket name.
    public static final String TEST_BUCKET_NAME_PATTERN = "^" + TEST_BUCKET_NAME_PREFIX + UUID_PATTERN + ".*";

    // Name of test buckets.
    public static String bucketName;
    public static String otherBucketName;

    // Time when test started. Used for determining which objects got
    // created after the test started.
    public static Instant testStartTime;

    // Name of the test object.
    protected static String objectName = "gcsio-test.txt";

    // List of buckets to delete during post-test cleanup.
    private static List<String> bucketsToDelete = new ArrayList<>();

    private static String projectId;
    private static Credential credential;

    /**
     * Perform initialization once before tests are run.
     */
    public static void beforeAllTests() throws IOException {
        if (gcs == null) {
            projectId = TestConfiguration.getInstance().getProjectId();
            Assert.assertNotNull(projectId);
            credential = GoogleCloudStorageTestHelper.getCredential();
            GoogleCloudStorageOptions.Builder optionsBuilder = GoogleCloudStorageOptions.newBuilder();
            optionsBuilder.setAppName(APP_NAME).setProjectId(projectId).getWriteChannelOptionsBuilder()
                    .setFileSizeLimitedTo250Gb(GoogleHadoopFileSystemBase.GCS_FILE_SIZE_LIMIT_250GB_DEFAULT)
                    .setUploadBufferSize(8 * 1024 * 1024); // 8MB upload buffer size for testing.

            gcs = new GoogleCloudStorageImpl(optionsBuilder.build(), credential);
            gcsit = new GoogleCloudStorageIntegrationTest() {
                // TODO(user): Remove with gcsit global variable.
                @Override
                protected SeekableReadableByteChannel open(String bucketName, String objectName)
                        throws IOException {
                    throw new IllegalStateException("Not implemented");
                }

                @Override
                protected WritableByteChannel create(String bucketName, String objectName,
                        CreateFileOptions options) throws IOException {
                    throw new IllegalStateException("Not implemented");
                }

                @Override
                protected void mkdir(String bucketName, String objectName) throws IOException {
                    throw new IllegalStateException("Not implemented");
                }

                @Override
                protected void mkdir(String bucketName) throws IOException {
                    throw new IllegalStateException("Not implemented");
                }

                @Override
                protected void delete(String bucketName) throws IOException {
                    throw new IllegalStateException("Not implemented");
                }

                @Override
                protected void delete(String bucketName, String objectName) throws IOException {
                    throw new IllegalStateException("Not implemented");
                }

                @Override
                protected void clearBucket(String bucketName) throws IOException {
                    throw new IllegalStateException("Not implemented");
                }
            };
            postCreateInit();
        }
    }

    /**
     * Perform initialization after creating test instances.
     */
    public static void postCreateInit() throws IOException {

        // Un-comment the following to enable logging during test run.
        //enableLogging();

        testStartTime = Instant.now();

        // Create a couple of buckets. The first one is used by most tests.
        // The second one is used by some tests (eg, copy()).
        // TODO(user): bucketName and otherBucketName are too generic names.
        // They should be renamed to testBucketName and otherTestBucketName.
        bucketName = createTempBucket();
        otherBucketName = createTempBucket();
    }

    /**
     * Perform clean-up once after all tests are turn.
     */
    public static void afterAllTests() throws IOException {

        // Delete buckets created during test.
        deleteBuckets();

        if (gcs != null) {
            gcs.close();
            gcs = null;
        }
    }

    /**
     * Writes a file with the give text at the given path.
     *
     * Note: This method takes non-trivial amount of time to complete.
     * If you use it in a test where multiple operations on the same file are performed,
     * then it is better to create the file once and perform multiple operations in
     * the same test rather than have multiple tests each creating its own test file.
     *
     * @param bucketName name of the bucket to create object in
     * @param objectName name of the object to create
     * @param text file contents
     * @return number of bytes written
     */
    protected int writeTextFile(String bucketName, String objectName, String text) throws IOException {
        int numBytesWritten = -1;
        byte[] textBytes = text.getBytes("UTF-8");
        ByteBuffer writeBuffer = ByteBuffer.wrap(textBytes);
        numBytesWritten = writeFile(bucketName, objectName, writeBuffer, 1);
        return numBytesWritten;
    }

    /**
     * Writes a file with the given buffer repeated numWrites times.
     *
     * @param bucketName name of the bucket to create object in
     * @param objectName name of the object to create
     * @param buffer Data to write
     * @param numWrites number of times to repeat the data
     * @return number of bytes written
     */
    protected int writeFile(String bucketName, String objectName, ByteBuffer buffer, int numWrites)
            throws IOException {
        int numBytesWritten = -1;
        int totalBytesWritten = 0;
        WritableByteChannel writeChannel = null;

        try {
            writeChannel = create(bucketName, objectName, new CreateFileOptions(false /* overwrite existing */));
            for (int i = 0; i < numWrites; i++) {
                buffer.clear();
                numBytesWritten = writeChannel.write(buffer);
                Assert.assertEquals("could not write the entire buffer", buffer.capacity(), numBytesWritten);
                totalBytesWritten += numBytesWritten;
            }

        } finally {
            if (writeChannel != null) {
                writeChannel.close();
            }
        }

        return totalBytesWritten;
    }

    /**
     * Helper which reads the entire file as a String.
     */
    protected String readTextFile(String bucketName, String objectName) throws IOException {
        SeekableReadableByteChannel readChannel = null;
        ByteBuffer readBuffer = ByteBuffer.allocate(1024);
        StringBuffer returnBuffer = new StringBuffer();

        try {
            readChannel = open(bucketName, objectName);
            int numBytesRead = readChannel.read(readBuffer);
            while (numBytesRead > 0) {
                readBuffer.flip();
                returnBuffer.append(StandardCharsets.UTF_8.decode(readBuffer));
                readBuffer.clear();
                numBytesRead = readChannel.read(readBuffer);
            }
        } finally {
            if (readChannel != null) {
                readChannel.close();
            }
        }
        return returnBuffer.toString();
    }

    /**
     * Helper that reads text from the given file at the given offset
     * and returns it. If checkOverflow is true, it will make sure that
     * no more than 'len' bytes were read.
     */
    protected String readTextFile(String bucketName, String objectName, int offset, int len, boolean checkOverflow)
            throws IOException {
        String text = null;
        SeekableReadableByteChannel readChannel = null;

        try {
            int bufferSize = len;
            bufferSize += checkOverflow ? 1 : 0;
            ByteBuffer readBuffer = ByteBuffer.allocate(bufferSize);
            readChannel = open(bucketName, objectName);
            if (offset > 0) {
                readChannel.position(offset);
            }
            int numBytesRead = readChannel.read(readBuffer);
            Assert.assertEquals("readTextFile: read size mismatch", len, numBytesRead);
            readBuffer.flip();
            text = StandardCharsets.UTF_8.decode(readBuffer).toString();
        } finally {
            if (readChannel != null) {
                readChannel.close();
            }
        }

        return text;
    }

    /**
     * Opens the given object for reading.
     */
    protected abstract SeekableReadableByteChannel open(String bucketName, String objectName) throws IOException;

    /**
     * Opens the given object for writing.
     */
    protected WritableByteChannel create(String bucketName, String objectName) throws IOException {
        return create(bucketName, objectName, CreateFileOptions.DEFAULT);
    }

    /**
     * Opens the given object for writing.
     */
    protected abstract WritableByteChannel create(String bucketName, String objectName, CreateFileOptions options)
            throws IOException;

    /**
     * Creates a directory like object.
     */
    protected abstract void mkdir(String bucketName, String objectName) throws IOException;

    /**
     * Creates the given bucket.
     */
    protected abstract void mkdir(String bucketName) throws IOException;

    /**
     * Deletes the given bucket.
     */
    protected abstract void delete(String bucketName) throws IOException;

    /**
     * Deletes the given object.
     */
    protected abstract void delete(String bucketName, String objectName) throws IOException;

    /**
     * Deletes all objects from the given bucket.
     */
    protected abstract void clearBucket(String bucketName) throws IOException;

    // -----------------------------------------------------------------
    // Misc helpers
    // -----------------------------------------------------------------

    /**
     * Gets the expected size of a test object. Subclasses are allowed to return Long.MIN_VALUE to
     * denote "don't care" or "don't know".
     *
     * This function assumes a certain behavior when we create test objects.
     * See {@link createObjects()} for details.
     */
    protected long getExpectedObjectSize(String objectName, boolean expectedToExist)
            throws UnsupportedEncodingException {
        // Determine the expected size.
        long expectedSize;
        if (expectedToExist) {
            if (Strings.isNullOrEmpty(objectName) || objectName.endsWith(GoogleCloudStorage.PATH_DELIMITER)) {
                expectedSize = 0;
            } else {
                expectedSize = objectName.getBytes("UTF-8").length;
            }
        } else {
            expectedSize = -1;
        }
        return expectedSize;
    }

    /**
     * Indicates whether the given object name looks like a directory path.
     *
     * @param objectName name of the object to inspect
     * @return whether the given object name looks like a directory path
     */
    public static boolean objectHasDirectoryPath(String objectName) {
        return FileInfo.objectHasDirectoryPath(objectName);
    }

    /**
     * Creates objects in the given bucket.
     * For objects whose name looks like a path (foo/bar/zoo), creates
     * objects for intermediate sub-paths.
     * for example,
     * foo/bar/zoo => creates: foo/, foo/bar/, foo/bar/zoo.
     */
    public void createObjectsWithSubdirs(String bucketName, String[] objectNames) throws IOException {
        List<String> allNames = new ArrayList<>();
        List<String> subdirs;
        Set<String> created = new HashSet<>();
        for (String objectName : objectNames) {
            subdirs = getSubdirs(objectName);
            for (String subdir : subdirs) {
                if (!created.contains(subdir)) {
                    created.add(subdir);
                    allNames.add(subdir);
                }
            }

            if (!created.contains(objectName)) {
                allNames.add(objectName);
            }
        }

        createObjects(bucketName, allNames.toArray(new String[0]));
    }

    /**
     * For objects whose name looks like a path (foo/bar/zoo),
     * returns intermediate sub-paths.
     * for example,
     * foo/bar/zoo => returns: (foo/, foo/bar/)
     * foo => returns: ()
     */
    private List<String> getSubdirs(String objectName) {
        List<String> subdirs = new ArrayList<>();
        // Create a list of all subdirs.
        // for example,
        // foo/bar/zoo => (foo/, foo/bar/)
        int currentIndex = 0;
        while (currentIndex < objectName.length()) {
            int index = objectName.indexOf('/', currentIndex);
            if (index < 0) {
                break;
            }
            subdirs.add(objectName.substring(0, index + 1));
            currentIndex = index + 1;
        }

        return subdirs;
    }

    /**
     * Creates objects with the given names in the given bucket.
     */
    private void createObjects(final String bucketName, String[] objectNames) throws IOException {

        final ExecutorService threadPool = Executors.newCachedThreadPool();
        final CountDownLatch counter = new CountDownLatch(objectNames.length);
        List<Future<?>> futures = new ArrayList<>();
        // Do each creation asynchronously.
        for (final String objectName : objectNames) {
            Future<?> future = threadPool.submit(new Runnable() {
                @Override
                public void run() {
                    try {
                        if (objectName.endsWith(GoogleCloudStorage.PATH_DELIMITER)) {
                            mkdir(bucketName, objectName);
                        } else {
                            // Just use objectName as file contents.
                            writeTextFile(bucketName, objectName, objectName);
                        }
                    } catch (Throwable ioe) {
                        throw new RuntimeException(
                                String.format("Exception creating %s/%s", bucketName, objectName), ioe);
                    } finally {
                        counter.countDown();
                    }
                }
            });
            futures.add(future);
        }

        try {
            counter.await();
        } catch (InterruptedException ie) {
            throw new IOException("Interrupted while awaiting object creation!", ie);
        } finally {
            threadPool.shutdown();
            try {
                if (!threadPool.awaitTermination(10L, TimeUnit.SECONDS)) {
                    System.err.println("Failed to awaitTermination! Forcing executor shutdown.");
                    threadPool.shutdownNow();
                }
            } catch (InterruptedException ie) {
                throw new IOException("Interrupted while shutting down threadpool!", ie);
            }
        }

        for (Future<?> future : futures) {
            try {
                // We should already be done.
                future.get(10, TimeUnit.MILLISECONDS);
            } catch (Exception e) {
                throw new IOException(String.format("Creation of file %s failed with exception", objectName), e);
            }
        }
    }

    /**
     * Gets the credential tests should use for accessing GCS.
     */
    static Credential getCredential() throws IOException {
        String clientId = System.getenv(GCS_TEST_CLIENT_ID);
        String clientSecret = System.getenv(GCS_TEST_CLIENT_SECRET);
        Assert.assertNotNull("clientId must not be null", clientId);
        Assert.assertNotNull("clientSecret must not be null", clientSecret);
        Credential credential;
        try {
            CredentialFactory credentialFactory = new CredentialFactory();
            credential = credentialFactory.getStorageCredential(clientId, clientSecret);
        } catch (GeneralSecurityException gse) {
            throw new IOException(gse);
        }
        return credential;
    }

    /**
     * Gets randomly generated name of a bucket.
     * The name is prefixed with an identifiable string. A bucket created by this method
     * can be identified by calling isTestBucketName() for that bucket.
     */
    public static String getUniqueBucketName() {
        return getUniqueBucketName("");
    }

    /**
     * Gets randomly generated name of a bucket with the given suffix.
     * The name is prefixed with an identifiable string. A bucket created by this method
     * can be identified by calling isTestBucketName() for that bucket.
     */
    public static String getUniqueBucketName(String suffix) {
        UUID uuid = UUID.randomUUID();
        String uniqueBucketName = TEST_BUCKET_NAME_PREFIX + uuid.toString() + suffix;
        // In most cases, caller creates a bucket with this name therefore we proactively add the name
        // to the list of buckets to be deleted. It is ok even if a bucket is not created with
        // this name because deleteBuckets() will ignore non-existent buckets.
        addToDeleteBucketList(uniqueBucketName);
        return uniqueBucketName;
    }

    /**
     * Indicates whether the given name matches the pattern used for
     * naming a test bucket.
     */
    static boolean isTestBucketName(String bucketName) {
        return bucketName.matches(TEST_BUCKET_NAME_PATTERN);
    }

    /**
     * Gets randomly generated name of a file like object.
     */
    static String getUniqueFileObjectName() {
        return "file-" + UUID.randomUUID().toString();
    }

    /**
     * Gets randomly generated name of a directory like object.
     */
    static String getUniqueDirectoryObjectName() {
        return "dir-" + UUID.randomUUID().toString();
    }

    /**
     * Creates a bucket and adds it to the list of buckets to delete
     * at the end of tests.
     */
    public static void createTempBucket(String tempBucketName) throws IOException {
        gcsit.mkdir(tempBucketName);
        addToDeleteBucketList(tempBucketName);
    }

    /**
     * Creates a bucket and adds it to the list of buckets to delete
     * at the end of tests.
     */
    static String createTempBucket() throws IOException {
        String tempBucketName = getUniqueBucketName();
        createTempBucket(tempBucketName);
        return tempBucketName;
    }

    /**
     * Add the given bucket name to the list of buckets to delete
     * during post-test cleanup.
     */
    static void addToDeleteBucketList(String bucketToDelete) {
        // We do not create more than a few buckets during tests, therefore linear search
        // to find duplicates is ok.
        if (!bucketsToDelete.contains(bucketToDelete)) {
            bucketsToDelete.add(bucketToDelete);
        }
    }

    /**
     * Delete buckets that were temporarily created during tests.
     */
    static void deleteBuckets() {
        for (String bucket : bucketsToDelete) {
            try {
                gcsit.clearBucket(bucket);
                gcsit.delete(bucket);
            } catch (IOException e) {
                log.warn("deleteBuckets: " + e.getLocalizedMessage());
            }
        }
        bucketsToDelete.clear();
    }

    /**
     * Enable debug logger.
     */
    static void enableLogging() {
        System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.Log4JLogger");
        ConsoleAppender consoleAppender = new ConsoleAppender();
        Logger rootLogger = Logger.getRootLogger();
        rootLogger.addAppender(consoleAppender);
        rootLogger.setLevel(Level.DEBUG);
    }
}