com.emc.atmos.api.test.AtmosApiClientTest.java Source code

Java tutorial

Introduction

Here is the source code for com.emc.atmos.api.test.AtmosApiClientTest.java

Source

/*
 * Copyright 2014 EMC Corporation. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 * http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package com.emc.atmos.api.test;

import com.emc.atmos.AtmosException;
import com.emc.atmos.StickyThreadAlgorithm;
import com.emc.atmos.api.*;
import com.emc.atmos.api.bean.*;
import com.emc.atmos.api.jersey.AtmosApiBasicClient;
import com.emc.atmos.api.jersey.AtmosApiClient;
import com.emc.atmos.api.multipart.MultipartEntity;
import com.emc.atmos.api.request.*;
import com.emc.atmos.util.AtmosClientFactory;
import com.emc.atmos.util.RandomInputStream;
import com.emc.atmos.util.ReorderedFormDataContentDisposition;
import com.emc.test.util.Concurrent;
import com.emc.test.util.ConcurrentJunitRunner;
import com.emc.util.StreamUtil;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientRequest;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.filter.ClientFilter;
import com.sun.jersey.core.header.FormDataContentDisposition;
import com.sun.jersey.multipart.BodyPart;
import com.sun.jersey.multipart.FormDataMultiPart;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import org.junit.*;
import org.junit.runner.RunWith;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.ws.rs.core.MediaType;
import java.io.*;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

@RunWith(ConcurrentJunitRunner.class)
@Concurrent
public class AtmosApiClientTest {
    public static Logger l4j = Logger.getLogger(AtmosApiClientTest.class);

    /**
     * Use this as a prefix for namespace object paths and you won't have to clean up after yourself.
     * This also keeps all test objects under one folder, which is easy to delete should something go awry.
     */
    protected static final String TEST_DIR_PREFIX = "/test_" + AtmosApiClientTest.class.getSimpleName();

    protected AtmosConfig config;
    protected AtmosApi api;
    protected boolean isVipr = false;

    protected List<ObjectIdentifier> cleanup = Collections.synchronizedList(new ArrayList<ObjectIdentifier>());
    protected List<ObjectPath> cleanupDirs = Collections.synchronizedList(new ArrayList<ObjectPath>());

    public AtmosApiClientTest() throws Exception {
        config = AtmosClientFactory.getAtmosConfig();
        Assume.assumeTrue("Could not load Atmos configuration", config != null);
        config.setDisableSslValidation(false);
        config.setEnableExpect100Continue(false);
        config.setEnableRetry(false);
        config.setLoadBalancingAlgorithm(new StickyThreadAlgorithm());
        api = new AtmosApiClient(config);
        isVipr = AtmosClientFactory.atmosIsVipr();
    }

    @After
    public void tearDown() {
        for (ObjectIdentifier cleanItem : cleanup) {
            try {
                api.delete(cleanItem);
            } catch (Throwable t) {
                System.out.println("Failed to delete " + cleanItem + ": " + t.getMessage());
            }
        }
        try { // if test directories exists, recursively delete them
            for (ObjectPath testDir : cleanupDirs) {
                deleteRecursively(testDir);
            }
        } catch (AtmosException e) {
            if (e.getHttpCode() != 404) {
                l4j.warn("Could not delete test dir: ", e);
            }
        }

        if (!isVipr) {
            try {
                ListAccessTokensResponse response = this.api.listAccessTokens(new ListAccessTokensRequest());
                if (response.getTokens() != null) {
                    for (AccessToken token : response.getTokens()) {
                        this.api.deleteAccessToken(token.getId());
                    }
                }
            } catch (Exception e) {
                System.out.println("Failed to delete access tokens: " + e.getMessage());
            }
        }
    }

    protected void deleteRecursively(ObjectPath path) {
        if (path.isDirectory()) {
            ListDirectoryRequest request = new ListDirectoryRequest().path(path);
            do {
                for (DirectoryEntry entry : this.api.listDirectory(request).getEntries()) {
                    deleteRecursively(new ObjectPath(path, entry));
                }
            } while (request.getToken() != null);
        }
        this.api.delete(path);
    }

    protected ObjectPath createTestDir(String name) {
        if (!name.endsWith("/"))
            name = name + "/";
        ObjectPath path = new ObjectPath(TEST_DIR_PREFIX + "_" + name);
        this.api.createDirectory(path);
        cleanupDirs.add(path);
        return path;
    }

    //
    // TESTS START HERE
    //

    @Test
    public void testUtf8JavaEncoding() throws Exception {
        String oneByteCharacters = "Hello";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String twoByteEscaped = "%D0%90%D0%91%D0%92%D0%93";
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        String fourByteEscaped = "%F0%A0%9C%8E%F0%A0%9C%B1%F0%A0%9D%B9%F0%A0%B1%93";
        Assert.assertEquals("2-byte characters failed", URLEncoder.encode(twoByteCharacters, "UTF-8"),
                twoByteEscaped);
        Assert.assertEquals("4-byte characters failed", URLEncoder.encode(fourByteCharacters, "UTF-8"),
                fourByteEscaped);
        Assert.assertEquals("2-byte/4-byte mix failed",
                URLEncoder.encode(twoByteCharacters + fourByteCharacters, "UTF-8"),
                twoByteEscaped + fourByteEscaped);
        Assert.assertEquals("1-byte/2-byte mix failed",
                URLEncoder.encode(oneByteCharacters + twoByteCharacters, "UTF-8"),
                oneByteCharacters + twoByteEscaped);
        Assert.assertEquals("1-4 byte mix failed",
                URLEncoder.encode(oneByteCharacters + twoByteCharacters + fourByteCharacters, "UTF-8"),
                oneByteCharacters + twoByteEscaped + fourByteEscaped);
    }

    /**
     * Test creating one empty object.  No metadata, no content.
     */
    @Test
    public void testCreateEmptyObject() throws Exception {
        ObjectId id = this.api.createObject(null, null);
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "", content);

    }

    /**
     * Test creating one empty object on a path.  No metadata, no content.
     */
    @Test
    public void testCreateEmptyObjectOnPath() throws Exception {
        ObjectPath op = new ObjectPath("/" + rand8char());
        ObjectId id = this.api.createObject(op, null, null);
        cleanup.add(op);
        l4j.debug("Path: " + op + " ID: " + id);
        Assert.assertNotNull(id);

        // Read back the content
        String content = new String(this.api.readObject(op, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "", content);
        content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong when reading by id", "", content);
    }

    /**
     * Tests using some extended characters when creating on a path.  This particular test
     * uses one cryllic, one accented, and one japanese character.
     */
    @Test
    public void testUnicodePath() throws Exception {
        String dirName = rand8char();
        ObjectPath path = new ObjectPath("/" + dirName + "/.txt");
        ObjectId id = this.api.createObject(path, null, null);
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        ObjectPath parent = new ObjectPath("/" + dirName + "/");
        ListDirectoryResponse response = this.api.listDirectory(new ListDirectoryRequest().path(parent));
        boolean found = false;
        for (DirectoryEntry ent : response.getEntries()) {
            if (new ObjectPath(parent, ent.getFilename()).equals(path)) {
                found = true;
            }
        }
        Assert.assertTrue("Did not find unicode file in dir", found);

        // Check read
        this.api.readObject(path, null, byte[].class);
    }

    /**
     * Tests using some extra characters that might break URIs
     */
    @Test
    public void testExtraPath() throws Exception {
        ObjectPath path = new ObjectPath("/" + rand8char() + "/a+=-  _!#$%^&*(),.z.txt");
        //ObjectPath path = new ObjectPath("/zimbramailbox/c8b4/511a-63c4-4ac9-8ff7+1c578de044be/stage/3r0sFrgUgL2ApCSkl3pobSX9D+k-1");
        byte[] data = "Hello World".getBytes("UTF-8");
        InputStream in = new ByteArrayInputStream(data);
        CreateObjectRequest request = new CreateObjectRequest().identifier(path).content(in)
                .contentLength(data.length);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);
    }

    @Test
    public void testUtf8Path() throws Exception {
        String oneByteCharacters = "Hello! ,";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        String crazyName = oneByteCharacters + twoByteCharacters + fourByteCharacters;
        byte[] content = "Crazy name creation test.".getBytes("UTF-8");
        ObjectPath parent = createTestDir("Utf8Path");
        ObjectPath path = new ObjectPath(parent, crazyName);

        // create crazy-name object
        this.api.createObject(path, content, "text/plain");

        cleanup.add(path);

        // verify name in directory list
        boolean found = false;
        ListDirectoryRequest request = new ListDirectoryRequest().path(parent);
        for (DirectoryEntry entry : this.api.listDirectory(request).getEntries()) {
            if (new ObjectPath(parent, entry.getFilename()).equals(path)) {
                found = true;
                break;
            }
        }
        Assert.assertTrue("crazyName not found in directory listing", found);

        // verify content
        Assert.assertTrue("content does not match",
                Arrays.equals(content, this.api.readObject(path, null, byte[].class)));
    }

    @Test
    public void testUtf8Content() throws Exception {
        String oneByteCharacters = "Hello! ,";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        byte[] content = (oneByteCharacters + twoByteCharacters + fourByteCharacters).getBytes("UTF-8");

        // create object with multi-byte UTF-8 content
        ObjectId oid = api.createObject(content, "text/plain");
        cleanup.add(oid);

        byte[] readContent = this.api.readObject(oid, null, byte[].class);

        // verify content
        Assert.assertTrue("content does not match", Arrays.equals(content, readContent));
    }

    /**
     * Test creating an object with content but without metadata
     */
    @Test
    public void testCreateObjectWithContent() throws Exception {
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);
    }

    @Test
    public void testCreateObjectWithSegment() throws Exception {
        byte[] content = "hello".getBytes("UTF-8");
        ObjectId id = api.createObject(new BufferSegment(content, 0, content.length), null);
        cleanup.add(id);

        // Read back the content
        String result = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", result);
    }

    @Test
    public void testCreateObjectWithContentStream() throws Exception {
        InputStream in = new ByteArrayInputStream("hello".getBytes("UTF-8"));
        CreateObjectRequest request = new CreateObjectRequest().content(in).contentLength(5)
                .contentType("text/plain");
        ObjectId id = this.api.createObject(request).getObjectId();
        in.close();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);
    }

    @Test
    public void testCreateObjectWithContentStreamOnPath() throws Exception {
        ObjectPath op = new ObjectPath("/" + rand8char() + ".tmp");
        InputStream in = new ByteArrayInputStream("hello".getBytes("UTF-8"));
        CreateObjectRequest request = new CreateObjectRequest();
        request.identifier(op).content(in).contentLength(5).contentType("text/plain");
        ObjectId id = this.api.createObject(request).getObjectId();
        in.close();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);
    }

    /**
     * Test creating an object with metadata but no content.
     */
    @Test
    public void testCreateObjectWithMetadataOnPath() {
        ObjectPath op = new ObjectPath("/" + rand8char() + ".tmp");
        CreateObjectRequest request = new CreateObjectRequest().identifier(op);
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        //this.esu.updateObject( op, null, mlist, null, null, null );
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(op);

        // Read and validate the metadata
        Map<String, Metadata> meta = this.api.getUserMetadata(op);
        Assert.assertNotNull("value of 'listable' missing", meta.get("listable"));
        Assert.assertNotNull("value of 'listable2' missing", meta.get("listable2"));
        Assert.assertNotNull("value of 'unlistable' missing", meta.get("unlistable"));
        Assert.assertNotNull("value of 'unlistable2' missing", meta.get("unlistable2"));

        Assert.assertEquals("value of 'listable' wrong", "foo", meta.get("listable").getValue());
        Assert.assertEquals("value of 'listable2' wrong", "foo2 foo2", meta.get("listable2").getValue());
        Assert.assertEquals("value of 'unlistable' wrong", "bar", meta.get("unlistable").getValue());
        Assert.assertEquals("value of 'unlistable2' wrong", "bar2 bar2", meta.get("unlistable2").getValue());
        // Check listable flags
        Assert.assertEquals("'listable' is not listable", true, meta.get("listable").isListable());
        Assert.assertEquals("'listable2' is not listable", true, meta.get("listable2").isListable());
        Assert.assertEquals("'unlistable' is listable", false, meta.get("unlistable").isListable());
        Assert.assertEquals("'unlistable2' is listable", false, meta.get("unlistable2").isListable());
    }

    /**
     * Test creating an object with metadata but no content.
     */
    @Test
    public void testCreateObjectWithMetadata() {
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        Metadata listable3 = new Metadata("listable3", null, true);
        Metadata quotes = new Metadata("ST_modalities", "\\US\\", false);
        //Metadata withCommas = new Metadata( "withcommas", "I, Robot", false );
        //Metadata withEquals = new Metadata( "withequals", "name=value", false );
        request.userMetadata(listable, unlistable, listable2, unlistable2, listable3, quotes);
        //request.addUserMetadata( withCommas );
        //request.addUserMetadata( withEquals );
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read and validate the metadata
        Map<String, Metadata> meta = this.api.getUserMetadata(id);
        Assert.assertEquals("value of 'listable' wrong", "foo", meta.get("listable").getValue());
        Assert.assertEquals("value of 'listable2' wrong", "foo2 foo2", meta.get("listable2").getValue());
        Assert.assertEquals("value of 'unlistable' wrong", "bar", meta.get("unlistable").getValue());
        Assert.assertEquals("value of 'unlistable2' wrong", "bar2 bar2", meta.get("unlistable2").getValue());
        Assert.assertNotNull("listable3 missing", meta.get("listable3"));
        Assert.assertTrue("Value of listable3 should be empty",
                meta.get("listable3").getValue() == null || meta.get("listable3").getValue().length() == 0);
        //Assert.assertEquals( "Value of withcommas wrong", "I, Robot", meta.get( "withcommas" ).getValue() );
        //Assert.assertEquals( "Value of withequals wrong", "name=value", meta.get( "withequals" ).getValue() );

        // Check listable flags
        Assert.assertEquals("'listable' is not listable", true, meta.get("listable").isListable());
        Assert.assertEquals("'listable2' is not listable", true, meta.get("listable2").isListable());
        Assert.assertEquals("'listable3' is not listable", true, meta.get("listable3").isListable());
        Assert.assertEquals("'unlistable' is listable", false, meta.get("unlistable").isListable());
        Assert.assertEquals("'unlistable2' is listable", false, meta.get("unlistable2").isListable());

    }

    /**
     * Test creating an object with metadata but no content.
     */
    @Test
    public void testMetadataNormalizeSpace() {
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata unlistable = new Metadata("unlistable", "bar  bar   bar    bar", false);
        Metadata leadingSpacesOdd = new Metadata("leadingodd", "   spaces", false);
        Metadata trailingSpacesOdd = new Metadata("trailingodd", "spaces   ", false);
        Metadata leadingSpacesEven = new Metadata("leadingeven", "    SPACES", false);
        Metadata trailingSpacesEven = new Metadata("trailingeven", "spaces    ", false);
        request.userMetadata(unlistable, leadingSpacesOdd, trailingSpacesOdd, leadingSpacesEven,
                trailingSpacesEven);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read and validate the metadata
        Map<String, Metadata> meta = this.api.getUserMetadata(id);
        Assert.assertEquals("value of 'unlistable' wrong", "bar  bar   bar    bar",
                meta.get("unlistable").getValue());
        // Check listable flags
        Assert.assertEquals("'unlistable' is listable", false, meta.get("unlistable").isListable());

    }

    /**
     * Test reading an object's content
     */
    @Test
    public void testReadObject() throws Exception {
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);

        // Read back only 2 bytes
        Range range = new Range(1, 2);
        content = new String(this.api.readObject(id, range, byte[].class), "UTF-8");
        Assert.assertEquals("partial object content wrong", "el", content);
    }

    @Test
    public void testResponseProperties() throws Exception {
        // Subtract a second since the HTTP dates only have 1s precision.
        Date now = new Date();
        CreateObjectRequest request = new CreateObjectRequest().content("hello".getBytes("UTF-8"))
                .contentType("text/plain");
        CreateObjectResponse response = this.api.createObject(request);
        Assert.assertNotNull("null ID returned", response.getObjectId());
        Assert.assertEquals("location wrong", "/rest/objects/" + response.getObjectId(), response.getLocation());
        cleanup.add(response.getObjectId());

        // Read back the content
        ReadObjectResponse<String> readResponse = api
                .readObject(new ReadObjectRequest().identifier(response.getObjectId()), String.class);
        Assert.assertEquals("object content wrong", "hello", readResponse.getObject());
        Assert.assertEquals("HTTP status wrong", 200, readResponse.getHttpStatus());
        Assert.assertEquals("HTTP message wrong", "OK", readResponse.getHttpMessage());
        Assert.assertFalse("HTTP headers empty", readResponse.getHeaders().isEmpty());
        Assert.assertTrue("HTTP content-type wrong",
                readResponse.getContentType().matches("text/plain(; charset=UTF-8)?"));
        Assert.assertEquals("HTTP content-length wrong", 5, readResponse.getContentLength());
        Assert.assertTrue("HTTP response date wrong",
                Math.abs(response.getDate().getTime() - now.getTime()) < (1000 * 60 * 5));
        // apparently last-modified isn't included in GET requests
        // Assert.assertTrue( "HTTP last modified date wrong", readResponse.getLastModified().after( now ) );
    }

    /**
     * Test reading an ACL back
     */
    @Test
    public void testReadAcl() {
        // Create an object with an ACL
        Acl acl = new Acl();
        acl.addUserGrant(stripUid(config.getTokenId()), Permission.FULL_CONTROL);
        acl.addGroupGrant(Acl.GROUP_OTHER, Permission.READ);
        ObjectId id = this.api.createObject(new CreateObjectRequest().acl(acl)).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the ACL and make sure it matches
        Acl newacl = this.api.getAcl(id);
        l4j.info("Comparing " + newacl + " with " + acl);

        Assert.assertEquals("ACLs don't match", acl, newacl);

    }

    /**
     * Inside an ACL, you use the UID only, not SubtenantID/UID
     */
    private String stripUid(String uid) {
        int slash = uid.indexOf('/');
        if (slash != -1) {
            return uid.substring(slash + 1);
        } else {
            return uid;
        }
    }

    @Test
    public void testReadAclByPath() {
        ObjectPath op = new ObjectPath("/" + rand8char() + ".tmp");
        // Create an object with an ACL
        Acl acl = new Acl();
        acl.addUserGrant(stripUid(config.getTokenId()), Permission.FULL_CONTROL);
        acl.addGroupGrant(Acl.GROUP_OTHER, Permission.READ);
        ObjectId id = this.api.createObject(new CreateObjectRequest().identifier(op).acl(acl)).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(op);

        // Read back the ACL and make sure it matches
        Acl newacl = this.api.getAcl(op);
        l4j.info("Comparing " + newacl + " with " + acl);

        Assert.assertEquals("ACLs don't match", acl, newacl);

    }

    /**
     * Test reading back user metadata
     */
    @Test
    public void testGetUserMetadata() {
        // Create an object with user metadata
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read only part of the metadata
        Map<String, Metadata> meta = this.api.getUserMetadata(id, "listable", "unlistable");
        Assert.assertEquals("value of 'listable' wrong", "foo", meta.get("listable").getValue());
        Assert.assertNull("value of 'listable2' should not have been returned", meta.get("listable2"));
        Assert.assertEquals("value of 'unlistable' wrong", "bar", meta.get("unlistable").getValue());
        Assert.assertNull("value of 'unlistable2' should not have been returned", meta.get("unlistable2"));

    }

    /**
     * Test deleting user metadata
     */
    @Test
    public void testDeleteUserMetadata() {
        // Create an object with metadata
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Delete a couple of the metadata entries
        this.api.deleteUserMetadata(id, "listable2", "unlistable2");

        // Read back the metadata for the object and ensure the deleted
        // entries don't exist
        Map<String, Metadata> meta = this.api.getUserMetadata(id);
        Assert.assertEquals("value of 'listable' wrong", "foo", meta.get("listable").getValue());
        Assert.assertNull("value of 'listable2' should not have been returned", meta.get("listable2"));
        Assert.assertEquals("value of 'unlistable' wrong", "bar", meta.get("unlistable").getValue());
        Assert.assertNull("value of 'unlistable2' should not have been returned", meta.get("unlistable2"));
    }

    /**
     * Test creating object versions
     */
    @Test
    public void testVersionObject() throws Exception {
        Assume.assumeFalse(isVipr);
        // Create an object
        String content = "Version Test";
        CreateObjectRequest request = new CreateObjectRequest().content(content);
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Version the object
        ObjectId vid = this.api.createVersion(id);
        cleanup.add(vid);
        Assert.assertNotNull("null version ID returned", vid);

        Assert.assertFalse("Version ID shoudn't be same as original ID", id.equals(vid));

        // Fetch the version and read its data
        Assert.assertEquals("Version content wrong", content, this.api.readObject(vid, null, String.class));

        Map<String, Metadata> meta = this.api.getUserMetadata(vid);
        Assert.assertEquals("value of 'listable' wrong", "foo", meta.get("listable").getValue());
        Assert.assertEquals("value of 'listable2' wrong", "foo2 foo2", meta.get("listable2").getValue());
        Assert.assertEquals("value of 'unlistable' wrong", "bar", meta.get("unlistable").getValue());
        Assert.assertEquals("value of 'unlistable2' wrong", "bar2 bar2", meta.get("unlistable2").getValue());

    }

    /**
     * Test listing the versions of an object
     */
    @Test
    public void testListVersions() {
        Assume.assumeFalse(isVipr);
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);
        Assert.assertNotNull("null ID returned", id);

        // Version the object
        ObjectId vid1 = this.api.createVersion(id);
        cleanup.add(vid1);
        Assert.assertNotNull("null version ID returned", vid1);
        ObjectId vid2 = this.api.createVersion(id);
        cleanup.add(vid2);
        Assert.assertNotNull("null version ID returned", vid2);

        // List the versions and ensure their IDs are correct
        ListVersionsResponse response = this.api.listVersions(new ListVersionsRequest().objectId(id));

        List<ObjectVersion> versions = response.getVersions();
        Assert.assertEquals("Wrong number of versions returned", 2, versions.size());
        Assert.assertTrue("Version number less than zero", versions.get(0).getVersionNumber() >= 0);
        Assert.assertNotNull("Version itime is null", versions.get(0).getItime());
        Assert.assertTrue("Version number less than zero", versions.get(1).getVersionNumber() >= 0);
        Assert.assertNotNull("Version itime is null", versions.get(1).getItime());

        List<ObjectId> versionIds = response.getVersionIds();
        Assert.assertEquals("Wrong number of versions returned", 2, versionIds.size());
        Assert.assertTrue("version 1 not found in version list", versionIds.contains(vid1));
        Assert.assertTrue("version 2 not found in version list", versionIds.contains(vid2));
    }

    /**
     * Test listing the versions of an object
     */
    @Test
    public void testListVersionsLong() {
        Assume.assumeFalse(isVipr);
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);
        Assert.assertNotNull("null ID returned", id);

        // Version the object
        ObjectId vid1 = this.api.createVersion(id);
        ObjectVersion v1 = new ObjectVersion(0, vid1, null);
        cleanup.add(vid1);
        Assert.assertNotNull("null version ID returned", vid1);
        ObjectId vid2 = this.api.createVersion(id);
        cleanup.add(vid2);
        ObjectVersion v2 = new ObjectVersion(1, vid2, null);
        Assert.assertNotNull("null version ID returned", vid2);

        // List the versions and ensure their IDs are correct
        ListVersionsRequest vRequest = new ListVersionsRequest();
        vRequest.objectId(id).setLimit(1);
        List<ObjectVersion> versions = new ArrayList<ObjectVersion>();
        do {
            ListVersionsResponse response = this.api.listVersions(vRequest);
            if (response.getVersions() != null)
                versions.addAll(response.getVersions());
        } while (vRequest.getToken() != null);
        Assert.assertEquals("Wrong number of versions returned", 2, versions.size());
        Assert.assertTrue("version 1 not found in version list", versions.contains(v1));
        Assert.assertTrue("version 2 not found in version list", versions.contains(v2));
        for (ObjectVersion v : versions) {
            Assert.assertNotNull("oid null in version", v.getVersionId());
            Assert.assertTrue("Invalid version number in version", v.getVersionNumber() > -1);
            Assert.assertNotNull("itime null in version", v.getItime());
        }
    }

    /**
     * Test listing the versions of an object
     */
    @Test
    public void testDeleteVersion() {
        Assume.assumeFalse(isVipr);
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);
        Assert.assertNotNull("null ID returned", id);

        // Version the object
        ObjectId vid1 = this.api.createVersion(id);
        Assert.assertNotNull("null version ID returned", vid1);
        ObjectId vid2 = this.api.createVersion(id);
        cleanup.add(vid2);
        Assert.assertNotNull("null version ID returned", vid2);

        // List the versions and ensure their IDs are correct
        List<ObjectId> versions = this.api.listVersions(new ListVersionsRequest().objectId(id)).getVersionIds();
        Assert.assertEquals("Wrong number of versions returned", 2, versions.size());
        Assert.assertTrue("version 1 not found in version list", versions.contains(vid1));
        Assert.assertTrue("version 2 not found in version list", versions.contains(vid2));

        // Delete a version
        this.api.deleteVersion(vid1);
        versions = this.api.listVersions(new ListVersionsRequest().objectId(id)).getVersionIds();
        Assert.assertEquals("Wrong number of versions returned", 1, versions.size());
        Assert.assertFalse("version 1 found in version list", versions.contains(vid1));
        Assert.assertTrue("version 2 not found in version list", versions.contains(vid2));

    }

    @Test
    public void testRestoreVersion() throws IOException {
        Assume.assumeFalse(isVipr);
        ObjectId id = this.api.createObject("Base Version Content".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Version the object
        ObjectId vId = this.api.createVersion(id);

        // Update the object content
        this.api.updateObject(id, "Child Version Content -- You should never see me".getBytes("UTF-8"));

        // Restore the original version
        this.api.restoreVersion(id, vId);

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "Base Version Content", content);
    }

    /**
     * Test listing the system metadata on an object
     */
    @Test
    public void testGetSystemMetadata() {
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read only part of the metadata
        Map<String, Metadata> meta = this.api.getSystemMetadata(id, "atime", "ctime");
        Assert.assertNotNull("value of 'atime' missing", meta.get("atime"));
        Assert.assertNull("value of 'mtime' should not have been returned", meta.get("mtime"));
        Assert.assertNotNull("value of 'ctime' missing", meta.get("ctime"));
        Assert.assertNull("value of 'gid' should not have been returned", meta.get("gid"));
        Assert.assertNull("value of 'listable' should not have been returned", meta.get("listable"));
    }

    @Test
    public void testObjectExists() {
        ObjectId oid = api.createObject("Hello exists!", "text/plain");
        Assert.assertTrue("object exists!", api.objectExists(oid));

        api.delete(oid);
        Assert.assertFalse("object does not exist!", api.objectExists(oid));
    }

    /**
     * Test listing objects by a tag that doesn't exist
     */
    @Test
    public void testListObjectsNoExist() {
        ListObjectsRequest request = new ListObjectsRequest().metadataName("this_tag_should_not_exist");
        List<ObjectEntry> objects = this.api.listObjects(request).getEntries();
        Assert.assertNotNull("object list should be not null", objects);
        Assert.assertEquals("No objects should be returned", 0, objects.size());
    }

    /**
     * Test listing objects by a tag
     */
    @Test
    public void testListObjects() {
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // List the objects.  Make sure the one we created is in the list
        ListObjectsRequest lRequest = new ListObjectsRequest().metadataName("listable");
        List<ObjectEntry> objects = this.api.listObjects(lRequest).getEntries();
        ObjectEntry toFind = new ObjectEntry();
        toFind.setObjectId(id);
        Assert.assertTrue("No objects returned", objects.size() > 0);
        Assert.assertTrue("object not found in list", objects.contains(toFind));

    }

    /**
     * Test listing objects by a tag
     */
    @Test
    public void testListObjectsWithMetadata() {
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // List the objects.  Make sure the one we created is in the list
        ListObjectsRequest lRequest = new ListObjectsRequest().metadataName("listable").includeMetadata(true);
        List<ObjectEntry> objects = this.api.listObjects(lRequest).getEntries();
        Assert.assertTrue("No objects returned", objects.size() > 0);

        // Find the item.
        boolean found = false;
        for (ObjectEntry or : objects) {
            if (or.getObjectId().equals(id)) {
                found = true;
                // check metadata
                Assert.assertEquals("Wrong value on metadata", or.getUserMetadataMap().get("listable").getValue(),
                        "foo");
            }
        }
        Assert.assertTrue("object not found in list", found);
    }

    /**
     * Test listing objects by a tag, with only some of the metadata
     */
    @Test
    public void testListObjectsWithSomeMetadata() {
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // List the objects.  Make sure the one we created is in the list
        ListObjectsRequest lRequest = new ListObjectsRequest();
        lRequest.metadataName("listable").includeMetadata(true).userMetadataNames("listable");
        List<ObjectEntry> objects = this.api.listObjects(lRequest).getEntries();
        Assert.assertTrue("No objects returned", objects.size() > 0);

        // Find the item.
        boolean found = false;
        for (ObjectEntry or : objects) {
            if (or.getObjectId().equals(id)) {
                found = true;
                // check metadata
                Assert.assertEquals("Wrong value on metadata", or.getUserMetadataMap().get("listable").getValue(),
                        "foo");

                // Other metadata should not be present
                Assert.assertNull("unlistable should be missing", or.getUserMetadataMap().get("unlistable"));
            }
        }
        Assert.assertTrue("object not found in list", found);
    }

    /**
     * Test listing objects by a tag, paging the results
     */
    @Test
    public void testListObjectsPaged() {
        // Create two objects.
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        request.userMetadata(listable);
        ObjectId id1 = this.api.createObject(request).getObjectId();
        ObjectId id2 = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id1);
        Assert.assertNotNull("null ID returned", id2);
        cleanup.add(id1);
        cleanup.add(id2);

        // List the objects.  Make sure the one we created is in the list
        ListObjectsRequest lRequest = new ListObjectsRequest().metadataName("listable");
        lRequest.setIncludeMetadata(true);
        lRequest.setLimit(1);
        List<ObjectEntry> objects = this.api.listObjects(lRequest).getEntries();
        Assert.assertTrue("No objects returned", objects.size() > 0);
        Assert.assertNotNull("Token should be present", lRequest.getToken());

        l4j.debug("listObjectsPaged, Token: " + lRequest.getToken());
        while (lRequest.getToken() != null) {
            // Subsequent pages
            objects.addAll(this.api.listObjects(lRequest).getEntries());
            l4j.debug("listObjectsPaged, Token: " + lRequest.getToken());
        }

        // Ensure our IDs exist
        ObjectEntry toFind1 = new ObjectEntry(), toFind2 = new ObjectEntry();
        toFind1.setObjectId(id1);
        toFind2.setObjectId(id2);
        Assert.assertTrue("First object not found", objects.contains(toFind1));
        Assert.assertTrue("Second object not found", objects.contains(toFind2));
    }

    /**
     * Test fetching listable tags
     */
    @Test
    public void testGetListableTags() {
        // Create an object
        ObjectId id = this.api.createObject(null, null);
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        UpdateObjectRequest request = new UpdateObjectRequest().identifier(id);
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);
        this.api.updateObject(request);

        // List tags.  Ensure our object's tags are in the list.
        Set<String> tags = this.api.listMetadata(null);
        Assert.assertTrue("listable tag not returned", tags.contains("listable"));
        Assert.assertTrue("list/able/2 root tag not returned", tags.contains("list"));
        Assert.assertFalse("list/able/not tag returned", tags.contains("list/able/not"));

        // List child tags
        tags = this.api.listMetadata("list/able");
        Assert.assertFalse("non-child returned", tags.contains("listable"));
        Assert.assertTrue("list/able/2 tag not returned", tags.contains("2"));
        Assert.assertFalse("list/able/not tag returned", tags.contains("not"));

    }

    /**
     * Test listing the user metadata tags on an object
     */
    @Test
    public void testListUserMetadataTags() {
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest();
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);

        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // List tags
        Map<String, Boolean> metaNames = this.api.getUserMetadataNames(id);
        Assert.assertTrue("listable tag not returned", metaNames.containsKey("listable"));
        Assert.assertTrue("list/able/2 tag not returned", metaNames.containsKey("list/able/2"));
        Assert.assertTrue("unlistable tag not returned", metaNames.containsKey("unlistable"));
        Assert.assertTrue("list/able/not tag not returned", metaNames.containsKey("list/able/not"));
        Assert.assertFalse("unknown tag returned", metaNames.containsKey("unknowntag"));

        // Check listable flag
        Assert.assertEquals("'listable' is not listable", true, metaNames.get("listable"));
        Assert.assertEquals("'list/able/2' is not listable", true, metaNames.get("list/able/2"));
        Assert.assertEquals("'unlistable' is listable", false, metaNames.get("unlistable"));
        Assert.assertEquals("'list/able/not' is listable", false, metaNames.get("list/able/not"));
    }

    /**
     * Tests updating an object's metadata
     */
    @Test
    public void testUpdateObjectMetadata() throws Exception {
        // Create an object
        CreateObjectRequest request = new CreateObjectRequest().content("hello".getBytes("UTF-8"));
        Metadata unlistable = new Metadata("unlistable", "foo", false);
        request.userMetadata(unlistable);
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Update the metadata
        unlistable.setValue("bar");
        this.api.setUserMetadata(id,
                request.getUserMetadata().toArray(new Metadata[request.getUserMetadata().size()]));

        // Re-read the metadata
        Map<String, Metadata> meta = this.api.getUserMetadata(id);
        Assert.assertEquals("value of 'unlistable' wrong", "bar", meta.get("unlistable").getValue());

        // Check that content was not modified
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);

    }

    @Test
    public void testUpdateObjectAcl() throws Exception {
        // Create an object with an ACL
        Acl acl = new Acl();
        acl.addUserGrant(stripUid(config.getTokenId()), Permission.FULL_CONTROL);
        acl.addGroupGrant(Acl.GROUP_OTHER, Permission.READ);
        ObjectId id = this.api.createObject(new CreateObjectRequest().acl(acl)).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the ACL and make sure it matches
        Acl newacl = this.api.getAcl(id);
        l4j.info("Comparing " + newacl + " with " + acl);

        Assert.assertEquals("ACLs don't match", acl, newacl);

        // Change the ACL and update the object.
        acl.removeGroupGrant(Acl.GROUP_OTHER);
        acl.addGroupGrant(Acl.GROUP_OTHER, Permission.NONE);
        this.api.setAcl(id, acl);

        // Read the ACL back and check it
        newacl = this.api.getAcl(id);
        l4j.info("Comparing " + newacl + " with " + acl);
        Assert.assertEquals("ACLs don't match", acl, newacl);
    }

    /**
     * Tests updating an object's contents
     */
    @Test
    public void testUpdateObjectContent() throws Exception {
        // Create an object
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Update part of the content
        Range range = new Range(1, 1);
        this.api.updateObject(id, "u".getBytes("UTF-8"), range);

        // Read back the content and check it
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hullo", content);
    }

    @Test
    public void testUpdateObjectContentStream() throws Exception {
        // Create an object
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Update part of the content
        InputStream in = new ByteArrayInputStream("u".getBytes("UTF-8"));
        UpdateObjectRequest request = new UpdateObjectRequest().identifier(id);
        request.range(new Range(1, 1)).content(in);
        request.setContentLength(1);
        this.api.updateObject(request);
        in.close();

        // Read back the content and check it
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hullo", content);
    }

    /**
     * Test replacing an object's entire contents
     */
    @Test
    public void testReplaceObjectContent() throws Exception {
        // Create an object
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Update all of the content
        this.api.updateObject(id, "bonjour".getBytes("UTF-8"));

        // Read back the content and check it
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "bonjour", content);
    }

    @Test
    public void testListDirectory() throws Exception {
        String dir = rand8char();
        String file = rand8char();
        String dir2 = rand8char();
        ObjectPath dirPath = new ObjectPath("/" + dir + "/");
        ObjectPath op = new ObjectPath("/" + dir + "/" + file);
        ObjectPath dirPath2 = new ObjectPath("/" + dir + "/" + dir2 + "/");

        ObjectId dirId = this.api.createDirectory(dirPath);
        ObjectId id = this.api.createObject(op, null, null);
        this.api.createDirectory(dirPath2);
        cleanup.add(op);
        cleanup.add(dirPath2);
        cleanup.add(dirPath);
        l4j.debug("Path: " + op + " ID: " + id);
        Assert.assertNotNull(id);
        Assert.assertNotNull(dirId);

        // Read back the content
        String content = new String(this.api.readObject(op, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "", content);
        content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong when reading by id", "", content);

        // List the parent path
        List<DirectoryEntry> dirList = api.listDirectory(new ListDirectoryRequest().path(dirPath)).getEntries();
        l4j.debug("Dir content: " + content);
        Assert.assertTrue("File not found in directory", directoryContains(dirList, op.getFilename()));
        Assert.assertTrue("subdirectory not found in directory",
                directoryContains(dirList, dirPath2.getFilename()));
    }

    @Test
    public void testListDirectoryPaged() throws Exception {
        String dir = rand8char();
        String file = rand8char();
        String dir2 = rand8char();
        ObjectPath dirPath = new ObjectPath("/" + dir + "/");
        ObjectPath op = new ObjectPath("/" + dir + "/" + file);
        ObjectPath dirPath2 = new ObjectPath("/" + dir + "/" + dir2 + "/");

        ObjectId dirId = this.api.createDirectory(dirPath);
        ObjectId id = this.api.createObject(op, null, null);
        this.api.createDirectory(dirPath2);
        cleanup.add(op);
        cleanup.add(dirPath2);
        cleanup.add(dirPath);
        l4j.debug("Path: " + op + " ID: " + id);
        Assert.assertNotNull(id);
        Assert.assertNotNull(dirId);

        // List the parent path
        ListDirectoryRequest request = new ListDirectoryRequest().path(dirPath);
        request.setLimit(1);
        List<DirectoryEntry> dirList = api.listDirectory(request).getEntries();

        Assert.assertNotNull("Token should have been returned", request.getToken());
        l4j.debug("listDirectoryPaged, token: " + request.getToken());
        while (request.getToken() != null) {
            dirList.addAll(api.listDirectory(request).getEntries());
        }

        Assert.assertTrue("File not found in directory", directoryContains(dirList, op.getFilename()));
        Assert.assertTrue("subdirectory not found in directory",
                directoryContains(dirList, dirPath2.getFilename()));
    }

    @Test
    public void testListDirectoryWithMetadata() throws Exception {
        String dir = rand8char();
        String file = rand8char();
        String dir2 = rand8char();
        ObjectPath dirPath = new ObjectPath("/" + dir + "/");
        ObjectPath op = new ObjectPath("/" + dir + "/" + file);
        ObjectPath dirPath2 = new ObjectPath("/" + dir + "/" + dir2 + "/");

        CreateObjectRequest request = new CreateObjectRequest().identifier(op);
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);

        ObjectId dirId = this.api.createDirectory(dirPath);
        ObjectId id = this.api.createObject(request).getObjectId();
        this.api.createDirectory(dirPath2);
        cleanup.add(op);
        cleanup.add(dirPath2);
        cleanup.add(dirPath);
        l4j.debug("Path: " + op + " ID: " + id);
        Assert.assertNotNull(id);
        Assert.assertNotNull(dirId);

        // List the parent path
        ListDirectoryRequest lRequest = new ListDirectoryRequest().path(dirPath).includeMetadata(true);
        List<DirectoryEntry> dirList = api.listDirectory(lRequest).getEntries();
        Assert.assertTrue("File not found in directory", directoryContains(dirList, op.getFilename()));
        Assert.assertTrue("subdirectory not found in directory",
                directoryContains(dirList, dirPath2.getFilename()));

        for (DirectoryEntry de : dirList) {
            if (new ObjectPath(dirPath, de.getFilename()).equals(op)) {
                // Check the metadata
                Assert.assertNotNull("missing metadata 'listable'", de.getUserMetadataMap().get("listable"));
                Assert.assertEquals("Wrong value on metadata", de.getUserMetadataMap().get("listable").getValue(),
                        "foo");

            }
        }
        Assert.assertTrue("File not found in directory", directoryContains(dirList, op.getFilename()));
        Assert.assertTrue("subdirectory not found in directory",
                directoryContains(dirList, dirPath2.getFilename()));
    }

    @Test
    public void testListDirectoryWithSomeMetadata() throws Exception {
        String dir = rand8char();
        String file = rand8char();
        String dir2 = rand8char();
        ObjectPath dirPath = new ObjectPath("/" + dir + "/");
        ObjectPath op = new ObjectPath("/" + dir + "/" + file);
        ObjectPath dirPath2 = new ObjectPath("/" + dir + "/" + dir2 + "/");

        CreateObjectRequest request = new CreateObjectRequest().identifier(op);
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("list/able/2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("list/able/not", "bar2 bar2", false);
        request.userMetadata(listable, unlistable, listable2, unlistable2);

        ObjectId dirId = this.api.createDirectory(dirPath);
        ObjectId id = this.api.createObject(request).getObjectId();
        this.api.createDirectory(dirPath2);
        cleanup.add(op);
        cleanup.add(dirPath2);
        cleanup.add(dirPath);
        l4j.debug("Path: " + op + " ID: " + id);
        Assert.assertNotNull(id);
        Assert.assertNotNull(dirId);

        // List the parent path
        ListDirectoryRequest lRequest = new ListDirectoryRequest().path(dirPath).includeMetadata(true);
        lRequest.userMetadataNames("listable");
        List<DirectoryEntry> dirList = api.listDirectory(lRequest).getEntries();
        Assert.assertTrue("File not found in directory", directoryContains(dirList, op.getFilename()));
        Assert.assertTrue("subdirectory not found in directory",
                directoryContains(dirList, dirPath2.getFilename()));

        for (DirectoryEntry de : dirList) {
            if (new ObjectPath(dirPath, de.getFilename()).equals(op)) {
                // Check the metadata
                Assert.assertNotNull("Missing metadata 'listable'", de.getUserMetadataMap().get("listable"));
                Assert.assertEquals("Wrong value on metadata", de.getUserMetadataMap().get("listable").getValue(),
                        "foo");
                // Other metadata should not be present
                Assert.assertNull("unlistable should be missing", de.getUserMetadataMap().get("unlistable"));
            }
        }
        Assert.assertTrue("File not found in directory", directoryContains(dirList, op.getFilename()));
        Assert.assertTrue("subdirectory not found in directory",
                directoryContains(dirList, dirPath2.getFilename()));
    }

    private boolean directoryContains(List<DirectoryEntry> dir, String filename) {
        for (DirectoryEntry de : dir) {
            if (de.getFilename().equals(filename)) {
                return true;
            }
        }

        return false;
    }

    /**
     * This method tests various legal and illegal pathnames
     *
     * @throws Exception
     */
    @Test
    public void testPathNaming() throws Exception {
        ObjectPath path = new ObjectPath("/some/file");
        Assert.assertFalse("File should not be directory", path.isDirectory());
        path = new ObjectPath("/some/file.txt");
        Assert.assertFalse("File should not be directory", path.isDirectory());
        ObjectPath path2 = new ObjectPath("/some/file.txt");
        Assert.assertEquals("Equal paths should be equal", path, path2);

        path = new ObjectPath("/some/file/with/long.path/extra.stuff.here.zip");
        Assert.assertFalse("File should not be directory", path.isDirectory());

        path = new ObjectPath("/");
        Assert.assertTrue("Directory should be directory", path.isDirectory());

        path = new ObjectPath("/long/path/with/lots/of/elements/");
        Assert.assertTrue("Directory should be directory", path.isDirectory());

    }

    /**
     * Tests dot directories (you should be able to create them even though they break the URL specification.)
     *
     * @throws Exception
     */
    @Test
    public void testDotDirectories() throws Exception {
        ObjectPath parentPath = createTestDir("DotDirectories");
        ObjectPath dotPath = new ObjectPath(parentPath, "./");
        ObjectPath dotdotPath = new ObjectPath(parentPath, "../");
        String filename = rand8char();
        byte[] content = "Hello World!".getBytes("UTF-8");

        // test single dot path (./)
        ObjectId dirId = this.api.createDirectory(dotPath);
        Assert.assertNotNull("null ID returned on dot path creation", dirId);
        ObjectId fileId = this.api.createObject(new ObjectPath(dotPath, filename), content, "text/plain");
        cleanup.add(fileId);
        cleanup.add(dirId);

        // make sure we only see one file (the "." path is its own directory and not a synonym for the current directory)
        List<DirectoryEntry> entries = this.api.listDirectory(new ListDirectoryRequest().path(dotPath))
                .getEntries();
        Assert.assertEquals("dot path listing was not 1", entries.size(), 1);
        Assert.assertEquals("dot path listing did not contain test file", entries.get(0).getFilename(), filename);

        // test double dot path (../)
        dirId = this.api.createDirectory(dotdotPath);
        Assert.assertNotNull("null ID returned on dotdot path creation", dirId);
        fileId = this.api.createObject(new ObjectPath(dotdotPath, filename), content, "text/plain");
        cleanup.add(fileId);
        cleanup.add(dirId);

        // make sure we only see one file (the ".." path is its own directory and not a synonym for the parent directory)
        entries = this.api.listDirectory(new ListDirectoryRequest().path(dotdotPath)).getEntries();
        Assert.assertEquals("dotdot path listing was not 1", entries.size(), 1);
        Assert.assertEquals("dotdot path listing did not contain test file", entries.get(0).getFilename(),
                filename);
    }

    /**
     * Tests the 'get all metadata' call using a path
     *
     * @throws Exception
     */
    @Test
    public void testGetAllMetadataByPath() throws Exception {
        ObjectPath op = new ObjectPath("/" + rand8char() + ".tmp");
        String mimeType = "test/mimetype";

        // Create an object with an ACL
        Acl acl = new Acl();
        acl.addUserGrant(stripUid(config.getTokenId()), Permission.FULL_CONTROL);
        acl.addGroupGrant(Acl.GROUP_OTHER, Permission.READ);
        CreateObjectRequest request = new CreateObjectRequest().identifier(op).acl(acl);
        request.content("test".getBytes("UTF-8")).contentType(mimeType);

        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(op);

        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);

        this.api.updateObject(new UpdateObjectRequest().identifier(op)
                .userMetadata(listable, unlistable, listable2, unlistable2).contentType(mimeType));

        // Read it back with HEAD call
        ObjectMetadata om = this.api.getObjectMetadata(op);
        Assert.assertNotNull("value of 'listable' missing", om.getMetadata().get("listable"));
        Assert.assertNotNull("value of 'unlistable' missing", om.getMetadata().get("unlistable"));
        Assert.assertNotNull("value of 'atime' missing", om.getMetadata().get("atime"));
        Assert.assertNotNull("value of 'ctime' missing", om.getMetadata().get("ctime"));
        Assert.assertEquals("value of 'listable' wrong", "foo", om.getMetadata().get("listable").getValue());
        Assert.assertEquals("value of 'unlistable' wrong", "bar", om.getMetadata().get("unlistable").getValue());
        Assert.assertEquals("Mimetype incorrect", mimeType, om.getContentType());

        // Check the ACL
        // not checking this by path because an extra groupid is added
        // during the create calls by path.
        //Assert.assertEquals( "ACLs don't match", acl, om.getAcl() );
    }

    @Test
    public void testGetAllMetadataById() throws Exception {
        // Create an object with an ACL
        CreateObjectRequest request = new CreateObjectRequest();

        Acl acl = new Acl();
        acl.addUserGrant(stripUid(config.getTokenId()), Permission.FULL_CONTROL);
        acl.addGroupGrant(Acl.GROUP_OTHER, Permission.READ);

        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);

        String mimeType = "test/mimetype";
        request.acl(acl).userMetadata(listable, unlistable, listable2, unlistable2);
        request.content("test".getBytes("UTF-8")).contentType(mimeType);

        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read it back with HEAD call
        ObjectMetadata om = this.api.getObjectMetadata(id);
        Assert.assertNotNull("value of 'listable' missing", om.getMetadata().get("listable"));
        Assert.assertNotNull("value of 'unlistable' missing", om.getMetadata().get("unlistable"));
        Assert.assertNotNull("value of 'atime' missing", om.getMetadata().get("atime"));
        Assert.assertNotNull("value of 'ctime' missing", om.getMetadata().get("ctime"));
        Assert.assertEquals("value of 'listable' wrong", "foo", om.getMetadata().get("listable").getValue());
        Assert.assertEquals("value of 'unlistable' wrong", "bar", om.getMetadata().get("unlistable").getValue());
        Assert.assertEquals("Mimetype incorrect", mimeType, om.getContentType());

        // Check the ACL
        Assert.assertEquals("ACLs don't match", acl, om.getAcl());
    }

    /**
     * Tests getting object replica information.
     */
    @Test
    public void testGetObjectReplicaInfo() throws Exception {
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        Map<String, Metadata> meta = this.api.getUserMetadata(id, "user.maui.lso");
        Assert.assertNotNull(meta.get("user.maui.lso"));
        l4j.debug("Replica info: " + meta.get("user.maui.lso"));
    }

    @Test
    public void testGetShareableUrl() throws Exception {
        Assume.assumeFalse(isVipr);
        // Create an object with content.
        String str = "Four score and twenty years ago";
        ObjectId id = this.api.createObject(str.getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        Calendar c = Calendar.getInstance();
        c.add(Calendar.HOUR, 4);
        Date expiration = c.getTime();
        URL u = api.getShareableUrl(id, expiration);

        l4j.debug("Sharable URL: " + u);

        InputStream stream = (InputStream) u.getContent();
        BufferedReader br = new BufferedReader(new InputStreamReader(stream));
        String content = br.readLine();
        l4j.debug("Content: " + content);
        Assert.assertEquals("URL does not contain proper content", str, content);
    }

    @Test
    public void testGetShareableUrlWithPath() throws Exception {
        Assume.assumeFalse(isVipr);
        // Create an object with content.
        String str = "Four score and twenty years ago";
        ObjectPath op = new ObjectPath("/" + rand8char() + ".txt");
        ObjectId id = this.api.createObject(op, str.getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(op);

        Calendar c = Calendar.getInstance();
        c.add(Calendar.HOUR, 4);
        Date expiration = c.getTime();
        URL u = api.getShareableUrl(op, expiration);

        l4j.debug("Sharable URL: " + u);

        InputStream stream = (InputStream) u.getContent();
        BufferedReader br = new BufferedReader(new InputStreamReader(stream));
        String content = br.readLine();
        l4j.debug("Content: " + content);
        Assert.assertEquals("URL does not contain proper content", str, content);
    }

    @Test
    public void testExpiredSharableUrl() throws Exception {
        Assume.assumeFalse(isVipr);
        // Create an object with content.
        String str = "Four score and twenty years ago";
        ObjectId id = this.api.createObject(str.getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        Calendar c = Calendar.getInstance();
        c.add(Calendar.HOUR, -4);
        Date expiration = c.getTime();
        URL u = api.getShareableUrl(id, expiration);

        l4j.debug("Sharable URL: " + u);

        try {
            InputStream stream = (InputStream) u.getContent();
            BufferedReader br = new BufferedReader(new InputStreamReader(stream));
            String content = br.readLine();
            l4j.debug("Content: " + content);
            Assert.fail("Request should have failed");
        } catch (Exception e) {
            l4j.debug("Error (expected): " + e);
        }
    }

    @Test
    public void testReadObjectStream() throws Exception {
        ObjectId id = this.api.createObject("hello".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read back the content
        InputStream in = this.api.readObjectStream(id, null).getObject();
        BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
        String content = br.readLine();
        br.close();
        Assert.assertEquals("object content wrong", "hello", content);

        // Read back only 2 bytes
        Range range = new Range(1, 2);
        in = this.api.readObjectStream(id, range).getObject();
        br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
        content = br.readLine();
        br.close();
        Assert.assertEquals("partial object content wrong", "el", content);
    }

    @Test
    public void testCreateChecksum() throws Exception {
        byte[] data = "hello".getBytes("UTF-8");
        RunningChecksum ck = new RunningChecksum(ChecksumAlgorithm.SHA0);
        ck.update(data, 0, data.length);

        CreateObjectRequest request = new CreateObjectRequest().content(data).contentType("text/plain");
        request.wsChecksum(ck);

        CreateObjectResponse response = this.api.createObject(request);
        cleanup.add(response.getObjectId());
        Assert.assertNotNull("Null object ID returned", response.getObjectId());
        Assert.assertEquals("Checksum doesn't match", ck, response.getWsChecksum());
    }

    /**
     * Note, to test read checksums, see comment in testReadChecksum
     *
     * @throws Exception
     */
    @Test
    public void testUploadDownloadChecksum() throws Exception {
        // Create a byte array to test
        int totalSize = 10 * 1024 * 1024; // 10MB
        int chunkSize = 4 * 1024 * 1024; // 4MB
        byte[] testData = new byte[totalSize]; // 10MB
        for (int i = 0; i < testData.length; i++) {
            testData[i] = (byte) (i % 0x93);
        }

        RunningChecksum sha0 = new RunningChecksum(ChecksumAlgorithm.SHA0);
        BufferSegment segment = new BufferSegment(testData, 0, chunkSize);

        // upload in chunks
        sha0.update(segment);
        l4j.debug("Create checksum: " + sha0);
        CreateObjectRequest request = new CreateObjectRequest();
        request.content(segment).userMetadata(new Metadata("policy", "erasure", false)).setWsChecksum(sha0);
        ObjectId id = api.createObject(request).getObjectId();
        cleanup.add(id);

        while (segment.getOffset() + segment.getSize() < totalSize) {
            segment.setOffset(segment.getOffset() + chunkSize);
            if (segment.getOffset() + chunkSize > totalSize)
                segment.setSize(totalSize - segment.getOffset());
            Range range = new Range(segment.getOffset(), segment.getOffset() + segment.getSize() - 1);
            sha0.update(segment.getBuffer(), segment.getOffset(), segment.getSize());
            l4j.debug("Update checksum: " + sha0);
            api.updateObject(
                    new UpdateObjectRequest().identifier(id).range(range).content(segment).wsChecksum(sha0));
        }

        // download in chunks
        totalSize = Integer.parseInt(api.getSystemMetadata(id).get("size").getValue());

        ByteArrayOutputStream out = new ByteArrayOutputStream();
        int first = 0, last = chunkSize - 1;
        Range range = new Range(first, last);
        ReadObjectResponse<byte[]> response;
        RunningChecksum readSha0 = new RunningChecksum(ChecksumAlgorithm.SHA0);
        do {
            response = api.readObject(new ReadObjectRequest().identifier(id).ranges(range), byte[].class);
            readSha0.update(response.getObject(), 0, response.getObject().length);
            out.write(response.getObject());
            first += chunkSize;
            last += chunkSize;
            if (last >= totalSize)
                last = totalSize - 1;
            range = new Range(first, last);
        } while (first < totalSize);

        byte[] outData = out.toByteArray();

        // verify checksum
        Assert.assertEquals("Write checksum doesn't match read checksum", sha0, readSha0);
        Assert.assertEquals("Read checksum doesn't match", readSha0, response.getWsChecksum());

        // Check the files
        Assert.assertEquals("File lengths differ", testData.length, outData.length);
        Assert.assertArrayEquals("Data contents differ", testData, outData);
    }

    @Ignore("TODO: Figure out why this fails")
    @Test
    public void testUtf8WhiteSpaceValues() throws Exception {
        String utf8String = "Hello ,\u0080 \r \u000B \t \n \t";

        CreateObjectRequest request = new CreateObjectRequest();
        request.userMetadata(new Metadata("utf8Key", utf8String, false));

        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);

        // get the user metadata and make sure all UTF8 characters are accurate
        Map<String, Metadata> metaMap = this.api.getUserMetadata(id);
        Assert.assertEquals("UTF8 value does not match", utf8String, metaMap.get("utf8Key").getValue());

        // test set metadata with UTF8
        this.api.setUserMetadata(id, new Metadata("newKey", utf8String + "2", false));

        // verify set metadata call (also testing getAllMetadata)
        ObjectMetadata objMeta = this.api.getObjectMetadata(id);
        metaMap = objMeta.getMetadata();
        //Assert.assertEquals( "UTF8 key does not match", meta.getName(), whiteSpaceString + "2" );
        //Assert.assertEquals( "UTF8 key value does not match", meta.getValue(), "newValue" );
        Assert.assertEquals("UTF8 value does not match", utf8String + "2", metaMap.get("newKey").getValue());
    }

    @Test
    public void testUnicodeMetadata() throws Exception {
        CreateObjectRequest request = new CreateObjectRequest();

        Metadata nbspValue = new Metadata("nbspvalue", "Nobreak\u00A0Value", false);
        Metadata nbspName = new Metadata("Nobreak\u00A0Name", "regular text here", false);
        Metadata cryllic = new Metadata("cryllic", "??", false);
        l4j.debug("NBSP Value: " + nbspValue);
        l4j.debug("NBSP Name: " + nbspName);

        request.userMetadata(nbspValue, nbspName, cryllic);

        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read and validate the metadata
        Map<String, Metadata> meta = this.api.getUserMetadata(id);
        l4j.debug("Read Back:");
        l4j.debug("NBSP Value: " + meta.get("nbspvalue"));
        l4j.debug("NBSP Name: " + meta.get("Nobreak\u00A0Name"));
        Assert.assertEquals("value of 'nobreakvalue' wrong", "Nobreak\u00A0Value",
                meta.get("nbspvalue").getValue());
        Assert.assertEquals("Value of cryllic wrong", "??", meta.get("cryllic").getValue());
    }

    @Test
    public void testUtf8Metadata() throws Exception {
        String oneByteCharacters = "Hello! ";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        String utf8String = oneByteCharacters + twoByteCharacters + fourByteCharacters;

        CreateObjectRequest request = new CreateObjectRequest();
        request.userMetadata(new Metadata("utf8Key", utf8String, false),
                new Metadata(utf8String, "utf8Value", false));

        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);

        // list all tags and make sure the UTF8 tag is in the list
        Map<String, Boolean> tags = this.api.getUserMetadataNames(id);
        Assert.assertTrue("UTF8 key not found in tag list", tags.containsKey(utf8String));

        // get the user metadata and make sure all UTF8 characters are accurate
        Map<String, Metadata> metaMap = this.api.getUserMetadata(id);
        Metadata meta = metaMap.get(utf8String);
        Assert.assertEquals("UTF8 key does not match", meta.getName(), utf8String);
        Assert.assertEquals("UTF8 key value does not match", meta.getValue(), "utf8Value");
        Assert.assertEquals("UTF8 value does not match", metaMap.get("utf8Key").getValue(), utf8String);

        // test set metadata with UTF8
        this.api.setUserMetadata(id, new Metadata("newKey", utf8String + "2", false),
                new Metadata(utf8String + "2", "newValue", false));

        // verify set metadata call (also testing getAllMetadata)
        ObjectMetadata objMeta = this.api.getObjectMetadata(id);
        metaMap = objMeta.getMetadata();
        meta = metaMap.get(utf8String + "2");
        Assert.assertEquals("UTF8 key does not match", meta.getName(), utf8String + "2");
        Assert.assertEquals("UTF8 key value does not match", meta.getValue(), "newValue");
        Assert.assertEquals("UTF8 value does not match", metaMap.get("newKey").getValue(), utf8String + "2");
    }

    @Test
    public void testUtf8MetadataFilter() throws Exception {
        String oneByteCharacters = "Hello! ";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        String utf8String = oneByteCharacters + twoByteCharacters + fourByteCharacters;

        CreateObjectRequest request = new CreateObjectRequest();
        request.userMetadata(new Metadata("utf8Key", utf8String, false))
                .userMetadata(new Metadata(utf8String, "utf8Value", false));

        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);

        // apply a filter that includes the UTF8 tag
        Map<String, Metadata> metaMap = this.api.getUserMetadata(id, utf8String);
        Assert.assertEquals("UTF8 filter was not honored", metaMap.size(), 1);
        Assert.assertNotNull("UTF8 key was not found in filtered results", metaMap.get(utf8String));
    }

    @Test
    public void testUtf8DeleteMetadata() throws Exception {
        String oneByteCharacters = "Hello! ";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        String utf8String = oneByteCharacters + twoByteCharacters + fourByteCharacters;

        CreateObjectRequest request = new CreateObjectRequest();
        request.userMetadata(new Metadata("utf8Key", utf8String, false))
                .userMetadata(new Metadata(utf8String, "utf8Value", false));

        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);

        // delete the UTF8 tag
        this.api.deleteUserMetadata(id, utf8String);

        // verify delete was successful
        Map<String, Boolean> nameMap = this.api.getUserMetadataNames(id);
        Assert.assertFalse("UTF8 key was not deleted", nameMap.containsKey(utf8String));
    }

    @Test
    public void testUtf8ListableMetadata() throws Exception {
        String oneByteCharacters = "Hello! ";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        String utf8String = oneByteCharacters + twoByteCharacters + fourByteCharacters;

        CreateObjectRequest request = new CreateObjectRequest();
        request.userMetadata(new Metadata(utf8String, "utf8Value", true));

        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);

        Map<String, Metadata> metaMap = this.api.getUserMetadata(id);
        Metadata meta = metaMap.get(utf8String);
        Assert.assertEquals("UTF8 key does not match", meta.getName(), utf8String);
        Assert.assertEquals("UTF8 key value does not match", meta.getValue(), "utf8Value");
        Assert.assertTrue("UTF8 metadata is not listable", meta.isListable());

        // verify we can list the tag and see our object
        boolean found = false;
        for (ObjectEntry result : this.api.listObjects(new ListObjectsRequest().metadataName(utf8String))
                .getEntries()) {
            if (result.getObjectId().equals(id)) {
                found = true;
                break;
            }
        }
        Assert.assertTrue("UTF8 tag listing did not contain the correct object ID", found);

        // verify we can list child tags of the UTF8 tag
        Set<String> tags = this.api.listMetadata(utf8String);
        Assert.assertNotNull("UTF8 child tag listing was null", tags);
    }

    @Test
    public void testUtf8ListableTagWithComma() {
        String stringWithComma = "Hello, you!";

        CreateObjectRequest request = new CreateObjectRequest();
        request.userMetadata(new Metadata(stringWithComma, "value", true));

        ObjectId id = this.api.createObject(request).getObjectId();
        cleanup.add(id);

        Map<String, Metadata> metaMap = this.api.getUserMetadata(id);
        Metadata meta = metaMap.get(stringWithComma);
        Assert.assertEquals("key does not match", meta.getName(), stringWithComma);
        Assert.assertTrue("metadata is not listable", meta.isListable());

        boolean found = false;
        for (ObjectEntry result : this.api.listObjects(new ListObjectsRequest().metadataName(stringWithComma))
                .getEntries()) {
            if (result.getObjectId().equals(id)) {
                found = true;
                break;
            }
        }
        Assert.assertTrue("listing did not contain the correct object ID", found);
    }

    @Test
    public void testRename() throws Exception {
        ObjectPath op1 = new ObjectPath("/" + rand8char() + ".tmp");
        ObjectPath op2 = new ObjectPath("/" + rand8char() + ".tmp");

        ObjectId id = this.api.createObject(op1, "Four score and seven years ago".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Rename
        this.api.move(op1, op2, false);

        // Read back the content
        String content = new String(this.api.readObject(op2, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "Four score and seven years ago", content);

    }

    @Test
    public void testRenameOverwrite() throws Exception {
        ObjectPath op1 = new ObjectPath("/" + rand8char() + ".tmp");
        ObjectPath op2 = new ObjectPath("/" + rand8char() + ".tmp");

        ObjectId id = this.api.createObject(op1, "Four score and seven years ago".getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        ObjectId id2 = this.api.createObject(op2, "You should not see this".getBytes("UTF-8"), "text/plain");
        cleanup.add(id2);

        // Rename
        this.api.move(op1, op2, true);

        // Wait for overwrite to complete
        Thread.sleep(5000);

        // Read back the content
        String content = new String(this.api.readObject(op2, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "Four score and seven years ago", content);

    }

    /**
     * Tests renaming a path to UTF-8 multi-byte characters.  This is a separate test from create as the characters are
     * passed in the headers instead of the URL itself.
     *
     * @throws Exception
     */
    @Test
    public void testUtf8Rename() throws Exception {
        ObjectPath parentDir = createTestDir("Utf8Rename");
        String oneByteCharacters = "Hello! ,";
        String twoByteCharacters = "\u0410\u0411\u0412\u0413"; // Cyrillic letters
        String fourByteCharacters = "\ud841\udf0e\ud841\udf31\ud841\udf79\ud843\udc53"; // Chinese symbols
        ObjectPath normalName = new ObjectPath(parentDir, rand8char() + ".tmp");
        String crazyName = oneByteCharacters + twoByteCharacters + fourByteCharacters;
        ObjectPath crazyPath = new ObjectPath(parentDir, crazyName);
        byte[] content = "This is a really crazy name.".getBytes("UTF-8");

        // normal name
        this.api.createObject(normalName, content, "text/plain");

        // crazy multi-byte character name
        this.api.move(normalName, crazyPath, true);

        // Wait for overwrite to complete
        Thread.sleep(5000);

        // verify name in directory list
        List<DirectoryEntry> entries = this.api.listDirectory(new ListDirectoryRequest().path(parentDir))
                .getEntries();

        Assert.assertTrue("crazyName not found in directory listing", directoryContains(entries, crazyName));

        // Read back the content
        Assert.assertTrue("object content wrong",
                Arrays.equals(content, this.api.readObject(crazyPath, null, byte[].class)));
    }

    @Test
    public void testPositiveChecksumValidation() throws Exception {
        byte[] data = "Hello Checksums!".getBytes("UTF-8");
        RunningChecksum md5 = new RunningChecksum(ChecksumAlgorithm.MD5);
        RunningChecksum sha0 = new RunningChecksum(ChecksumAlgorithm.SHA0);
        RunningChecksum sha1 = new RunningChecksum(ChecksumAlgorithm.SHA1);
        md5.update(data, 0, data.length);
        sha0.update(data, 0, data.length);
        sha1.update(data, 0, data.length);

        CreateObjectRequest request = new CreateObjectRequest().content(data);
        ObjectId md5Id = api.createObject(request.wsChecksum(md5)).getObjectId();
        ObjectId sha0Id = api.createObject(request.wsChecksum(sha0)).getObjectId();
        ObjectId sha1Id = api.createObject(request.wsChecksum(sha1)).getObjectId();
        cleanup.add(md5Id);
        cleanup.add(sha0Id);
        cleanup.add(sha1Id);

        Assert.assertEquals("MD5 checksum was not equal", md5,
                api.readObject(new ReadObjectRequest().identifier(md5Id), byte[].class).getWsChecksum());
        Assert.assertEquals("SHA0 checksum was not equal", sha0,
                api.readObject(new ReadObjectRequest().identifier(sha0Id), byte[].class).getWsChecksum());
        Assert.assertEquals("SHA1 checksum was not equal", sha1,
                api.readObject(new ReadObjectRequest().identifier(sha1Id), byte[].class).getWsChecksum());

        // do a bunch of calls to make sure we don't try to validate
        api.getSystemMetadata(md5Id);
        api.getObjectMetadata(sha1Id);
        api.getObjectInfo(sha0Id);
        api.readObject(md5Id, new Range(1, 8), byte[].class);
        api.listVersions(new ListVersionsRequest().objectId(sha1Id));
        api.getAcl(sha0Id);

        Assert.assertTrue("object stream is not a ChecksummedInputStream",
                api.readObjectStream(sha0Id, null).getObject() instanceof ChecksummedInputStream);

        // test update
        byte[] appendData = " and stuff!".getBytes("UTF-8");
        md5.update(appendData, 0, appendData.length);
        UpdateObjectRequest uRequest = new UpdateObjectRequest().identifier(md5Id).content(appendData)
                .wsChecksum(md5);
        uRequest.setRange(new Range(data.length, data.length + appendData.length - 1));
        api.updateObject(uRequest);

        Assert.assertTrue("object stream is not a ChecksummedInputStream",
                api.readObjectStream(md5Id, null).getObject() instanceof ChecksummedInputStream);

        api.readObject(md5Id, byte[].class);
    }

    /**
     * Tests readback with checksum verification.  In order to test this, create a policy
     * with erasure coding and then set a policy selector with "policy=erasure" to invoke
     * the erasure coding policy.
     *
     * @throws Exception
     */
    @Test
    public void testReadChecksum() throws Exception {
        byte[] data = "hello".getBytes("UTF-8");
        Metadata policy = new Metadata("policy", "erasure", false);
        RunningChecksum wsChecksum = new RunningChecksum(ChecksumAlgorithm.SHA0);
        wsChecksum.update(data, 0, data.length);

        CreateObjectRequest request = new CreateObjectRequest().content(data).contentType("text/plain");
        request.userMetadata(policy).wsChecksum(wsChecksum);

        CreateObjectResponse response = this.api.createObject(request);
        Assert.assertNotNull("null ID returned", response.getObjectId());
        cleanup.add(response.getObjectId());
        Assert.assertNotNull("null ID returned", response.getObjectId());
        Assert.assertEquals("create checksums don't match", wsChecksum, response.getWsChecksum());

        // Read back the content
        ReadObjectRequest readRequest = new ReadObjectRequest().identifier(response.getObjectId());
        ReadObjectResponse<byte[]> readResponse = this.api.readObject(readRequest, byte[].class);
        Assert.assertArrayEquals("object content wrong", data, readResponse.getObject());
        Assert.assertEquals("read checksums don't match", wsChecksum, readResponse.getWsChecksum());
    }

    /**
     * Tests getting the service information
     */
    @Test
    public void testGetServiceInformation() throws Exception {
        ServiceInformation si = this.api.getServiceInformation();

        Assert.assertNotNull("Atmos version is null", si.getAtmosVersion());
    }

    /**
     * Test getting object info.  Note to fully run this testcase, you should
     * create a policy named 'retaindelete' that keys off of the metadata
     * policy=retaindelete that includes a retention and deletion criteria.
     */
    @Test
    public void testGetObjectInfo() throws Exception {
        CreateObjectRequest request = new CreateObjectRequest();
        request.content("hello".getBytes("UTF-8")).contentType("text/plain");
        ObjectId id = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        api.setUserMetadata(id, new Metadata("policy", "retain", false));

        // Read back the content
        String content = new String(this.api.readObject(id, null, byte[].class), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);

        // Check policyname
        Map<String, Metadata> sysmeta = this.api.getSystemMetadata(id, "policyname");
        Assert.assertNotNull("Missing system metadata 'policyname'", sysmeta.get("policyname"));

        // Get the object info
        ObjectInfo oi = this.api.getObjectInfo(id);
        Assert.assertNotNull("ObjectInfo null", oi);
        Assert.assertNotNull("ObjectInfo objectid null", oi.getObjectId());
        Assert.assertTrue("ObjectInfo numReplicas is 0", oi.getNumReplicas() > 0);
        Assert.assertNotNull("ObjectInfo replicas null", oi.getReplicas());
        Assert.assertNotNull("ObjectInfo selection null", oi.getSelection());
        Assert.assertTrue("ObjectInfo should have at least one replica", oi.getReplicas().size() > 0);

        // only run these tests if the policy configuration is valid
        Assume.assumeTrue("policyname != retaindelete",
                "retaindelete".equals(sysmeta.get("policyname").getValue()));
        Assert.assertNotNull("ObjectInfo expiration null", oi.getExpiration().getEndAt());
        Assert.assertNotNull("ObjectInfo retention null", oi.getRetention().getEndAt());
        api.setUserMetadata(id, new Metadata("user.maui.retentionEnable", "false", false));
    }

    @Test
    public void testHmac() throws Exception {
        // Compute the signature hash
        String input = "Hello World";
        byte[] secret = Base64.decodeBase64("D7qsp4j16PBHWSiUbc/bt3lbPBY=".getBytes("UTF-8"));
        Mac mac = Mac.getInstance("HmacSHA1");
        SecretKeySpec key = new SecretKeySpec(secret, "HmacSHA1");
        mac.init(key);
        l4j.debug("Hashing: \n" + input);

        byte[] hashData = mac.doFinal(input.getBytes("ISO-8859-1"));

        // Encode the hash in Base64.
        String hashOut = new String(Base64.encodeBase64(hashData), "UTF-8");

        l4j.debug("Hash: " + hashOut);
    }

    @Test
    public void testDirectoryMetadata() throws Exception {
        ObjectPath dir = new ObjectPath("/" + rand8char() + "/");
        Metadata listable = new Metadata("listable", "foo", true);
        Metadata unlistable = new Metadata("unlistable", "bar", false);
        Metadata listable2 = new Metadata("listable2", "foo2 foo2", true);
        Metadata unlistable2 = new Metadata("unlistable2", "bar2 bar2", false);
        Metadata listable3 = new Metadata("listable3", null, true);
        Metadata withCommas = new Metadata("withcommas", "I, Robot", false);
        Metadata withEquals = new Metadata("withequals", "name=value", false);
        ObjectId id = this.api.createDirectory(dir, null, listable, unlistable, listable2, unlistable2, listable3,
                withCommas, withEquals);
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        // Read and validate the metadata
        Map<String, Metadata> metaMap = this.api.getObjectMetadata(dir).getMetadata();
        Assert.assertNotNull("Missing metadata 'listable'", metaMap.get("listable"));
        Assert.assertNotNull("Missing metadata 'listable2'", metaMap.get("listable2"));
        Assert.assertNotNull("Missing metadata 'unlistable'", metaMap.get("unlistable"));
        Assert.assertNotNull("Missing metadata 'unlistable2'", metaMap.get("unlistable2"));
        Assert.assertEquals("value of 'listable' wrong", "foo", metaMap.get("listable").getValue());
        Assert.assertEquals("value of 'listable2' wrong", "foo2 foo2", metaMap.get("listable2").getValue());
        Assert.assertEquals("value of 'unlistable' wrong", "bar", metaMap.get("unlistable").getValue());
        Assert.assertEquals("value of 'unlistable2' wrong", "bar2 bar2", metaMap.get("unlistable2").getValue());
        Assert.assertNotNull("listable3 missing", metaMap.get("listable3"));
        Assert.assertTrue("Value of listable3 should be empty",
                metaMap.get("listable3").getValue() == null || metaMap.get("listable3").getValue().length() == 0);
        Assert.assertEquals("Value of withcommas wrong", "I, Robot", metaMap.get("withcommas").getValue());
        Assert.assertEquals("Value of withequals wrong", "name=value", metaMap.get("withequals").getValue());

        // Check listable flags
        Assert.assertEquals("'listable' is not listable", true, metaMap.get("listable").isListable());
        Assert.assertEquals("'listable2' is not listable", true, metaMap.get("listable2").isListable());
        Assert.assertEquals("'listable3' is not listable", true, metaMap.get("listable3").isListable());
        Assert.assertEquals("'unlistable' is listable", false, metaMap.get("unlistable").isListable());
        Assert.assertEquals("'unlistable2' is listable", false, metaMap.get("unlistable2").isListable());
    }

    /**
     * Tests fetching data with multiple ranges.
     */
    @Test
    public void testMultipleRanges() throws Exception {
        String input = "Four score and seven years ago";
        ObjectId id = api.createObject(input.getBytes("UTF-8"), "text/plain");
        cleanup.add(id);
        Assert.assertNotNull("Object null", id);

        Range[] ranges = new Range[5];
        ranges[0] = new Range(27, 28); //ag
        ranges[1] = new Range(9, 9); // e
        ranges[2] = new Range(5, 5); // s
        ranges[3] = new Range(4, 4); // ' '
        ranges[4] = new Range(27, 29); // ago

        ReadObjectResponse<MultipartEntity> response = api
                .readObject(new ReadObjectRequest().identifier(id).ranges(ranges), MultipartEntity.class);
        String out = new String(response.getObject().aggregateBytes(), "UTF-8");
        Assert.assertEquals("Content incorrect", "ages ago", out);
    }

    //---------- Features supported by the Atmos 2.0 REST API. ----------\\

    @Test
    public void testGetShareableUrlAndDisposition() throws Exception {
        Assume.assumeFalse(isVipr);
        // Create an object with content.
        String str = "Four score and twenty years ago";
        ObjectId id = this.api.createObject(str.getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        cleanup.add(id);

        String disposition = "attachment; filename=\"foo bar.txt\"";

        Calendar c = Calendar.getInstance();
        c.add(Calendar.HOUR, 4);
        Date expiration = c.getTime();
        URL u = this.api.getShareableUrl(id, expiration, disposition);

        l4j.debug("Sharable URL: " + u);

        InputStream stream = (InputStream) u.getContent();
        BufferedReader br = new BufferedReader(new InputStreamReader(stream));
        String content = br.readLine();
        l4j.debug("Content: " + content);
        Assert.assertEquals("URL does not contain proper content", str, content);
    }

    @Test
    public void testGetShareableUrlWithPathAndDisposition() throws Exception {
        Assume.assumeFalse(isVipr);
        // Create an object with content.
        String str = "Four score and twenty years ago";
        ObjectPath op = new ObjectPath("/" + rand8char() + ".txt");
        ObjectId id = this.api.createObject(op, str.getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        //cleanup.add( op );

        String disposition = "attachment; filename=\"foo bar.txt\"";

        Calendar c = Calendar.getInstance();
        c.add(Calendar.HOUR, 4);
        Date expiration = c.getTime();
        URL u = this.api.getShareableUrl(op, expiration, disposition);

        l4j.debug("Sharable URL: " + u);

        InputStream stream = (InputStream) u.getContent();
        BufferedReader br = new BufferedReader(new InputStreamReader(stream));
        String content = br.readLine();
        l4j.debug("Content: " + content);
        Assert.assertEquals("URL does not contain proper content", str, content);
    }

    @Test
    public void testGetShareableUrlWithPathAndUTF8Disposition() throws Exception {
        // Create an object with content.
        Assume.assumeFalse(isVipr);
        String str = "Four score and twenty years ago";
        ObjectPath op = new ObjectPath("/" + rand8char() + ".txt");
        ObjectId id = this.api.createObject(op, str.getBytes("UTF-8"), "text/plain");
        Assert.assertNotNull("null ID returned", id);
        //cleanup.add( op );

        // One cryllic, one accented, and one japanese character
        // RFC5987
        String disposition = "attachment; filename=\"no UTF support.txt\"; filename*=UTF-8''"
                + URLEncoder.encode(".txt", "UTF-8");

        Calendar c = Calendar.getInstance();
        c.add(Calendar.HOUR, 4);
        Date expiration = c.getTime();
        URL u = this.api.getShareableUrl(op, expiration, disposition);

        l4j.debug("Sharable URL: " + u);

        InputStream stream = (InputStream) u.getContent();
        BufferedReader br = new BufferedReader(new InputStreamReader(stream));
        String content = br.readLine();
        l4j.debug("Content: " + content);
        Assert.assertEquals("URL does not contain proper content", str, content);
    }

    @Test
    public void testGetServiceInformationFeatures() throws Exception {
        ServiceInformation info = this.api.getServiceInformation();
        l4j.info("Supported features: " + info.getFeatures());

        Assert.assertTrue("Expected at least one feature", info.getFeatures().size() > 0);

    }

    @Test
    public void testBug23750() throws Exception {
        byte[] data = new byte[1000];
        Arrays.fill(data, (byte) 0);
        Metadata meta = new Metadata("test", null, true);

        RunningChecksum sha1 = new RunningChecksum(ChecksumAlgorithm.SHA1);
        sha1.update(data, 0, data.length);
        CreateObjectResponse response = this.api
                .createObject(new CreateObjectRequest().content(data).wsChecksum(sha1).userMetadata(meta));

        try {
            Range range = new Range(1000, 1999);
            RunningChecksum sha0 = new RunningChecksum(ChecksumAlgorithm.SHA0);
            sha0.update(data, 0, 1000);
            this.api.updateObject(new UpdateObjectRequest().identifier(response.getObjectId()).content(data)
                    .range(range).wsChecksum(sha0).userMetadata(meta));

            Assert.fail("Should have triggered an exception");
        } catch (AtmosException e) {
            // expected
        }
    }

    @Test
    public void testCrudKeys() throws Exception {
        Assume.assumeFalse(isVipr);
        ObjectKey key = new ObjectKey("Test_key-pool#@!$%^..", "KEY_TEST");
        String content = "Hello World!";

        CreateObjectRequest request = new CreateObjectRequest().identifier(key);
        request.content(content.getBytes("UTF-8")).contentType("text/plain");

        ObjectId oid = this.api.createObject(request).getObjectId();
        Assert.assertNotNull("Null object ID returned", oid);
        cleanup.add(oid);

        String readContent = new String(this.api.readObject(key, null, byte[].class), "UTF-8");
        Assert.assertEquals("content mismatch", content, readContent);

        content = "Hello Waldo!";
        this.api.updateObject(new UpdateObjectRequest().identifier(key).content(content.getBytes("UTF-8")));

        readContent = new String(this.api.readObject(key, null, byte[].class), "UTF-8");
        Assert.assertEquals("content mismatch", content, readContent);

        this.api.delete(key);

        try {
            this.api.readObject(key, null, byte[].class);
            Assert.fail("Object still exists");
        } catch (AtmosException e) {
            if (e.getHttpCode() != 404)
                throw e;
        }
    }

    @Test
    public void testIssue9() throws Exception {
        int threadCount = 10;

        final int objectSize = 10 * 1000 * 1000; // not a power of 2
        final AtmosApi atmosApi = api;
        final List<ObjectIdentifier> cleanupList = new ArrayList<ObjectIdentifier>();
        ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 0, TimeUnit.SECONDS,
                new LinkedBlockingQueue<Runnable>());
        try {
            for (int i = 0; i < threadCount; i++) {
                executor.execute(new Thread() {
                    public void run() {
                        CreateObjectRequest request = new CreateObjectRequest();
                        request.content(new RandomInputStream(objectSize)).contentLength(objectSize)
                                .userMetadata(new Metadata("test-data", null, true));
                        ObjectId oid = atmosApi.createObject(request).getObjectId();
                        cleanupList.add(oid);
                    }
                });
            }
            while (true) {
                Thread.sleep(1000);
                if (executor.getActiveCount() < 1)
                    break;
            }
        } finally {
            executor.shutdown();
            cleanup.addAll(cleanupList);
            if (cleanupList.size() < threadCount)
                Assert.fail("At least one thread failed");
        }
    }

    /**
     * Test handling signature failures.  Should throw an exception with
     * error code 1032.
     */
    @Test
    public void testSignatureFailure() throws Exception {
        byte[] goodSecret = config.getSecretKey();
        try {

            // Fiddle with the secret key
            byte[] badSecret = Arrays.copyOf(goodSecret, goodSecret.length);
            Arrays.fill(badSecret, 5, 10, (byte) 128); // indexes 5-9 will be 10000000 (binary)
            config.setSecretKey(badSecret);
            testCreateEmptyObject();
            Assert.fail("Expected exception to be thrown");
        } catch (AtmosException e) {
            Assert.assertEquals("Expected error code 1032 for signature failure", 1032, e.getErrorCode());
        } finally {
            config.setSecretKey(goodSecret);
        }
    }

    /**
     * Test general HTTP errors by generating a 404.
     */
    @Test
    public void testFourOhFour() throws Exception {
        try {
            // Fiddle with the context
            config.setContext("/restttttttttt");
            testCreateEmptyObject();
            Assert.fail("Expected exception to be thrown");
        } catch (AtmosException e) {
            Assert.assertEquals("Expected error code 404 for bad context root", 404, e.getHttpCode());
        } finally {
            config.setContext(AtmosConfig.DEFAULT_CONTEXT);
        }
    }

    @Test
    public void testServerOffset() throws Exception {
        long offset = api.calculateServerClockSkew();
        l4j.info("Server offset: " + offset + " milliseconds");
        testCreateEmptyObject(); // make sure requests still work after setting clock skew
    }

    /**
     * NOTE: This method does not actually test that the custom headers are sent over the wire. Run tcpmon or wireshark
     * to verify
     */
    @Test
    public void testCustomHeaders() throws Exception {
        final Map<String, String> customHeaders = new HashMap<String, String>();
        customHeaders.put("myCustomHeader", "Hello World!");

        ((AtmosApiClient) api).addClientFilter(new ClientFilter() {
            @Override
            public ClientResponse handle(ClientRequest clientRequest) throws ClientHandlerException {
                for (String name : customHeaders.keySet())
                    clientRequest.getHeaders().add(name, customHeaders.get(name));
                return getNext().handle(clientRequest);
            }
        });

        api.getServiceInformation();
    }

    @Test
    public void testServerGeneratedChecksum() throws Exception {
        Assume.assumeFalse(isVipr);
        byte[] data = "hello".getBytes("UTF-8");

        // generate our own checksum
        RunningChecksum md5 = new RunningChecksum(ChecksumAlgorithm.MD5);
        md5.update(data, 0, data.length);

        CreateObjectRequest request = new CreateObjectRequest().content(data).contentType("text/plain");
        request.setServerGeneratedChecksumAlgorithm(ChecksumAlgorithm.MD5);
        CreateObjectResponse response = this.api.createObject(request);
        Assert.assertNotNull("null ID returned", response.getObjectId());
        cleanup.add(response.getObjectId());

        // verify checksum
        Assert.assertEquals(md5.toString(false), response.getServerGeneratedChecksum().toString(false));

        // Read back the content
        ReadObjectRequest readRequest = new ReadObjectRequest().identifier(response.getObjectId());
        ReadObjectResponse<byte[]> readResponse = api.readObject(readRequest, byte[].class);
        String content = new String(readResponse.getObject(), "UTF-8");
        Assert.assertEquals("object content wrong", "hello", content);

        // verify checksum
        Assert.assertEquals(md5.toString(false), readResponse.getServerGeneratedChecksum().toString(false));
    }

    @Ignore("Blocked by Bug 30073")
    @Test
    public void testReadAccessToken() throws Exception {
        Assume.assumeFalse(isVipr);
        ObjectPath parentDir = createTestDir("ReadAccessToken");
        ObjectPath path = new ObjectPath(parentDir, "read_token \n,<x> test");
        ObjectId id = api.createObject(path, "hello", "text/plain");

        Calendar expiration = Calendar.getInstance();
        expiration.add(Calendar.MINUTE, 5); // 5 minutes from now

        AccessTokenPolicy.Source source = new AccessTokenPolicy.Source();
        source.setAllowList(Arrays.asList("10.0.0.0/8"));
        source.setDenyList(Arrays.asList("1.1.1.1"));

        AccessTokenPolicy.ContentLengthRange range = new AccessTokenPolicy.ContentLengthRange();
        range.setFrom(0);
        range.setTo(1024); // 1KB

        AccessTokenPolicy policy = new AccessTokenPolicy();
        policy.setExpiration(expiration.getTime());
        policy.setSource(source);
        policy.setMaxDownloads(2);
        policy.setMaxUploads(0);
        policy.setContentLengthRange(range);

        CreateAccessTokenRequest request = new CreateAccessTokenRequest().identifier(id).policy(policy);
        CreateAccessTokenResponse response = api.createAccessToken(request);

        String content = StreamUtil.readAsString(response.getTokenUrl().openStream());
        Assert.assertEquals("content from *id* access token doesn't match", content, "hello");

        api.deleteAccessToken(response.getTokenUrl());

        response = api.createAccessToken(new CreateAccessTokenRequest().identifier(path).policy(policy));

        content = StreamUtil.readAsString(response.getTokenUrl().openStream());
        Assert.assertEquals("content from *path* access token doesn't match", content, "hello");

        GetAccessTokenResponse getResponse = api.getAccessToken(response.getTokenUrl());
        AccessToken token = getResponse.getToken();

        api.deleteAccessToken(token.getId());

        Assert.assertEquals("token ID doesn't match", RestUtil.lastPathElement(response.getTokenUrl().getPath()),
                token.getId());
        policy.setMaxDownloads(policy.getMaxDownloads() - 1); // we already used one
        Assert.assertEquals("policy differs", policy, token);
    }

    @Ignore("Blocked by Bug 30073")
    @Test
    public void testWriteAccessToken() throws Exception {
        Assume.assumeFalse(isVipr);
        ObjectPath parentDir = createTestDir("WriteAccessToken");
        ObjectPath path = new ObjectPath(parentDir, "write_token_test");

        Calendar expiration = Calendar.getInstance();
        expiration.add(Calendar.MINUTE, 10); // 10 minutes from now

        AccessTokenPolicy.Source source = new AccessTokenPolicy.Source();
        source.setAllowList(Arrays.asList("10.0.0.0/8"));
        source.setDenyList(Arrays.asList("1.1.1.1"));

        AccessTokenPolicy.ContentLengthRange range = new AccessTokenPolicy.ContentLengthRange();
        range.setFrom(0);
        range.setTo(1024); // 1KB

        List<AccessTokenPolicy.FormField> formFields = new ArrayList<AccessTokenPolicy.FormField>();
        AccessTokenPolicy.FormField formField = new AccessTokenPolicy.FormField();
        formField.setName("x-emc-meta");
        formField.setOptional(true);
        formFields.add(formField);
        formField = new AccessTokenPolicy.FormField();
        formField.setName("x-emc-listable-meta");
        formField.setOptional(true);
        formFields.add(formField);

        AccessTokenPolicy policy = new AccessTokenPolicy();
        policy.setExpiration(expiration.getTime());
        policy.setSource(source);
        policy.setMaxDownloads(2);
        policy.setMaxUploads(1);
        policy.setContentLengthRange(range);
        policy.setFormFieldList(formFields);

        CreateAccessTokenRequest request = new CreateAccessTokenRequest().identifier(path).policy(policy);
        URL tokenUrl = api.createAccessToken(request).getTokenUrl();

        Client client = Client.create();

        // prepare upload form
        String content = "Anonymous Upload Test";

        // note we have to specify content-disposition parameters in a specific order due to bug 27005
        FormDataContentDisposition contentDisposition = new ReorderedFormDataContentDisposition(
                "form-data; name=\"data\"; filename=\"test.txt\"");
        BodyPart bodyPart = new BodyPart(content, MediaType.TEXT_PLAIN_TYPE).contentDisposition(contentDisposition);

        FormDataMultiPart form = new FormDataMultiPart();
        form.field("x-emc-meta", "color=gray,size=3,foo=bar").field("x-emc-listable-meta", "listable=")
                .bodyPart(bodyPart);

        // upload
        ClientResponse clientResponse = client.resource(tokenUrl.toURI()).type(MediaType.MULTIPART_FORM_DATA_TYPE)
                .post(ClientResponse.class, form);
        Assert.assertEquals("http status from upload is wrong", 201, clientResponse.getStatus());
        ObjectId oid = new ObjectId(RestUtil.lastPathElement(clientResponse.getLocation().getPath()));
        cleanup.add(oid);

        clientResponse = client.resource(tokenUrl.toURI()).get(ClientResponse.class);
        Assert.assertEquals(content, clientResponse.getEntity(String.class));

        // verify upload/download counts changed
        AccessToken token = api.getAccessToken(tokenUrl).getToken();
        Assert.assertEquals("upload count is wrong", 0, token.getMaxUploads());
        Assert.assertEquals("download count is wrong", 1, token.getMaxDownloads());

        // read object via standard api (namespace) - just make sure it's there
        api.readObject(new ReadObjectRequest().identifier(path), String.class);

        // " " (objectspace)
        ReadObjectResponse<String> response = api.readObject(new ReadObjectRequest().identifier(oid), String.class);
        Assert.assertEquals("content is wrong", content, response.getObject());
        Assert.assertNotNull("metadata is null", response.getMetadata());
        Assert.assertEquals("content-type is wrong", "text/plain", response.getMetadata().getContentType());

        Map<String, Metadata> meta = response.getMetadata().getMetadata();
        Assert.assertTrue("color missing from metadata", meta.containsKey("color"));
        Assert.assertTrue("size missing from metadata", meta.containsKey("size"));
        Assert.assertTrue("foo missing from metadata", meta.containsKey("foo"));

        api.deleteAccessToken(tokenUrl);
    }

    @Ignore("Blocked by Bug 30073")
    @Test
    public void testListAccessTokens() throws Exception {
        Assume.assumeFalse(isVipr);
        ObjectPath parentDir = createTestDir("ListAccessTokens");
        ObjectPath path = new ObjectPath(parentDir, "read_token_test");
        ObjectId id = api.createObject(path, "hello", "text/plain");

        Calendar expiration = Calendar.getInstance();
        expiration.add(Calendar.MINUTE, 5); // 5 minutes from now

        AccessTokenPolicy.Source source = new AccessTokenPolicy.Source();
        source.setAllowList(Arrays.asList("10.0.0.0/8"));
        source.setDenyList(Arrays.asList("1.1.1.1"));

        AccessTokenPolicy.ContentLengthRange range = new AccessTokenPolicy.ContentLengthRange();
        range.setFrom(0);
        range.setTo(1024); // 1KB

        AccessTokenPolicy policy = new AccessTokenPolicy();
        policy.setExpiration(expiration.getTime());
        policy.setSource(source);
        policy.setMaxDownloads(2);
        policy.setMaxUploads(0);
        policy.setContentLengthRange(range);

        CreateAccessTokenRequest request = new CreateAccessTokenRequest().identifier(id).policy(policy);
        URL tokenUrl1 = api.createAccessToken(request).getTokenUrl();

        request = new CreateAccessTokenRequest().identifier(path).policy(policy);
        URL tokenUrl2 = api.createAccessToken(request).getTokenUrl();

        ListAccessTokensResponse response = api.listAccessTokens(new ListAccessTokensRequest());
        Assert.assertNotNull("access token list is null", response.getTokens());
        Assert.assertEquals("access token count wrong", 2, response.getTokens().size());

        AccessToken token = response.getTokens().get(0);
        Assert.assertEquals("token ID doesn't match", RestUtil.lastPathElement(tokenUrl1.getPath()), token.getId());
        Assert.assertEquals("policy differs", policy, token);

        token = response.getTokens().get(1);
        Assert.assertEquals("token ID doesn't match", RestUtil.lastPathElement(tokenUrl2.getPath()), token.getId());
        Assert.assertEquals("policy differs", policy, token);
    }

    @Test
    public void testDisableSslValidation() throws Exception {
        Assume.assumeFalse(isVipr);
        config.setDisableSslValidation(true);
        api = new AtmosApiClient(config);
        List<URI> sslUris = new ArrayList<URI>();
        for (URI uri : config.getEndpoints()) {
            sslUris.add(new URI("https", uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(),
                    uri.getQuery(), uri.getFragment()));
        }
        config.setEndpoints(sslUris.toArray(new URI[sslUris.size()]));

        cleanup.add(api.createObject("Hello SSL!", null));
    }

    @Test
    public void testRetryFilter() throws Exception {
        final int retries = 3, delay = 500;
        final String flagMessage = "XXXXX";

        config.setEnableRetry(true);
        config.setMaxRetries(retries);
        config.setRetryDelayMillis(delay);
        api = new AtmosApiClient(config);

        CreateObjectRequest request = new CreateObjectRequest().contentLength(1).contentType("text/plain");
        try {
            api.createObject(request.content(new RetryInputStream(config, flagMessage)));
            Assert.fail("Retried more than maxRetries times");
        } catch (ClientHandlerException e) {
            Assert.assertEquals("Wrong exception thrown", flagMessage, e.getCause().getMessage());
        }

        config.setMaxRetries(retries + 1);

        ObjectId oid = api.createObject(request.content(new RetryInputStream(config, flagMessage))).getObjectId();
        cleanup.add(oid);
        byte[] content = api.readObject(oid, null, byte[].class);
        Assert.assertEquals("Content wrong size", 1, content.length);
        Assert.assertEquals("Wrong content", (byte) 65, content[0]);

        try {
            api.createObject(request.content(new RetryInputStream(null, null) {
                @Override
                public int read() throws IOException {
                    switch (callCount++) {
                    case 0:
                        throw new AtmosException("should not retry", 400);
                    case 1:
                        return 65;
                    }
                    return -1;
                }
            }));
            Assert.fail("HTTP 400 was retried and should not be");
        } catch (ClientHandlerException e) {
            Assert.assertEquals("Wrong http code", 400, ((AtmosException) e.getCause()).getHttpCode());
        }

        try {
            api.createObject(request.content(new RetryInputStream(null, null) {
                @Override
                public int read() throws IOException {
                    switch (callCount++) {
                    case 0:
                        throw new RuntimeException(flagMessage);
                    case 1:
                        return 65;
                    }
                    return -1;
                }
            }));
            Assert.fail("RuntimeException was retried and should not be");
        } catch (ClientHandlerException e) {
            Assert.assertEquals("Wrong exception message", flagMessage, e.getCause().getMessage());
        }
    }

    @Test
    public void testExpect100Continue() throws Exception {
        config.setEnableExpect100Continue(true);

        InputStream is = new RandomInputStream(5);
        CreateObjectRequest request = new CreateObjectRequest().content(is).contentLength(5);

        // test success first since some load-balancers screw up the next request after an E: 100-C failure
        cleanup.add(api.createObject(request).getObjectId());

        // now test failure
        String tokenId = config.getTokenId();
        config.setTokenId("bogustokenid");
        is = new RandomInputStream(5);
        try {
            api.createObject(request);
        } catch (AtmosException e) {
            Assert.assertEquals("wrong error code", 1033, e.getErrorCode());
            Assert.assertEquals("input stream was read", 5, is.available());
        } finally {
            config.setTokenId(tokenId);
            is.close();
        }
    }

    @Test
    public void testMultiThreadedBufferedWriter() throws Exception {
        int threadCount = 20;
        ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 5000, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>());

        // test with String
        List<Throwable> errorList = Collections.synchronizedList(new ArrayList<Throwable>());
        for (int i = 0; i < threadCount; i++) {
            executor.execute(
                    new ObjectTestThread<String>("Test thread " + i, "text/plain", String.class, errorList));
        }
        do {
            Thread.sleep(500);
        } while (executor.getActiveCount() > 0);
        if (!errorList.isEmpty()) {
            for (Throwable t : errorList)
                t.printStackTrace();
            Assert.fail("At least one thread failed");
        }

        // test with JAXB bean
        try {
            for (int i = 0; i < threadCount; i++) {
                executor.execute(new ObjectTestThread<AccessTokenPolicy>(
                        createTestTokenPolicy("Test thread " + i, "x.x.x." + i), "text/xml",
                        AccessTokenPolicy.class, errorList));
            }
            do {
                Thread.sleep(500);
            } while (executor.getActiveCount() > 0);
        } finally {
            executor.shutdown();
        }
        if (!errorList.isEmpty()) {
            for (Throwable t : errorList)
                t.printStackTrace();
            Assert.fail("At least one thread failed");
        }
    }

    @Test
    public void testProxyConfiguration() {
        AtmosConfig config = AtmosClientFactory.getAtmosConfig();
        URI proxyUri = config.getProxyUri();

        // don't run this test without a proxy config
        Assume.assumeNotNull(proxyUri);

        // capture existing system props for safety
        String oldProxyHost = System.getProperty("http.proxyHost");
        String oldProxyPort = System.getProperty("http.proxyPort");
        try {
            // just create and delete an object in each scenario
            // 1) URLConnection - no proxy
            config.setProxyUri(null);
            AtmosApi atmos = new AtmosApiBasicClient(config);
            ObjectId oid = atmos.createObject("URLConnection with no proxy", "text/plain");
            atmos.delete(oid);

            // 2) Apache - no proxy
            atmos = new AtmosApiClient(config);
            oid = atmos.createObject("Apache with no proxy", "text/plain");
            atmos.delete(oid);

            // 3) URLConnection - with config proxy
            config.setProxyUri(proxyUri);
            atmos = new AtmosApiBasicClient(config);
            oid = atmos.createObject("URLConnection with config proxy", "text/plain");
            atmos.delete(oid);

            // 4) Apache - with config proxy
            atmos = new AtmosApiClient(config);
            oid = atmos.createObject("Apache with config proxy", "text/plain");
            atmos.delete(oid);

            // 5) URLConnection - with system props (old school) proxy
            config.setProxyUri(null);
            System.setProperty("http.proxyHost", proxyUri.getHost());
            System.setProperty("http.proxyPort", "" + proxyUri.getPort());
            atmos = new AtmosApiBasicClient(config);
            oid = atmos.createObject("URLConnection with sysprop proxy", "text/plain");
            atmos.delete(oid);

            // 6) Apache - with system props (old school) proxy
            // can't specify proxy user/pass in system props
            atmos = new AtmosApiClient(config);
            oid = atmos.createObject("Apache with sysprop proxy", "text/plain");
            atmos.delete(oid);
        } finally {
            // now play nice and reset old props
            if (oldProxyHost != null)
                System.setProperty("http.proxyHost", oldProxyHost);
            if (oldProxyPort != null)
                System.setProperty("http.proxyPort", oldProxyPort);
        }
    }

    @Test
    public void testRetention() throws Exception {
        Metadata retention = new Metadata("retentionperiod", "1year", false);
        CreateObjectRequest request = new CreateObjectRequest().content(null).userMetadata(retention);
        ObjectId oid = api.createObject(request.contentType("text/plain")).getObjectId();
        cleanup.add(oid);

        Thread.sleep(2000);

        // make sure retention is enabled
        ObjectInfo info = api.getObjectInfo(oid);
        Assume.assumeTrue(info.getRetention().isEnabled());
        Calendar newEnd = Calendar.getInstance();
        newEnd.setTime(info.getRetainedUntil());

        DateFormat iso8601Format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
        String retentionEnd = "user.maui.retentionEnd";

        newEnd.set(Calendar.HOUR, 0);
        newEnd.set(Calendar.MINUTE, 0);
        newEnd.set(Calendar.SECOND, 0);
        newEnd.set(Calendar.MILLISECOND, 0);
        newEnd.add(Calendar.DATE, -1);
        try {
            api.setUserMetadata(oid, new Metadata(retentionEnd, iso8601Format.format(newEnd.getTime()), false));
            Assert.fail("should not be able to shorten retention period");
        } catch (AtmosException e) {
            Assert.assertEquals("Wrong error code", 1002, e.getErrorCode());
        }

        // disable retention so we can delete (won't work on compliant subtenants!)
        api.setUserMetadata(oid, new Metadata("user.maui.retentionEnable", "false", false));
    }

    protected String rand8char() {
        Random r = new Random();
        StringBuilder sb = new StringBuilder(8);
        for (int i = 0; i < 8; i++) {
            sb.append((char) ('a' + r.nextInt(26)));
        }
        return sb.toString();
    }

    private AccessTokenPolicy createTestTokenPolicy(String allow, String deny) {
        AccessTokenPolicy.Source source = new AccessTokenPolicy.Source();
        source.setAllowList(Arrays.asList(allow));
        source.setDenyList(Arrays.asList(deny));
        AccessTokenPolicy policy = new AccessTokenPolicy();
        policy.setExpiration(new Date(1355897000000L));
        policy.setMaxDownloads(5);
        policy.setMaxUploads(10);
        policy.setSource(source);
        return policy;
    }

    private class RetryInputStream extends InputStream {
        protected int callCount = 0;
        private long now;
        private long lastTime;
        private AtmosConfig config;
        private String flagMessage;

        public RetryInputStream(AtmosConfig config, String flagMessage) {
            this.config = config;
            this.flagMessage = flagMessage;
        }

        @Override
        public int read() throws IOException {
            switch (callCount++) {
            case 0:
                lastTime = System.currentTimeMillis();
                throw new AtmosException("foo", 500);
            case 1:
                now = System.currentTimeMillis();
                Assert.assertTrue("Retry delay for 500 error was not honored",
                        now - lastTime >= config.getRetryDelayMillis());
                lastTime = now;
                throw new AtmosException("bar", 500, 1040);
            case 2:
                now = System.currentTimeMillis();
                Assert.assertTrue("Retry delay for 1040 error was not honored",
                        now - lastTime >= config.getRetryDelayMillis() + 300);
                lastTime = now;
                throw new IOException("baz");
            case 3:
                now = System.currentTimeMillis();
                Assert.assertTrue("Retry delay for IOException was not honored",
                        now - lastTime >= config.getRetryDelayMillis());
                lastTime = now;
                throw new AtmosException(flagMessage, 500);
            case 4:
                return 65;
            }
            return -1;
        }

        @Override
        public synchronized void reset() throws IOException {
        }

        @Override
        public boolean markSupported() {
            return true;
        }
    }

    private class ObjectTestThread<T> implements Runnable {
        private T content;
        private String contentType;
        private Class<T> objectType;
        private List<Throwable> errorList;

        public ObjectTestThread(T content, String contentType, Class<T> objectType, List<Throwable> errorList) {
            this.content = content;
            this.contentType = contentType;
            this.objectType = objectType;
            this.errorList = errorList;
        }

        @Override
        public void run() {
            try {
                ObjectId oid = api.createObject(content, contentType);
                cleanup.add(oid);
                T readContent = api.readObject(oid, null, objectType);
                Assert.assertEquals("Content for object " + oid + " not equal", content, readContent);
            } catch (Throwable t) {
                errorList.add(t);
            }
        }
    }
}