com.github.ambry.admin.AdminIntegrationTest.java Source code

Java tutorial

Introduction

Here is the source code for com.github.ambry.admin.AdminIntegrationTest.java

Source

/**
 * Copyright 2016 LinkedIn Corp. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */
package com.github.ambry.admin;

import com.github.ambry.clustermap.ClusterMap;
import com.github.ambry.clustermap.MockClusterMap;
import com.github.ambry.clustermap.PartitionId;
import com.github.ambry.commons.BlobId;
import com.github.ambry.commons.LoggingNotificationSystem;
import com.github.ambry.config.VerifiableProperties;
import com.github.ambry.messageformat.BlobProperties;
import com.github.ambry.rest.NettyClient;
import com.github.ambry.rest.RestServer;
import com.github.ambry.rest.RestTestUtils;
import com.github.ambry.rest.RestUtils;
import com.github.ambry.utils.Utils;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.DefaultHttpHeaders;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpContent;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpObject;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.HttpVersion;
import io.netty.handler.codec.http.LastHttpContent;
import io.netty.util.ReferenceCountUtil;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Properties;
import java.util.Queue;
import java.util.Random;
import java.util.concurrent.ExecutionException;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import static org.junit.Assert.*;

/**
 * Integration tests for Admin.
 */
public class AdminIntegrationTest {
    private static final int SERVER_PORT = 16503;
    private static final ClusterMap CLUSTER_MAP;

    static {
        try {
            CLUSTER_MAP = new MockClusterMap();
        } catch (IOException e) {
            throw new IllegalStateException(e);
        }
    }

    private static RestServer adminRestServer = null;
    private static NettyClient nettyClient = null;

    /**
     * Sets up an admin server.
     * @throws InstantiationException
     * @throws InterruptedException
     */
    @BeforeClass
    public static void setup() throws InstantiationException, InterruptedException {
        adminRestServer = new RestServer(buildAdminVProps(), CLUSTER_MAP, new LoggingNotificationSystem());
        adminRestServer.start();
        nettyClient = new NettyClient("localhost", SERVER_PORT);
    }

    /**
     * Shuts down the admin server.
     */
    @AfterClass
    public static void teardown() {
        if (nettyClient != null) {
            nettyClient.close();
        }
        if (adminRestServer != null) {
            adminRestServer.shutdown();
        }
    }

    /**
     * Tests the {@link AdminBlobStorageService#ECHO} operation. Checks to see that the echo matches input text.
     * @throws ExecutionException
     * @throws InterruptedException
     * @throws JSONException
     */
    @Test
    public void echoTest() throws ExecutionException, InterruptedException, JSONException {
        String inputText = "loremIpsum";
        String uri = AdminBlobStorageService.ECHO + "?" + EchoHandler.TEXT_KEY + "=" + inputText;
        FullHttpRequest httpRequest = buildRequest(HttpMethod.GET, uri, null, null);
        Queue<HttpObject> responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
        HttpResponse response = (HttpResponse) responseParts.poll();
        assertEquals("Unexpected status", HttpResponseStatus.OK, response.getStatus());
        assertEquals("Unexpected Content-Type", "application/json",
                HttpHeaders.getHeader(response, HttpHeaders.Names.CONTENT_TYPE));
        ByteBuffer buffer = getContent(response, responseParts);
        String echoedText = new JSONObject(new String(buffer.array())).getString(EchoHandler.TEXT_KEY);
        assertEquals("Did not get expected response", inputText, echoedText);
    }

    /**
     * Tests the {@link AdminBlobStorageService#GET_REPLICAS_FOR_BLOB_ID} operation.
     * <p/>
     * For a random {@link PartitionId} in the {@link ClusterMap}, a {@link BlobId} is created. The string representation
     * is sent to the server as a part of request. The returned replica list is checked for equality against a locally
     * obtained replica list.
     * @throws ExecutionException
     * @throws InterruptedException
     * @throws JSONException
     */
    @Test
    public void getReplicasForBlobIdTest() throws ExecutionException, InterruptedException, JSONException {
        List<PartitionId> partitionIds = CLUSTER_MAP.getWritablePartitionIds();
        PartitionId partitionId = partitionIds.get(new Random().nextInt(partitionIds.size()));
        String originalReplicaStr = partitionId.getReplicaIds().toString().replace(", ", ",");
        BlobId blobId = new BlobId(partitionId);
        String uri = AdminBlobStorageService.GET_REPLICAS_FOR_BLOB_ID + "?"
                + GetReplicasForBlobIdHandler.BLOB_ID_KEY + "=" + blobId;
        FullHttpRequest httpRequest = buildRequest(HttpMethod.GET, uri, null, null);
        Queue<HttpObject> responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
        HttpResponse response = (HttpResponse) responseParts.poll();
        assertEquals("Unexpected status", HttpResponseStatus.OK, response.getStatus());
        assertEquals("Unexpected Content-Type", "application/json",
                HttpHeaders.getHeader(response, HttpHeaders.Names.CONTENT_TYPE));
        ByteBuffer buffer = getContent(response, responseParts);
        JSONObject responseObj = new JSONObject(new String(buffer.array()));
        String returnedReplicasStr = responseObj.getString(GetReplicasForBlobIdHandler.REPLICAS_KEY).replace("\"",
                "");
        assertEquals("Returned response for the BlobId do no match with the replicas IDs of partition",
                originalReplicaStr, returnedReplicasStr);
    }

    /**
     * Tests blob POST, GET, HEAD and DELETE operations.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    @Test
    public void postGetHeadDeleteTest() throws ExecutionException, InterruptedException {
        ByteBuffer content = ByteBuffer.wrap(RestTestUtils.getRandomBytes(1024));
        String serviceId = "postGetHeadDeleteServiceID";
        String contentType = "application/octet-stream";
        String ownerId = "postGetHeadDeleteOwnerID";
        HttpHeaders headers = new DefaultHttpHeaders();
        setAmbryHeaders(headers, content.capacity(), 7200, false, serviceId, contentType, ownerId);
        headers.set(HttpHeaders.Names.CONTENT_LENGTH, content.capacity());
        headers.add(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key1", "value1");
        headers.add(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + "key2", "value2");

        String blobId = postBlobAndVerify(headers, content);
        getBlobAndVerify(blobId, headers, content);
        getHeadAndVerify(blobId, headers);
        deleteBlobAndVerify(blobId);

        // check GET, HEAD and DELETE after delete.
        verifyOperationsAfterDelete(blobId);
    }

    // helpers
    // general

    /**
     * Method to easily create a request.
     * @param httpMethod the {@link HttpMethod} desired.
     * @param uri string representation of the desired URI.
     * @param headers any associated headers as a {@link HttpHeaders} object. Can be null.
     * @param content the content that accompanies the request. Can be null.
     * @return A {@link FullHttpRequest} object that defines the request required by the input.
     */
    private FullHttpRequest buildRequest(HttpMethod httpMethod, String uri, HttpHeaders headers,
            ByteBuffer content) {
        ByteBuf contentBuf;
        if (content != null) {
            contentBuf = Unpooled.wrappedBuffer(content);
        } else {
            contentBuf = Unpooled.buffer(0);
        }
        FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, httpMethod, uri, contentBuf);
        if (headers != null) {
            httpRequest.headers().set(headers);
        }
        return httpRequest;
    }

    /**
     * Combines all the parts in {@code contents} into one {@link ByteBuffer}.
     * @param response the {@link HttpResponse} containing headers.
     * @param contents the content of the response.
     * @return a {@link ByteBuffer} that contains all the data in {@code contents}.
     */
    private ByteBuffer getContent(HttpResponse response, Queue<HttpObject> contents) {
        long contentLength = HttpHeaders.getContentLength(response, -1);
        if (contentLength == -1) {
            contentLength = HttpHeaders.getIntHeader(response, RestUtils.Headers.BLOB_SIZE, 0);
        }
        ByteBuffer buffer = ByteBuffer.allocate((int) contentLength);
        for (HttpObject object : contents) {
            HttpContent content = (HttpContent) object;
            buffer.put(content.content().nioBuffer());
            ReferenceCountUtil.release(content);
        }
        return buffer;
    }

    /**
     * Discards all the content in {@code contents}.
     * @param contents the content to discard.
     * @param expectedDiscardCount the number of {@link HttpObject}s that are expected to discarded.
     */
    private void discardContent(Queue<HttpObject> contents, int expectedDiscardCount) {
        assertEquals("Objects that will be discarded differ from expected", expectedDiscardCount, contents.size());
        boolean endMarkerFound = false;
        for (HttpObject object : contents) {
            assertFalse("There should have been only a single end marker", endMarkerFound);
            endMarkerFound = object instanceof LastHttpContent;
            ReferenceCountUtil.release(object);
        }
        assertTrue("There should have been an end marker", endMarkerFound);
    }

    // BeforeClass helpers

    /**
     * Builds properties required to start a {@link RestServer} as an Admin server.
     * @return a {@link VerifiableProperties} with the parameters for an Admin server.
     */
    private static VerifiableProperties buildAdminVProps() {
        Properties properties = new Properties();
        properties.put("rest.server.blob.storage.service.factory",
                "com.github.ambry.admin.AdminBlobStorageServiceFactory");
        properties.put("rest.server.router.factory", "com.github.ambry.router.InMemoryRouterFactory");
        properties.put("netty.server.port", Integer.toString(SERVER_PORT));
        return new VerifiableProperties(properties);
    }

    // postGetHeadDeleteTest() helpers

    /**
     * Sets headers that helps build {@link BlobProperties} on the server. See argument list for the headers that are set.
     * Any other headers have to be set explicitly.
     * @param httpHeaders the {@link HttpHeaders} where the headers should be set.
     * @param contentLength sets the {@link RestUtils.Headers#BLOB_SIZE} header. Required.
     * @param ttlInSecs sets the {@link RestUtils.Headers#TTL} header. Set to {@link Utils#Infinite_Time} if no
     *                  expiry.
     * @param isPrivate sets the {@link RestUtils.Headers#PRIVATE} header. Allowed values: true, false.
     * @param serviceId sets the {@link RestUtils.Headers#SERVICE_ID} header. Required.
     * @param contentType sets the {@link RestUtils.Headers#AMBRY_CONTENT_TYPE} header. Required and has to be a valid MIME
     *                    type.
     * @param ownerId sets the {@link RestUtils.Headers#OWNER_ID} header. Optional - if not required, send null.
     * @throws IllegalArgumentException if any of {@code headers}, {@code serviceId}, {@code contentType} is null or if
     *                                  {@code contentLength} < 0 or if {@code ttlInSecs} < -1.
     */
    private void setAmbryHeaders(HttpHeaders httpHeaders, long contentLength, long ttlInSecs, boolean isPrivate,
            String serviceId, String contentType, String ownerId) {
        if (httpHeaders != null && contentLength >= 0 && ttlInSecs >= -1 && serviceId != null
                && contentType != null) {
            httpHeaders.add(RestUtils.Headers.BLOB_SIZE, contentLength);
            httpHeaders.add(RestUtils.Headers.TTL, ttlInSecs);
            httpHeaders.add(RestUtils.Headers.PRIVATE, isPrivate);
            httpHeaders.add(RestUtils.Headers.SERVICE_ID, serviceId);
            httpHeaders.add(RestUtils.Headers.AMBRY_CONTENT_TYPE, contentType);
            if (ownerId != null) {
                httpHeaders.add(RestUtils.Headers.OWNER_ID, ownerId);
            }
        } else {
            throw new IllegalArgumentException("Some required arguments are null. Cannot set ambry headers");
        }
    }

    /**
     * Posts a blob with the given {@code headers} and {@code content}.
     * @param headers the headers required.
     * @param content the content of the blob.
     * @return the blob ID of the blob.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    private String postBlobAndVerify(HttpHeaders headers, ByteBuffer content)
            throws ExecutionException, InterruptedException {
        FullHttpRequest httpRequest = buildRequest(HttpMethod.POST, "/", headers, content);
        Queue<HttpObject> responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
        HttpResponse response = (HttpResponse) responseParts.poll();
        discardContent(responseParts, 1);
        assertEquals("Unexpected response status", HttpResponseStatus.CREATED, response.getStatus());
        assertTrue("No Date header", HttpHeaders.getDateHeader(response, HttpHeaders.Names.DATE, null) != null);
        assertTrue("No " + RestUtils.Headers.CREATION_TIME,
                HttpHeaders.getHeader(response, RestUtils.Headers.CREATION_TIME, null) != null);
        assertEquals("Content-Length is not 0", 0, HttpHeaders.getContentLength(response));
        String blobId = HttpHeaders.getHeader(response, HttpHeaders.Names.LOCATION, null);

        if (blobId == null) {
            fail("postBlobAndVerify did not return a blob ID");
        }
        return blobId;
    }

    /**
     * Gets the blob with blob ID {@code blobId} and verifies that the headers and content match with what is expected.
     * @param blobId the blob ID of the blob to GET.
     * @param expectedHeaders the expected headers in the response.
     * @param expectedContent the expected content of the blob.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    private void getBlobAndVerify(String blobId, HttpHeaders expectedHeaders, ByteBuffer expectedContent)
            throws ExecutionException, InterruptedException {
        FullHttpRequest httpRequest = buildRequest(HttpMethod.GET, blobId, null, null);
        Queue<HttpObject> responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
        HttpResponse response = (HttpResponse) responseParts.poll();
        assertEquals("Unexpected response status", HttpResponseStatus.OK, response.getStatus());
        checkCommonGetHeadHeaders(response.headers(), expectedHeaders);
        ByteBuffer responseContent = getContent(response, responseParts);
        assertArrayEquals("GET content does not match original content", expectedContent.array(),
                responseContent.array());
    }

    /**
     * Gets the headers of the blob with blob ID {@code blobId} and verifies them against what is expected.
     * @param blobId the blob ID of the blob to HEAD.
     * @param expectedHeaders the expected headers in the response.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    private void getHeadAndVerify(String blobId, HttpHeaders expectedHeaders)
            throws ExecutionException, InterruptedException {
        FullHttpRequest httpRequest = buildRequest(HttpMethod.HEAD, blobId, null, null);
        Queue<HttpObject> responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
        HttpResponse response = (HttpResponse) responseParts.poll();
        discardContent(responseParts, 1);
        assertEquals("Unexpected response status", HttpResponseStatus.OK, response.getStatus());
        checkCommonGetHeadHeaders(response.headers(), expectedHeaders);
        assertEquals("Content-Length does not match blob size",
                Long.parseLong(expectedHeaders.get(RestUtils.Headers.BLOB_SIZE)),
                HttpHeaders.getContentLength(response));
        assertEquals("Blob size does not match", expectedHeaders.get(RestUtils.Headers.BLOB_SIZE),
                HttpHeaders.getHeader(response, RestUtils.Headers.BLOB_SIZE));
        assertEquals(RestUtils.Headers.SERVICE_ID + " does not match",
                expectedHeaders.get(RestUtils.Headers.SERVICE_ID),
                HttpHeaders.getHeader(response, RestUtils.Headers.SERVICE_ID));
        assertEquals(RestUtils.Headers.PRIVATE + " does not match", expectedHeaders.get(RestUtils.Headers.PRIVATE),
                HttpHeaders.getHeader(response, RestUtils.Headers.PRIVATE));
        assertEquals(RestUtils.Headers.AMBRY_CONTENT_TYPE + " does not match",
                expectedHeaders.get(RestUtils.Headers.AMBRY_CONTENT_TYPE),
                HttpHeaders.getHeader(response, RestUtils.Headers.AMBRY_CONTENT_TYPE));
        assertTrue("No " + RestUtils.Headers.CREATION_TIME,
                HttpHeaders.getHeader(response, RestUtils.Headers.CREATION_TIME, null) != null);
        if (Long.parseLong(expectedHeaders.get(RestUtils.Headers.TTL)) != Utils.Infinite_Time) {
            assertEquals(RestUtils.Headers.TTL + " does not match", expectedHeaders.get(RestUtils.Headers.TTL),
                    HttpHeaders.getHeader(response, RestUtils.Headers.TTL));
        }
        if (expectedHeaders.contains(RestUtils.Headers.OWNER_ID)) {
            assertEquals(RestUtils.Headers.OWNER_ID + " does not match",
                    expectedHeaders.get(RestUtils.Headers.OWNER_ID),
                    HttpHeaders.getHeader(response, RestUtils.Headers.OWNER_ID));
        }
    }

    /**
     * Deletes the blob with blob ID {@code blobId} and verifies the response returned.
     * @param blobId the blob ID of the blob to DELETE.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    private void deleteBlobAndVerify(String blobId) throws ExecutionException, InterruptedException {
        FullHttpRequest httpRequest = buildRequest(HttpMethod.DELETE, blobId, null, null);
        verifyDeleted(httpRequest, HttpResponseStatus.ACCEPTED);
    }

    /**
     * Verifies that the right response code is returned for GET, HEAD and DELETE once a blob is deleted.
     * @param blobId the ID of the blob that was deleted.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    private void verifyOperationsAfterDelete(String blobId) throws ExecutionException, InterruptedException {
        FullHttpRequest httpRequest = buildRequest(HttpMethod.GET, blobId, null, null);
        verifyDeleted(httpRequest, HttpResponseStatus.GONE);

        httpRequest = buildRequest(HttpMethod.HEAD, blobId, null, null);
        verifyDeleted(httpRequest, HttpResponseStatus.GONE);

        httpRequest = buildRequest(HttpMethod.DELETE, blobId, null, null);
        verifyDeleted(httpRequest, HttpResponseStatus.ACCEPTED);
    }

    /**
     * Verifies that a request returns the right response code  once the blob has been deleted.
     * @param httpRequest the {@link FullHttpRequest} to send to the server.
     * @param expectedStatusCode the expected {@link HttpResponseStatus}.
     * @throws ExecutionException
     * @throws InterruptedException
     */
    private void verifyDeleted(FullHttpRequest httpRequest, HttpResponseStatus expectedStatusCode)
            throws ExecutionException, InterruptedException {
        Queue<HttpObject> responseParts = nettyClient.sendRequest(httpRequest, null, null).get();
        HttpResponse response = (HttpResponse) responseParts.poll();
        discardContent(responseParts, 1);
        assertEquals("Unexpected response status", expectedStatusCode, response.getStatus());
        assertTrue("No Date header", HttpHeaders.getDateHeader(response, HttpHeaders.Names.DATE, null) != null);
    }

    /**
     * Checks headers that are common to HEAD and GET.
     * @param receivedHeaders the {@link HttpHeaders} that were received.
     * @param expectedHeaders the expected headers.
     */
    private void checkCommonGetHeadHeaders(HttpHeaders receivedHeaders, HttpHeaders expectedHeaders) {
        assertEquals("Content-Type does not match", expectedHeaders.get(RestUtils.Headers.AMBRY_CONTENT_TYPE),
                receivedHeaders.get(HttpHeaders.Names.CONTENT_TYPE));
        assertTrue("No Date header", receivedHeaders.get(HttpHeaders.Names.DATE) != null);
        assertTrue("No Last-Modified header", receivedHeaders.get(HttpHeaders.Names.LAST_MODIFIED) != null);
        assertEquals(RestUtils.Headers.BLOB_SIZE + " does not match",
                expectedHeaders.get(RestUtils.Headers.BLOB_SIZE), receivedHeaders.get(RestUtils.Headers.BLOB_SIZE));
    }
}