co.cask.cdap.test.internal.AppFabricClient.java Source code

Java tutorial

Introduction

Here is the source code for co.cask.cdap.test.internal.AppFabricClient.java

Source

/*
 * Copyright  2014 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.test.internal;

import co.cask.cdap.app.program.ManifestFields;
import co.cask.cdap.common.conf.Constants;
import co.cask.cdap.gateway.handlers.AppFabricHttpHandler;
import co.cask.cdap.gateway.handlers.ServiceHttpHandler;
import co.cask.cdap.internal.app.BufferFileInputStream;
import co.cask.cdap.proto.RunRecord;
import co.cask.http.BodyConsumer;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
import org.apache.twill.internal.ApplicationBundler;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.handler.codec.http.DefaultHttpRequest;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.zip.ZipException;

/**
 * Client tool for AppFabricHttpHandler.
 */
public class AppFabricClient {
    private static final Logger LOG = LoggerFactory.getLogger(AppFabricClient.class);
    private static final Gson GSON = new Gson();

    private final AppFabricHttpHandler httpHandler;
    private final ServiceHttpHandler serviceHttpHandler;
    private final LocationFactory locationFactory;

    public AppFabricClient(AppFabricHttpHandler httpHandler, ServiceHttpHandler serviceHttpHandler,
            LocationFactory locationFactory) {
        this.httpHandler = httpHandler;
        this.serviceHttpHandler = serviceHttpHandler;
        this.locationFactory = locationFactory;
    }

    public void reset() {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/unrecoverable/reset");
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.PUT, uri);
        httpHandler.resetCDAP(request, responder);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "reset application failed");
    }

    public void startProgram(String appId, String flowId, String type, Map<String, String> args) {

        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/%s/%s/start", appId, type, flowId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri);
        String argString = GSON.toJson(args);
        if (argString != null) {
            request.setContent(ChannelBuffers.wrappedBuffer(argString.getBytes(Charsets.UTF_8)));
        }
        httpHandler.startProgram(request, responder, appId, type, flowId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "start " + type + " failed");
    }

    public void stopProgram(String appId, String flowId, String type) {

        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/%s/%s/stop", appId, type, flowId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri);
        httpHandler.stopProgram(request, responder, appId, type, flowId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "stop " + type + " failed");
    }

    public String getStatus(String appId, String flowId, String type) {

        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/%s/%s/status", appId, type, flowId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, uri);
        httpHandler.getStatus(request, responder, appId, type, flowId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "get status " + type + " failed");
        Map<String, String> json = responder.decodeResponseContent(new TypeToken<Map<String, String>>() {
        });
        return json.get("status");
    }

    public void setRunnableInstances(String applicationId, String serviceName, String runnableName, int instances) {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/services/%s/runnables/%s/instances", applicationId, serviceName,
                runnableName);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.PUT, uri);
        JsonObject json = new JsonObject();
        json.addProperty("instances", instances);
        request.setContent(ChannelBuffers.wrappedBuffer(json.toString().getBytes()));
        serviceHttpHandler.setInstances(request, responder, applicationId, serviceName, runnableName);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "set runnable instances failed");
    }

    public int getRunnableInstances(String applicationId, String serviceName, String runnableName) {

        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/services/%s/runnables/%s/instances", applicationId, serviceName,
                runnableName);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
        serviceHttpHandler.getInstances(request, responder, applicationId, serviceName, runnableName);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "get runnable instances failed");
        Map<String, String> instances = responder.decodeResponseContent(new TypeToken<Map<String, String>>() {
        });
        return Integer.parseInt(instances.get("provisioned"));
    }

    public void setFlowletInstances(String applicationId, String flowId, String flowletName, int instances) {

        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/flows/%s/flowlets/%s/instances/%s", applicationId, flowId,
                flowletName, instances);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.PUT, uri);
        JsonObject json = new JsonObject();
        json.addProperty("instances", instances);
        request.setContent(ChannelBuffers.wrappedBuffer(json.toString().getBytes()));
        httpHandler.setFlowletInstances(request, responder, applicationId, flowId, flowletName);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, "set flowlet instances failed");
    }

    public List<String> getSchedules(String appId, String wflowId) {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/workflows/%s/schedules", appId, wflowId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
        httpHandler.workflowSchedules(request, responder, appId, wflowId);

        List<String> schedules = responder.decodeResponseContent(new TypeToken<List<String>>() {
        });
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, " getting workflow schedules failed");
        return schedules;
    }

    public List<RunRecord> getHistory(String appId, String wflowId) {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/workflows/%s/history", appId, wflowId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
        httpHandler.runnableHistory(request, responder, appId, "workflows", wflowId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, " getting workflow schedules failed");

        return responder.decodeResponseContent(new TypeToken<List<RunRecord>>() {
        });
    }

    public void suspend(String appId, String wflowId, String schedId) {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/workflows/%s/schedules/%s/suspend", appId, wflowId, schedId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
        httpHandler.workflowScheduleSuspend(request, responder, appId, wflowId, schedId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, " getting workflow schedules failed");
    }

    public void resume(String appId, String wflowId, String schedId) {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/workflows/%s/schedules/%s/resume", appId, wflowId, schedId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
        httpHandler.workflowScheduleResume(request, responder, appId, wflowId, schedId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, " getting workflow schedules failed");
    }

    public String scheduleStatus(String appId, String wflowId, String schedId) {
        MockResponder responder = new MockResponder();
        String uri = String.format("/v2/apps/%s/workflows/%s/schedules/%s/status", appId, wflowId, schedId);
        HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
        httpHandler.getScheuleState(request, responder, appId, wflowId, schedId);
        Preconditions.checkArgument(responder.getStatus().getCode() == 200, " getting workflow schedules failed");
        Map<String, String> json = responder.decodeResponseContent(new TypeToken<Map<String, String>>() {
        });
        return json.get("status");
    }

    /**
     * Given a class generates a manifest file with main-class as class.
     *
     * @param klass to set as Main-Class in manifest file.
     * @return An instance {@link java.util.jar.Manifest}
     */
    public static Manifest getManifestWithMainClass(Class<?> klass) {
        Manifest manifest = new Manifest();
        manifest.getMainAttributes().put(ManifestFields.MANIFEST_VERSION, "1.0");
        manifest.getMainAttributes().put(ManifestFields.MAIN_CLASS, klass.getName());
        return manifest;
    }

    public Location deployApplication(final String applicationId, Class<?> applicationClz,
            File... bundleEmbeddedJars) throws Exception {

        Preconditions.checkNotNull(applicationClz, "Application cannot be null.");

        Location deployedJar = locationFactory
                .create(createDeploymentJar(locationFactory, applicationClz, bundleEmbeddedJars).toURI());
        LOG.info("Created deployedJar at {}", deployedJar.toURI().toASCIIString());

        DefaultHttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/v2/apps");
        request.setHeader(Constants.Gateway.API_KEY, "api-key-example");
        request.setHeader("X-Archive-Name", applicationId + ".jar");
        MockResponder mockResponder = new MockResponder();
        BodyConsumer bodyConsumer = httpHandler.deploy(request, mockResponder);

        BufferFileInputStream is = new BufferFileInputStream(deployedJar.getInputStream(), 100 * 1024);
        try {
            byte[] chunk = is.read();
            while (chunk.length > 0) {
                mockResponder = new MockResponder();
                bodyConsumer.chunk(ChannelBuffers.wrappedBuffer(chunk), mockResponder);
                Preconditions.checkState(mockResponder.getStatus() == null, "failed to deploy app");
                chunk = is.read();
            }
            mockResponder = new MockResponder();
            bodyConsumer.finished(mockResponder);
            Preconditions.checkState(mockResponder.getStatus().getCode() == 200, "failed to deploy app");
        } catch (Exception e) {
            throw Throwables.propagate(e);
        } finally {
            is.close();
        }
        return deployedJar;
    }

    private static File createDeploymentJar(LocationFactory locationFactory, Class<?> clz,
            File... bundleEmbeddedJars) throws IOException {

        ApplicationBundler bundler = new ApplicationBundler(
                ImmutableList.of("co.cask.cdap.api", "org.apache.hadoop", "org.apache.hbase", "org.apache.hive"));
        Location jarLocation = locationFactory.create(clz.getName()).getTempFile(".jar");
        bundler.createBundle(jarLocation, clz);

        Location deployJar = locationFactory.create(clz.getName()).getTempFile(".jar");

        // Creates Manifest
        Manifest manifest = new Manifest();
        manifest.getMainAttributes().put(ManifestFields.MANIFEST_VERSION, "1.0");
        manifest.getMainAttributes().put(ManifestFields.MAIN_CLASS, clz.getName());

        // Create the program jar for deployment. It removes the "classes/" prefix as that's the convention taken
        // by the ApplicationBundler inside Twill.
        JarOutputStream jarOutput = new JarOutputStream(deployJar.getOutputStream(), manifest);
        try {
            JarInputStream jarInput = new JarInputStream(jarLocation.getInputStream());
            try {
                JarEntry jarEntry = jarInput.getNextJarEntry();
                while (jarEntry != null) {
                    boolean isDir = jarEntry.isDirectory();
                    String entryName = jarEntry.getName();
                    if (!entryName.equals("classes/")) {
                        if (entryName.startsWith("classes/")) {
                            jarEntry = new JarEntry(entryName.substring("classes/".length()));
                        } else {
                            jarEntry = new JarEntry(entryName);
                        }

                        // TODO: this is due to manifest possibly already existing in the jar, but we also
                        // create a manifest programatically so it's possible to have a duplicate entry here
                        if ("META-INF/MANIFEST.MF".equalsIgnoreCase(jarEntry.getName())) {
                            jarEntry = jarInput.getNextJarEntry();
                            continue;
                        }

                        jarOutput.putNextEntry(jarEntry);
                        if (!isDir) {
                            ByteStreams.copy(jarInput, jarOutput);
                        }
                    }

                    jarEntry = jarInput.getNextJarEntry();
                }
            } finally {
                jarInput.close();
            }

            for (File embeddedJar : bundleEmbeddedJars) {
                JarEntry jarEntry = new JarEntry("lib/" + embeddedJar.getName());
                jarOutput.putNextEntry(jarEntry);
                Files.copy(embeddedJar, jarOutput);
            }

        } finally {
            jarOutput.close();
        }

        return new File(deployJar.toURI());
    }

}