org.apache.gobblin.service.modules.orchestration.AzkabanAjaxAPIClient.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.gobblin.service.modules.orchestration.AzkabanAjaxAPIClient.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.gobblin.service.modules.orchestration;

import java.io.File;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Random;

import org.apache.commons.codec.EncoderException;
import org.apache.commons.codec.net.URLCodec;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.collect.Maps;

import lombok.Cleanup;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@Deprecated
/**
 * This format of azkaban client is obsolete. Please use {@link AzkabanClient} as the new alternative.
 */
public class AzkabanAjaxAPIClient {
    private static Splitter SPLIT_ON_COMMA = Splitter.on(",").omitEmptyStrings().trimResults();

    // TODO: Ensure GET call urls do not grow too big
    private static final int LOW_NETWORK_TRAFFIC_BEGIN_HOUR = 17;
    private static final int LOW_NETWORK_TRAFFIC_END_HOUR = 22;
    private static final int JOB_START_DELAY_MINUTES = 5;
    private static final long MILLISECONDS_IN_HOUR = 60 * 60 * 1000;
    private static final URLCodec codec = new URLCodec();

    /***
     * Authenticate a user and obtain a session.id from response. Once a session.id has been obtained,
     * until the session expires, this id can be used to do any API requests with a proper permission granted.
     * A session expires if user log's out, changes machine, browser or location, if Azkaban is restarted,
     * or if the session expires. The default session timeout is 24 hours (one day). User can re-login irrespective
     * of wheter the session has expired or not. For the same user, a new session will always override the old one.
     * @param username Username.
     * @param password Password.
     * @param azkabanServerUrl Azkaban Server Url.
     * @return Session Id.
     * @throws IOException
     * @throws EncoderException
     */
    public static String authenticateAndGetSessionId(String username, String password, String azkabanServerUrl)
            throws IOException, EncoderException {
        // Create post request
        Map<String, String> params = Maps.newHashMap();
        params.put("action", "login");
        params.put("username", username);
        params.put("password", codec.encode(password));

        return executePostRequest(preparePostRequest(azkabanServerUrl, null, params)).get("session.id");
    }

    /***
     * Get project.id for a Project Name.
     * @param sessionId Session Id.
     * @param azkabanProjectConfig Azkaban Project Config.
     * @return Project Id.
     * @throws IOException
     */
    public static String getProjectId(String sessionId, AzkabanProjectConfig azkabanProjectConfig)
            throws IOException {
        // Note: Every get call to Azkaban provides a projectId in response, so we have are using fetchProjectFlows call
        // .. because it does not need any additional params other than project name
        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "fetchprojectflows");
        params.put("project", azkabanProjectConfig.getAzkabanProjectName());

        return executeGetRequest(
                prepareGetRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/manager", sessionId, params))
                        .get("projectId");
    }

    /***
     * Creates an Azkaban project and uploads the zip file. If proxy user and group permissions are specified in
     * Azkaban Project Config, then this method also adds it to the project configuration.
     * @param sessionId Session Id.
     * @param zipFilePath Zip file to upload.
     * @param azkabanProjectConfig Azkaban Project Config.
     * @return Project Id.
     * @throws IOException
     */
    public static String createAzkabanProject(String sessionId, String zipFilePath,
            AzkabanProjectConfig azkabanProjectConfig) throws IOException {
        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "executeFlow");
        params.put("name", azkabanProjectConfig.getAzkabanProjectName());
        params.put("description", azkabanProjectConfig.getAzkabanProjectDescription());

        executePostRequest(preparePostRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/manager?action=create",
                sessionId, params));

        // Add proxy user if any
        if (azkabanProjectConfig.getAzkabanUserToProxy().isPresent()) {
            Iterable<String> proxyUsers = SPLIT_ON_COMMA.split(azkabanProjectConfig.getAzkabanUserToProxy().get());
            for (String user : proxyUsers) {
                addProxyUser(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                        azkabanProjectConfig.getAzkabanProjectName(), user);
            }
        }

        // Add group permissions if any
        // TODO: Support users (not just groups), and different permission types
        // (though we can add users, we only support groups at the moment and award them with admin permissions)
        if (StringUtils.isNotBlank(azkabanProjectConfig.getAzkabanGroupAdminUsers())) {
            String[] groups = StringUtils.split(azkabanProjectConfig.getAzkabanGroupAdminUsers(), ",");
            for (String group : groups) {
                addUserPermission(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                        azkabanProjectConfig.getAzkabanProjectName(), group, true, true, false, false, false,
                        false);
            }
        }

        // Upload zip file to azkaban and return projectId
        return uploadZipFileToAzkaban(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                azkabanProjectConfig.getAzkabanProjectName(), zipFilePath);
    }

    /***
     * Deletes an Azkaban project.
     * @param sessionId Session Id.
     * @param azkabanProjectConfig Azkaban Project Config.
     * @throws IOException
     */
    public static void deleteAzkabanProject(String sessionId, AzkabanProjectConfig azkabanProjectConfig)
            throws IOException {
        Map<String, String> params = Maps.newHashMap();
        params.put("delete", "true");
        params.put("project", azkabanProjectConfig.getAzkabanProjectName());

        executeGetRequest(
                prepareGetRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/manager", sessionId, params));
    }

    /***
     * Replace an existing Azkaban Project. If proxy user and group permissions are specified in
     * Azkaban Project Config, then this method also adds it to the project configuration.
     * @param sessionId Session Id.
     * @param zipFilePath Zip file to upload.
     * @param azkabanProjectConfig Azkaban Project Config.
     * @return Project Id.
     * @throws IOException
     */
    public static String replaceAzkabanProject(String sessionId, String zipFilePath,
            AzkabanProjectConfig azkabanProjectConfig) throws IOException {
        // Change project description
        changeProjectDescription(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                azkabanProjectConfig.getAzkabanProjectName(), azkabanProjectConfig.getAzkabanProjectDescription());

        // Add proxy user if any
        // Note: 1. We cannot remove previous proxy-user because there is no way to read it from Azkaban
        //       2. Adding same proxy user multiple times is a non-issue
        // Add proxy user if any
        if (azkabanProjectConfig.getAzkabanUserToProxy().isPresent()) {
            Iterable<String> proxyUsers = SPLIT_ON_COMMA.split(azkabanProjectConfig.getAzkabanUserToProxy().get());
            for (String user : proxyUsers) {
                addProxyUser(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                        azkabanProjectConfig.getAzkabanProjectName(), user);
            }
        }

        // Add group permissions if any
        // TODO: Support users (not just groups), and different permission types
        // Note: 1. We cannot remove previous group-user because there is no way to read it from Azkaban
        //       2. Adding same group-user will return an error message, but we will ignore it
        // (though we can add users, we only support groups at the moment and award them with admin permissions)
        if (StringUtils.isNotBlank(azkabanProjectConfig.getAzkabanGroupAdminUsers())) {
            String[] groups = StringUtils.split(azkabanProjectConfig.getAzkabanGroupAdminUsers(), ",");
            for (String group : groups) {
                try {
                    addUserPermission(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                            azkabanProjectConfig.getAzkabanProjectName(), group, true, true, false, false, false,
                            false);
                } catch (IOException e) {
                    // Ignore if group already exists, we cannot list existing groups; so its okay to attempt adding exiting
                    // .. groups
                    if (!"Group permission already exists.".equalsIgnoreCase(e.getMessage())) {
                        throw e;
                    }
                }
            }
        }

        // Upload zip file to azkaban and return projectId
        return uploadZipFileToAzkaban(sessionId, azkabanProjectConfig.getAzkabanServerUrl(),
                azkabanProjectConfig.getAzkabanProjectName(), zipFilePath);
    }

    private static void addProxyUser(String sessionId, String azkabanServerUrl, String azkabanProjectName,
            String proxyUser) throws IOException {
        // Create get request (adding same proxy user multiple times is a non-issue, Azkaban handles it)
        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "addProxyUser");
        params.put("project", azkabanProjectName);
        params.put("name", proxyUser);

        executeGetRequest(prepareGetRequest(azkabanServerUrl + "/manager", sessionId, params));
    }

    private static void addUserPermission(String sessionId, String azkabanServerUrl, String azkabanProjectName,
            String name, boolean isGroup, boolean adminPermission, boolean readPermission, boolean writePermission,
            boolean executePermission, boolean schedulePermission) throws IOException {

        // NOTE: We are not listing the permissions before adding them, because Azkaban in its current state only
        // .. returns user permissions and not group permissions

        // Create get request (adding same normal user permission multiple times will throw an error, but we cannot
        // list whole list of permissions anyways)
        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "addPermission");
        params.put("project", azkabanProjectName);
        params.put("name", name);
        params.put("group", Boolean.toString(isGroup));
        params.put("permissions[admin]", Boolean.toString(adminPermission));
        params.put("permissions[read]", Boolean.toString(readPermission));
        params.put("permissions[write]", Boolean.toString(writePermission));
        params.put("permissions[execute]", Boolean.toString(executePermission));
        params.put("permissions[schedule]", Boolean.toString(schedulePermission));

        executeGetRequest(prepareGetRequest(azkabanServerUrl + "/manager", sessionId, params));
    }

    /***
     * Schedule the Azkaban Project to run with a schedule.
     * @param sessionId Session Id.
     * @param azkabanProjectId Project Id.
     * @param azkabanProjectConfig Azkaban Project Config.
     * @throws IOException
     */
    public static void scheduleAzkabanProject(String sessionId, String azkabanProjectId,
            AzkabanProjectConfig azkabanProjectConfig) throws IOException {
        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "scheduleFlow");
        params.put("projectName", azkabanProjectConfig.getAzkabanProjectName());
        params.put("flow", azkabanProjectConfig.getAzkabanProjectFlowName());
        params.put("projectId", azkabanProjectId);
        params.put("scheduleTime", getScheduledTimeInAzkabanFormat(LOW_NETWORK_TRAFFIC_BEGIN_HOUR,
                LOW_NETWORK_TRAFFIC_END_HOUR, JOB_START_DELAY_MINUTES));
        params.put("scheduleDate", getScheduledDateInAzkabanFormat());
        params.put("is_recurring", "off");

        // Run once OR push down schedule (TODO: Enable when push down is finalized)
        //    if (azkabanProjectConfig.isScheduled()) {
        //      params.put("is_recurring", "on");
        //      params.put("period", "1d");
        //    } else {
        //      params.put("is_recurring", "off");
        //    }

        executePostRequest(
                preparePostRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/schedule", sessionId, params));
    }

    private static void changeProjectDescription(String sessionId, String azkabanServerUrl,
            String azkabanProjectName, String projectDescription) throws IOException {
        String encodedProjectDescription;
        try {
            encodedProjectDescription = new URLCodec().encode(projectDescription);
        } catch (EncoderException e) {
            throw new IOException("Could not encode Azkaban project description", e);
        }

        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "changeDescription");
        params.put("project", azkabanProjectName);
        params.put("description", encodedProjectDescription);

        executeGetRequest(prepareGetRequest(azkabanServerUrl + "/manager", sessionId, params));
    }

    /***
     * Execute an existing Azkaban project.
     * @param sessionId Session Id.
     * @param azkabanProjectId Project Id.
     * @param azkabanProjectConfig Azkaban Project Config.
     * @throws IOException
     */
    public static void executeAzkabanProject(String sessionId, String azkabanProjectId,
            AzkabanProjectConfig azkabanProjectConfig) throws IOException {
        Map<String, String> params = Maps.newHashMap();
        params.put("ajax", "executeFlow");
        params.put("project", azkabanProjectConfig.getAzkabanProjectName());
        params.put("flow", azkabanProjectConfig.getAzkabanProjectFlowName());

        executePostRequest(
                preparePostRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/executor", sessionId, params));
    }

    private static HttpGet prepareGetRequest(String requestUrl, String sessionId, Map<String, String> params)
            throws IOException {
        // Create get request
        StringBuilder stringEntityBuilder = new StringBuilder();
        stringEntityBuilder.append(String.format("?session.id=%s", sessionId));
        for (Map.Entry<String, String> entry : params.entrySet()) {
            stringEntityBuilder.append(String.format("&%s=%s", entry.getKey(), entry.getValue()));
        }

        return new HttpGet(requestUrl + stringEntityBuilder);
    }

    private static HttpPost preparePostRequest(String requestUrl, String sessionId, Map<String, String> params)
            throws IOException {
        // Create post request
        HttpPost postRequest = new HttpPost(requestUrl);
        StringBuilder stringEntityBuilder = new StringBuilder();
        stringEntityBuilder.append(String.format("session.id=%s", sessionId));
        for (Map.Entry<String, String> entry : params.entrySet()) {
            if (stringEntityBuilder.length() > 0) {
                stringEntityBuilder.append("&");
            }
            stringEntityBuilder.append(String.format("%s=%s", entry.getKey(), entry.getValue()));
        }
        StringEntity input = new StringEntity(stringEntityBuilder.toString());
        input.setContentType("application/x-www-form-urlencoded");
        postRequest.setEntity(input);
        postRequest.setHeader("X-Requested-With", "XMLHttpRequest");

        return postRequest;
    }

    @VisibleForTesting
    protected static Map<String, String> executeGetRequest(HttpGet getRequest) throws IOException {
        // Make the call, get response
        @Cleanup
        CloseableHttpClient httpClient = getHttpClient();
        HttpResponse response = httpClient.execute(getRequest);
        return AzkabanClient.handleResponse(response);
    }

    @VisibleForTesting
    protected static Map<String, String> executePostRequest(HttpPost postRequest) throws IOException {
        // Make the call, get response
        @Cleanup
        CloseableHttpClient httpClient = getHttpClient();
        HttpResponse response = httpClient.execute(postRequest);
        return AzkabanClient.handleResponse(response);
    }

    private static String uploadZipFileToAzkaban(String sessionId, String azkabanServerUrl,
            String azkabanProjectName, String jobZipFile) throws IOException {

        // Create post request
        HttpPost postRequest = new HttpPost(azkabanServerUrl + "/manager");
        HttpEntity entity = MultipartEntityBuilder.create().addTextBody("session.id", sessionId)
                .addTextBody("ajax", "upload").addBinaryBody("file", new File(jobZipFile),
                        ContentType.create("application/zip"), azkabanProjectName + ".zip")
                .addTextBody("project", azkabanProjectName).build();
        postRequest.setEntity(entity);

        // Make the call, get response
        @Cleanup
        CloseableHttpClient httpClient = getHttpClient();
        HttpResponse response = httpClient.execute(postRequest);

        // Obtaining projectId is hard. Uploading zip file is one avenue to get it from Azkaban
        return AzkabanClient.handleResponse(response).get("projectId");
    }

    private static CloseableHttpClient getHttpClient() throws IOException {
        try {
            // Self sign SSL
            SSLContextBuilder builder = new SSLContextBuilder();
            builder.loadTrustMaterial(null, (TrustStrategy) new TrustSelfSignedStrategy());
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(builder.build());

            // Create client
            return HttpClients.custom().setSSLSocketFactory(sslsf).setDefaultCookieStore(new BasicCookieStore())
                    .build();
        } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
            throw new IOException("Issue with creating http client", e);
        }
    }

    /***
     * Generate a random scheduled time between specified execution time window in the Azkaban compatible format
     * which is: hh,mm,a,z Eg. ScheduleTime=12,00,PM,PDT
     *
     * @param windowStartHour Window start hour in 24 hr (HH) format (inclusive)
     * @param windowEndHour Window end hour in 24 hr (HH) format (exclusive)
     * @param delayMinutes If current time is within window, then additional delay for bootstrapping if desired
     * @return Scheduled time string of the format hh,mm,a,z
     */
    @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DMI_RANDOM_USED_ONLY_ONCE", justification = "As expected for randomization")
    public static String getScheduledTimeInAzkabanFormat(int windowStartHour, int windowEndHour, int delayMinutes) {
        // Validate
        if (windowStartHour < 0 || windowEndHour > 23 || windowStartHour >= windowEndHour) {
            throw new IllegalArgumentException(
                    "Window start should be less than window end, and both should be between " + "0 and 23");
        }
        if (delayMinutes < 0 || delayMinutes > 59) {
            throw new IllegalArgumentException("Delay in minutes should be between 0 and 59 (inclusive)");
        }

        // Setup window
        Calendar windowStartTime = Calendar.getInstance();
        windowStartTime.set(Calendar.HOUR_OF_DAY, windowStartHour);
        windowStartTime.set(Calendar.MINUTE, 0);
        windowStartTime.set(Calendar.SECOND, 0);

        Calendar windowEndTime = Calendar.getInstance();
        windowEndTime.set(Calendar.HOUR_OF_DAY, windowEndHour);
        windowEndTime.set(Calendar.MINUTE, 0);
        windowEndTime.set(Calendar.SECOND, 0);

        // Check if current time is between windowStartTime and windowEndTime, then let the execution happen
        // after delayMinutes minutes
        Calendar now = Calendar.getInstance();
        if (now.after(windowStartTime) && now.before(windowEndTime)) {
            // Azkaban takes a few seconds / a minute to bootstrap,
            // so extra few minutes get the first execution to run instantly
            now.add(Calendar.MINUTE, delayMinutes);

            return new SimpleDateFormat("hh,mm,a,z").format(now.getTime());
        }

        // Current time is not between windowStartTime and windowEndTime, so get random execution time for next day
        int allowedSchedulingWindow = (int) ((windowEndTime.getTimeInMillis() - windowStartTime.getTimeInMillis())
                / MILLISECONDS_IN_HOUR);
        int randomHourInWindow = new Random(System.currentTimeMillis()).nextInt(allowedSchedulingWindow);
        int randomMinute = new Random(System.currentTimeMillis()).nextInt(60);
        windowStartTime.add(Calendar.HOUR, randomHourInWindow);
        windowStartTime.set(Calendar.MINUTE, randomMinute);

        return new SimpleDateFormat("hh,mm,a,z").format(windowStartTime.getTime());
    }

    private static String getScheduledDateInAzkabanFormat() {
        // Eg. ScheduleDate=07/22/2014"
        return new SimpleDateFormat("MM/dd/yyyy").format(new Date());
    }
}