org.apache.ranger.services.sqoop.client.SqoopClient.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.ranger.services.sqoop.client.SqoopClient.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.ranger.services.sqoop.client;

import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.security.auth.Subject;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.apache.http.HttpStatus;
import org.apache.log4j.Logger;
import org.apache.ranger.plugin.client.BaseClient;
import org.apache.ranger.plugin.client.HadoopException;
import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorResponse;
import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorsResponse;
import org.apache.ranger.services.sqoop.client.json.model.SqoopJobResponse;
import org.apache.ranger.services.sqoop.client.json.model.SqoopJobsResponse;
import org.apache.ranger.services.sqoop.client.json.model.SqoopLinkResponse;
import org.apache.ranger.services.sqoop.client.json.model.SqoopLinksResponse;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;

public class SqoopClient extends BaseClient {

    private static final Logger LOG = Logger.getLogger(SqoopClient.class);

    private static final String EXPECTED_MIME_TYPE = "application/json";

    private static final String SQOOP_CONNECTOR_API_ENDPOINT = "/sqoop/v1/connector/all";

    private static final String SQOOP_LINK_API_ENDPOINT = "/sqoop/v1/link/all";

    private static final String SQOOP_JOB_API_ENDPOINT = "/sqoop/v1/job/all";

    private static final String ERROR_MESSAGE = " You can still save the repository and start creating "
            + "policies, but you would not be able to use autocomplete for "
            + "resource names. Check ranger_admin.log for more info.";

    private String sqoopUrl;
    private String userName;

    public SqoopClient(String serviceName, Map<String, String> configs) {

        super(serviceName, configs, "sqoop-client");
        this.sqoopUrl = configs.get("sqoop.url");
        this.userName = configs.get("username");

        if (StringUtils.isEmpty(this.sqoopUrl)) {
            LOG.error("No value found for configuration 'sqoop.url'. Sqoop resource lookup will fail.");
        }
        if (StringUtils.isEmpty(this.userName)) {
            LOG.error("No value found for configuration 'username'. Sqoop resource lookup will fail.");
        }

        if (LOG.isDebugEnabled()) {
            LOG.debug("Sqoop Client is build with url [" + this.sqoopUrl + "], user: [" + this.userName + "].");
        }
    }

    public List<String> getConnectorList(final String connectorMatching, final List<String> existingConnectors) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Get sqoop connector list for connectorMatching: " + connectorMatching
                    + ", existingConnectors: " + existingConnectors);
        }
        Subject subj = getLoginSubject();
        if (subj == null) {
            return Collections.emptyList();
        }

        List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() {

            @Override
            public List<String> run() {

                ClientResponse response = getClientResponse(sqoopUrl, SQOOP_CONNECTOR_API_ENDPOINT, userName);

                SqoopConnectorsResponse sqoopConnectorsResponse = getSqoopResourceResponse(response,
                        SqoopConnectorsResponse.class);
                if (sqoopConnectorsResponse == null
                        || CollectionUtils.isEmpty(sqoopConnectorsResponse.getConnectors())) {
                    return Collections.emptyList();
                }
                List<String> connectorResponses = new ArrayList<>();
                for (SqoopConnectorResponse sqoopConnectorResponse : sqoopConnectorsResponse.getConnectors()) {
                    connectorResponses.add(sqoopConnectorResponse.getName());
                }

                List<String> connectors = null;
                if (CollectionUtils.isNotEmpty(connectorResponses)) {
                    connectors = filterResourceFromResponse(connectorMatching, existingConnectors,
                            connectorResponses);
                }
                return connectors;
            }
        });

        if (LOG.isDebugEnabled()) {
            LOG.debug("Get sqoop connector list result: " + ret);
        }
        return ret;
    }

    public List<String> getLinkList(final String linkMatching, final List<String> existingLinks) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(
                    "Get sqoop link list for linkMatching: " + linkMatching + ", existingLinks: " + existingLinks);
        }
        Subject subj = getLoginSubject();
        if (subj == null) {
            return Collections.emptyList();
        }

        List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() {

            @Override
            public List<String> run() {

                ClientResponse response = getClientResponse(sqoopUrl, SQOOP_LINK_API_ENDPOINT, userName);

                SqoopLinksResponse sqoopLinksResponse = getSqoopResourceResponse(response,
                        SqoopLinksResponse.class);
                if (sqoopLinksResponse == null || CollectionUtils.isEmpty(sqoopLinksResponse.getLinks())) {
                    return Collections.emptyList();
                }
                List<String> linkResponses = new ArrayList<>();
                for (SqoopLinkResponse sqoopLinkResponse : sqoopLinksResponse.getLinks()) {
                    linkResponses.add(sqoopLinkResponse.getName());
                }

                List<String> links = null;
                if (CollectionUtils.isNotEmpty(linkResponses)) {
                    links = filterResourceFromResponse(linkMatching, existingLinks, linkResponses);
                }
                return links;
            }
        });

        if (LOG.isDebugEnabled()) {
            LOG.debug("Get sqoop link list result: " + ret);
        }
        return ret;
    }

    public List<String> getJobList(final String jobMatching, final List<String> existingJobs) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Get sqoop job list for jobMatching: " + jobMatching + ", existingJobs: " + existingJobs);
        }
        Subject subj = getLoginSubject();
        if (subj == null) {
            return Collections.emptyList();
        }

        List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() {

            @Override
            public List<String> run() {

                ClientResponse response = getClientResponse(sqoopUrl, SQOOP_JOB_API_ENDPOINT, userName);

                SqoopJobsResponse sqoopJobsResponse = getSqoopResourceResponse(response, SqoopJobsResponse.class);
                if (sqoopJobsResponse == null || CollectionUtils.isEmpty(sqoopJobsResponse.getJobs())) {
                    return Collections.emptyList();
                }
                List<String> jobResponses = new ArrayList<>();
                for (SqoopJobResponse sqoopJobResponse : sqoopJobsResponse.getJobs()) {
                    jobResponses.add(sqoopJobResponse.getName());
                }

                List<String> jobs = null;
                if (CollectionUtils.isNotEmpty(jobResponses)) {
                    jobs = filterResourceFromResponse(jobMatching, existingJobs, jobResponses);
                }
                return jobs;
            }
        });

        if (LOG.isDebugEnabled()) {
            LOG.debug("Get sqoop job list result: " + ret);
        }
        return ret;
    }

    private static ClientResponse getClientResponse(String sqoopUrl, String sqoopApi, String userName) {
        ClientResponse response = null;
        String[] sqoopUrls = sqoopUrl.trim().split("[,;]");
        if (ArrayUtils.isEmpty(sqoopUrls)) {
            return null;
        }

        Client client = Client.create();

        for (String currentUrl : sqoopUrls) {
            if (StringUtils.isBlank(currentUrl)) {
                continue;
            }

            String url = currentUrl.trim() + sqoopApi + "?" + PseudoAuthenticator.USER_NAME + "=" + userName;
            try {
                response = getClientResponse(url, client);

                if (response != null) {
                    if (response.getStatus() == HttpStatus.SC_OK) {
                        break;
                    } else {
                        response.close();
                    }
                }
            } catch (Throwable t) {
                String msgDesc = "Exception while getting sqoop response, sqoopUrl: " + url;
                LOG.error(msgDesc, t);
            }
        }
        client.destroy();

        return response;
    }

    private static ClientResponse getClientResponse(String url, Client client) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("getClientResponse():calling " + url);
        }

        WebResource webResource = client.resource(url);

        ClientResponse response = webResource.accept(EXPECTED_MIME_TYPE).get(ClientResponse.class);

        if (response != null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("getClientResponse():response.getStatus()= " + response.getStatus());
            }
            if (response.getStatus() != HttpStatus.SC_OK) {
                LOG.warn("getClientResponse():response.getStatus()= " + response.getStatus() + " for URL " + url
                        + ", failed to get sqoop resource list.");
                String jsonString = response.getEntity(String.class);
                LOG.warn(jsonString);
            }
        }
        return response;
    }

    private <T> T getSqoopResourceResponse(ClientResponse response, Class<T> classOfT) {
        T resource = null;
        try {
            if (response != null && response.getStatus() == HttpStatus.SC_OK) {
                String jsonString = response.getEntity(String.class);
                Gson gson = new GsonBuilder().setPrettyPrinting().create();

                resource = gson.fromJson(jsonString, classOfT);
            } else {
                String msgDesc = "Unable to get a valid response for " + "expected mime type : ["
                        + EXPECTED_MIME_TYPE + "], sqoopUrl: " + sqoopUrl + " - got null response.";
                LOG.error(msgDesc);
                HadoopException hdpException = new HadoopException(msgDesc);
                hdpException.generateResponseDataMap(false, msgDesc, msgDesc + ERROR_MESSAGE, null, null);
                throw hdpException;
            }
        } catch (HadoopException he) {
            throw he;
        } catch (Throwable t) {
            String msgDesc = "Exception while getting sqoop resource response, sqoopUrl: " + sqoopUrl;
            HadoopException hdpException = new HadoopException(msgDesc, t);

            LOG.error(msgDesc, t);

            hdpException.generateResponseDataMap(false, BaseClient.getMessage(t), msgDesc + ERROR_MESSAGE, null,
                    null);
            throw hdpException;

        } finally {
            if (response != null) {
                response.close();
            }
        }
        return resource;
    }

    private static List<String> filterResourceFromResponse(String resourceMatching, List<String> existingResources,
            List<String> resourceResponses) {
        List<String> resources = new ArrayList<String>();
        for (String resourceResponse : resourceResponses) {
            if (CollectionUtils.isNotEmpty(existingResources) && existingResources.contains(resourceResponse)) {
                continue;
            }
            if (StringUtils.isEmpty(resourceMatching) || resourceMatching.startsWith("*")
                    || resourceResponse.toLowerCase().startsWith(resourceMatching.toLowerCase())) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("filterResourceFromResponse(): Adding sqoop resource " + resourceResponse);
                }
                resources.add(resourceResponse);
            }
        }
        return resources;
    }

    public static Map<String, Object> connectionTest(String serviceName, Map<String, String> configs) {
        SqoopClient sqoopClient = getSqoopClient(serviceName, configs);
        List<String> strList = sqoopClient.getConnectorList(null, null);

        boolean connectivityStatus = false;
        if (CollectionUtils.isNotEmpty(strList)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("ConnectionTest list size " + strList.size() + " sqoop connectors.");
            }
            connectivityStatus = true;
        }

        Map<String, Object> responseData = new HashMap<String, Object>();
        if (connectivityStatus) {
            String successMsg = "ConnectionTest Successful.";
            BaseClient.generateResponseDataMap(connectivityStatus, successMsg, successMsg, null, null,
                    responseData);
        } else {
            String failureMsg = "Unable to retrieve any sqoop connectors using given parameters.";
            BaseClient.generateResponseDataMap(connectivityStatus, failureMsg, failureMsg + ERROR_MESSAGE, null,
                    null, responseData);
        }

        return responseData;
    }

    public static SqoopClient getSqoopClient(String serviceName, Map<String, String> configs) {
        SqoopClient sqoopClient = null;
        if (LOG.isDebugEnabled()) {
            LOG.debug("Getting SqoopClient for datasource: " + serviceName);
        }
        if (MapUtils.isEmpty(configs)) {
            String msgDesc = "Could not connect sqoop as Connection ConfigMap is empty.";
            LOG.error(msgDesc);
            HadoopException hdpException = new HadoopException(msgDesc);
            hdpException.generateResponseDataMap(false, msgDesc, msgDesc + ERROR_MESSAGE, null, null);
            throw hdpException;
        } else {
            sqoopClient = new SqoopClient(serviceName, configs);
        }
        return sqoopClient;
    }
}