io.hops.hopsworks.api.admin.HDFSUIProxyServlet.java Source code

Java tutorial

Introduction

Here is the source code for io.hops.hopsworks.api.admin.HDFSUIProxyServlet.java

Source

/*
 * Changes to this file committed after and not including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * This file is part of Hopsworks
 * Copyright (C) 2018, Logical Clocks AB. All rights reserved
 *
 * Hopsworks is free software: you can redistribute it and/or modify it under the terms of
 * the GNU Affero General Public License as published by the Free Software Foundation,
 * either version 3 of the License, or (at your option) any later version.
 *
 * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 * PURPOSE.  See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this program.
 * If not, see <https://www.gnu.org/licenses/>.
 *
 * Changes to this file committed before and including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * Copyright (C) 2013 - 2018, Logical Clocks AB and RISE SICS AB. All rights reserved
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
 * persons to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS  OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package io.hops.hopsworks.api.admin;

import io.hops.hopsworks.api.kibana.ProxyServlet;
import io.hops.hopsworks.api.util.CustomSSLProtocolSocketFactory;
import io.hops.hopsworks.common.security.BaseHadoopClientsService;
import io.hops.hopsworks.common.util.Settings;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashSet;
import javax.ejb.EJB;
import javax.ejb.Stateless;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.Response;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.client.utils.URIUtils;

@Stateless
public class HDFSUIProxyServlet extends ProxyServlet {

    @EJB
    private Settings settings;
    @EJB
    private BaseHadoopClientsService baseHadoopClientsService;

    private static final HashSet<String> PASS_THROUGH_HEADERS = new HashSet<String>(
            Arrays.asList("User-Agent", "user-agent", "Accept", "accept", "Accept-Encoding", "accept-encoding",
                    "Accept-Language", "accept-language", "Accept-Charset", "accept-charset"));

    protected void initTarget() throws ServletException {
        // TODO - should get the Kibana URI from Settings.java
        //    targetUri = Settings.getKibanaUri();
        targetUri = settings.getHDFSWebUIAddress();

        if (targetUri == null) {
            throw new ServletException(P_TARGET_URI + " is required.");
        }

        if (settings.getHopsRpcTls()) {
            if (!targetUri.contains("https://")) {
                targetUri = "https://" + targetUri;
            }
        } else {
            if (!targetUri.contains("http://")) {
                targetUri = "http://" + targetUri;
            }
        }

        //test it's valid
        try {
            targetUriObj = new URI(targetUri);
        } catch (Exception e) {
            throw new ServletException("Trying to process targetUri init parameter: " + e, e);
        }
        targetHost = URIUtils.extractHost(targetUriObj);
    }

    @Override
    protected void service(HttpServletRequest servletRequest, HttpServletResponse servletResponse)
            throws ServletException, IOException {

        if (servletRequest.getUserPrincipal() == null) {
            servletResponse.sendError(403, "User is not logged in");
            return;
        }
        if (!servletRequest.isUserInRole("HOPS_ADMIN")) {
            servletResponse.sendError(Response.Status.BAD_REQUEST.getStatusCode(),
                    "You don't have the access right for this service");
            return;
        }
        if (servletRequest.getAttribute(ATTR_TARGET_URI) == null) {
            servletRequest.setAttribute(ATTR_TARGET_URI, targetUri);
        }
        if (servletRequest.getAttribute(ATTR_TARGET_HOST) == null) {
            servletRequest.setAttribute(ATTR_TARGET_HOST, targetHost);
        }

        // Make the Request
        // note: we won't transfer the protocol version because I'm not 
        // sure it would truly be compatible
        String proxyRequestUri = rewriteUrlFromRequest(servletRequest);

        try {
            String[] targetHost_port = settings.getHDFSWebUIAddress().split(":");
            File keyStore = new File(baseHadoopClientsService.getSuperKeystorePath());
            File trustStore = new File(baseHadoopClientsService.getSuperTrustStorePath());
            // Assume that KeyStore password and Key password are the same
            Protocol httpsProto = new Protocol("https",
                    new CustomSSLProtocolSocketFactory(keyStore,
                            baseHadoopClientsService.getSuperKeystorePassword(),
                            baseHadoopClientsService.getSuperKeystorePassword(), trustStore,
                            baseHadoopClientsService.getSuperTrustStorePassword()),
                    Integer.parseInt(targetHost_port[1]));
            Protocol.registerProtocol("https", httpsProto);
            // Execute the request
            HttpClientParams params = new HttpClientParams();
            params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
            params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true);
            HttpClient client = new HttpClient(params);
            HostConfiguration config = new HostConfiguration();
            InetAddress localAddress = InetAddress.getLocalHost();
            config.setLocalAddress(localAddress);

            HttpMethod m = new GetMethod(proxyRequestUri);
            Enumeration<String> names = servletRequest.getHeaderNames();
            while (names.hasMoreElements()) {
                String headerName = names.nextElement();
                String value = servletRequest.getHeader(headerName);
                if (PASS_THROUGH_HEADERS.contains(headerName)) {
                    //hdfs does not send back the js if encoding is not accepted
                    //but we don't want to accept encoding for the html because we
                    //need to be able to parse it
                    if (headerName.equalsIgnoreCase("accept-encoding") && (servletRequest.getPathInfo() == null
                            || !servletRequest.getPathInfo().contains(".js"))) {
                        continue;
                    } else {
                        m.setRequestHeader(headerName, value);
                    }
                }
            }
            String user = servletRequest.getRemoteUser();
            if (user != null && !user.isEmpty()) {
                m.setRequestHeader("Cookie", "proxy-user" + "=" + URLEncoder.encode(user, "ASCII"));
            }

            client.executeMethod(config, m);

            // Process the response
            int statusCode = m.getStatusCode();

            // Pass the response code. This method with the "reason phrase" is 
            //deprecated but it's the only way to pass the reason along too.
            //noinspection deprecation
            servletResponse.setStatus(statusCode, m.getStatusLine().getReasonPhrase());

            copyResponseHeaders(m, servletRequest, servletResponse);

            // Send the content to the client
            copyResponseEntity(m, servletResponse);

        } catch (Exception e) {
            if (e instanceof RuntimeException) {
                throw (RuntimeException) e;
            }
            if (e instanceof ServletException) {
                throw (ServletException) e;
            }
            //noinspection ConstantConditions
            if (e instanceof IOException) {
                throw (IOException) e;
            }
            throw new RuntimeException(e);

        }
    }

    protected void copyResponseEntity(HttpMethod method, HttpServletResponse servletResponse) throws IOException {
        InputStream entity = method.getResponseBodyAsStream();
        if (entity != null) {
            OutputStream servletOutputStream = servletResponse.getOutputStream();
            if (servletResponse.getHeader("Content-Type") == null
                    || servletResponse.getHeader("Content-Type").contains("html")) {
                String inputLine;
                BufferedReader br = new BufferedReader(new InputStreamReader(entity));

                try {
                    int contentSize = 0;
                    while ((inputLine = br.readLine()) != null) {
                        String outputLine = hopify(inputLine, targetUri) + "\n";
                        byte[] output = outputLine.getBytes(Charset.forName("UTF-8"));
                        servletOutputStream.write(output);
                        contentSize += output.length;
                    }
                    br.close();
                    servletResponse.setHeader("Content-Length", Integer.toString(contentSize));
                } catch (IOException e) {
                    e.printStackTrace();
                }
            } else {
                org.apache.hadoop.io.IOUtils.copyBytes(entity, servletOutputStream, 4096, doLog);
            }
        }
    }

    protected void copyResponseHeaders(HttpMethod method, HttpServletRequest servletRequest,
            HttpServletResponse servletResponse) {
        for (org.apache.commons.httpclient.Header header : method.getResponseHeaders()) {
            if (hopByHopHeaders.containsHeader(header.getName())) {
                continue;
            }
            if (header.getName().equalsIgnoreCase("Content-Length")
                    && (method.getResponseHeader("Content-Type") == null
                            || method.getResponseHeader("Content-Type").getValue().contains("html"))) {
                continue;
            }
            if (header.getName().equalsIgnoreCase(org.apache.http.cookie.SM.SET_COOKIE)
                    || header.getName().equalsIgnoreCase(org.apache.http.cookie.SM.SET_COOKIE2)) {
                copyProxyCookie(servletRequest, servletResponse, header.getValue());
            } else {
                servletResponse.addHeader(header.getName(), header.getValue());
            }
        }
    }

    private String hopify(String ui, String source) {

        ui = ui.replaceAll("<a href='http://hadoop.apache.org/core'>Hadoop</a>, 2018.", "");
        ui = ui.replaceAll("(?<=(url=))(?=[a-zA-Z])", "/hopsworks-api/hdfsui/");
        ui = ui.replaceAll("(?<=(href|src)=\")/(?=[a-zA-Z])", "/hopsworks-api/hdfsui/" + source + "/");
        ui = ui.replaceAll("(?<=(href|src)=\')/(?=[a-zA-Z])", "/hopsworks-api/hdfsui/" + source + "/");
        ui = ui.replaceAll("(?<=(href|src)=\")//", "/hopsworks-api/hdfsui/");
        ui = ui.replaceAll("(?<=(href|src)=\')//", "/hopsworks-api/hdfsui/");
        ui = ui.replaceAll("(?<=(href|src)=\")(?=(http|https))", "/hopsworks-api/hdfsui/");
        ui = ui.replaceAll("(?<=(href|src)=\')(?=(http|https))", "/hopsworks-api/hdfsui/");
        ui = ui.replaceAll("(?<=(href|src)=\")(?=[a-zA-Z])", "/hopsworks-api/hdfsui/" + source + "/");
        ui = ui.replaceAll("(?<=(href|src)=\')(?=[a-zA-Z])", "/hopsworks-api/hdfsui/" + source + "/");
        ui = ui.replaceAll("(?<=(href|src)=)/(?=[a-zA-Z])", "/hopsworks-api/hdfsui/" + source + "/");
        ui = ui.replaceAll("(?<=(action)=\")(?=[a-zA-Z/]*.jsp)", "/hopsworks-api/hdfsui/" + source + "/");
        return ui;

    }

    //  @Override
    //  protected String rewriteUrlFromResponse(HttpServletRequest servletRequest,
    //      String theUrl) {
    //    //TODO document example paths
    //    final String targetUri = getTargetUri(servletRequest);
    //    String curUrl = servletRequest.getRequestURL().toString();//no query
    //
    //    String pathInfo = servletRequest.getPathInfo();
    //    if (pathInfo != null) {
    //      assert curUrl.endsWith(pathInfo);
    //      curUrl = curUrl.substring(0, curUrl.length() - pathInfo.length());//take pathInfo off
    //    }
    //    if (theUrl.startsWith(targetUri)) {
    //      if (curUrl.endsWith("/") || theUrl.substring(targetUri.length()).startsWith("/")) {
    //        theUrl = curUrl + theUrl.substring(targetUri.length());
    //      } else {
    //        theUrl = curUrl + "/" + theUrl.substring(targetUri.length());
    //      }
    //    } else if (curUrl.endsWith("/") || theUrl.substring(targetUri.length()).startsWith("/")) {
    //      theUrl = curUrl + theUrl;
    //    } else {
    //      theUrl = curUrl + "/" + theUrl;
    //    }
    //    return theUrl;
    //  }
    protected String rewriteUrlFromRequest(HttpServletRequest servletRequest) {
        StringBuilder uri = new StringBuilder(500);
        if (servletRequest.getPathInfo() != null
                && servletRequest.getPathInfo().matches("/http([a-zA-Z,:,/,.,0-9,-])+:([0-9])+(.)+")) {

            // Remove '/' from the beginning of the path
            String target = servletRequest.getPathInfo().substring(1);
            if (target.startsWith("https")) {
                target = "https:/" + target.substring(6);
            } else {
                target = "http:/" + target.substring(5);
            }
            servletRequest.setAttribute(ATTR_TARGET_URI, target);
            uri.append(target);
        } else {
            uri.append(getTargetUri(servletRequest));
            // Handle the path given to the servlet
            if (servletRequest.getPathInfo() != null) {//ex: /my/path.html
                uri.append(encodeUriQuery(servletRequest.getPathInfo()));
            }
        }
        // Handle the query string & fragment
        //ex:(following '?'): name=value&foo=bar#fragment
        String queryString = servletRequest.getQueryString();
        String fragment = null;
        //split off fragment from queryString, updating queryString if found
        if (queryString != null) {
            int fragIdx = queryString.indexOf('#');
            if (fragIdx >= 0) {
                fragment = queryString.substring(fragIdx + 2); // '#!', not '#'
                //        fragment = queryString.substring(fragIdx + 1);
                queryString = queryString.substring(0, fragIdx);
            }
        }

        queryString = rewriteQueryStringFromRequest(servletRequest, queryString);
        if (queryString != null && queryString.length() > 0) {
            uri.append('?');
            uri.append(encodeUriQuery(queryString));
        }

        if (doSendUrlFragment && fragment != null) {
            uri.append('#');
            uri.append(encodeUriQuery(fragment));
        }
        return uri.toString();
    }

    @Override
    protected void copyRequestHeaders(HttpServletRequest servletRequest, HttpRequest proxyRequest) {
        // Get an Enumeration of all of the header names sent by the client
        Enumeration enumerationOfHeaderNames = servletRequest.getHeaderNames();
        while (enumerationOfHeaderNames.hasMoreElements()) {
            String headerName = (String) enumerationOfHeaderNames.nextElement();
            //Instead the content-length is effectively set via InputStreamEntity
            if (headerName.equalsIgnoreCase(HttpHeaders.CONTENT_LENGTH)) {
                continue;
            }
            if (hopByHopHeaders.containsHeader(headerName)) {
                continue;
            }
            if (headerName.equalsIgnoreCase("accept-encoding")
                    && (servletRequest.getPathInfo() == null || !servletRequest.getPathInfo().contains(".js"))) {
                continue;
            }
            Enumeration headers = servletRequest.getHeaders(headerName);
            while (headers.hasMoreElements()) {//sometimes more than one value
                String headerValue = (String) headers.nextElement();
                // In case the proxy host is running multiple virtual servers,
                // rewrite the Host header to ensure that we get content from
                // the correct virtual server
                if (headerName.equalsIgnoreCase(HttpHeaders.HOST)) {
                    HttpHost host = getTargetHost(servletRequest);
                    headerValue = host.getHostName();
                    if (host.getPort() != -1) {
                        headerValue += ":" + host.getPort();
                    }
                } else if (headerName.equalsIgnoreCase(org.apache.http.cookie.SM.COOKIE)) {
                    headerValue = getRealCookie(headerValue);
                }
                proxyRequest.addHeader(headerName, headerValue);
            }
        }
    }

}