edu.umn.cs.spatialHadoop.visualization.HadoopvizServer.java Source code

Java tutorial

Introduction

Here is the source code for edu.umn.cs.spatialHadoop.visualization.HadoopvizServer.java

Source

/***********************************************************************
 * Copyright (c) 2015 by Regents of the University of Minnesota.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Apache License, Version 2.0 which 
 * accompanies this distribution and is available at
 * http://www.opensource.org/licenses/apache2.0.php.
 *
 *************************************************************************/
package edu.umn.cs.spatialHadoop.visualization;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URLConnection;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;

import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Task;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.handler.AbstractHandler;

import edu.umn.cs.spatialHadoop.OperationsParams;
import edu.umn.cs.spatialHadoop.nasa.HDFPlot;
import edu.umn.cs.spatialHadoop.nasa.HDFRecordReader;

public class HadoopvizServer extends AbstractHandler {

    private static final Log LOG = LogFactory.getLog(HadoopvizServer.class);

    /** Common parameters for all queries */
    private OperationsParams commonParams;
    private Path datasetPath;
    private Path outputPath;
    private Path shapePath;
    private Path watermaskPath;

    /** Job Number **/
    private int jobNumber;

    /** Shape Map **/
    private HashMap<String, String> shapeMap;

    /**
     * A constructor that starts the Jetty server
     * @param datasetPath
     * @param outputPath
     * @param shapePath
     * @param watermaskPath
     * @param params
     */
    public HadoopvizServer(Path datasetPath, Path outputPath, Path shapePath, Path watermaskPath,
            OperationsParams params) {
        this.commonParams = new OperationsParams(params);
        this.datasetPath = datasetPath;
        this.outputPath = outputPath;
        this.jobNumber = Integer.MAX_VALUE;
        this.shapeMap = new HashMap<String, String>();
        this.shapePath = shapePath;
        this.watermaskPath = watermaskPath;
        readShapeFile();
    }

    /**
     * Create an HTTP web server (using Jetty) that will stay running to answer
     * all queries
     * 
     * @throws Exception
     */
    private static void startServer(Path datasetPath, Path outputPath, Path shapePath, Path watermaskPath,
            OperationsParams params) throws Exception {
        int port = params.getInt("port", 8889);
        Server server = new Server(port);
        server.setHandler(new HadoopvizServer(datasetPath, outputPath, shapePath, watermaskPath, params));
        server.start();
        server.join();
    }

    /**
     * Read the shape file from dataset path + shape.txt
     */
    private void readShapeFile() {
        try {
            LOG.info("Reading Shape File");
            FileSystem fs = datasetPath.getFileSystem(commonParams);
            BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(shapePath)));

            String line;
            while ((line = br.readLine()) != null) {
                shapeMap.put(line.split(":")[0], line.split(":")[1]);
            }
            br.close();
            LOG.info(shapeMap.size() + " shapes available.");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * The handler for all requests.
     */
    @Override
    public void handle(String target, HttpServletRequest request, HttpServletResponse response, int dispatch)
            throws IOException, ServletException {
        // Bypass cross-site scripting (XSS)
        response.addHeader("Access-Control-Allow-Origin", "*");
        response.addHeader("Access-Control-Allow-Credentials", "true");
        ((Request) request).setHandled(true);

        try {
            LOG.info("Received request: '" + request.getRequestURL() + "'");
            if (target.endsWith("/generate_dataset.cgi")) {
                handleGenerateDataset(request, response);
            } else if (target.endsWith("/visualize.cgi")) {
                handlePlot(request, response);
            } else if (target.endsWith("/generate_output_list.cgi")) {
                handleGenerateOutputList(request, response);
            } else if (target.endsWith("/fetch_result.cgi")) {
                handleOutput(request, response);
            } else if (target.startsWith("/hdfs/")) {
                handleHDFSFetch(request, response);
            } else if (target.endsWith("/get_output_info.cgi")) {
                handleOutputInfo(request, response);
            } else {
                if (target.equals("/"))
                    target = "/index.html";
                tryToLoadStaticResource(target, response);
            }
        } catch (Exception e) {
            e.printStackTrace();
            reportError(response, "Error placing the request", e);
        }
    }

    /**
     * This will be called when the index.html loaded. It will return the
     * avialable datasets.
     * 
     * @param request
     * @param response
     * @throws ParseException
     * @throws IOException
     */
    private void handleGenerateDataset(HttpServletRequest request, HttpServletResponse response)
            throws ParseException, IOException {

        try {
            FileSystem fs = datasetPath.getFileSystem(commonParams);
            ArrayList<String> datasetList = new ArrayList<String>();
            // Get the file list.
            for (FileStatus fileStatus : fs.listStatus(datasetPath)) {
                if (fileStatus.isDir()) {
                    String fileName = fileStatus.getPath().getName();
                    datasetList.add(fileName);
                }
            }

            // Report the answer.
            LOG.info("Reporting the answer.");
            response.setContentType("application/json;charset=utf-8");
            PrintWriter writer = response.getWriter();
            writer.print("[");
            for (int i = 0; i < datasetList.size(); i++) {
                if (i == datasetList.size() - 1) {
                    writer.print("\"" + datasetList.get(i) + "\"");
                } else {
                    writer.print("\"" + datasetList.get(i) + "\",");
                }
            }
            writer.print("]");
            writer.close();
            response.setStatus(HttpServletResponse.SC_OK);
        } catch (Exception e) {
            System.out.println("error happened");
            response.setContentType("text/plain;charset=utf-8");
            PrintWriter writer = response.getWriter();
            e.printStackTrace(writer);
            writer.close();
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Handle all plotting requests.
     * 
     * @param request
     * @param response
     * @throws ParseException
     * @throws IOException
     */
    private void handlePlot(HttpServletRequest request, HttpServletResponse response)
            throws ParseException, IOException {

        try {
            LOG.info(request.getQueryString());
            // Get the parameter.
            String dataset = request.getParameter("dataset");
            String width = request.getParameter("width");
            String height = request.getParameter("height");
            String partition = request.getParameter("partition");
            String vizType = request.getParameter("viztype");
            String plotType = request.getParameter("plottype");
            String noMerge = request.getParameter("no-merge");

            // Check the dataset and get the shape.
            String shape = "";
            if (shapeMap.get(dataset) != null) {
                shape = shapeMap.get(dataset);
            }

            // Create the path.
            Path[] inputPath = new Path[1];
            inputPath[0] = new Path(datasetPath.toString() + "/" + dataset);
            // First run. Need to check the next available output path.
            if (jobNumber == Integer.MAX_VALUE) {
                jobNumber = 0;
                FileSystem fs = outputPath.getFileSystem(commonParams);
                // Get the file list.
                for (FileStatus fileStatus : fs.listStatus(outputPath)) {
                    if (fileStatus.isDir()) {
                        jobNumber++;
                    }
                }
            }
            Path outputFolder = new Path(outputPath.toString() + "/" + jobNumber);
            Path outputRoot = new Path(outputFolder.toString());
            // Create the query parameters
            OperationsParams params = new OperationsParams(commonParams);
            if (plotType.equals("gplot")) {
                params.set("color", "red");
            }
            if (!shape.isEmpty()) {
                params.set("shape", shape);
            }

            if (vizType.equals("single_level")) {
                params.set("width", width);
                params.set("height", height);
                params.set("partition", partition);
                if (noMerge.equals("true")) {
                    params.setBoolean("merge", false);
                }

                if (noMerge.equals("false")) {
                    outputFolder = new Path(outputFolder.toString() + "/result.png");
                } else {
                    outputFolder = new Path(outputFolder.toString() + "_no-merge");
                    outputRoot = new Path(outputFolder.toString());
                }
            } else {
                params.setBoolean("pyramid", true);
                params.set("tileWidth", width);
                params.set("tileHeight", height);

                String min_zoom = request.getParameter("min_zoom");
                String max_zoom = request.getParameter("max_zoom");

                if (min_zoom.equals("null") && !max_zoom.equals("null")) {
                    params.set("levels", "" + (Integer.parseInt(max_zoom) + 1));
                } else if (!min_zoom.equals("null") && max_zoom.equals("null")) {
                    params.set("levels", min_zoom + ".." + min_zoom);
                } else {
                    params.set("levels", min_zoom + ".." + max_zoom);
                }

                // Checked the partition. If null, use default.
                if (!partition.equals("null")) {
                    params.set("partition", partition);
                }
            }

            params.setBoolean("background", true);

            Job job = null;
            if (plotType.equals("gplot")) {
                job = GeometricPlot.plot(inputPath, outputFolder, params);
            } else if (plotType.equals("hplot")) {
                job = HeatMapPlot.plot(inputPath, outputFolder, params);
            } else if (plotType.equals("hdfplot")) {
                params.set("mbr", "-180,-90,180,90");
                params.set("dataset", "LST_Day_1km");
                params.set(HDFRecordReader.WATER_MASK_PATH, watermaskPath.toString());
                params.set("recover", request.getParameter("recover"));
                params.set("valuerange", "12000..17000");
                job = HDFPlot.plotHeatMap(inputPath, outputFolder, params);
            }
            // Report the answer and time
            response.setContentType("application/json;charset=utf-8");
            PrintWriter writer = response.getWriter();
            writer.print("{\"job\":\"" + job.getJobID() + "\",");
            writer.print("\"url\":\"" + job.getTrackingURL() + "\",");
            if (noMerge.equals("true")) {
                writer.print("\"output\":" + "\"" + jobNumber + "_no-merge\"}");
            } else {
                writer.print("\"output\":" + "\"" + jobNumber + "\"}");
            }
            writer.close();
            response.setStatus(HttpServletResponse.SC_OK);
            jobNumber++;
            // Write the job id.
            writeJobInfo(outputRoot, job.getJobID().toString());
        } catch (Exception e) {
            response.setContentType("text/plain;charset=utf-8");
            PrintWriter writer = response.getWriter();
            e.printStackTrace(writer);
            writer.close();
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * This method will be called when display.html loaded. It will show the
     * available output to be shown.
     * 
     * @param request
     * @param response
     * @throws ParseException
     * @throws IOException
     */
    private void handleGenerateOutputList(HttpServletRequest request, HttpServletResponse response)
            throws ParseException, IOException {

        try {
            FileSystem fs = outputPath.getFileSystem(commonParams);
            ArrayList<String> outputList = new ArrayList<String>();
            // Get the file list.
            for (FileStatus fileStatus : fs.listStatus(outputPath)) {
                if (fileStatus.isDir()) {
                    String fileName = fileStatus.getPath().getName();
                    outputList.add(fileName);
                }
            }
            // Report the answer.
            LOG.info("Reporting the answer.");
            response.setContentType("application/json;charset=utf-8");
            PrintWriter writer = response.getWriter();
            writer.print("[");
            for (int i = 0; i < outputList.size(); i++) {
                if (i == outputList.size() - 1) {
                    writer.print("\"" + outputList.get(i) + "\"");
                } else {
                    writer.print("\"" + outputList.get(i) + "\",");
                }
            }
            writer.print("]");
            writer.close();
            response.setStatus(HttpServletResponse.SC_OK);
        } catch (Exception e) {
            System.out.println("error happened");
            response.setContentType("text/plain;charset=utf-8");
            PrintWriter writer = response.getWriter();
            e.printStackTrace(writer);
            writer.close();
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * This method will handle the output result.
     * 
     * @param request
     * @param response
     */
    private void handleOutput(HttpServletRequest request, HttpServletResponse response) {
        try {
            FileSystem fs = outputPath.getFileSystem(commonParams);
            Path directoryPath = new Path(outputPath.toString() + "/" + request.getParameter("path"));

            PrintWriter writer = response.getWriter();
            response.setContentType("text/html");
            // Get the file list.
            Path fileName = null;
            if (request.getParameter("path").contains("no-merge")) {
                boolean isSpacePartitioning = isSpacePartitioning(directoryPath);
                if (isSpacePartitioning) {
                    fileName = new Path(directoryPath.toString() + "/_master.html");
                    writer.print("<iframe src=\"/hdfs" + fileName.toUri().getPath()
                            + "\" width=\"99%\" height=\"99%\">");
                } else {
                    int count = 0;
                    for (FileStatus fileStatus : fs.listStatus(directoryPath)) {
                        if (count >= 10) {
                            break;
                        }
                        if (fileStatus.getPath().getName().endsWith("png")) {
                            fileName = fileStatus.getPath();
                            writer.print("<img src=\"/hdfs" + fileName.toUri().getPath()
                                    + "\" width=\"99%\" height=\"99%\"> <br>\n");
                            count++;
                        }
                    }
                }
            } else {
                for (FileStatus fileStatus : fs.listStatus(directoryPath)) {
                    if (fileStatus.getPath().getName().endsWith("png")
                            || fileStatus.getPath().getName().endsWith("html")) {
                        fileName = fileStatus.getPath();
                        break;
                    }
                }
                // Single Level.
                if (fileName.getName().endsWith("png")) {
                    writer.print("<img src=\"/hdfs" + fileName.toUri().getPath() + "\">");
                    // Multi Level.
                } else {
                    writer.print("<iframe src=\"/hdfs" + fileName.toUri().getPath()
                            + "\" width=\"99%\" height=\"99%\">");
                }
            }

            writer.close();
            response.setStatus(HttpServletResponse.SC_OK);
        } catch (Exception e) {
            System.out.println("error happened");
            e.printStackTrace();
            response.setContentType("text/plain;charset=utf-8");
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * This method will handle the output result.
     * 
     * @param request
     * @param response
     */
    private void handleOutputInfo(HttpServletRequest request, HttpServletResponse response) {
        try {
            FileSystem fs = outputPath.getFileSystem(commonParams);
            Path infoPath = new Path(outputPath.toString() + "/" + request.getParameter("path") + "/info.txt");
            BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(infoPath)));
            JobID jobID = JobID.forName(br.readLine());
            br.close();

            JobClient jobClient = new JobClient(new JobConf(commonParams));
            RunningJob runningJob = jobClient.getJob(jobID);
            Counters counters = runningJob.getCounters();

            response.setContentType("application/json;charset=utf-8");
            PrintWriter writer = response.getWriter();
            writer.print("{\"inputSize\":\"" + humanReadable(
                    runningJob.getCounters().findCounter("FileSystemCounters", "HDFS_BYTES_READ").getCounter())
                    + "\",");
            writer.print("\"intermediateSize\":\""
                    + humanReadable(counters.getCounter(Task.Counter.MAP_OUTPUT_BYTES)) + "\",");
            writer.printf("\"jobID\":\"%s\",\n", jobID);
            writer.printf("\"jobURL\":\"%s\",\n", runningJob.getTrackingURL());
            writer.print("\"intermediateGroup\":" + "\"" + counters.getCounter(Task.Counter.REDUCE_INPUT_GROUPS)
                    + "\"}");
            writer.close();
            response.setStatus(HttpServletResponse.SC_OK);
        } catch (Exception e) {
            System.out.println("error happened");
            e.printStackTrace();
            response.setContentType("text/plain;charset=utf-8");
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * This method will handle each time a file need to be fetched from HDFS.
     * 
     * @param request
     * @param response
     */
    private void handleHDFSFetch(HttpServletRequest request, HttpServletResponse response) {
        try {
            FileSystem fs = outputPath.getFileSystem(commonParams);
            String path = request.getRequestURI().replace("/hdfs", "");
            Path filePath = new Path(path);

            LOG.info("Fetching from " + path);

            FSDataInputStream resource;

            resource = fs.open(filePath);

            if (resource == null) {
                reportError(response, "Cannot load resource '" + filePath + "'", null);
                return;
            }
            byte[] buffer = new byte[1024 * 1024];
            ServletOutputStream outResponse = response.getOutputStream();
            int size;
            while ((size = resource.read(buffer)) != -1) {
                outResponse.write(buffer, 0, size);
            }
            resource.close();
            outResponse.close();
            response.setStatus(HttpServletResponse.SC_OK);
            if (filePath.toString().endsWith("png")) {
                response.setContentType("image/png");
            }
        } catch (Exception e) {
            System.out.println("error happened");
            e.printStackTrace();
            response.setContentType("text/plain;charset=utf-8");
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Tries to load the given resource name frmo class path if it exists. Used to
     * serve static files such as HTML pages, images and JavaScript files.
     * 
     * @param target
     * @param response
     * @throws IOException
     */
    private void tryToLoadStaticResource(String target, HttpServletResponse response) throws IOException {
        LOG.info("Loading resource " + target);
        // Try to load this resource as a static page
        InputStream resource = getClass().getResourceAsStream("/webapps/static/hadoopviz" + target);
        if (resource == null) {
            reportError(response, "Cannot load resource '" + target + "'", null);
            return;
        }
        byte[] buffer = new byte[1024 * 1024];
        ServletOutputStream outResponse = response.getOutputStream();
        int size;
        while ((size = resource.read(buffer)) != -1) {
            outResponse.write(buffer, 0, size);
        }
        resource.close();
        outResponse.close();
        response.setStatus(HttpServletResponse.SC_OK);
        if (target.endsWith(".js")) {
            response.setContentType("application/javascript");
        } else if (target.endsWith(".css")) {
            response.setContentType("text/css");
        } else {
            response.setContentType(URLConnection.guessContentTypeFromName(target));
        }
    }

    private void writeJobInfo(Path path, String jobId) {
        try {
            Path infoFile = new Path(path.toString() + "/info.txt");
            FileSystem fs = outputPath.getFileSystem(commonParams);
            BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(infoFile, true)));

            bw.write(jobId);
            bw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private boolean isSpacePartitioning(Path path) {
        boolean spacePartitioning = false;

        try {
            Path masterHeap = new Path(path.toString() + "/_master.heap");
            FileSystem fs = outputPath.getFileSystem(commonParams);
            BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(masterHeap)));

            String line;
            while ((line = br.readLine()) != null) {
                if (!line.split(",")[0].equals("0") || !line.split(",")[1].equals("0")) {
                    spacePartitioning = true;
                    break;
                }
            }
            br.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return spacePartitioning;
    }

    private void reportError(HttpServletResponse response, String msg, Exception e) throws IOException {
        if (e != null)
            e.printStackTrace();
        LOG.error(msg);
        response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
        response.getWriter().println("{\"message\": '" + msg + "',");
        if (e != null) {
            response.getWriter().println("\"error\": '" + e.getMessage() + "',");
            response.getWriter().println("\"stacktrace\": [");
            for (StackTraceElement trc : e.getStackTrace()) {
                response.getWriter().println("'" + trc.toString() + "',");
            }
            response.getWriter().println("]");
        }
        response.getWriter().println("}");
    }

    private String humanReadable(double size) {
        final String[] units = { "", "KB", "MB", "GB", "TB", "PB" };
        int unit = 0;
        while (unit < units.length && size > 1024) {
            size /= 1024;
            unit++;
        }
        return String.format("%.2f %s", size, units[unit]);
    }

    /**
     * Prints the usage of starting the server.
     */
    public static void printUsage() {
        System.out.println("Starts a server which will handle visualization requests");
        System.out.println("Parameters: (* marks required parameters)");
        System.out.println("<dataset> - (*) The path of the dataset.");
        System.out.println("<output> - (*) The output path.");
        System.out.println("<shape> - (*) The shape file.");
        System.out.println("<watermask> - (*) The watermask folder.");
        System.out.println("port:<p> - The port to start listening to. Default: 8889");
        GenericOptionsParser.printGenericCommandUsage(System.out);
    }

    /**
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        final OperationsParams params = new OperationsParams(new GenericOptionsParser(args), false);

        if (params.getPaths().length < 4) {
            System.err.println("Please specify the path of the dataset.");
            printUsage();
            System.exit(1);
        }
        Path datasetPath = params.getPaths()[0];
        Path outputPath = params.getPaths()[1];
        Path shapePath = params.getPaths()[2];
        Path watermaskPath = params.getPaths()[3];
        startServer(datasetPath, outputPath, shapePath, watermaskPath, params);
    }
}