Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ambari.eventdb.webservice; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.TreeSet; import javax.servlet.ServletContext; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import org.apache.ambari.eventdb.db.MySQLConnector; import org.apache.ambari.eventdb.db.OracleConnector; import org.apache.ambari.eventdb.db.PostgresConnector; import org.apache.ambari.eventdb.model.DataTable; import org.apache.ambari.eventdb.model.Jobs; import org.apache.ambari.eventdb.model.Jobs.JobDBEntry; import org.apache.ambari.eventdb.model.TaskAttempt; import org.apache.ambari.eventdb.model.TaskData; import org.apache.ambari.eventdb.model.TaskData.Point; import org.apache.ambari.eventdb.model.TaskLocalityData; import org.apache.ambari.eventdb.model.TaskLocalityData.DataPoint; import org.apache.ambari.eventdb.model.Workflows; import org.apache.ambari.eventdb.model.Workflows.WorkflowDBEntry; import org.apache.ambari.eventdb.model.Workflows.WorkflowDBEntry.WorkflowFields; import org.apache.ambari.server.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Path("/jobhistory") public class WorkflowJsonService { private static final String PREFIX = "eventdb."; private static final String HOSTNAME = PREFIX + "db.hostname"; private static final String DBNAME = PREFIX + "db.name"; private static final String USERNAME = PREFIX + "db.user"; private static final String PASSWORD = PREFIX + "db.password"; private static String DEFAULT_DRIVER; private static String DEFAULT_URL; private static String DEFAULT_USERNAME = "mapred"; private static String DEFAULT_PASSWORD = "mapred"; private static final Workflows EMPTY_WORKFLOWS = new Workflows(); private static final List<JobDBEntry> EMPTY_JOBS = Collections.emptyList(); { List<WorkflowDBEntry> emptyWorkflows = Collections.emptyList(); EMPTY_WORKFLOWS.setWorkflows(emptyWorkflows); } private static final Logger LOG = LoggerFactory.getLogger(WorkflowJsonService.class); PostgresConnector getConnector() throws IOException { //TODO fix temp hack if (StringUtils.contains(DEFAULT_DRIVER, "oracle")) { return new OracleConnector(DEFAULT_URL, DEFAULT_DRIVER, DEFAULT_USERNAME, DEFAULT_PASSWORD); } else if (StringUtils.contains(DEFAULT_DRIVER, "mysql")) { return new MySQLConnector(DEFAULT_URL, DEFAULT_DRIVER, DEFAULT_USERNAME, DEFAULT_PASSWORD); } else { return new PostgresConnector(DEFAULT_URL, DEFAULT_DRIVER, DEFAULT_USERNAME, DEFAULT_PASSWORD); } } public static void setDBProperties(Configuration configuration) { DEFAULT_DRIVER = configuration.getRcaDatabaseDriver(); DEFAULT_URL = configuration.getRcaDatabaseUrl(); if (DEFAULT_URL.contains(Configuration.HOSTNAME_MACRO)) { DEFAULT_URL = DEFAULT_URL.replace(Configuration.HOSTNAME_MACRO, "localhost"); } DEFAULT_USERNAME = configuration.getRcaDatabaseUser(); DEFAULT_PASSWORD = configuration.getRcaDatabasePassword(); } @Context ServletContext servletContext; @GET @Produces(MediaType.APPLICATION_JSON) @Path("/workflow") public Workflows getWorkflows(@QueryParam("orderBy") String field, @DefaultValue(PostgresConnector.SORT_ASC) @QueryParam("sortDir") String sortDir, @DefaultValue("0") @QueryParam("offset") int offset, @DefaultValue("-1") @QueryParam("limit") int limit) { Workflows workflows = EMPTY_WORKFLOWS; PostgresConnector conn = null; try { conn = getConnector(); if (field == null) workflows = conn.fetchWorkflows(); else { field = field.toUpperCase(); if ("ELAPSEDTIME".equals(field)) field = "DURATION"; workflows = conn.fetchWorkflows(WorkflowFields.valueOf(field), sortDir.toUpperCase().equals(PostgresConnector.SORT_ASC), offset, limit); } } catch (IOException e) { LOG.error("Error interacting with RCA database ", e); workflows = EMPTY_WORKFLOWS; } finally { if (conn != null) { conn.close(); } } return workflows; } @GET @Produces(MediaType.APPLICATION_JSON) @Path("/datatable") public DataTable getWorkflowDataTable(@DefaultValue("0") @QueryParam("iDisplayStart") int start, @DefaultValue("10") @QueryParam("iDisplayLength") int amount, @QueryParam("sSearch") String searchTerm, @DefaultValue("0") @QueryParam("sEcho") int echo, @DefaultValue("0") @QueryParam("iSortCol_0") int col, @DefaultValue(PostgresConnector.SORT_ASC) @QueryParam("sSortDir_0") String sdir, @QueryParam("sSearch_0") String workflowId, @QueryParam("sSearch_1") String workflowName, @QueryParam("sSearch_2") String workflowType, @QueryParam("sSearch_3") String userName, @DefaultValue("-1") @QueryParam("minJobs") int minJobs, @DefaultValue("-1") @QueryParam("maxJobs") int maxJobs, @DefaultValue("-1") @QueryParam("minInputBytes") long minInputBytes, @DefaultValue("-1") @QueryParam("maxInputBytes") long maxInputBytes, @DefaultValue("-1") @QueryParam("minOutputBytes") long minOutputBytes, @DefaultValue("-1") @QueryParam("maxOutputBytes") long maxOutputBytes, @DefaultValue("-1") @QueryParam("minDuration") long minDuration, @DefaultValue("-1") @QueryParam("maxDuration") long maxDuration, @DefaultValue("-1") @QueryParam("minStartTime") long minStartTime, @DefaultValue("-1") @QueryParam("maxStartTime") long maxStartTime, @DefaultValue("-1") @QueryParam("minFinishTime") long minFinishTime, @DefaultValue("-1") @QueryParam("maxFinishTime") long maxFinishTime) { if (start < 0) start = 0; if (amount < 10 || amount > 100) amount = 10; boolean sortAscending = true; if (!sdir.toUpperCase().equals(PostgresConnector.SORT_ASC)) sortAscending = false; WorkflowFields field = null; switch (col) { case 0: // workflowId field = WorkflowFields.WORKFLOWID; break; case 1: // workflowName field = WorkflowFields.WORKFLOWNAME; break; case 2: // workflowType field = WorkflowFields.WORKFLOWID; break; case 3: // userName field = WorkflowFields.USERNAME; break; case 4: // numJobsTotal field = WorkflowFields.NUMJOBSTOTAL; break; case 5: // inputBytes field = WorkflowFields.INPUTBYTES; break; case 6: // outputBytes field = WorkflowFields.OUTPUTBYTES; break; case 7: // duration field = WorkflowFields.DURATION; break; case 8: // startTime field = WorkflowFields.STARTTIME; break; case 9: // lastUpdateTime field = WorkflowFields.LASTUPDATETIME; break; default: field = WorkflowFields.WORKFLOWID; } DataTable table = null; PostgresConnector conn = null; try { conn = getConnector(); table = conn.fetchWorkflows(start, amount, searchTerm, echo, field, sortAscending, workflowId, workflowName, workflowType, userName, minJobs, maxJobs, minInputBytes, maxInputBytes, minOutputBytes, maxOutputBytes, minDuration, maxDuration, minStartTime, maxStartTime, minFinishTime, maxFinishTime); } catch (IOException e) { LOG.error("Error interacting with RCA database ", e); } finally { if (conn != null) { conn.close(); } } return table; } @GET @Produces(MediaType.APPLICATION_JSON) @Path("/job") public Jobs getJobs(@QueryParam("workflowId") String workflowId, @DefaultValue("-1") @QueryParam("startTime") long minFinishTime, @DefaultValue("-1") @QueryParam("endTime") long maxStartTime) { Jobs jobs = new Jobs(); PostgresConnector conn = null; try { conn = getConnector(); if (workflowId != null) jobs.setJobs(conn.fetchJobDetails(workflowId)); else if (maxStartTime >= minFinishTime) jobs.setJobs(conn.fetchJobDetails(minFinishTime, maxStartTime)); } catch (IOException e) { LOG.error("Error interacting with RCA database ", e); jobs.setJobs(EMPTY_JOBS); } finally { if (conn != null) { conn.close(); } } return jobs; } @GET @Produces(MediaType.APPLICATION_JSON) @Path("/task") public TaskData getTaskSummary(@QueryParam("jobId") String jobId, @QueryParam("width") int steps, @QueryParam("workflowId") String workflowId, @DefaultValue("-1") @QueryParam("startTime") long minFinishTime, @DefaultValue("-1") @QueryParam("endTime") long maxStartTime) { TaskData points = new TaskData(); PostgresConnector conn = null; try { conn = getConnector(); List<TaskAttempt> taskAttempts = null; long startTime = -1; long endTime = -1; if (jobId != null) { long[] times = conn.fetchJobStartStopTimes(jobId); if (times != null) { startTime = times[0]; endTime = times[1]; taskAttempts = conn.fetchJobTaskAttempts(jobId); } } else { startTime = minFinishTime; endTime = maxStartTime; if (workflowId != null) taskAttempts = conn.fetchWorkflowTaskAttempts(workflowId); else taskAttempts = conn.fetchTaskAttempts(minFinishTime, maxStartTime); } if (startTime > 0 && endTime > 0 && endTime >= startTime) { double submitTimeSecs = startTime / 1000.0; double finishTimeSecs = endTime / 1000.0; double step = (finishTimeSecs - submitTimeSecs) / steps; if (step < 1) step = 1; if (taskAttempts != null) getTaskDetails(taskAttempts, points, submitTimeSecs, finishTimeSecs, step); } } catch (IOException e) { LOG.error("Error interacting with RCA database ", e); } finally { if (conn != null) { conn.close(); } } return points; } @GET @Produces(MediaType.APPLICATION_JSON) @Path("/taskdetails") public List<TaskAttempt> getTaskDetails(@QueryParam("jobId") String jobId, @QueryParam("workflowId") String workflowId) { List<TaskAttempt> taskAttempts = new ArrayList<TaskAttempt>(); PostgresConnector conn = null; try { conn = getConnector(); if (jobId != null) { taskAttempts = conn.fetchJobTaskAttempts(jobId); } else if (workflowId != null) { taskAttempts = conn.fetchWorkflowTaskAttempts(workflowId); } } catch (IOException e) { LOG.error("Error interacting with RCA database ", e); } finally { if (conn != null) { conn.close(); } } return taskAttempts; } @GET @Produces(MediaType.APPLICATION_JSON) @Path("/tasklocality") public TaskLocalityData getTaskLocalitySummary(@QueryParam("jobId") String jobId, @DefaultValue("4") @QueryParam("minr") int minr, @DefaultValue("24") @QueryParam("maxr") int maxr, @QueryParam("workflowId") String workflowId) { if (maxr < minr) maxr = minr; TaskLocalityData data = new TaskLocalityData(); PostgresConnector conn = null; try { conn = getConnector(); if (jobId != null) { long[] times = conn.fetchJobStartStopTimes(jobId); if (times != null) { getApproxTaskAttemptsByLocality(conn.fetchJobTaskAttempts(jobId), times[0], times[1], data, minr, maxr); } } else if (workflowId != null) { getExactTaskAttemptsByLocality(conn.fetchWorkflowTaskAttempts(workflowId), data, minr, maxr); } } catch (IOException e) { LOG.error("Error interacting with RCA database ", e); } finally { if (conn != null) { conn.close(); } } return data; } private static void getTaskDetails(List<TaskAttempt> taskAttempts, TaskData points, double submitTimeSecs, double finishTimeSecs, double step) throws IOException { List<Point> mapPoints = new ArrayList<Point>(); List<Point> shufflePoints = new ArrayList<Point>(); List<Point> reducePoints = new ArrayList<Point>(); for (double time = submitTimeSecs; time < finishTimeSecs; time += step) { int numTasks = 0; int numShuffleTasks = 0; int numReduceTasks = 0; for (TaskAttempt taskAttempt : taskAttempts) { if (taskAttempt.getTaskType().equals("MAP")) { if ((taskAttempt.getStartTime() / 1000.0) <= (time + step) && (taskAttempt.getFinishTime() / 1000.0) >= time) numTasks++; } else if (taskAttempt.getTaskType().equals("REDUCE")) { if ((taskAttempt.getStartTime() / 1000.0) <= (time + step) && (taskAttempt.getShuffleFinishTime() / 1000.0) >= time) { numShuffleTasks++; } else if ((taskAttempt.getShuffleFinishTime() / 1000.0) < (time + step) && (taskAttempt.getFinishTime() / 1000.0) >= time) { numReduceTasks++; } } } mapPoints.add(new Point(Math.round(time), numTasks)); shufflePoints.add(new Point(Math.round(time), numShuffleTasks)); reducePoints.add(new Point(Math.round(time), numReduceTasks)); } points.setMapData(mapPoints); points.setShuffleData(shufflePoints); points.setReduceData(reducePoints); } private static void getExactTaskAttemptsByLocality(List<TaskAttempt> taskAttempts, TaskLocalityData data, int minr, int maxr) throws IOException { MinMax io = new MinMax(); data.setMapNodeLocal(processExactLocalityData(taskAttempts, "MAP", "NODE_LOCAL", io)); data.setMapRackLocal(processExactLocalityData(taskAttempts, "MAP", "RACK_LOCAL", io)); data.setMapOffSwitch(processExactLocalityData(taskAttempts, "MAP", "OFF_SWITCH", io)); data.setReduceOffSwitch(processExactLocalityData(taskAttempts, "REDUCE", null, io)); setRValues(data.getMapNodeLocal(), minr, maxr, io.max); setRValues(data.getMapRackLocal(), minr, maxr, io.max); setRValues(data.getMapOffSwitch(), minr, maxr, io.max); setRValues(data.getReduceOffSwitch(), minr, maxr, io.max); } private static void getApproxTaskAttemptsByLocality(List<TaskAttempt> taskAttempts, long submitTime, long finishTime, TaskLocalityData data, int minr, int maxr) throws IOException { long submitTimeX = transformX(submitTime); long finishTimeX = transformX(finishTime); Set<Long> xPoints = getXPoints(taskAttempts, submitTimeX, finishTimeX); Long[] xList = xPoints.toArray(new Long[xPoints.size()]); MinMax io = new MinMax(); data.setMapNodeLocal(processLocalityData(taskAttempts, "MAP", "NODE_LOCAL", xList, io)); data.setMapRackLocal(processLocalityData(taskAttempts, "MAP", "RACK_LOCAL", xList, io)); data.setMapOffSwitch(processLocalityData(taskAttempts, "MAP", "OFF_SWITCH", xList, io)); data.setReduceOffSwitch(processLocalityData(taskAttempts, "REDUCE", "OFF_SWITCH", xList, io)); setRValues(data.getMapNodeLocal(), minr, maxr, io.max); setRValues(data.getMapRackLocal(), minr, maxr, io.max); setRValues(data.getMapOffSwitch(), minr, maxr, io.max); setRValues(data.getReduceOffSwitch(), minr, maxr, io.max); data.setSubmitTime(submitTimeX); data.setFinishTime(finishTimeX); } private static class MinMax { private long min = Long.MAX_VALUE; private long max = 0; } private static long transformX(long time) { return Math.round(time / 1000.0); } private static long untransformX(long x) { return x * 1000; } private static long transformY(long time) { return time; } private static Set<Long> getXPoints(List<TaskAttempt> taskAttempts, long submitTimeX, long finishTimeX) { TreeSet<Long> xPoints = new TreeSet<Long>(); TreeSet<TaskAttempt> sortedAttempts = new TreeSet<TaskAttempt>(new Comparator<TaskAttempt>() { @Override public int compare(TaskAttempt t1, TaskAttempt t2) { if (t1.getStartTime() < t2.getStartTime()) return -1; else if (t1.getStartTime() > t2.getStartTime()) return 1; return t1.getTaskAttemptId().compareTo(t2.getTaskAttemptId()); } }); sortedAttempts.addAll(taskAttempts); getXPoints(sortedAttempts, xPoints); xPoints.add(submitTimeX); xPoints.add(finishTimeX); return xPoints; } private static void getXPoints(Iterable<TaskAttempt> taskAttempts, Set<Long> xPoints) { for (TaskAttempt taskAttempt : taskAttempts) { long x = transformX(taskAttempt.getStartTime()); while (xPoints.contains(x)) x += 1; xPoints.add(x); taskAttempt.setStartTime(untransformX(x)); } } private static int addDataPoint(List<DataPoint> data, DataPoint point, int index, Long[] xPoints) { while (index < xPoints.length) { if (point.getX() == xPoints[index]) { index++; break; } else if (point.getX() > xPoints[index]) { data.add(new DataPoint(xPoints[index++])); } } data.add(point); return index; } private static List<DataPoint> processExactLocalityData(List<TaskAttempt> taskAttempts, String taskType, String locality, MinMax io) { List<DataPoint> data = new ArrayList<DataPoint>(); for (TaskAttempt taskAttempt : taskAttempts) { if (taskType.equals(taskAttempt.getTaskType()) && (locality == null || locality.equals(taskAttempt.getLocality()))) { DataPoint point = new DataPoint(); point.setX(taskAttempt.getStartTime()); point.setY(taskAttempt.getFinishTime() - taskAttempt.getStartTime()); point.setIO(taskAttempt.getInputBytes() + taskAttempt.getOutputBytes()); point.setLabel(taskAttempt.getTaskAttemptId()); point.setStatus(taskAttempt.getStatus()); data.add(point); io.max = Math.max(io.max, point.getIO()); io.min = Math.min(io.min, point.getIO()); } } return data; } private static List<DataPoint> processLocalityData(List<TaskAttempt> taskAttempts, String taskType, String locality, Long[] xPoints, MinMax io) { List<DataPoint> data = new ArrayList<DataPoint>(); int i = 0; for (TaskAttempt taskAttempt : taskAttempts) { if (taskType.equals(taskAttempt.getTaskType()) && locality.equals(taskAttempt.getLocality())) { DataPoint point = new DataPoint(); point.setX(transformX(taskAttempt.getStartTime())); point.setY(transformY(taskAttempt.getFinishTime() - taskAttempt.getStartTime())); point.setIO(taskAttempt.getInputBytes() + taskAttempt.getOutputBytes()); point.setLabel(taskAttempt.getTaskAttemptId()); point.setStatus(taskAttempt.getStatus()); i = addDataPoint(data, point, i, xPoints); io.max = Math.max(io.max, point.getIO()); io.min = Math.min(io.min, point.getIO()); } } while (i < xPoints.length) data.add(new DataPoint(xPoints[i++])); return data; } private static void setRValues(List<DataPoint> data, int minr, int maxr, long maxIO) { for (DataPoint point : data) { if (point.getY() == 0) { continue; } if (maxIO == 0 || maxr == minr) { point.setR(minr); continue; } point.setR(Math.round(Math.sqrt(point.getIO() * 1.0 / maxIO) * (maxr - minr) + minr)); } } }