Java tutorial
/* * Copyright 2015 Shazam Entertainment Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * either express or implied. See the License for the specific * language governing permissions and limitations under the License */ package com.shazam.dataengineering.pipelinebuilder; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.services.datapipeline.DataPipelineClient; import com.amazonaws.services.datapipeline.model.ValidatePipelineDefinitionResult; import com.amazonaws.services.datapipeline.model.ValidationError; import com.amazonaws.services.datapipeline.model.ValidationWarning; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import hudson.FilePath; import hudson.model.*; import net.sf.json.JSONObject; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; import javax.servlet.ServletException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.*; public class DeploymentAction implements Action { private static final String LOG_FILENAME = "deployment.log"; private AbstractProject project; private AbstractBuild build; private Map<S3Environment, String> s3Urls; private List<Run.Artifact> artifacts; private AWSCredentials credentials; private String pipelineToRemoveId; private String pipelineFile; private PipelineObject pipelineObject; private DeploymentException lastException; private List<String> clientMessages = new ArrayList<String>(); public DeploymentAction(AbstractBuild build, Map<S3Environment, String> s3Urls, AWSCredentials awsCredentials) { this.project = build.getProject(); this.build = build; this.s3Urls = s3Urls; this.artifacts = build.getArtifacts(); this.credentials = awsCredentials; } public String getIconFileName() { return "/plugin/pipeline-builder/icons/pipeline-22x22.png"; } public String getDisplayName() { return "Deploy Pipeline"; } public String getUrlName() { return "pipeline"; } public AbstractBuild getBuild() { return build; } public String getPipelineToRemoveId() { return pipelineToRemoveId; } public boolean hasPipelineToRemove() { return pipelineToRemoveId != null && !pipelineToRemoveId.isEmpty(); } public List<String> getClientMessages() { return clientMessages; } public String getPipelineFile() { return pipelineFile; } public DeploymentException getLastException() { return lastException; } public String getUrl() { return build.getUpUrl(); } public List<Deployment> getDeployments() { FilePath newPath = new FilePath(new FilePath(build.getArtifactsDir()), LOG_FILENAME); try { String logContent = newPath.readToString(); if (!logContent.isEmpty()) { DeploymentLog dto = new DeploymentLog(logContent); return dto.getAll(); } } catch (IOException e) { // Ignore } return Collections.EMPTY_LIST; } public BallColor getBallColorRed() { return BallColor.RED; } public BallColor getBallColorBlue() { return BallColor.BLUE; } public boolean isStartDatePast() { return pipelineObject != null && PipelineObject.isPast(pipelineObject.getScheduleDate()); } public boolean hasScriptsToDeploy() { return s3Urls.size() > 0; } public boolean oldPipelineHasRunningTasks() { DataPipelineClient client = new DataPipelineClient(credentials); AWSProxy proxy = new AWSProxy(client); return proxy.hasRunningTasks(pipelineToRemoveId); } public List<String> getPipelines() { ArrayList<String> pipelines = new ArrayList<String>(); if (artifacts != null && artifacts.size() > 0) { for (Run.Artifact artifact : artifacts) { try { PipelineObject object = new PipelineObject(new FilePath(artifact.getFile()).readToString()); if (object.isValid()) { pipelines.add(artifact.getFileName()); } } catch (IOException e) { // Ignore } } } return pipelines; } // TODO: Multiple schedule objects per pipeline public String getScheduledDate() throws IOException { PipelineObject pipelineObject = this.pipelineObject; // TODO: Change based on the value of pipeline selector if (pipelineObject == null && artifacts.size() > 0) { if (pipelineFile != null) { pipelineObject = getPipelineByName(pipelineFile); } else { Run.Artifact artifact = null; // Get the pipeline file, any pipeline file // TODO: This makes a lot of assumptions, need to fix. for (Run.Artifact allegedPipeline : artifacts) { if (allegedPipeline.getFileName().endsWith(".json")) { artifact = allegedPipeline; } } if (artifact != null) { pipelineObject = new PipelineObject(new FilePath(artifact.getFile()).readToString()); } } } if (pipelineObject != null) { return pipelineObject.getScheduleDate(); } else { return ""; } } public void doConfirmProcess(StaplerRequest req, StaplerResponse resp) throws IOException, ServletException { // Clear out previous warnings clientMessages.clear(); JSONObject formData = req.getSubmittedForm(); pipelineFile = formData.getString("pipeline"); String startDate = formData.getString("scheduleDate"); // Validate start date, and warn if its in the past. if (!PipelineObject.validateDate(startDate)) { clientMessages.add( "[ERROR] Passed start date was not in expected format: " + PipelineObject.PIPELINE_DATE_FORMAT); req.getView(this, "error").forward(req, resp); } else if (PipelineObject.isPast(startDate)) { clientMessages.add("[WARN] Passed start date is in the past. Backfill may occur."); } // Validate chosen pipeline pipelineObject = getPipelineByName(pipelineFile); if (pipelineObject == null) { clientMessages.add("[ERROR] Pipeline not found"); req.getView(this, "error").forward(req, resp); } else { pipelineObject.setScheduleDate(startDate); } // Find previously deployed pipeline. try { DataPipelineClient client = new DataPipelineClient(credentials); pipelineToRemoveId = getPipelineId(pipelineFile, client); if (!pipelineToRemoveId.isEmpty() && oldPipelineHasRunningTasks()) { clientMessages.add("[WARN] Old pipeline is currently running. Execution will be terminated."); } } catch (DeploymentException e) { pipelineToRemoveId = ""; } req.getView(this, "confirm").forward(req, resp); } public synchronized void doDeploy(StaplerRequest req, StaplerResponse resp) throws ServletException, IOException { DataPipelineClient client = new DataPipelineClient(credentials); Date start = new Date(); try { String pipelineId = createNewPipeline(client); validateNewPipeline(pipelineId, client); uploadNewPipeline(pipelineId, client); deployScriptsToS3(); removeOldPipeline(client); activateNewPipeline(pipelineId, client); writeReport(start, pipelineId, true); req.getView(this, "report").forward(req, resp); } catch (DeploymentException e) { if (e.getCause() != null) { clientMessages.add("[ERROR] " + e.getCause().getMessage()); } writeReport(start, "", false); req.getView(this, "error").forward(req, resp); } } private void deployScriptsToS3() throws DeploymentException { String pathPrefix = build.getArtifactsDir().getPath() + "/scripts/"; AmazonS3 s3Client = new AmazonS3Client(credentials); for (S3Environment env : s3Urls.keySet()) { if (env.pipelineName.equals(pipelineFile)) { String filename = env.scriptName; File file = new File(pathPrefix + filename); if (file.exists()) { String url = s3Urls.get(env); clientMessages.add(String.format("[INFO] Uploading %s to %s", filename, url)); boolean result = AWSProxy.uploadFileToS3Url(s3Client, url, file); if (result) { clientMessages.add(String.format("[INFO] Upload successful!")); } else { clientMessages.add(String.format("[ERROR] Upload failed!")); throw new DeploymentException(); } } else { clientMessages.add(String.format("[ERROR] Unable to find %s in artifacts", filename)); throw new DeploymentException(); } } } } private PipelineObject getPipelineByName(String pipelineName) throws IOException { if (!pipelineName.isEmpty() && artifacts != null && artifacts.size() > 0) { for (Run.Artifact artifact : artifacts) { if (artifact.getFileName().equals(pipelineName)) { return new PipelineObject(new FilePath(artifact.getFile()).readToString()); } } } return null; } private String getPipelineName() { return pipelineFile.substring(0, pipelineFile.lastIndexOf(".json")); } private void activateNewPipeline(String pipelineId, DataPipelineClient client) throws DeploymentException { AWSProxy proxy = new AWSProxy(client); proxy.activatePipeline(pipelineId); clientMessages.add("[INFO] Pipeline has been activated!"); clientMessages.add("[INFO] New pipeline ID: " + pipelineId); } private void uploadNewPipeline(String pipelineId, DataPipelineClient client) throws DeploymentException { AWSProxy proxy = new AWSProxy(client); boolean success = proxy.putPipeline(pipelineId, pipelineObject); if (!success) { clientMessages.add("[ERROR] Unable to upload new pipeline definition."); throw new DeploymentException(); } else { clientMessages.add("[INFO] Upload of pipeline definition completed successfully"); } } private void validateNewPipeline(String pipelineId, DataPipelineClient client) throws DeploymentException { AWSProxy proxy = new AWSProxy(client); ValidatePipelineDefinitionResult validation = proxy.validatePipeline(pipelineId, pipelineObject); List<ValidationError> errors = validation.getValidationErrors(); List<ValidationWarning> warnings = validation.getValidationWarnings(); for (ValidationError error : errors) { for (String errorMessage : error.getErrors()) { clientMessages.add("[ERROR] " + errorMessage); } } for (ValidationWarning warning : warnings) { for (String warningMessage : warning.getWarnings()) { clientMessages.add("[WARN] " + warningMessage); } } if (validation.isErrored()) { clientMessages.add("[ERROR] Critical errors detected in validation."); throw new DeploymentException(); } else { clientMessages.add("[INFO] No critical errors for the pipeline detected in validation."); } } private String createNewPipeline(DataPipelineClient client) throws DeploymentException { AWSProxy proxy = new AWSProxy(client); return proxy.createPipeline(getPipelineName()); } private void removeOldPipeline(DataPipelineClient client) throws DeploymentException { if (pipelineToRemoveId == null || !pipelineToRemoveId.isEmpty()) { AWSProxy proxy = new AWSProxy(client); boolean result = proxy.removePipeline(pipelineToRemoveId); if (result) { clientMessages.add("[INFO] Successfully removed pipeline " + pipelineToRemoveId); } else { clientMessages.add("[WARN] Failed to remove pipeline " + pipelineToRemoveId); } } else { clientMessages.add("[INFO] No old pipeline to remove"); } } private String getPipelineId(String pipelineName, DataPipelineClient client) throws DeploymentException { String pipelineRegex = pipelineName.substring(0, pipelineName.lastIndexOf("-")) + "-\\d+"; AWSProxy proxy = new AWSProxy(client); return proxy.getPipelineId(pipelineRegex); } private void writeReport(Date date, String pipelineId, boolean success) { User currentUser = User.current(); String username; if (currentUser != null) { username = currentUser.getFullName(); } else { username = "Anonymous"; } FilePath newPath = new FilePath(new FilePath(build.getArtifactsDir()), LOG_FILENAME); try { String logContent = ""; DeploymentLog dto; try { logContent = newPath.readToString(); } catch (FileNotFoundException e) { // Ignore } if (logContent.isEmpty()) { dto = new DeploymentLog(); } else { dto = new DeploymentLog(logContent); } dto.add(username, success, pipelineId, date, clientMessages); newPath.write(dto.toString(), StandardCharsets.UTF_8.name()); } catch (IOException e) { clientMessages.add("[ERROR] Failed to write deployment report!"); } catch (InterruptedException e) { clientMessages.add("[ERROR] Failed to write deployment report!"); } } }