Java tutorial
/** * Copyright 2014 Lockheed Martin Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package streamflow.service; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.google.inject.Singleton; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; import streamflow.datastore.core.TopologyDao; import streamflow.engine.StormEngine; import streamflow.model.Cluster; import streamflow.model.Component; import streamflow.model.ComponentConfig; import streamflow.model.ComponentProperty; import streamflow.model.Resource; import streamflow.model.ResourceConfig; import streamflow.model.ResourceEntry; import streamflow.model.ResourceProperty; import streamflow.model.Serialization; import streamflow.model.Topology; import streamflow.model.TopologyComponent; import streamflow.model.TopologyConfig; import streamflow.model.TopologyLog; import streamflow.model.TopologyLogCriteria; import streamflow.model.TopologyLogPage; import streamflow.model.TopologyResourceEntry; import streamflow.model.TopologySerialization; import streamflow.model.config.StreamflowConfig; import streamflow.model.storm.TopologyInfo; import streamflow.service.exception.EntityConflictException; import streamflow.service.exception.EntityInvalidException; import streamflow.service.exception.EntityNotFoundException; import streamflow.service.exception.ServiceException; import streamflow.service.util.IDUtils; import streamflow.service.util.JarBuilder; import streamflow.util.environment.StreamflowEnvironment; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.SerializationUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton public class TopologyService { public static final Logger LOG = LoggerFactory.getLogger(TopologyService.class); private final TopologyDao topologyDao; private final ComponentService componentService; private final ResourceService resourceService; private final ResourceEntryService resourceEntryService; private final SerializationService serializationService; private final FileService fileService; private final FrameworkService frameworkService; private final ClusterService clusterService; private final LogService logService; private final StormEngine stormEngine; private final StreamflowConfig streamflowConfig; private final ObjectMapper objectMapper = new ObjectMapper(); @Inject public TopologyService(TopologyDao topologyDao, ComponentService componentService, ResourceService resourceService, ResourceEntryService resourceEntryService, SerializationService serializationService, FileService fileService, FrameworkService frameworkService, ClusterService clusterService, LogService logService, StormEngine stormEngine, StreamflowConfig streamflowConfig) { this.topologyDao = topologyDao; this.componentService = componentService; this.resourceService = resourceService; this.resourceEntryService = resourceEntryService; this.serializationService = serializationService; this.fileService = fileService; this.frameworkService = frameworkService; this.clusterService = clusterService; this.logService = logService; this.stormEngine = stormEngine; this.streamflowConfig = streamflowConfig; } public List<Topology> listAllTopologies() { return topologyDao.findAll(); } public List<Topology> listTopologies(String userId) { List<Topology> topologies = topologyDao.findAll(userId); // Get the live status for all topologies for (Topology topology : topologies) { // Get live topology status from the storm engine TopologyInfo topologyInfo = stormEngine.getTopologyInfo(topology); topology.setStatus(topologyInfo.getStatus()); } return topologies; } public Topology createTopology(Topology topology, String userId) { if (topology == null) { throw new EntityInvalidException("The provided topology is NULL"); } if (topology.getName() == null || topology.getType() == null) { throw new EntityInvalidException("The topology is missing required fields"); } if (hasTopology(topology.getName(), userId)) { throw new EntityConflictException( "Topology with the specified name already exists: Name = " + topology.getName()); } // Generate a topology id if none is present topology.setId(IDUtils.formatId(topology.getName())); topology.setCreated(new Date()); topology.setModified(topology.getCreated()); topology.setUserId(userId); return topologyDao.save(topology, userId); } public Topology getTopology(String topologyId, String userId) { Topology topology = topologyDao.findById(topologyId, userId); if (topology == null) { throw new EntityNotFoundException( "Topology with the specified ID could not be found: ID = " + topologyId); } // Get the updated status for the topology TopologyInfo topologyInfo = stormEngine.getTopologyInfo(topology); topology.setStatus(topologyInfo.getStatus()); //topologyDao.update(topology); return topology; } public boolean hasTopology(String topologyName, String userId) { return topologyDao.findByName(topologyName, userId) != null; } public void deleteTopology(String topologyId, String userId) { Topology topology = getTopology(topologyId, userId); // Clear the topology before deletion to make sure it is killed and all temp data removed clearTopology(topologyId, userId); topologyDao.delete(topology); } public void updateTopology(String topologyId, Topology topology, String userId) { Topology oldTopology = getTopology(topologyId, userId); if (topology == null) { throw new EntityInvalidException("The provided topology is NULL"); } if (topology.getName() == null || topology.getType() == null) { throw new EntityInvalidException("The topology is missing required fields"); } if (!oldTopology.getName().equals(topology.getName())) { if (hasTopology(topology.getName(), userId)) { throw new EntityConflictException( "Topology with the specified name already exists: Name = " + topology.getName()); } } topology.setId(topologyId); topology.setCreated(oldTopology.getCreated()); topology.setModified(new Date()); topologyDao.update(topology); } public Topology submitTopology(String topologyId, String userId, String clusterId, String logLevel, String classLoaderPolicy) { LOG.info("Submitting topology: Topology ID = " + topologyId + ", Cluster ID = " + clusterId + ", Log Level = " + logLevel + ", Class Loader Policy = " + classLoaderPolicy); // Always attempt to kill the topology before submitting it in case it is already active killTopology(topologyId, 0, false, userId); // Clear out all previous project artifacts before building a new one clearTopology(topologyId, userId); // Intialize the deployed topology config with current settings Topology topology = initializeTopologyObject(getTopology(topologyId, userId)); if (topology != null) { Cluster cluster = clusterService.getCluster(clusterId); // Generate the jar for the new topology String projectId = generateTopologyJar(topology, cluster); if (projectId != null) { // Update the topology entity with the deployed project info topology.setProjectId(projectId); topology.setSubmitted(new Date()); topology.setClusterId(cluster.getId()); topology.setClusterName(cluster.getDisplayName()); topology.setLogLevel(logLevel); topology.setClassLoaderPolicy(classLoaderPolicy); topology.setStatus("ACTIVE"); topology = stormEngine.submitTopology(topology, cluster); topologyDao.update(topology); } else { LOG.error("There was an error generating the topology jar"); } } else { LOG.error("There was an error initializing the topology object"); } return topology; } public void killTopology(String topologyId, int waitTimeSecs, boolean async, String userId) { Topology topology = getTopology(topologyId, userId); if (stormEngine.killTopology(topology, waitTimeSecs, async)) { // Update the topology entity status and update the killed date to now topology.setStatus("KILLED"); topology.setKilled(new Date()); topologyDao.update(topology); } } public void clearTopology(String topologyId, String userId) { // Reset the topology status in the database Topology topology = getTopology(topologyId, userId); // Clear the generated topology jar and local log file clearTopologyProject(topology.getProjectId()); // Delete local and cluster log files from the log server clearTopologyLog(topology.getId(), userId); // Reset the status of the topology back to IDLE to indicate no deployed status topology.setStatus("IDLE"); topology.setProjectId(null); topology.setClusterId(null); topology.setClusterName(null); topology.setLogLevel(null); topology.setClassLoaderPolicy(null); topology.setSubmitted(null); topology.setKilled(null); topologyDao.update(topology); } public TopologyInfo getTopologyInfo(String topologyId, String userId) { return stormEngine.getTopologyInfo(getTopology(topologyId, userId)); } public TopologyLog getTopologyLogLocal(String topologyId, String userId, long offset, long limit) { return logService.getTopologyLogLocal(getTopology(topologyId, userId), offset, limit); } public TopologyLogPage getTopologyLogCluster(String topologyId, String userId, TopologyLogCriteria criteria) { Topology topology = getTopology(topologyId, userId); Cluster cluster = clusterService.getCluster(topology.getClusterId()); return logService.getTopologyLogCluster(topology, cluster, criteria); } public void clearTopologyLog(String topologyId, String userId) { Topology topology = getTopology(topologyId, userId); Cluster cluster = null; if (topology.getClusterId() != null) { cluster = clusterService.getCluster(topology.getClusterId()); } logService.clearTopologyLog(topology, cluster); } public Topology initializeTopologyObject(Topology topology) { try { TopologyConfig topologyConfig = (TopologyConfig) SerializationUtils.clone(topology.getCurrentConfig()); for (TopologyComponent topologyComponent : topologyConfig.getComponents().values()) { Component component = componentService.getComponent(topologyComponent.getFramework(), topologyComponent.getName()); if (component == null) { throw new ServiceException("A component with the specified " + " framework and name could not be found: " + "Framework = " + topologyComponent.getFramework() + "Name = " + topologyComponent.getName()); } ComponentConfig componentConfig = component.getConfig(); // Iterate over the resource config to build a mapping of names to types HashMap<String, String> componentPropertyTypes = new HashMap<String, String>(); for (ComponentProperty componentProperty : componentConfig.getProperties()) { // Get the property type using "text" as default if not specified String propertyType = componentProperty.getType(); if (propertyType == null) { propertyType = "text"; } componentPropertyTypes.put(componentProperty.getName(), propertyType); } topologyComponent.setPropertyTypes(componentPropertyTypes); topologyComponent.setMainClass(componentConfig.getMainClass()); topologyComponent.setFrameworkHash( frameworkService.getFrameworkFileInfo(component.getFramework()).getContentHash()); topologyComponent.setVersion(component.getVersion()); topologyComponent.getResources().clear(); // Iterate over each of the properties looking for resource types for (ComponentProperty componentProperty : componentConfig.getProperties()) { // The property type is a resource type so fetch the resource entry details if (componentProperty.getType() != null && componentProperty.getType().equalsIgnoreCase("resource")) { String resourceFramework = componentProperty.getOptions().getResourceFramework(); String resourceName = componentProperty.getOptions().getResourceName(); Resource resource = resourceService.getResource(resourceFramework, resourceName); ResourceConfig resourceConfig = resource.getConfig(); // Iterate over the resource config to build a mapping of names to types HashMap<String, String> resourcePropertyTypes = new HashMap<String, String>(); for (ResourceProperty resourceProperty : resourceConfig.getProperties()) { resourcePropertyTypes.put(resourceProperty.getName(), resourceProperty.getType()); } String resourceEntryId = topologyComponent.getProperties().get(componentProperty.getName()); if (resourceEntryId != null) { ResourceEntry resourceEntry = resourceEntryService.getResourceEntry(resourceEntryId); // Create a new topology resource to store with the topology node TopologyResourceEntry topologyResource = new TopologyResourceEntry(); topologyResource.setId(resourceEntryId); topologyResource.setVersion(resource.getVersion()); topologyResource.setFramework(resourceFramework); topologyResource.setFrameworkHash(frameworkService .getFrameworkFileInfo(resource.getFramework()).getContentHash()); topologyResource.setResource(resourceName); topologyResource.setName(resourceEntry.getName()); topologyResource.setDescription(resourceEntry.getDescription()); topologyResource.setResourceClass(resourceConfig.getResourceClass()); topologyResource.setProperties(resourceEntry.getConfig().getProperties()); topologyResource.setPropertyTypes(resourcePropertyTypes); // Add the newly built topology resource to the topology ode topologyComponent.getResources().add(topologyResource); } } } } // Rewrite the modified topology config to the topology so the additional metadata is included topology.setDeployedConfig(topologyConfig); return topology; } catch (ServiceException ex) { LOG.error("Exception while initializing the topology config object", ex); throw new ServiceException( "Exception while initializing the " + "Topology config object: " + ex.getMessage()); } } public String generateTopologyJar(Topology topology, Cluster cluster) { // Generate a unique artifact ID for the topology String projectId = "topology_" + topology.getId() + "_" + System.currentTimeMillis(); try { File topologyJarFile = new File(StreamflowEnvironment.getTopologiesDir(), projectId + ".jar"); // Copy the template jar file to the topologies directory as a base for the topology FileUtils.writeByteArrayToFile(topologyJarFile, loadTopologyTemplate()); JarBuilder jarBuilder = new JarBuilder(topologyJarFile); jarBuilder.open(); // Keep track of already added dependencies HashSet<String> frameworkDependencies = new HashSet<>(); HashSet<String> processedSerializations = new HashSet<>(); TopologyConfig topologyConfig = topology.getDeployedConfig(); // Iterate over all of the nodes and add the dependencies to the lib dir (only once) for (TopologyComponent topologyComponent : topologyConfig.getComponents().values()) { // Save the node coordinates so each node is added only once frameworkDependencies.add(topologyComponent.getFramework()); // Iterate over the property types for the component to see if files are specified for (Map.Entry<String, String> propertyType : topologyComponent.getPropertyTypes().entrySet()) { // Check if the resource property type is a file if (propertyType.getValue() != null && propertyType.getValue().equalsIgnoreCase("file")) { // Get the upload ID from the property value String fileId = topologyComponent.getProperties().get(propertyType.getKey()); // Make sure the user actually specified a file for the field if (StringUtils.isNotBlank(fileId)) { // Embed the upload inside the topology jar if (embedTopologyFile(jarBuilder, fileId)) { // Update the file property to use the file path instead of the file ID topologyComponent.getProperties().put(propertyType.getKey(), "/files/" + fileId); } } } else if (propertyType.getValue() != null && propertyType.getValue().equalsIgnoreCase("serialization")) { String typeClass = topologyComponent.getProperties().get(propertyType.getKey()); // Only add the serialization if it has not already been added if (StringUtils.isNotBlank(typeClass) && !processedSerializations.contains(typeClass)) { Serialization serialization = serializationService .getSerializationWithTypeClass(typeClass); TopologySerialization topologySerialization = new TopologySerialization(); topologySerialization.setTypeClass(serialization.getTypeClass()); topologySerialization.setSerializerClass(serialization.getSerializerClass()); topologySerialization.setFramework(serialization.getFramework()); topologySerialization.setFrameworkHash(frameworkService .getFrameworkFileInfo(serialization.getFramework()).getContentHash()); topologySerialization.setVersion(serialization.getVersion()); topologyConfig.getSerializations().add(topologySerialization); processedSerializations.add(serialization.getTypeClass()); // Save the node coordinates so each node is added only once frameworkDependencies.add(serialization.getFramework()); } } } // Iterate over the resources to add the required dependencies to the build for (TopologyResourceEntry topologyResource : topologyComponent.getResources()) { // Save the node coordinates so each node is added only once frameworkDependencies.add(topologyResource.getFramework()); // Iterate over the property types for the resoruce to see if files are specified for (Map.Entry<String, String> resourceType : topologyResource.getPropertyTypes().entrySet()) { // Check if the resource property type is a file if (resourceType.getValue() != null && resourceType.getValue().equalsIgnoreCase("file")) { // Get the upload ID from the property value String fileId = topologyResource.getProperties().get(resourceType.getKey()); // Make sure the user actually specified a file for the field if (StringUtils.isNotBlank(fileId)) { // Embed the upload inside the topology jar if (embedTopologyFile(jarBuilder, fileId)) { // Update the resource property to use the file path instead of the upload ID topologyResource.getProperties().put(resourceType.getKey(), "/files/" + fileId); } } } } } List<Serialization> serializations = serializationService .listSerializationsWithFramework(topologyComponent.getFramework()); // Iterate over each of the serializations for the current framework for (Serialization serialization : serializations) { // Iterate over the serializations for the current node and add them only once if (!processedSerializations.contains(serialization.getTypeClass())) { TopologySerialization topologySerialization = new TopologySerialization(); topologySerialization.setTypeClass(serialization.getTypeClass()); topologySerialization.setSerializerClass(serialization.getSerializerClass()); topologySerialization.setFramework(topologyComponent.getFramework()); topologySerialization.setFrameworkHash(frameworkService .getFrameworkFileInfo(serialization.getFramework()).getContentHash()); topologySerialization.setVersion(topologyComponent.getVersion()); topologyConfig.getSerializations().add(topologySerialization); processedSerializations.add(serialization.getTypeClass()); } } } // Build the relam framework dir as a temporary storage for files File realmFrameworkDir = new File(StreamflowEnvironment.getFrameworksDir(), projectId); realmFrameworkDir.mkdirs(); // Iterate over all of the identified dependencies and add them to topology jar for (String frameworkDependency : frameworkDependencies) { String frameworkHash = frameworkService.getFrameworkFileInfo(frameworkDependency).getContentHash(); if (cluster.getId().equalsIgnoreCase(Cluster.LOCAL)) { File frameworkJarFile = new File(StreamflowEnvironment.getFrameworksDir(), frameworkHash + ".jar"); // Write out the file to disk only if it is not already there if (!frameworkJarFile.exists()) { byte[] frameworkJarData = frameworkService.getFrameworkJar(frameworkDependency); FileUtils.writeByteArrayToFile(frameworkJarFile, frameworkJarData); } } else { // Only need to embed files within the topology jar for cluster deploys String frameworkJarPath = "STREAMFLOW-INF" + File.separator + "lib" + File.separator + frameworkHash + ".jar"; byte[] frameworkJarData = frameworkService.getFrameworkJar(frameworkDependency); if (!jarBuilder.addFile(frameworkJarPath, frameworkJarData)) { LOG.error("Error while writing the framework jar dependency to the topology jar"); } // Check each framework jar for inbuilt resources to copy to the topology resources embedFrameworkResources(jarBuilder, frameworkJarData); } } // Write out the topology file to the topology jar with the modified changes String topologyJsonPath = "STREAMFLOW-INF" + File.separator + "topology.json"; if (!jarBuilder.addFile(topologyJsonPath, objectMapper.writeValueAsBytes(topology))) { LOG.error("Error while writing the topology.json file to the topology jar"); } // Write out the config file to the topology jar with the modified changes String configJsonPath = "STREAMFLOW-INF" + File.separator + "config.json"; streamflowConfig.setSelectedCluster(cluster); if (!jarBuilder.addFile(configJsonPath, objectMapper.writeValueAsBytes(streamflowConfig))) { LOG.error("Error while writing the config.json file to the topology jar"); } // Close the new topology jar to move it to the topology directory jarBuilder.close(); } catch (IOException ex) { // If an exception occurs, delete the topology project from disk clearTopologyProject(projectId); LOG.error("Exception while generating the topology jar", ex); throw new ServiceException("Error while generating the topology jar: " + ex.getMessage()); } return projectId; } private boolean embedTopologyFile(JarBuilder jarBuilder, String fileId) { boolean success = false; // Retrieve the upload metadata from the service byte[] fileContent = fileService.getFileContent(fileId); if (fileContent != null) { String targetFilePath1 = "files" + File.separator + fileId; String targetFilePath2 = "resources" + File.separator + "files" + File.separator + fileId; // Copy the file upload to the files directory of the jar if (jarBuilder.addFile(targetFilePath1, fileContent)) { // Also copy the file to a special /resources folder which Storm will automatically explode to disk if (jarBuilder.addFile(targetFilePath2, fileContent)) { success = true; } else { LOG.error("Error while writing the file upload to the topology jar: " + targetFilePath2); } } else { LOG.error("Error while writing the file upload to the topology jar: " + targetFilePath1); } } else { LOG.error("File was not found for the topology: ID = " + fileId); } return success; } private void embedFrameworkResources(JarBuilder jarBuilder, byte[] frameworkJar) { JarEntry jarEntry; JarInputStream jarInputStream; try { jarInputStream = new JarInputStream(new ByteArrayInputStream(frameworkJar)); while ((jarEntry = jarInputStream.getNextJarEntry()) != null) { // A resource file is defined as a non *.class file that is a real file if (!jarEntry.isDirectory() && !jarEntry.getName().endsWith(".class")) { // Ignore the resource files in the META-INF folder if (!jarEntry.getName().startsWith("META-INF") && !jarEntry.getName().startsWith("STREAMFLOW-INF")) { // Create the handle to the target resource file in the topology jar if (!jarBuilder.addFile(jarEntry.getName(), IOUtils.toByteArray(jarInputStream))) { LOG.error("Error occurred while writing a framework resource to the topology jar: " + jarEntry.getName()); } } } } } catch (IOException ex) { LOG.error("Exception while embedding framework resource: " + ex.getMessage()); } } private void clearTopologyProject(String projectId) { if (projectId != null) { try { // Delete the project folder from the server FileUtils.forceDelete(new File(StreamflowEnvironment.getTopologiesDir(), projectId + ".jar")); } catch (IOException ex) { //LOG.error("Exception while clearing the topology project: ", ex); } } } private byte[] loadTopologyTemplate() { byte[] engineJarData = null; try { InputStream engineStream = null; try { // Copy the inbuilt streamflow-engine.jar template to the streamflow lib directory engineStream = Thread.currentThread().getContextClassLoader() .getResourceAsStream("STREAMFLOW-INF/lib/streamflow-engine.jar"); engineJarData = IOUtils.toByteArray(engineStream); } finally { if (engineStream != null) { engineStream.close(); } } } catch (IOException ex) { LOG.error("Exception occurred while loading the streamflow-engine jar: ", ex); } return engineJarData; } }