Java tutorial
package com.thinkbiganalytics.feedmgr.service.template; /*- * #%L * thinkbig-feed-manager-controller * %% * Copyright (C) 2017 ThinkBig Analytics * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * #L% */ import com.fasterxml.jackson.annotation.JsonIgnore; import com.thinkbiganalytics.feedmgr.nifi.NifiTemplateParser; import com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver; import com.thinkbiganalytics.feedmgr.nifi.cache.NifiFlowCache; import com.thinkbiganalytics.feedmgr.rest.ImportComponent; import com.thinkbiganalytics.feedmgr.rest.ImportSection; import com.thinkbiganalytics.feedmgr.rest.ImportType; import com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption; import com.thinkbiganalytics.feedmgr.rest.model.ImportOptions; import com.thinkbiganalytics.feedmgr.rest.model.ImportTemplateOptions; import com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate; import com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest; import com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo; import com.thinkbiganalytics.feedmgr.rest.model.UploadProgress; import com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage; import com.thinkbiganalytics.feedmgr.rest.support.SystemNamingService; import com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl; import com.thinkbiganalytics.feedmgr.service.MetadataService; import com.thinkbiganalytics.feedmgr.service.UploadProgressService; import com.thinkbiganalytics.feedmgr.support.ZipFileUtil; import com.thinkbiganalytics.feedmgr.util.ImportUtil; import com.thinkbiganalytics.json.ObjectMapperSerializer; import com.thinkbiganalytics.metadata.api.MetadataAccess; import com.thinkbiganalytics.metadata.api.template.security.TemplateAccessControl; import com.thinkbiganalytics.nifi.feedmgr.ReusableTemplateCreationCallback; import com.thinkbiganalytics.nifi.feedmgr.TemplateCreationHelper; import com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient; import com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException; import com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException; import com.thinkbiganalytics.nifi.rest.model.NiFiComponentErrors; import com.thinkbiganalytics.nifi.rest.model.NiFiPropertyDescriptorTransform; import com.thinkbiganalytics.nifi.rest.model.NifiError; import com.thinkbiganalytics.nifi.rest.model.NifiProcessGroup; import com.thinkbiganalytics.nifi.rest.model.NifiProperty; import com.thinkbiganalytics.nifi.rest.support.NifiConnectionUtil; import com.thinkbiganalytics.nifi.rest.support.NifiProcessUtil; import com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil; import com.thinkbiganalytics.security.AccessController; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.TemplateDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.xml.sax.SAXException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import javax.inject.Inject; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathExpressionException; /** * Export or Import a template */ public class ExportImportTemplateService { public static final Logger log = LoggerFactory.getLogger(ExportImportTemplateService.class); public static final String NIFI_CONNECTING_REUSABLE_TEMPLATE_XML_FILE = "nifiConnectingReusableTemplate"; public static final String NIFI_TEMPLATE_XML_FILE = "nifiTemplate.xml"; public static final String TEMPLATE_JSON_FILE = "template.json"; @Autowired PropertyExpressionResolver propertyExpressionResolver; @Autowired MetadataService metadataService; @Inject MetadataAccess metadataAccess; @Autowired LegacyNifiRestClient nifiRestClient; @Inject NifiFlowCache nifiFlowCache; @Inject private AccessController accessController; @Inject private RegisteredTemplateService registeredTemplateService; @Inject private NiFiPropertyDescriptorTransform propertyDescriptorTransform; @Inject private UploadProgressService uploadProgressService; @Inject private RegisteredTemplateCache registeredTemplateCache; //Export Methods /** * Check to ensure the user just has the EXPORT_FEEDS functional access permission before exporting * * @param templateId the registered template id * @return the exported template */ public ExportTemplate exportTemplateForFeedExport(String templateId) { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_FEEDS); registeredTemplateService.checkTemplatePermission(templateId, TemplateAccessControl.ACCESS_TEMPLATE); return export(templateId); } /** * Check to ensure the user has the EXPORT_TEMPLATES functional access permission before exporting * * @param templateId the registered template id * @return the exported template */ public ExportTemplate exportTemplate(String templateId) { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_TEMPLATES); registeredTemplateService.checkTemplatePermission(templateId, TemplateAccessControl.EXPORT); return export(templateId); } private ExportTemplate export(String templateId) { RegisteredTemplate template = registeredTemplateService .findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId) .nifiTemplateId(templateId).includeSensitiveProperties(true).build()); if (template != null) { List<String> connectingReusableTemplates = new ArrayList<>(); Set<String> connectedTemplateIds = new HashSet<>(); //if this template uses any reusable templates then export those reusable ones as well if (template.usesReusableTemplate()) { List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = template .getReusableTemplateConnections(); for (ReusableTemplateConnectionInfo reusableTemplateConnectionInfo : reusableTemplateConnectionInfos) { String inputName = reusableTemplateConnectionInfo.getReusableTemplateInputPortName(); //find the template that has the input port name? Map<String, String> map = nifiRestClient.getTemplatesAsXmlMatchingInputPortName(inputName); if (map != null && !map.isEmpty()) { //get the first one?? for (Map.Entry<String, String> entry : map.entrySet()) { String portTemplateId = entry.getKey(); if (!connectedTemplateIds.contains(portTemplateId)) { connectedTemplateIds.add(portTemplateId); connectingReusableTemplates.add(entry.getValue()); } } } } } String templateXml = null; try { if (template != null) { try { templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId()); } catch (NifiClientRuntimeException e) { TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName()); if (templateDTO != null) { templateXml = nifiRestClient.getTemplateXml(templateDTO.getId()); } } } } catch (Exception e) { throw new UnsupportedOperationException("Unable to find Nifi Template for " + templateId); } //create a zip file with the template and xml byte[] zipFile = zip(template, templateXml, connectingReusableTemplates); return new ExportTemplate( SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", zipFile); } else { throw new UnsupportedOperationException("Unable to find Template for " + templateId); } } private byte[] zip(RegisteredTemplate template, String nifiTemplateXml, List<String> reusableTemplateXmls) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zos = new ZipOutputStream(baos)) { ZipEntry entry = new ZipEntry(NIFI_TEMPLATE_XML_FILE); zos.putNextEntry(entry); zos.write(nifiTemplateXml.getBytes()); zos.closeEntry(); int reusableTemplateNumber = 0; for (String reusableTemplateXml : reusableTemplateXmls) { entry = new ZipEntry(String.format("%s_%s.xml", NIFI_CONNECTING_REUSABLE_TEMPLATE_XML_FILE, reusableTemplateNumber++)); zos.putNextEntry(entry); zos.write(reusableTemplateXml.getBytes()); zos.closeEntry(); } entry = new ZipEntry(TEMPLATE_JSON_FILE); zos.putNextEntry(entry); String json = ObjectMapperSerializer.serialize(template); zos.write(json.getBytes()); zos.closeEntry(); } catch (IOException ioe) { throw new RuntimeException(ioe); } return baos.toByteArray(); } //Validation Methods /** * Is the file a valid file for importing * * @param fileName the name of the file * @return true if valid, false if not valid */ private boolean isValidFileImport(String fileName) { return fileName.endsWith(".zip") || fileName.endsWith(".xml"); } //validate public ImportTemplate validateTemplateForImport(final String fileName, byte[] content, ImportOptions importOptions) { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_TEMPLATES); InputStream inputStream = new ByteArrayInputStream(content); UploadProgressMessage overallStatusMessage = uploadProgressService .addUploadStatus(importOptions.getUploadKey(), "Validating template for import"); UploadProgressMessage statusMessage = overallStatusMessage; ImportTemplateOptions options = new ImportTemplateOptions(); options.setUploadKey(importOptions.getUploadKey()); ImportTemplate template = null; if (!isValidFileImport(fileName)) { statusMessage.complete(false); throw new UnsupportedOperationException( "Unable to import " + fileName + ". The file must be a zip file or a Nifi Template xml file"); } try { if (fileName.endsWith(".zip")) { template = openZip(fileName, inputStream); template.setValid(true); Set<ImportComponentOption> componentOptions = ImportUtil.inspectZipComponents(content, ImportType.TEMPLATE); options.setImportComponentOptions(importOptions.getImportComponentOptions()); options.addOptionsIfNotExists(componentOptions); template.setImportOptions(options); validateReusableTemplate(template, options); validateRegisteredTemplate(template, options); if (template.isValid()) { validateNiFiTemplateImport(template); } } else { template = getNewNiFiTemplateImport(fileName, inputStream); template.setImportOptions(options); //deal with reusable templates?? validateNiFiTemplateImport(template); } } catch (Exception e) { statusMessage.update("An Error occurred " + e.getMessage(), false); throw new UnsupportedOperationException( "Error importing template " + fileName + ". " + e.getMessage()); } overallStatusMessage.update("Validated template for import ", template.isValid()); return template; } /** * Validate the NiFi template is valid. This method will validate the {@link ImportTemplate#nifiTemplateXml} * * @param template the template data to validate before importing */ private void validateNiFiTemplateImport(ImportTemplate template) throws IOException { ImportOptions options = template.getImportOptions(); ImportComponentOption nifiTemplateOption = options.findImportComponentOption(ImportComponent.NIFI_TEMPLATE); //if the options of the TEMPLATE_DATA are marked to import and overwrite this should be as well ImportComponentOption templateData = options.findImportComponentOption(ImportComponent.TEMPLATE_DATA); if (templateData.isUserAcknowledged()) { nifiTemplateOption.setUserAcknowledged(true); } if (templateData.isShouldImport()) { nifiTemplateOption.setShouldImport(true); } if (templateData.isOverwrite()) { nifiTemplateOption.setOverwrite(true); } if (nifiTemplateOption.isShouldImport()) { UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(), "Validating the NiFi template"); String templateName = null; TemplateDTO dto = null; try { templateName = NifiTemplateParser.getTemplateName(template.getNifiTemplateXml()); template.setTemplateName(templateName); dto = nifiRestClient.getTemplateByName(templateName); if (dto != null) { template.setNifiTemplateId(dto.getId()); //if the template incoming is an XML template and it already exists, or if its a zip file and it exists and the user has not acknowledge to overwrite then error out if ((!options.isUserAcknowledged(ImportComponent.NIFI_TEMPLATE) || options.isUserAcknowledged(ImportComponent.NIFI_TEMPLATE) && !template.isZipFile()) && !options.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && !options.isContinueIfExists(ImportComponent.NIFI_TEMPLATE)) { template.setValid(false); String msg = "Unable to import Template " + templateName + ". It already exists in NiFi."; template.getImportOptions().addErrorMessage(ImportComponent.NIFI_TEMPLATE, msg); statusMessage.update("Validation Error: Unable to import Template " + templateName + ". It already exists in NiFi."); statusMessage.complete(false); } else { statusMessage.update("Validated the NiFi template. "); statusMessage.complete(true); } } else { statusMessage.update("Validated the NiFi template. The template " + templateName + " will be created in NiFi"); statusMessage.complete(true); } } catch (ParserConfigurationException | XPathExpressionException | SAXException e) { template.setValid(false); template.getTemplateResults().addError(NifiError.SEVERITY.WARN, "The xml file you are trying to import is not a valid NiFi template. Please try again. " + e.getMessage(), ""); statusMessage.complete(false); } nifiTemplateOption.setValidForImport(!nifiTemplateOption.hasErrorMessages()); } completeSection(options, ImportSection.Section.VALIDATE_NIFI_TEMPLATE); } /** * Validate the Registered Template is valid for importing * * @param template the template data to validate before importing * @param options user options about what/how it should be imported */ private void validateRegisteredTemplate(ImportTemplate template, ImportOptions options) { ImportComponentOption registeredTemplateOption = options .findImportComponentOption(ImportComponent.TEMPLATE_DATA); //validate template boolean validForImport = true; if (!registeredTemplateOption.isUserAcknowledged()) { template.setValid(false); template.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "A template exists. Do you want to import it?"); } if (registeredTemplateOption.isUserAcknowledged() && registeredTemplateOption.isShouldImport()) { UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(), "Validating feed template for import"); RegisteredTemplate registeredTemplate = template.getTemplateToImport(); //validate unique //1 find if the template exists in the system running as a service account RegisteredTemplate existingTemplate = registeredTemplateService .findRegisteredTemplate(new RegisteredTemplateRequest.Builder() .templateName(registeredTemplate.getTemplateName()).isFeedEdit(true).build()); if (existingTemplate != null) { if (options.stopProcessingAlreadyExists(ImportComponent.TEMPLATE_DATA)) { template.setValid(false); String msg = "Unable to import the template " + registeredTemplate.getTemplateName() + " It is already registered."; template.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, msg); statusMessage.update("Validation error: The template " + registeredTemplate.getTemplateName() + " is already registered", false); } else { //skip importing if user doesnt want it. if (!registeredTemplateOption.isOverwrite()) { validForImport = false; } statusMessage.complete(true); } registeredTemplate.setId(existingTemplate.getId()); //validate entity access if (accessController.isEntityAccessControlled() && registeredTemplateOption.isOverwrite()) { //requery as the currently logged in user statusMessage = uploadProgressService.addUploadStatus(options.getUploadKey(), "Validating template entity access"); existingTemplate = registeredTemplateService.findRegisteredTemplate( RegisteredTemplateRequest.requestByTemplateName(registeredTemplate.getTemplateName())); //ensure the user can Edit this template boolean valid = existingTemplate != null && existingTemplate.getAllowedActions() .hasAction(TemplateAccessControl.EDIT_TEMPLATE.getSystemName()); if (!valid) { template.setValid(false); validForImport = false; statusMessage.update("Access Denied: You do not have edit access for the template " + registeredTemplate.getTemplateName(), false); template.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "Access Denied: You do not have edit access for the template "); } else { statusMessage.complete(valid); } } } else { //template doesnt exist.. it is new ensure the user is allowed to create templates boolean editAccess = accessController.hasPermission(AccessController.SERVICES, FeedServicesAccessControl.EDIT_TEMPLATES); if (!editAccess) { statusMessage.update("Access Denied: You are not allowed to create the template " + registeredTemplate.getTemplateName(), false); template.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "Access Denied: You are not allowed to create the template "); } registeredTemplate.setId(null); statusMessage.complete(editAccess); } validForImport &= !registeredTemplateOption.hasErrorMessages(); if (validForImport) { boolean isValid = validateTemplateProperties(template, options); if (template.isValid()) { template.setValid(isValid); } validForImport &= isValid; } registeredTemplateOption.setValidForImport(validForImport); } completeSection(options, ImportSection.Section.VALIDATE_REGISTERED_TEMPLATE); } /** * Validate any reusable templates as part of a zip file upload are valid for importing * * @param template the template data to validate before importing * @param options user options about what/how it should be imported */ private void validateReusableTemplate(ImportTemplate template, ImportTemplateOptions options) throws Exception { //validate the reusable template if (template.hasConnectingReusableTemplate()) { ImportComponentOption reusableTemplateOption = options .findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE); UploadProgressMessage reusableTemplateStatusMessage = uploadProgressService .addUploadStatus(options.getUploadKey(), "Validating Reusable Template. "); if (reusableTemplateOption.isShouldImport()) { boolean validForImport = true; //for each of the connecting template for (String reusableTemplateXml : template.getNifiConnectingReusableTemplateXmls()) { String templateName = NifiTemplateParser.getTemplateName(reusableTemplateXml); UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus( options.getUploadKey(), "Validating Reusable Template. " + templateName); TemplateDTO dto = nifiRestClient.getTemplateByName(templateName); //if there is a match and it has not been acknowledged by the user to overwrite or not, error out if (dto != null && !reusableTemplateOption.isUserAcknowledged()) { //error out it exists template.getImportOptions().addErrorMessage(ImportComponent.REUSABLE_TEMPLATE, "A reusable template with the same name " + templateName + " exists."); template.setValid(false); statusMessage.update("Reusable template, " + templateName + ", already exists.", false); validForImport = false; } else if (dto != null && reusableTemplateOption.isUserAcknowledged() && !reusableTemplateOption.isOverwrite()) { validForImport = false; //user has asked to not import the template. uploadProgressService.removeMessage(options.getUploadKey(), statusMessage); } else { uploadProgressService.removeMessage(options.getUploadKey(), statusMessage); //statusMessage.update("Validated Reusable Template", true); validForImport &= true; } } reusableTemplateOption.setValidForImport(validForImport); reusableTemplateStatusMessage.update("Validated Reusable Templates ", !reusableTemplateOption.hasErrorMessages()); } else if (!reusableTemplateOption.isUserAcknowledged()) { template.getImportOptions().addErrorMessage(ImportComponent.REUSABLE_TEMPLATE, "The file " + template.getFileName() + " has a reusable template to import."); template.setValid(false); reusableTemplateStatusMessage.update("A reusable template was found. Additional input needed.", false); } else { reusableTemplateStatusMessage .update("Reusable template found in import, but it is not marked for importing", true); } } completeSection(options, ImportSection.Section.VALIDATE_REUSABLE_TEMPLATE); } /** * Validate the Template doesnt have any sensitive properties needing additional user input before importing * * @param importTemplate the template data to validate before importing * @param importOptions user options about what/how it should be imported * @return true if valid, false if not */ private boolean validateTemplateProperties(ImportTemplate importTemplate, ImportOptions importOptions) { RegisteredTemplate template = importTemplate.getTemplateToImport(); //detect any sensitive properties and prompt for input before proceeding List<NifiProperty> sensitiveProperties = template.getSensitiveProperties(); ImportUtil.addToImportOptionsSensitiveProperties(importOptions, sensitiveProperties, ImportComponent.TEMPLATE_DATA); boolean valid = ImportUtil.applyImportPropertiesToTemplate(template, importTemplate, ImportComponent.TEMPLATE_DATA); if (!valid) { importTemplate.getImportOptions().addErrorMessage(ImportComponent.TEMPLATE_DATA, "Additional properties are required for the Template"); } importOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA).setAnalyzed(true); return valid; } /** * Validate the sensitive properties are set for the reusable templates. * * @param importTemplate the template data to validate before importing * @param importOptions user options about what/how it should be imported * @return true if valid, false if not */ private boolean analyzeReusableTemplateForImport(ImportTemplate importTemplate, ImportOptions importOptions) throws Exception { boolean valid = true; if (importTemplate.hasConnectingReusableTemplate()) { log.info("analyzing reusable templates in file"); for (String reusableTemplateXml : importTemplate.getNifiConnectingReusableTemplateXmls()) { //create a temp instance of this template to assess if it has any sensitive properties String templateName = NifiTemplateParser.getTemplateName(importTemplate.getNifiTemplateXml()); String reusableTemplateName = templateName; templateName += "_" + System.currentTimeMillis(); String templateXml = NifiTemplateParser.updateTemplateName(importTemplate.getNifiTemplateXml(), templateName); TemplateDTO temporaryTemplate = nifiRestClient.importTemplate(templateName, templateXml); log.info("created temp reusable template {} ", temporaryTemplate.getName() + ", for " + reusableTemplateName); TemplateCreationHelper helper = new TemplateCreationHelper(nifiRestClient); if (importTemplate.getTemplateToImport() != null) { helper.setTemplateProperties(importTemplate.getTemplateToImport().getProperties()); } ProcessGroupDTO flow = helper.createTemporaryTemplateFlow(temporaryTemplate.getId()); nifiRestClient.deleteTemplate(temporaryTemplate.getId()); List<NifiProperty> sensitiveProperties = NifiPropertyUtil .getProperties(flow, propertyDescriptorTransform).stream() .filter(nifiProperty -> nifiProperty.isSensitive()).collect(Collectors.toList()); if (sensitiveProperties != null && !sensitiveProperties.isEmpty()) { ImportUtil.addToImportOptionsSensitiveProperties(importOptions, sensitiveProperties, ImportComponent.REUSABLE_TEMPLATE); importTemplate.getImportOptions().addErrorMessage(ImportComponent.REUSABLE_TEMPLATE, "Additional Properties are required for input on the reusable template"); valid = false; } } importOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).setAnalyzed(true); } return valid; } private void completeSection(ImportOptions options, ImportSection.Section section) { UploadProgress progress = uploadProgressService.getUploadStatus(options.getUploadKey()); progress.completeSection(section.name()); } /// IMPORT methods public ImportTemplate importTemplateForFeed(final String fileName, final byte[] content, ImportTemplateOptions importOptions) { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EXPORT_TEMPLATES); return importTemplate(fileName, content, importOptions); } /** * Import a xml or zip file. * * @param fileName the name of the file * @param content the file * @param importOptions user options about what/how it should be imported * @return the template data to import along with status/messages/error information if it was valid and if was successfully imported */ public ImportTemplate importTemplate(final String fileName, final byte[] content, ImportTemplateOptions importOptions) { // return metadataAccess.commit(() -> { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_TEMPLATES); ImportTemplate template = null; if (!isValidFileImport(fileName)) { throw new UnsupportedOperationException( "Unable to import " + fileName + ". The file must be a zip file or a Nifi Template xml file"); } try { if (fileName.endsWith(".zip")) { UploadProgress progress = uploadProgressService.getUploadStatus(importOptions.getUploadKey()); progress.setSections(ImportSection.sectionsForImportAsString(ImportType.TEMPLATE)); template = validateAndImportZip(fileName, content, importOptions); //dont allow exported reusable flows to become registered templates } else if (fileName.endsWith(".xml")) { UploadProgress progress = uploadProgressService.getUploadStatus(importOptions.getUploadKey()); progress.setSections(ImportSection.sectionsForImportAsString(ImportType.TEMPLATE_XML)); template = importNifiTemplate(fileName, content, importOptions, true); } } catch (IOException e) { throw new UnsupportedOperationException( "Error importing template " + fileName + ". " + e.getMessage()); } template.setImportOptions(importOptions); return template; // }); } /** * Imports the data and various components based upon the supplied {@link ImportTemplate#importOptions} as part * of a feed import operation. Validates whether the user has import feed permission. * * Note. This method will not call any validation routines. It will just import. * If you want to validate before to ensure the import will be correct call this.importTemplate(final String fileName, final byte[] content, ImportTemplateOptions importOptions) * * @param importTemplate the template data to validate before importing * @return the template data to validate before importing */ public ImportTemplate importZipForFeedImport(ImportTemplate importTemplate) throws Exception { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_FEEDS); return importTemplate(importTemplate); } /** * Imports the data and various components based upon the supplied {@link ImportTemplate#importOptions}. * Validates whether the user has import template permission. * * Note. This method will not call any validation routines. It will just import. * If you want to validate before to ensure the import will be correct call this.importTemplate(final String fileName, final byte[] content, ImportTemplateOptions importOptions) * * @param importTemplate the template data to validate before importing * @return the template data to validate before importing */ public ImportTemplate importZip(ImportTemplate importTemplate) throws Exception { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_TEMPLATES); return importTemplate(importTemplate); } private ImportTemplate importTemplate(ImportTemplate importTemplate) throws Exception { ImportTemplateOptions importOptions = importTemplate.getImportOptions(); log.info("Importing Zip file template {}, overwrite: {}, reusableFlow: {}", importTemplate.getFileName(), importOptions.isImportAndOverwrite(ImportComponent.TEMPLATE_DATA), importOptions.isImport(ImportComponent.REUSABLE_TEMPLATE)); if (importTemplate.isValid()) { // UploadProgressMessage startingStatusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Starting import of the template"); // startingStatusMessage.complete(true); UploadProgressMessage statusMessage = null; //Get information about the import RegisteredTemplate template = importTemplate.getTemplateToImport(); // validateTemplateProperties(template, importTemplate, importOptions); //1 ensure this template doesnt already exist importTemplate.setTemplateName(template.getTemplateName()); RegisteredTemplate existingTemplate = registeredTemplateService.findRegisteredTemplate( RegisteredTemplateRequest.requestByTemplateName(template.getTemplateName())); if (existingTemplate != null) { template.setId(existingTemplate.getId()); } else { template.setId(null); } List<ImportTemplate> connectingTemplates = importReusableTemplate(importTemplate, importOptions); if (importOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).hasErrorMessages()) { //return if invalid return importTemplate; } NiFiTemplateImport niFiTemplateImport = importNiFiXmlTemplate(importTemplate, importOptions); String oldTemplateXml = niFiTemplateImport.getOldTemplateXml(); TemplateDTO dto = niFiTemplateImport.getDto(); template.setNifiTemplateId(dto.getId()); importTemplate.setNifiTemplateId(dto.getId()); ImportComponentOption registeredTemplateImport = importOptions .findImportComponentOption(ImportComponent.TEMPLATE_DATA); if (existingTemplate != null) { importTemplate.setTemplateId(existingTemplate.getId()); } if (registeredTemplateImport.isShouldImport() && registeredTemplateImport.isValidForImport()) { NifiProcessGroup newTemplateInstance = createTemplateInstance(importTemplate, importOptions); if (newTemplateInstance.isSuccess()) { importTemplate.setSuccess(true); RegisteredTemplate savedTemplate = registerTemplate(importTemplate, importOptions); if (savedTemplate != null) { template = savedTemplate; } } if (!importTemplate.isSuccess()) { //ROLLBACK.. errrors statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Errors were found registering the template. Attempting to rollback."); rollbackTemplateImportInNifi(importTemplate, dto, oldTemplateXml); //also restore existing registered template with metadata //restore old registered template if (existingTemplate != null) { try { metadataService.registerTemplate(existingTemplate); } catch (Exception e) { Throwable root = ExceptionUtils.getRootCause(e); String msg = root != null ? root.getMessage() : e.getMessage(); importTemplate.getTemplateResults() .addError( NifiError.SEVERITY.WARN, "Error while restoring the template " + template.getTemplateName() + " in the Kylo metadata. " + msg, ""); } } statusMessage.update( "Errors were found registering the template. Rolled back to previous version.", true); } //remove the temporary Process Group we created removeTemporaryProcessGroup(importTemplate); } else { importTemplate.setSuccess(true); } //if we also imported the reusable template make sure that is all running properly for (ImportTemplate connectingTemplate : connectingTemplates) { //enable it nifiRestClient.markConnectionPortsAsRunning( connectingTemplate.getTemplateResults().getProcessGroupEntity()); } completeSection(importOptions, ImportSection.Section.IMPORT_REGISTERED_TEMPLATE); } return importTemplate; } /** * Import a template string into NiFi as a template to use */ private NiFiTemplateImport importNiFiXmlTemplate(ImportTemplate template, ImportTemplateOptions importOptions) throws IOException { TemplateDTO dto = null; String templateName = null; String oldTemplateXml = null; ImportComponentOption nifiTemplateImport = importOptions .findImportComponentOption(ImportComponent.NIFI_TEMPLATE); if (nifiTemplateImport.isValidForImport()) { try { templateName = NifiTemplateParser.getTemplateName(template.getNifiTemplateXml()); template.setTemplateName(templateName); dto = nifiRestClient.getTemplateByName(templateName); if (dto != null) { oldTemplateXml = nifiRestClient.getTemplateXml(dto.getId()); template.setNifiTemplateId(dto.getId()); if (importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && nifiTemplateImport.isValidForImport()) { nifiRestClient.deleteTemplate(dto.getId()); } else if (!template.isZipFile()) { //if its not a zip file we need to error out if the user has decided not to overwrite when it already exists uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "The template " + templateName + " already exists in NiFi. Please choose the option to replace the template and try again.", true, false); template.setValid(false); } } } catch (ParserConfigurationException | XPathExpressionException | SAXException e) { throw new UnsupportedOperationException( "The xml file you are trying to import is not a valid NiFi template. Please try again. " + e.getMessage()); } boolean register = (dto == null || (importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && nifiTemplateImport.isValidForImport())); //attempt to import the xml into NiFi if its new, or if the user said to overwrite if (register) { UploadProgressMessage statusMessage = uploadProgressService .addUploadStatus(importOptions.getUploadKey(), "Importing " + templateName + " into NiFi"); log.info("Attempting to import Nifi Template: {} for file {}", templateName, template.getFileName()); dto = nifiRestClient.importTemplate(template.getTemplateName(), template.getNifiTemplateXml()); template.setNifiTemplateId(dto.getId()); statusMessage.update("Imported " + templateName + " into NiFi", true); } } completeSection(importOptions, ImportSection.Section.IMPORT_NIFI_TEMPLATE); return new NiFiTemplateImport(oldTemplateXml, dto); } private ImportTemplate validateAndImportZip(String fileName, byte[] content, ImportTemplateOptions importOptions) { this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.IMPORT_TEMPLATES); ImportTemplate importTemplate = validateTemplateForImport(fileName, content, importOptions); return metadataAccess.commit(() -> importZip(importTemplate)); } private NifiProcessGroup createTemplateInstance(ImportTemplate importTemplate, ImportTemplateOptions importOptions) { UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Starting to import RegisteredTemplate. Attempting to verify NiFi flow "); //Create the new instance of the template in NiFi Map<String, Object> configProperties = propertyExpressionResolver.getStaticConfigProperties(); List<NifiProperty> templateProperties = importTemplate.getTemplateToImport() != null ? importTemplate.getTemplateToImport().getProperties() : Collections.emptyList(); NifiProcessGroup newTemplateInstance = nifiRestClient.createNewTemplateInstance( importTemplate.getNifiTemplateId(), templateProperties, configProperties, false, new NifiFlowCacheReusableTemplateCreationCallback(false)); importTemplate.setTemplateResults(newTemplateInstance); if (newTemplateInstance.isSuccess()) { statusMessage.update("Verified NiFi flow.", true); } else { statusMessage.update("Error creating NiFi template instance ", false); } completeSection(importOptions, ImportSection.Section.CREATE_NIFI_INSTANCE); return newTemplateInstance; } /** * Register the template with the metadata and save it * * @param importTemplate the template data to import * @param importOptions user options about what/how it should be imported * @return the registered template that was saved */ private RegisteredTemplate registerTemplate(ImportTemplate importTemplate, ImportTemplateOptions importOptions) { RegisteredTemplate template = importTemplate.getTemplateToImport(); ImportComponentOption registeredTemplateOption = importOptions .findImportComponentOption(ImportComponent.TEMPLATE_DATA); if (registeredTemplateOption.isValidForImport()) { UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus( importOptions.getUploadKey(), "Registering template " + template.getTemplateName() + " with Kylo metadata."); try { importTemplate.setNifiTemplateId(template.getNifiTemplateId()); //register it in the system metadataService.registerTemplate(template); //get the new template template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder() .templateId(template.getId()).templateName(template.getTemplateName()).build()); importTemplate.setTemplateId(template.getId()); statusMessage.update("Registered template with Kylo metadata.", true); } catch (Exception e) { importTemplate.setSuccess(false); Throwable root = ExceptionUtils.getRootCause(e); String msg = root != null ? root.getMessage() : e.getMessage(); importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "Error registering the template " + template.getTemplateName() + " in the Kylo metadata. " + msg, ""); statusMessage.update("Error registering template with Kylo metadata. " + msg, false); } } return template; } private List<ImportTemplate> importReusableTemplate(ImportTemplate importTemplate, ImportTemplateOptions importOptions) throws Exception { List<ImportTemplate> connectingTemplates = new ArrayList<>(); //start the import ImportComponentOption reusableComponentOption = importOptions .findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE); if (importTemplate.hasConnectingReusableTemplate() && reusableComponentOption.isShouldImport() && reusableComponentOption.isValidForImport()) { UploadProgressMessage statusMessage = uploadProgressService .addUploadStatus(importOptions.getUploadKey(), "Import Reusable Template. Starting import"); //import the reusable templates boolean valid = true; ImportTemplate lastReusableTemplate = null; log.info("Importing Zip file template {}. first importing reusable flow from zip"); for (String reusableTemplateXml : importTemplate.getNifiConnectingReusableTemplateXmls()) { String name = NifiTemplateParser.getTemplateName(reusableTemplateXml); ImportTemplate connectingTemplate = importNifiTemplateWithTemplateString(name, reusableTemplateXml, importOptions, false); lastReusableTemplate = connectingTemplate; if (!connectingTemplate.isSuccess()) { //return with exception //add in the error messages connectingTemplate.getTemplateResults().getAllErrors().stream().forEach(nifiError -> { importTemplate.getTemplateResults().addError(nifiError); }); //error out importTemplate.setSuccess(false); reusableComponentOption.getErrorMessages().add("Error importing Reusable Template"); //exit valid = false; break; } else { connectingTemplates.add(connectingTemplate); } } if (valid) { statusMessage.update("Successfully imported Reusable Templates " + (connectingTemplates.stream() .map(t -> t.getTemplateName()).collect(Collectors.joining(","))), true); } else { statusMessage.update("Errors importing reusable template: Imported Reusable Template. " + lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : ""); } } completeSection(importOptions, ImportSection.Section.IMPORT_REUSABLE_TEMPLATE); return connectingTemplates; } private ImportTemplate importNifiTemplateWithTemplateString(String fileName, String xmlFile, ImportTemplateOptions importOptions, boolean xmlImport) throws IOException { byte[] content = xmlFile.getBytes("UTF-8"); return importNifiTemplate(fileName, content, importOptions, xmlImport); } /** * * @param fileName * @param xmlFile * @param importOptions * @param xmlImport * @return * @throws IOException */ private ImportTemplate importNifiTemplate(String fileName, byte[] xmlFile, ImportTemplateOptions importOptions, boolean xmlImport) throws IOException { InputStream inputStream = new ByteArrayInputStream(xmlFile); ImportTemplate importTemplate = getNewNiFiTemplateImport(fileName, inputStream); importTemplate.setImportOptions(importOptions); validateNiFiTemplateImport(importTemplate); if (importTemplate.isValid()) { UploadProgressMessage importStatusMessage = uploadProgressService .addUploadStatus(importOptions.getUploadKey(), "Importing the NiFi flow "); boolean createReusableFlow = importOptions.isImport(ImportComponent.REUSABLE_TEMPLATE); boolean overwrite = importOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE); log.info("Importing XML file template {}, overwrite: {}, reusableFlow: {}", fileName, overwrite, createReusableFlow); NiFiTemplateImport niFiTemplateImport = importNiFiXmlTemplate(importTemplate, importOptions); String oldTemplateXml = niFiTemplateImport.getOldTemplateXml(); TemplateDTO dto = niFiTemplateImport.getDto(); String templateName = importTemplate.getTemplateName(); importStatusMessage.update("Importing the NiFi flow, " + templateName); log.info("validate NiFi Flow by creating a template instance in nifi Nifi Template: {} for file {}", templateName, fileName); Map<String, Object> configProperties = propertyExpressionResolver.getStaticConfigProperties(); NifiFlowCacheReusableTemplateCreationCallback reusableTemplateCreationCallback = new NifiFlowCacheReusableTemplateCreationCallback( xmlImport); if (createReusableFlow && importTemplate.isValid()) { importStatusMessage.update("Creating reusable flow instance for " + templateName); } if (importTemplate.isValid()) { List<NifiProperty> templateProperties = importTemplate.getTemplateToImport() != null ? importTemplate.getTemplateToImport().getProperties() : Collections.emptyList(); NifiProcessGroup newTemplateInstance = nifiRestClient.createNewTemplateInstance(dto.getId(), templateProperties, configProperties, createReusableFlow, reusableTemplateCreationCallback); importTemplate.setTemplateResults(newTemplateInstance); log.info("Import finished for {}, {}... verify results", templateName, fileName); if (newTemplateInstance.isSuccess()) { log.info("SUCCESS! This template is valid Nifi Template: {} for file {}", templateName, fileName); importTemplate.setSuccess(true); if (createReusableFlow) { importStatusMessage.update("Finished creating reusable flow instance for " + templateName, true); importReusableTemplateSuccess(importTemplate); registeredTemplateCache.invalidateAllProcessors(); } else { importStatusMessage.update("Validated " + templateName, true); } } else { rollbackTemplateImportInNifi(importTemplate, dto, oldTemplateXml); importStatusMessage.update("An error occurred importing the template, " + templateName, false); } if (!createReusableFlow) { removeTemporaryProcessGroup(importTemplate); try { nifiRestClient.deleteProcessGroup(newTemplateInstance.getProcessGroupEntity()); } catch (NifiComponentNotFoundException e) { //its ok if its not found } log.info("Success cleanup: Successfully cleaned up Nifi"); } } log.info("Import NiFi Template for {} finished", fileName); } return importTemplate; } /** * Restore the previous Template back to Nifi */ private void rollbackTemplateImportInNifi(ImportTemplate importTemplate, TemplateDTO dto, String oldTemplateXml) throws IOException { log.error("ERROR! This template is NOT VALID Nifi Template: {} for file {}. Errors are: {} ", importTemplate.getTemplateName(), importTemplate.getFileName(), importTemplate.getTemplateResults().getAllErrors()); //delete this template importTemplate.setSuccess(false); //delete the template from NiFi nifiRestClient.deleteTemplate(dto.getId()); //restore old template if (oldTemplateXml != null) { log.info("Rollback Nifi: Attempt to restore old template xml "); nifiRestClient.importTemplate(oldTemplateXml); log.info("Rollback Nifi: restored old template xml "); } log.info("Rollback Nifi: Deleted the template: {} from Nifi ", importTemplate.getTemplateName()); } /** * Cleanup and remove the temporary process group associated with this import * * This should be called after the import to cleanup NiFi */ private void removeTemporaryProcessGroup(ImportTemplate importTemplate) { UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus( importTemplate.getImportOptions().getUploadKey(), "Cleaning up the temporary process group in NiFi"); String processGroupName = importTemplate.getTemplateResults().getProcessGroupEntity().getName(); String processGroupId = importTemplate.getTemplateResults().getProcessGroupEntity().getId(); String parentProcessGroupId = importTemplate.getTemplateResults().getProcessGroupEntity() .getParentGroupId(); log.info("About to cleanup the temporary process group {} ({})", processGroupName, processGroupId); try { //Now try to remove the processgroup associated with this template import ProcessGroupDTO e = null; try { e = nifiRestClient.getProcessGroup(processGroupId, false, false); } catch (NifiComponentNotFoundException notFound) { //if its not there then we already cleaned up :) } if (e != null) { try { nifiRestClient.stopAllProcessors(processGroupId, parentProcessGroupId); //remove connections nifiRestClient.removeConnectionsToProcessGroup(parentProcessGroupId, processGroupId); } catch (Exception e2) { //this is ok. we are cleaning up the template so if an error occurs due to no connections it is fine since we ultimately want to remove this temp template. } try { nifiRestClient.deleteProcessGroup(importTemplate.getTemplateResults().getProcessGroupEntity()); } catch (NifiComponentNotFoundException nfe) { //this is ok } } log.info("Successfully cleaned up Nifi and deleted the process group {} ", importTemplate.getTemplateResults().getProcessGroupEntity().getName()); statusMessage.update("Cleaned up NiFi", true); } catch (NifiClientRuntimeException e) { log.error("error attempting to cleanup and remove the temporary process group (" + processGroupId + " during the import of template " + importTemplate.getTemplateName()); importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "Issues found in cleaning up the template import: " + importTemplate.getTemplateName() + ". The Process Group : " + processGroupName + " (" + processGroupId + ")" + " may need to be manually cleaned up in NiFi ", ""); statusMessage.update( "Error cleaning up NiFi. The Process Group : " + processGroupName + " (" + processGroupId + ")", false); } } /** * Open the zip file and populate the {@link ImportTemplate} object with the components in the file/archive * * @param fileName the file name * @param inputStream the file * @return the template data to import */ private ImportTemplate openZip(String fileName, InputStream inputStream) throws IOException { byte[] buffer = new byte[1024]; ZipInputStream zis = new ZipInputStream(inputStream); ZipEntry zipEntry; ImportTemplate importTemplate = new ImportTemplate(fileName); while ((zipEntry = zis.getNextEntry()) != null) { String zipEntryContents = ZipFileUtil.zipEntryToString(buffer, zis, zipEntry); if (zipEntry.getName().startsWith(NIFI_TEMPLATE_XML_FILE)) { importTemplate.setNifiTemplateXml(zipEntryContents); } else if (zipEntry.getName().startsWith(TEMPLATE_JSON_FILE)) { importTemplate.setTemplateJson(zipEntryContents); } else if (zipEntry.getName().startsWith(NIFI_CONNECTING_REUSABLE_TEMPLATE_XML_FILE)) { importTemplate.addNifiConnectingReusableTemplateXml(zipEntryContents); } } zis.closeEntry(); zis.close(); if (!importTemplate.hasValidComponents()) { throw new UnsupportedOperationException( " The file you uploaded is not a valid archive. Please ensure the Zip file has been exported from the system and has 2 valid files named: " + NIFI_TEMPLATE_XML_FILE + ", and " + TEMPLATE_JSON_FILE); } importTemplate.setZipFile(true); return importTemplate; } private ImportTemplate getNewNiFiTemplateImport(String fileName, InputStream inputStream) throws IOException { ImportTemplate template = new ImportTemplate(fileName); template.setValid(true); StringWriter writer = new StringWriter(); IOUtils.copy(inputStream, writer, "UTF-8"); String xmlTemplate = writer.toString(); template.setNifiTemplateXml(xmlTemplate); return template; } /** * Called when the system imports a Reusable template either from a ZIP file or an xml file uploaded in kylo. */ private void importReusableTemplateSuccess(ImportTemplate importTemplate) { } //Internal classes public static class ImportTemplate { String fileName; String templateName; boolean success; private boolean valid; private NifiProcessGroup templateResults; private List<NiFiComponentErrors> controllerServiceErrors; private String templateId; private String nifiTemplateId; private boolean zipFile; private String nifiTemplateXml; private String templateJson; private List<String> nifiConnectingReusableTemplateXmls = new ArrayList<>(); private boolean verificationToReplaceConnectingResuableTemplateNeeded; private ImportTemplateOptions importOptions; @JsonIgnore private RegisteredTemplate templateToImport; public ImportTemplate() { } public ImportTemplate(String fileName) { this.fileName = fileName; } public ImportTemplate(String templateName, boolean success) { this.templateName = templateName; this.success = success; } public boolean isValid() { return valid; } public void setValid(boolean valid) { this.valid = valid; } @JsonIgnore public boolean hasValidComponents() { return StringUtils.isNotBlank(templateJson) && StringUtils.isNotBlank(nifiTemplateXml); } public String getTemplateName() { return templateName; } public void setTemplateName(String templateName) { this.templateName = templateName; } public boolean isSuccess() { return success; } public void setSuccess(boolean success) { this.success = success; } public String getNifiTemplateXml() { return nifiTemplateXml; } public void setNifiTemplateXml(String nifiTemplateXml) { this.nifiTemplateXml = nifiTemplateXml; } public String getTemplateJson() { return templateJson; } public void setTemplateJson(String templateJson) { this.templateJson = templateJson; } public NifiProcessGroup getTemplateResults() { if (templateResults == null) { templateResults = new NifiProcessGroup(); } return templateResults; } public void setTemplateResults(NifiProcessGroup templateResults) { this.templateResults = templateResults; inspectForControllerServiceErrors(); } public String getFileName() { return fileName; } public boolean isZipFile() { return zipFile; } public void setZipFile(boolean zipFile) { this.zipFile = zipFile; } private void inspectForControllerServiceErrors() { if (templateResults != null) { List<NiFiComponentErrors> errors = templateResults.getControllerServiceErrors(); this.controllerServiceErrors = errors; } } public List<NiFiComponentErrors> getControllerServiceErrors() { return controllerServiceErrors; } public String getTemplateId() { return templateId; } public void setTemplateId(String templateId) { this.templateId = templateId; } public String getNifiTemplateId() { return nifiTemplateId; } public void setNifiTemplateId(String nifiTemplateId) { this.nifiTemplateId = nifiTemplateId; } public List<String> getNifiConnectingReusableTemplateXmls() { return nifiConnectingReusableTemplateXmls; } public void addNifiConnectingReusableTemplateXml(String nifiConnectingReusableTemplateXml) { this.nifiConnectingReusableTemplateXmls.add(nifiConnectingReusableTemplateXml); } public boolean hasConnectingReusableTemplate() { return !nifiConnectingReusableTemplateXmls.isEmpty(); } public boolean isVerificationToReplaceConnectingResuableTemplateNeeded() { return verificationToReplaceConnectingResuableTemplateNeeded; } public void setVerificationToReplaceConnectingResuableTemplateNeeded( boolean verificationToReplaceConnectingResuableTemplateNeeded) { this.verificationToReplaceConnectingResuableTemplateNeeded = verificationToReplaceConnectingResuableTemplateNeeded; } public ImportTemplateOptions getImportOptions() { return importOptions; } public void setImportOptions(ImportTemplateOptions importOptions) { this.importOptions = importOptions; } @JsonIgnore public RegisteredTemplate getTemplateToImport() { if (templateToImport == null && StringUtils.isNotBlank(templateJson)) { templateToImport = ObjectMapperSerializer.deserialize(getTemplateJson(), RegisteredTemplate.class); } return templateToImport; } @JsonIgnore public void setTemplateToImport(RegisteredTemplate templateToImport) { this.templateToImport = templateToImport; } } public class ExportTemplate { private String fileName; private byte[] file; public ExportTemplate(String fileName, byte[] file) { this.fileName = fileName; this.file = file; } public String getFileName() { return fileName; } public byte[] getFile() { return file; } } /** * Store data about a NiFi import */ private class NiFiTemplateImport { private String oldTemplateXml; private TemplateDTO dto; public NiFiTemplateImport(String oldTemplateXml, TemplateDTO dto) { this.oldTemplateXml = oldTemplateXml; this.dto = dto; } public String getOldTemplateXml() { return oldTemplateXml; } public void setOldTemplateXml(String oldTemplateXml) { this.oldTemplateXml = oldTemplateXml; } public TemplateDTO getDto() { return dto; } public void setDto(TemplateDTO dto) { this.dto = dto; } } public class NifiFlowCacheReusableTemplateCreationCallback implements ReusableTemplateCreationCallback { private boolean isXmlImport; public NifiFlowCacheReusableTemplateCreationCallback(boolean isXmlImport) { this.isXmlImport = isXmlImport; } /** * Update the NiFi Flow Cache with the new processors information * * @param templateName the name of the template * @param processGroupDTO the group where this template resides (under the reusable_templates) group */ @Override public void beforeMarkAsRunning(String templateName, ProcessGroupDTO processGroupDTO) { //update the cache Collection<ProcessorDTO> processors = NifiProcessUtil.getProcessors(processGroupDTO); nifiFlowCache.updateProcessorIdNames(templateName, processors); nifiFlowCache.updateConnectionMap(templateName, NifiConnectionUtil.getAllConnections(processGroupDTO)); } } }