Java tutorial
/*! * * This program is free software; you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License, version 2.1 as published by the Free Software * Foundation. * * You should have received a copy of the GNU Lesser General Public License along with this * program; if not, you can obtain a copy at http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html * or from the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * * * Copyright (c) 2002-2018 Hitachi Vantara. All rights reserved. * */ package org.pentaho.platform.plugin.action.kettle; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.dom4j.Node; import org.pentaho.commons.connection.memory.MemoryMetaData; import org.pentaho.commons.connection.memory.MemoryResultSet; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.exception.KettleValueException; import org.pentaho.di.core.logging.KettleLogStore; import org.pentaho.di.core.logging.LogChannel; import org.pentaho.di.core.logging.LogLevel; import org.pentaho.di.core.logging.LoggingObjectInterface; import org.pentaho.di.core.logging.LoggingRegistry; import org.pentaho.di.core.parameters.UnknownParamException; import org.pentaho.di.core.plugins.PluginRegistry; import org.pentaho.di.core.plugins.RepositoryPluginType; import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.xml.XMLHandlerCache; import org.pentaho.di.job.Job; import org.pentaho.di.job.JobMeta; import org.pentaho.di.repository.RepositoriesMeta; import org.pentaho.di.repository.Repository; import org.pentaho.di.repository.RepositoryDirectoryInterface; import org.pentaho.di.repository.RepositoryMeta; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.RowListener; import org.pentaho.di.trans.step.StepMetaDataCombi; import org.pentaho.platform.api.engine.IActionSequenceResource; import org.pentaho.platform.api.engine.ILogger; import org.pentaho.platform.engine.core.system.PentahoSystem; import org.pentaho.platform.engine.services.solution.ComponentBase; import org.pentaho.platform.plugin.action.messages.Messages; import org.pentaho.platform.util.xml.w3c.XmlW3CHelper; import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; /** * KettleComponent shows a list of available transformations in the root of the choosen repository. * * @author Matt */ /* * Legitimate outputs: EXECUTION_STATUS_OUTPUT - (execution-status) [JOB | TRANS] Returns the resultant execution status * * EXECUTION_LOG_OUTPUT - (execution-log) [JOB | TRANS] Returns the resultant log * * TRANSFORM_SUCCESS_OUTPUT - (transformation-written) [Requires MONITORSTEP to be defined] [TRANS] Returns a * "result-set" for all successful rows written (Unless error handling is not defined for the specified step, in which * case ALL rows are returned here) * * TRANSFORM_ERROR_OUTPUT - (transformation-errors) [Requires MONITORSTEP to be defined] [TRANS] Returns a "result-set" * for all rows written that have caused an error * * TRANSFORM_SUCCESS_COUNT_OUTPUT - (transformation-written-count) [Requires MONITORSTEP to be defined] [TRANS] Returns * a count of all rows returned in TRANSFORM_SUCCESS_OUTPUT * * TRANSFORM_ERROR_COUNT_OUTPUT - (transformation-errors-count) [Requires MONITORSTEP to be defined] [TRANS] Returns a * count of all rows returned in TRANSFORM_ERROR_OUTPUT * * Legitimate inputs: MONITORSTEP Takes the name of the step from which success and error rows can be detected * * KETTLELOGLEVEL Sets the logging level to be used in the EXECUTION_LOG_OUTPUT Valid settings: basic detail error debug * minimal rowlevel */ public class KettleComponent extends ComponentBase implements RowListener { private static final long serialVersionUID = 8217343898202366129L; private static final String DIRECTORY = "directory"; //$NON-NLS-1$ private static final String TRANSFORMATION = "transformation"; //$NON-NLS-1$ private static final String JOB = "job"; //$NON-NLS-1$ private static final String TRANSFORMFILE = "transformation-file"; //$NON-NLS-1$ private static final String JOBFILE = "job-file"; //$NON-NLS-1$ // IMPORTSTEP here for backwards compatibility; Superceded by MONITORSTEP private static final String IMPORTSTEP = "importstep"; //$NON-NLS-1$ private static final String MONITORSTEP = "monitor-step"; //$NON-NLS-1$ private static final String KETTLELOGLEVEL = "kettle-logging-level"; //$NON-NLS-1$ private static final String EXECUTION_STATUS_OUTPUT = "kettle-execution-status"; //$NON-NLS-1$ private static final String EXECUTION_LOG_OUTPUT = "kettle-execution-log"; //$NON-NLS-1$ private static final String TRANSFORM_SUCCESS_OUTPUT = "transformation-output-rows"; //$NON-NLS-1$ private static final String TRANSFORM_ERROR_OUTPUT = "transformation-output-error-rows"; //$NON-NLS-1$ private static final String TRANSFORM_SUCCESS_COUNT_OUTPUT = "transformation-output-rows-count"; //$NON-NLS-1$ private static final String TRANSFORM_ERROR_COUNT_OUTPUT = "transformation-output-error-rows-count"; //$NON-NLS-1$ public static final String PARAMETER_MAP_CMD_ARG = "set-argument"; //$NON-NLS-1$ public static final String PARAMETER_MAP_VARIABLE = "set-variable"; //$NON-NLS-1$ public static final String PARAMETER_MAP_PARAMETER = "set-parameter"; //$NON-NLS-1$ private static final ArrayList<String> outputParams = new ArrayList<String>( Arrays.asList(EXECUTION_STATUS_OUTPUT, EXECUTION_LOG_OUTPUT, TRANSFORM_SUCCESS_OUTPUT, TRANSFORM_ERROR_OUTPUT, TRANSFORM_SUCCESS_COUNT_OUTPUT, TRANSFORM_ERROR_COUNT_OUTPUT)); /** * The repositories.xml file location, if empty take the default $HOME/.kettle/repositories.xml */ private String repositoriesXMLFile; /** * The name of the repository to use */ private String repositoryName; /** * The username to login with */ private String username; private MemoryResultSet results; private MemoryResultSet errorResults; private String executionStatus; private String executionLog; /** * The password to login with */ private String password; /** * The log channel ID of the executing transformation or job */ private String logChannelId; @Override public Log getLogger() { return LogFactory.getLog(KettleComponent.class); } @Override protected boolean validateSystemSettings() { // set pentaho.solutionpath so that it can be used in file paths boolean useRepository = PentahoSystem.getSystemSetting("kettle/settings.xml", "repository.type", "files") //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ .equals("rdbms"); //$NON-NLS-1$ if (useRepository) { repositoriesXMLFile = PentahoSystem.getSystemSetting("kettle/settings.xml", "repositories.xml.file", //$NON-NLS-1$//$NON-NLS-2$ null); repositoryName = PentahoSystem.getSystemSetting("kettle/settings.xml", "repository.name", null); //$NON-NLS-1$ //$NON-NLS-2$ username = PentahoSystem.getSystemSetting("kettle/settings.xml", "repository.userid", ""); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ password = PentahoSystem.getSystemSetting("kettle/settings.xml", "repository.password", ""); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ // Check the Kettle settings... if ("".equals(repositoryName) || username.equals("")) { //$NON-NLS-1$ //$NON-NLS-2$ // looks like the Kettle stuff is not configured yet... // see if we can provide feedback to the user... error(Messages.getInstance().getErrorString("Kettle.ERROR_0001_SERVER_SETTINGS_NOT_SET")); //$NON-NLS-1$ return false; } boolean ok = ((repositoryName != null) && (repositoryName.length() > 0)); ok = ok || ((username != null) && (username.length() > 0)); return ok; } return true; } @Override public boolean init() { LogChannel kettleComponentChannel = new LogChannel("Kettle platform component"); logChannelId = kettleComponentChannel.getLogChannelId(); return true; } private boolean checkMapping(Node name, Node mapping) { if (name == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0031_NAME_ELEMENT_MISSING_FROM_MAPPING")); //$NON-NLS-1$ return false; } if (mapping == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0032_MAPPING_ELEMENT_MISSING_FROM_MAPPING")); //$NON-NLS-1$ return false; } // Make sure the mapping field is available as an input if (!isDefinedInput(mapping.getText())) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0033_MAPPING_NOT_FOUND_IN_ACTION_INPUTS", //$NON-NLS-1$ mapping.getText())); return false; } return true; } @SuppressWarnings("unchecked") @Override public boolean validateAction() { // If there are any mappings, validate their xml and values if (getComponentDefinition() .selectNodes( PARAMETER_MAP_CMD_ARG + " | " + PARAMETER_MAP_VARIABLE + " | " + PARAMETER_MAP_PARAMETER) //$NON-NLS-1$//$NON-NLS-2$ .size() > 0) { Map<String, String> argumentMap = null; Node name = null, mapping = null; // Extract all mapping elements from component-definition and verify // they have a 'name' and 'mapping' child element for (Node n : (List<Node>) getComponentDefinition().selectNodes(PARAMETER_MAP_CMD_ARG)) { name = n.selectSingleNode("name"); //$NON-NLS-1$ mapping = n.selectSingleNode("mapping"); //$NON-NLS-1$ if (checkMapping(name, mapping)) { if (argumentMap == null) { argumentMap = new HashMap<String, String>(); } argumentMap.put(name.getText(), applyInputsToFormat(getInputStringValue(mapping.getText()))); } else { return false; } } for (Node n : (List<Node>) getComponentDefinition().selectNodes(PARAMETER_MAP_VARIABLE)) { name = n.selectSingleNode("name"); //$NON-NLS-1$ mapping = n.selectSingleNode("mapping"); //$NON-NLS-1$ if (!checkMapping(name, mapping)) { return false; } } for (Node n : (List<Node>) getComponentDefinition().selectNodes(PARAMETER_MAP_PARAMETER)) { name = n.selectSingleNode("name"); //$NON-NLS-1$ mapping = n.selectSingleNode("mapping"); //$NON-NLS-1$ if (!checkMapping(name, mapping)) { return false; } } // Make sure all of the arguments are present, correctly labeled and // that there are not more then 10 (currently supported by Kettle) if (argumentMap != null) { String val = null; for (int i = 1; i <= argumentMap.size(); i++) { val = argumentMap.get(Integer.toString(i)); if (val == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0030_INVALID_ARGUMENT_MAPPING")); //$NON-NLS-1$ return false; } } } } if (isDefinedResource(KettleComponent.TRANSFORMFILE) || isDefinedResource(KettleComponent.JOBFILE)) { return true; } boolean useRepository = PentahoSystem.getSystemSetting("kettle/settings.xml", "repository.type", "files") //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ .equals("rdbms"); //$NON-NLS-1$ if (!useRepository) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0019_REPOSITORY_TYPE_FILES")); //$NON-NLS-1$ return false; } if (isDefinedInput(KettleComponent.DIRECTORY) && (isDefinedInput(KettleComponent.TRANSFORMATION) || isDefinedInput(KettleComponent.JOB))) { return true; } if (!isDefinedInput(KettleComponent.DIRECTORY)) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0002_DIR_OR_FILE__NOT_DEFINED", //$NON-NLS-1$ getActionName())); return false; } else { if (!isDefinedInput(KettleComponent.TRANSFORMATION)) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0003_TRANS_NOT_DEFINED", //$NON-NLS-1$ getActionName())); return false; } } return false; } /** * Execute the specified transformation in the chosen repository. */ @SuppressWarnings("unchecked") @Override public boolean executeAction() { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_START")); //$NON-NLS-1$ } TransMeta transMeta = null; JobMeta jobMeta = null; // Build lists of parameters, variables and command line arguments Map<String, String> argumentMap = new HashMap<String, String>(); Map<String, String> variableMap = new HashMap<String, String>(); Map<String, String> parameterMap = new HashMap<String, String>(); for (Node n : (List<Node>) getComponentDefinition().selectNodes(PARAMETER_MAP_CMD_ARG)) { argumentMap.put(n.selectSingleNode("name").getText(), applyInputsToFormat(getInputStringValue(n.selectSingleNode("mapping").getText()))); //$NON-NLS-1$ //$NON-NLS-2$ } for (Node n : (List<Node>) getComponentDefinition().selectNodes(PARAMETER_MAP_VARIABLE)) { variableMap.put(n.selectSingleNode("name").getText(), applyInputsToFormat(getInputStringValue(n.selectSingleNode("mapping").getText()))); //$NON-NLS-1$ //$NON-NLS-2$ } for (Node n : (List<Node>) getComponentDefinition().selectNodes(PARAMETER_MAP_PARAMETER)) { parameterMap.put(n.selectSingleNode("name").getText(), applyInputsToFormat(getInputStringValue(n.selectSingleNode("mapping").getText()))); //$NON-NLS-1$ //$NON-NLS-2$ } String[] arguments = null; // If no mappings are provided, assume all inputs are command line // arguments (This supports the legacy method) if (argumentMap.size() <= 0 && variableMap.size() <= 0 && parameterMap.size() <= 0) { // this use is now considered obsolete, as we prefer the // action-sequence inputs since they // now maintain order boolean running = true; int index = 1; ArrayList<String> argumentList = new ArrayList<String>(); while (running) { if (isDefinedInput("parameter" + index)) { //$NON-NLS-1$ String value = null; String inputName = getInputStringValue("parameter" + index); //$NON-NLS-1$ // see if we have an input with this name if (isDefinedInput(inputName)) { value = getInputStringValue(inputName); } argumentList.add(value); } else { running = false; } index++; } // this is the preferred way to provide inputs to the // KetteComponent, the order of inputs is now preserved Iterator<?> inputNamesIter = getInputNames().iterator(); while (inputNamesIter.hasNext()) { String name = (String) inputNamesIter.next(); argumentList.add(getInputStringValue(name)); } arguments = (String[]) argumentList.toArray(new String[argumentList.size()]); } else { // Extract arguments from argumentMap (Throw an error if the // sequential ordering is broken) arguments = new String[argumentMap.size()]; for (int i = 0; i < argumentMap.size(); i++) { arguments[i] = argumentMap.get(Integer.toString(i + 1)); // Mapping // is // 1 // based // to // match // Kettle // UI if (arguments[i] == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0030_INVALID_ARGUMENT_MAPPING")); //$NON-NLS-1$ } } } // initialize environment variables try { KettleSystemListener.environmentInit(getSession()); } catch (KettleException ke) { error(ke.getMessage(), ke); } String solutionPath = "solution:"; Repository repository = connectToRepository(); boolean result = false; try { if (isDefinedInput(KettleComponent.DIRECTORY)) { String directoryName = getInputStringValue(KettleComponent.DIRECTORY); if (repository == null) { return false; } if (isDefinedInput(KettleComponent.TRANSFORMATION)) { String transformationName = getInputStringValue(KettleComponent.TRANSFORMATION); transMeta = loadTransformFromRepository(directoryName, transformationName, repository); if (transMeta != null) { try { for (String key : parameterMap.keySet()) { transMeta.setParameterValue(key, parameterMap.get(key)); } for (String key : variableMap.keySet()) { transMeta.setVariable(key, variableMap.get(key)); } } catch (UnknownParamException e) { error(e.getMessage()); } transMeta.setArguments(arguments); } else { return false; } } else if (isDefinedInput(KettleComponent.JOB)) { String jobName = getInputStringValue(KettleComponent.JOB); jobMeta = loadJobFromRepository(directoryName, jobName, repository); if (jobMeta != null) { try { for (String key : parameterMap.keySet()) { jobMeta.setParameterValue(key, parameterMap.get(key)); } for (String key : variableMap.keySet()) { jobMeta.setVariable(key, variableMap.get(key)); } } catch (UnknownParamException e) { error(e.getMessage()); } jobMeta.setArguments(arguments); } else { return false; } } } else if (isDefinedResource(KettleComponent.TRANSFORMFILE)) { IActionSequenceResource transformResource = getResource(KettleComponent.TRANSFORMFILE); String fileAddress = getActualFileName(transformResource); try { if (fileAddress != null) { // We have an actual loadable // filesystem and file transMeta = new TransMeta(fileAddress, repository, true); transMeta.setFilename(fileAddress); } else if (repository != null && repository.isConnected()) { fileAddress = transformResource.getAddress(); // load transformation resource from kettle/settings.xml configured repository transMeta = loadTransformFromRepository(FilenameUtils.getPathNoEndSeparator(fileAddress), FilenameUtils.getBaseName(fileAddress), repository); } else { String jobXmlStr = getResourceAsString(getResource(KettleComponent.TRANSFORMFILE)); jobXmlStr = jobXmlStr.replaceAll("\\$\\{pentaho.solutionpath\\}", solutionPath); //$NON-NLS-1$ jobXmlStr = jobXmlStr.replaceAll("\\%\\%pentaho.solutionpath\\%\\%", solutionPath); //$NON-NLS-1$ org.w3c.dom.Document doc = XmlW3CHelper.getDomFromString(jobXmlStr); // create a tranformation from the document transMeta = new TransMeta(doc.getFirstChild(), repository); } } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0015_BAD_RESOURCE", //$NON-NLS-1$ KettleComponent.TRANSFORMFILE, fileAddress), e); return false; } /* * Unreachable code below... if (transMeta == null) { * error(Messages.getInstance().getErrorString("Kettle.ERROR_0015_BAD_RESOURCE", KettleComponent.TRANSFORMFILE, * fileAddress)); //$NON-NLS-1$ debug(getKettleLog(true)); return false; } */ // Don't forget to set the parameters here as well... try { for (String key : parameterMap.keySet()) { transMeta.setParameterValue(key, parameterMap.get(key)); } for (String key : variableMap.keySet()) { transMeta.setVariable(key, variableMap.get(key)); } } catch (UnknownParamException e) { error(e.getMessage()); } transMeta.setArguments(arguments); /* * We do not need to concatenate the solutionPath info as the fileAddress has the complete location of the file * from start to end. This is to resolve BISERVER-502. */ transMeta.setFilename(fileAddress); } else if (isDefinedResource(KettleComponent.JOBFILE)) { String fileAddress = ""; //$NON-NLS-1$ try { fileAddress = getResource(KettleComponent.JOBFILE).getAddress(); if (repository != null && repository.isConnected()) { solutionPath = StringUtils.EMPTY; // load job resource from kettle/settings.xml configured repository jobMeta = loadJobFromRepository(FilenameUtils.getPathNoEndSeparator(fileAddress), FilenameUtils.getBaseName(fileAddress), repository); } else { String jobXmlStr = getResourceAsString(getResource(KettleComponent.JOBFILE)); // String jobXmlStr = // XmlW3CHelper.getContentFromSolutionResource(fileAddress); jobXmlStr = jobXmlStr.replaceAll("\\$\\{pentaho.solutionpath\\}", solutionPath); //$NON-NLS-1$ jobXmlStr = jobXmlStr.replaceAll("\\%\\%pentaho.solutionpath\\%\\%", solutionPath); //$NON-NLS-1$ org.w3c.dom.Document doc = XmlW3CHelper.getDomFromString(jobXmlStr); if (doc == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0015_BAD_RESOURCE", //$NON-NLS-1$ KettleComponent.JOBFILE, fileAddress)); debug(getKettleLog(true)); return false; } // create a job from the document try { repository = connectToRepository(); // if we get a valid repository its great, if not try it // without jobMeta = new JobMeta(solutionPath + fileAddress, repository); } catch (Exception e) { error(Messages.getInstance().getString("Kettle.ERROR_0023_NO_META"), e); //$NON-NLS-1$ } finally { if (repository != null) { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_DISCONNECTING")); //$NON-NLS-1$ } repository.disconnect(); } } } } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0015_BAD_RESOURCE", //$NON-NLS-1$ KettleComponent.JOBFILE, fileAddress), e); return false; } if (jobMeta == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0015_BAD_RESOURCE", //$NON-NLS-1$ KettleComponent.JOBFILE, fileAddress)); debug(getKettleLog(true)); return false; } else { try { for (String key : parameterMap.keySet()) { jobMeta.setParameterValue(key, parameterMap.get(key)); } for (String key : variableMap.keySet()) { jobMeta.setVariable(key, variableMap.get(key)); } } catch (UnknownParamException e) { error(e.getMessage()); } jobMeta.setArguments(arguments); jobMeta.setFilename(solutionPath + fileAddress); } } // OK, we have the information, let's load and execute the // transformation or job if (transMeta != null) { result = executeTransformation(transMeta); } if (jobMeta != null) { result = executeJob(jobMeta, repository); } } finally { if (repository != null) { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_DISCONNECTING")); //$NON-NLS-1$ } try { repository.disconnect(); } catch (Exception ignored) { //ignore } } if (transMeta != null) { try { cleanLogChannel(transMeta); transMeta.clear(); } catch (Exception ignored) { //ignore } transMeta = null; } if (jobMeta != null) { try { cleanLogChannel(jobMeta); jobMeta.clear(); } catch (Exception ignored) { //ignored } // Can't do anything about an exception here. jobMeta = null; } } if (isDefinedOutput(EXECUTION_LOG_OUTPUT)) { setOutputValue(EXECUTION_LOG_OUTPUT, executionLog); } if (isDefinedOutput(EXECUTION_STATUS_OUTPUT)) { setOutputValue(EXECUTION_STATUS_OUTPUT, executionStatus); } XMLHandlerCache.getInstance().clear(); return result; } private void cleanLogChannel(LoggingObjectInterface loi) { try { cleanLogChannelFromMap(loi); KettleLogStore.getAppender().removeChannelFromBuffer(loi.getLogChannelId()); } catch (Exception ignored) { //ignored } // Nothing I can do here... } private void cleanLogChannelFromMap(LoggingObjectInterface loi) { String logChannelId = loi.getLogChannelId(); Map<String, LoggingObjectInterface> logChannelMap = LoggingRegistry.getInstance().getMap(); if ((logChannelMap != null)) { List<String> logKids = LoggingRegistry.getInstance().getLogChannelChildren(logChannelId); if (logKids != null) { for (int i = 0; i < logKids.size(); i++) { logChannelMap.remove(logKids.get(i)); } } logChannelMap.remove(logChannelId); } } private String getActualFileName(final IActionSequenceResource resource) { String fileAddress = null; // Is it a hardcoded path? if ((resource.getSourceType() == IActionSequenceResource.FILE_RESOURCE)) { fileAddress = resource.getAddress(); } else if (resource.getSourceType() == IActionSequenceResource.SOLUTION_FILE_RESOURCE) { fileAddress = resource.getAddress(); } // Can it be loaded? this may not be true if using the DB Based repos if (fileAddress != null) { File file = new File(fileAddress); if (!file.exists() || !file.isFile()) { fileAddress = null; } } return (fileAddress); } protected boolean customizeTrans(Trans trans) { // override this to customize the transformation before it runs // by default there is no transformation return true; } private boolean executeTransformation(final TransMeta transMeta) { boolean success = true; Trans trans = null; try { if (transMeta != null) { try { trans = new Trans(transMeta); } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0010_BAD_TRANSFORMATION_METADATA"), //$NON-NLS-1$ e); } } if (trans == null) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0010_BAD_TRANSFORMATION_METADATA")); //$NON-NLS-1$ } // Remember where to get our execution logging from // logChannelId = trans.getLogChannelId(); // OK, we have the transformation, now run it! if (!customizeTrans(trans)) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0028_CUSTOMIZATION_FUNCITON_FAILED")); //$NON-NLS-1$ } debug(Messages.getInstance().getString("Kettle.DEBUG_PREPARING_TRANSFORMATION")); //$NON-NLS-1$ try { LogLevel lvl = getLogLevel(); trans.setLogLevel(lvl); trans.prepareExecution(transMeta.getArguments()); } catch (Exception e) { throw new KettleComponentException(Messages.getInstance() .getErrorString("Kettle.ERROR_0011_TRANSFORMATION_PREPARATION_FAILED"), e); //$NON-NLS-1$ } String stepName = null; String outputName = null; try { debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_STEP_IMPORTER")); //$NON-NLS-1$ stepName = getMonitorStepName(); outputName = getTransformSuccessOutputName(); if (outputName != null) { registerAsStepListener(stepName, trans); } } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0012_ROW_LISTENER_CREATE_FAILED"), e); //$NON-NLS-1$ } try { debug(Messages.getInstance().getString("Kettle.DEBUG_STARTING_TRANSFORMATION")); //$NON-NLS-1$ trans.startThreads(); } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0013_TRANSFORMATION_START_FAILED"), e); //$NON-NLS-1$ } try { // It's running in a separate thread to allow monitoring, // etc. debug(Messages.getInstance().getString("Kettle.DEBUG_TRANSFORMATION_RUNNING")); //$NON-NLS-1$ trans.waitUntilFinished(); cleanLogChannel(trans); trans.cleanup(); } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0014_ERROR_DURING_EXECUTE"), e); //$NON-NLS-1$ } // Dump the Kettle log... debug(getKettleLog(false)); // Build written row output if (results != null) { if (outputName != null) { setOutputValue(outputName, results); } if (isDefinedOutput(TRANSFORM_SUCCESS_COUNT_OUTPUT)) { setOutputValue(TRANSFORM_SUCCESS_COUNT_OUTPUT, results.getRowCount()); } } // Build error row output if (errorResults != null) { if (isDefinedOutput(TRANSFORM_ERROR_OUTPUT)) { setOutputValue(TRANSFORM_ERROR_OUTPUT, errorResults); } if (isDefinedOutput(TRANSFORM_ERROR_COUNT_OUTPUT)) { setOutputValue(TRANSFORM_ERROR_COUNT_OUTPUT, errorResults.getRowCount()); } } } catch (KettleComponentException e) { success = false; error(Messages.getInstance().getErrorString("Kettle.ERROR_0008_ERROR_RUNNING", e.toString()), e); //$NON-NLS-1$ } prepareKettleOutput(trans); return success; } private boolean registerAsStepListener(String stepName, Trans trans) throws KettleComponentException { boolean success = false; try { if (trans != null) { List<StepMetaDataCombi> stepList = trans.getSteps(); // find the specified step for (StepMetaDataCombi step : stepList) { if (step.stepname.equals(stepName)) { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_FOUND_STEP_IMPORTER")); //$NON-NLS-1$ } // this is the step we are looking for if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_GETTING_STEP_METADATA")); //$NON-NLS-1$ } RowMetaInterface row = trans.getTransMeta().getStepFields(stepName); // create the metadata that the Pentaho result sets need String[] fieldNames = row.getFieldNames(); String[][] columns = new String[1][fieldNames.length]; for (int column = 0; column < fieldNames.length; column++) { columns[0][column] = fieldNames[column]; } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_CREATING_RESULTSET_METADATA")); //$NON-NLS-1$ } MemoryMetaData metaData = new MemoryMetaData(columns, null); results = new MemoryResultSet(metaData); errorResults = new MemoryResultSet(metaData); // add ourself as a row listener step.step.addRowListener(this); success = true; break; } } } } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getString("Kettle.ERROR_0027_ERROR_INIT_STEP", stepName), e); //$NON-NLS-1$ } return success; } private String getMonitorStepName() { String result = null; // Supporting "importstep" for backwards compatibility if (isDefinedInput(KettleComponent.IMPORTSTEP)) { result = getInputStringValue(KettleComponent.IMPORTSTEP); } else if (isDefinedInput(KettleComponent.MONITORSTEP)) { result = getInputStringValue(KettleComponent.MONITORSTEP); } return result; } private LogLevel getLogLevel() { if (isDefinedInput(KettleComponent.KETTLELOGLEVEL)) { String logLevelStr = getInputStringValue(KettleComponent.KETTLELOGLEVEL); try { return LogLevel.valueOf(logLevelStr); } catch (Exception ex) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0024_BAD_LOGGING_LEVEL", logLevelStr)); //$NON-NLS-1$ return LogLevel.BASIC; } } else { // If not defined in the component, translate // xaction logging level to PDI Logging Level switch (loggingLevel) { case ILogger.DEBUG: return LogLevel.DEBUG; case ILogger.ERROR: return LogLevel.ERROR; case ILogger.FATAL: return LogLevel.ERROR; case ILogger.INFO: return LogLevel.MINIMAL; case ILogger.WARN: return LogLevel.BASIC; case ILogger.TRACE: return LogLevel.ROWLEVEL; default: return LogLevel.BASIC; } } } @SuppressWarnings("unchecked") private String getTransformSuccessOutputName() { String result = null; // Supporting "importstep" for backwards compatibility if (isDefinedInput(KettleComponent.IMPORTSTEP)) { if (getOutputNames().size() == 1) { result = (String) getOutputNames().iterator().next(); } else { // Need to find the name that does not match one of the // predefined output parameters result = getUndefinedOutputParameter(getOutputNames().iterator()); if (result == null) { // Use the new TRANSFORM_SUCCESS_OUTPUT to send the output // to, if it is present if (isDefinedOutput(TRANSFORM_SUCCESS_OUTPUT)) { result = TRANSFORM_SUCCESS_OUTPUT; } } } } else if (isDefinedOutput(TRANSFORM_SUCCESS_OUTPUT)) { result = TRANSFORM_SUCCESS_OUTPUT; } return result; } private void prepareKettleOutput(Trans trans) { extractKettleStatus(trans); extractKettleLog(); } private void prepareKettleOutput(Job job) { extractKettleStatus(job); extractKettleLog(); } private void extractKettleStatus(Trans trans) { if (trans != null) { executionStatus = trans.getStatus(); } else { executionStatus = Messages.getInstance().getErrorString("Kettle.ERROR_0025_TRANSFORMATION_NOT_LOADED"); //$NON-NLS-1$ } } private void extractKettleStatus(Job job) { if (job != null) { executionStatus = job.getStatus(); } else { executionStatus = Messages.getInstance().getErrorString("Kettle.ERROR_0026_JOB_NOT_LOADED"); //$NON-NLS-1$ } } private String getKettleLog(boolean includeGeneral) { StringBuffer logText = KettleLogStore.getAppender().getBuffer(logChannelId, includeGeneral); return logText.toString(); } private void extractKettleLog() { executionLog = getKettleLog(false); } private String getUndefinedOutputParameter(Iterator<String> outputNames) { String tempName = null; while (outputNames.hasNext()) { tempName = (String) outputNames.next(); if (!outputParams.contains(tempName)) { // Found user defined named return (tempName); } } return null; } private boolean executeJob(final JobMeta jobMeta, final Repository repository) { boolean success = true; Job job = null; try { if (jobMeta != null) { try { job = new Job(repository, jobMeta); } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0021_BAD_JOB_METADATA"), e); //$NON-NLS-1$ } } if (job == null) { debug(getKettleLog(true)); throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0021_BAD_JOB_METADATA")); //$NON-NLS-1$ } // Remember where to get our execution logging from // logChannelId = job.getLogChannelId(); try { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_STARTING_JOB")); //$NON-NLS-1$ } LogLevel lvl = getLogLevel(); job.setLogLevel(lvl); job.start(); } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0022_JOB_START_FAILED"), e); //$NON-NLS-1$ } try { // It's running in a separate tread to allow monitoring, // etc. if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_JOB_RUNNING")); //$NON-NLS-1$ } job.waitUntilFinished(); if (job.getResult().getNrErrors() > 0) { debug(getKettleLog(true)); throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0014_ERROR_DURING_EXECUTE")); //$NON-NLS-1$ } } catch (Exception e) { throw new KettleComponentException( Messages.getInstance().getErrorString("Kettle.ERROR_0014_ERROR_DURING_EXECUTE"), e); //$NON-NLS-1$ } finally { if (job != null) { cleanLogChannel(job); } } // Dump the Kettle log... debug(getKettleLog(false)); } catch (KettleComponentException e) { success = false; error(Messages.getInstance().getErrorString("Kettle.ERROR_0008_ERROR_RUNNING", e.toString()), e); //$NON-NLS-1$ } prepareKettleOutput(job); return success; } private TransMeta loadTransformFromRepository(final String directoryName, final String transformationName, final Repository repository) { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_DIRECTORY", directoryName)); //$NON-NLS-1$ } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_TRANSFORMATION", transformationName)); //$NON-NLS-1$ } TransMeta transMeta = null; try { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_DIRECTORY")); //$NON-NLS-1$ } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_GETTING_TRANSFORMATION_METADATA")); //$NON-NLS-1$ } try { // Load the transformation from the repository RepositoryDirectoryInterface repositoryDirectory = repository.loadRepositoryDirectoryTree() .findDirectory(directoryName); transMeta = repository.loadTransformation(transformationName, repositoryDirectory, null, true, null); } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0009_TRANSFROMATION_METADATA_NOT_FOUND", //$NON-NLS-1$ directoryName + "/" + transformationName), e); //$NON-NLS-1$ return null; } if (transMeta == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0009_TRANSFROMATION_METADATA_NOT_FOUND", //$NON-NLS-1$ directoryName + "/" + transformationName)); //$NON-NLS-1$ debug(getKettleLog(true)); return null; } else { return transMeta; } } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0008_ERROR_RUNNING", e.toString()), e); //$NON-NLS-1$ } return null; } private JobMeta loadJobFromRepository(final String directoryName, final String jobName, final Repository repository) { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_DIRECTORY", directoryName)); //$NON-NLS-1$ } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_JOB", jobName)); //$NON-NLS-1$ } JobMeta jobMeta = null; try { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_DIRECTORY")); //$NON-NLS-1$ } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_GETTING_JOB_METADATA")); //$NON-NLS-1$ } try { // Load the job from the repository RepositoryDirectoryInterface repositoryDirectory = repository.loadRepositoryDirectoryTree() .findDirectory(directoryName); jobMeta = repository.loadJob(jobName, repositoryDirectory, null, null); } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0020_JOB_METADATA_NOT_FOUND", //$NON-NLS-1$ directoryName + "/" + jobName), e); //$NON-NLS-1$ return null; } if (jobMeta == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0020_JOB_METADATA_NOT_FOUND", //$NON-NLS-1$ directoryName + "/" + jobName)); //$NON-NLS-1$ debug(getKettleLog(true)); return null; } else { return jobMeta; } } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0008_ERROR_RUNNING", e.toString()), e); //$NON-NLS-1$ } return null; } private Repository connectToRepository() { boolean useRepository = PentahoSystem.getSystemSetting("kettle/settings.xml", "repository.type", "files") //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ .equals("rdbms"); //$NON-NLS-1$ if (!useRepository) { return null; } try { if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_META_REPOSITORY")); //$NON-NLS-1$ } RepositoriesMeta repositoriesMeta = null; try { repositoriesMeta = new RepositoriesMeta(); } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0007_BAD_META_REPOSITORY"), e); //$NON-NLS-1$ return null; } /* * Unreachable code below if (repositoriesMeta == null) { * error(Messages.getInstance().getErrorString("Kettle.ERROR_0007_BAD_META_REPOSITORY")); //$NON-NLS-1$ * debug(getKettleLog(true)); return null; } */ if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_POPULATING_META")); //$NON-NLS-1$ } try { // TODO: add support for specified repositories.xml files... repositoriesMeta.readData(); // Read from the default // $HOME/.kettle/repositories.xml // file. } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0018_META_REPOSITORY_NOT_POPULATED"), e); //$NON-NLS-1$ return null; } if ((repositoriesXMLFile != null) && !"".equals(repositoriesXMLFile)) { //$NON-NLS-1$ error(Messages.getInstance().getErrorString("Kettle.ERROR_0017_XML_REPOSITORY_NOT_SUPPORTED")); //$NON-NLS-1$ debug(getKettleLog(true)); return null; } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_REPOSITORY")); //$NON-NLS-1$ } // Find the specified repository. RepositoryMeta repositoryMeta = null; try { repositoryMeta = repositoriesMeta.findRepository(repositoryName); } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0004_REPOSITORY_NOT_FOUND", //$NON-NLS-1$ repositoryName), e); return null; } if (repositoryMeta == null) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0004_REPOSITORY_NOT_FOUND", //$NON-NLS-1$ repositoryName)); debug(getKettleLog(true)); return null; } if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_GETTING_REPOSITORY")); //$NON-NLS-1$ } Repository repository = null; try { repository = PluginRegistry.getInstance().loadClass(RepositoryPluginType.class, repositoryMeta.getId(), Repository.class); repository.init(repositoryMeta); } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0016_COULD_NOT_GET_REPOSITORY_INSTANCE"), //$NON-NLS-1$ e); return null; } // OK, now try the username and password if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_CONNECTING")); //$NON-NLS-1$ } repository.connect(username, password); // OK, the repository is open and ready to use. if (ComponentBase.debug) { debug(Messages.getInstance().getString("Kettle.DEBUG_FINDING_DIRECTORY")); //$NON-NLS-1$ } return repository; } catch (Exception e) { error(Messages.getInstance().getErrorString("Kettle.ERROR_0008_ERROR_RUNNING", e.toString()), e); //$NON-NLS-1$ } return null; } @Override public void done() { } public void rowReadEvent(final RowMetaInterface row, final Object[] values) { } public void rowWrittenEvent(final RowMetaInterface rowMeta, final Object[] row) throws KettleStepException { processRow(results, rowMeta, row); } public void errorRowWrittenEvent(final RowMetaInterface rowMeta, final Object[] row) throws KettleStepException { processRow(errorResults, rowMeta, row); } public void processRow(MemoryResultSet memResults, final RowMetaInterface rowMeta, final Object[] row) throws KettleStepException { if (memResults == null) { return; } try { Object[] pentahoRow = new Object[memResults.getColumnCount()]; for (int columnNo = 0; columnNo < memResults.getColumnCount(); columnNo++) { ValueMetaInterface valueMeta = rowMeta.getValueMeta(columnNo); switch (valueMeta.getType()) { case ValueMetaInterface.TYPE_BIGNUMBER: pentahoRow[columnNo] = rowMeta.getBigNumber(row, columnNo); break; case ValueMetaInterface.TYPE_BOOLEAN: pentahoRow[columnNo] = rowMeta.getBoolean(row, columnNo); break; case ValueMetaInterface.TYPE_DATE: pentahoRow[columnNo] = rowMeta.getDate(row, columnNo); break; case ValueMetaInterface.TYPE_INTEGER: pentahoRow[columnNo] = rowMeta.getInteger(row, columnNo); break; case ValueMetaInterface.TYPE_NONE: pentahoRow[columnNo] = rowMeta.getString(row, columnNo); break; case ValueMetaInterface.TYPE_NUMBER: pentahoRow[columnNo] = rowMeta.getNumber(row, columnNo); break; case ValueMetaInterface.TYPE_STRING: pentahoRow[columnNo] = rowMeta.getString(row, columnNo); break; default: pentahoRow[columnNo] = rowMeta.getString(row, columnNo); } } memResults.addRow(pentahoRow); } catch (KettleValueException e) { throw new KettleStepException(e); } } }