org.pentaho.di.trans.dataservice.DataServiceExecutor.java Source code

Java tutorial

Introduction

Here is the source code for org.pentaho.di.trans.dataservice.DataServiceExecutor.java

Source

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.trans.dataservice;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.MultimapBuilder;
import org.apache.commons.lang.StringUtils;
import org.pentaho.di.core.Condition;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.logging.LogLevel;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaAndData;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.sql.SQL;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.RowProducer;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransAdapter;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.dataservice.clients.TransMutators;
import org.pentaho.di.trans.dataservice.execution.CopyParameters;
import org.pentaho.di.trans.dataservice.execution.DefaultTransWiring;
import org.pentaho.di.trans.dataservice.execution.PrepareExecution;
import org.pentaho.di.trans.dataservice.execution.TransStarter;
import org.pentaho.di.trans.dataservice.optimization.PushDownOptimizationMeta;
import org.pentaho.di.trans.dataservice.optimization.ValueMetaResolver;
import org.pentaho.di.trans.step.RowAdapter;
import org.pentaho.di.trans.step.RowListener;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.metastore.api.IMetaStore;

import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;

public class DataServiceExecutor {
    private static final Class<?> PKG = DataServiceExecutor.class;

    private static final String ROW_LIMIT_PROPERTY = "det.dataservice.dynamic.limit";
    private static final int ROW_LIMIT_DEFAULT = 50000;

    private final Trans serviceTrans;
    private final Trans genTrans;

    private final DataServiceMeta service;
    private final SQL sql;
    private final Map<String, String> parameters;
    private final SqlTransGenerator sqlTransGenerator;
    private final ListMultimap<ExecutionPoint, Runnable> listenerMap;

    private DataServiceExecutor(Builder builder) {
        sql = builder.sql;
        service = builder.service;
        parameters = Maps.newHashMap(builder.parameters);
        parameters.putAll(getWhereConditionParameters());
        serviceTrans = builder.serviceTrans;
        sqlTransGenerator = builder.sqlTransGenerator;
        genTrans = builder.genTrans;

        listenerMap = MultimapBuilder.enumKeys(ExecutionPoint.class).linkedListValues().build();
    }

    public static class Builder {
        private final SQL sql;
        private final DataServiceMeta service;
        private final DataServiceContext context;
        private Trans serviceTrans;
        private Trans genTrans;
        private int rowLimit = 0;
        private Map<String, String> parameters = Collections.emptyMap();
        private LogLevel logLevel;
        private SqlTransGenerator sqlTransGenerator;

        private boolean normalizeConditions = true;
        private boolean prepareExecution = true;
        private boolean enableMetrics = false;
        private IMetaStore metastore;
        private BiConsumer<String, TransMeta> transMutator = (stepName, transMeta) -> TransMutators
                .disableAllUnrelatedHops(stepName, transMeta, true);

        public Builder(SQL sql, DataServiceMeta service, DataServiceContext context) {
            this.sql = Preconditions.checkNotNull(sql, "SQL must not be null.");
            this.service = Preconditions.checkNotNull(service, "Service must not be null.");
            this.context = context;
        }

        public Builder parameters(Map<String, String> parameters) {
            this.parameters = parameters;
            return this;
        }

        public Builder rowLimit(int rowLimit) {
            this.rowLimit = rowLimit;
            return this;
        }

        public Builder logLevel(LogLevel logLevel) {
            this.logLevel = logLevel;
            return this;
        }

        public Builder metastore(final IMetaStore metastore) {
            this.metastore = metastore;
            return this;
        }

        public Builder serviceTrans(Trans serviceTrans) {
            this.serviceTrans = serviceTrans;
            return this;
        }

        Builder serviceTransMutator(BiConsumer<String, TransMeta> transMutator) {
            this.transMutator = transMutator;
            return this;
        }

        public Builder serviceTrans(TransMeta serviceTransMeta) {
            // Copy TransMeta, we don't want to persist any changes to the meta during execution
            serviceTransMeta = (TransMeta) serviceTransMeta.realClone(false);
            serviceTransMeta.setName(calculateTransname(sql, true));
            serviceTransMeta.activateParameters();
            transMutator.accept(service.getStepname(), serviceTransMeta);
            return serviceTrans(new Trans(serviceTransMeta));
        }

        public Builder sqlTransGenerator(SqlTransGenerator sqlTransGenerator) {
            this.sqlTransGenerator = sqlTransGenerator;
            return this;
        }

        public Builder genTrans(Trans trans) {
            this.genTrans = trans;
            return this;
        }

        public Builder enableMetrics(boolean enable) {
            enableMetrics = enable;
            return this;
        }

        public Builder normalizeConditions(boolean enable) {
            normalizeConditions = enable;
            return this;
        }

        public Builder prepareExecution(boolean enable) {
            prepareExecution = enable;
            return this;
        }

        public DataServiceExecutor build() throws KettleException {
            RowMetaInterface serviceFields;

            if (sql.getServiceName() != null && !sql.getServiceName().equals(service.getName())) {
                throw new KettleException(BaseMessages.getString(PKG,
                        "DataServiceExecutor.Error.TableNameAndDataServiceNameDifferent", sql.getServiceName(),
                        service.getName()));
            }

            if (serviceTrans != null) {
                serviceFields = serviceTrans.getTransMeta().getStepFields(service.getStepname());
            } else if (service.getServiceTrans() != null) {
                serviceTrans(service.getServiceTrans());
                serviceFields = serviceTrans.getTransMeta().getStepFields(service.getStepname());
            } else {
                serviceFields = new RowMeta();
            }
            ValueMetaResolver resolver = new ValueMetaResolver(serviceFields);

            sql.parse(resolver.getRowMeta());

            if (normalizeConditions) {
                if (sql.getWhereCondition() != null && sql.getWhereCondition().getCondition() != null) {
                    convertCondition(sql.getWhereCondition().getCondition(), resolver);
                }
                if (sql.getHavingCondition() != null && sql.getHavingCondition().getCondition() != null) {
                    convertCondition(sql.getHavingCondition().getCondition(), resolver);
                }
            }

            int serviceRowLimit = getServiceRowLimit(service);
            if (sqlTransGenerator == null) {
                sqlTransGenerator = new SqlTransGenerator(sql, rowLimit, serviceRowLimit);
            }
            if (genTrans == null) {
                genTrans = new Trans(sqlTransGenerator.generateTransMeta());
            }

            serviceTrans.setContainerObjectId(UUID.randomUUID().toString());
            genTrans.setContainerObjectId(UUID.randomUUID().toString());
            serviceTrans.setMetaStore(metastore);
            genTrans.setMetaStore(metastore);

            DataServiceExecutor dataServiceExecutor = new DataServiceExecutor(this);

            context.addExecutor(dataServiceExecutor);

            if (logLevel != null) {
                dataServiceExecutor.setLogLevel(logLevel);
            }

            genTrans.setGatheringMetrics(enableMetrics);
            if (serviceTrans != null) {
                serviceTrans.setGatheringMetrics(enableMetrics);
            }

            if (prepareExecution) {
                dataServiceExecutor.prepareExecution();
            }

            return dataServiceExecutor;
        }

        private int getServiceRowLimit(DataServiceMeta service) throws KettleException {
            if (service.getRowLimit() != null && service.getRowLimit() > 0) {
                return service.getRowLimit();
            } else if (!service.isUserDefined()) {
                String limit = getKettleProperty(ROW_LIMIT_PROPERTY);
                if (!Utils.isEmpty(limit)) {
                    try {
                        return Integer.parseInt(limit);
                    } catch (NumberFormatException e) {
                        if (context != null && context.getLogChannel() != null) {
                            context.getLogChannel().logError(String.format("%s: %s ", ROW_LIMIT_PROPERTY, e));
                        }
                    }
                }
                return ROW_LIMIT_DEFAULT;
            }
            return 0;
        }

        private String getKettleProperty(String propertyName) throws KettleException {
            // loaded in system properties at startup
            return System.getProperty(propertyName);
        }
    }

    private void setLogLevel(LogLevel logLevel) {
        if (serviceTrans != null) {
            serviceTrans.setLogLevel(logLevel);
            getServiceTransMeta().setLogLevel(logLevel);
        }
        if (genTrans != null) {
            genTrans.setLogLevel(logLevel);
            getGenTransMeta().setLogLevel(logLevel);
        }
    }

    private static void convertCondition(Condition condition, ValueMetaResolver resolver) {
        if (condition.isAtomic()) {
            if (condition.getFunction() == Condition.FUNC_IN_LIST) {
                convertListCondition(condition, resolver);
            } else {
                convertAtomicCondition(condition, resolver);
            }
        } else {
            for (Condition child : condition.getChildren()) {
                convertCondition(child, resolver);
            }
        }
    }

    private static void convertAtomicCondition(Condition condition, ValueMetaResolver resolver) {
        // No need to convert conditions with no right side argument
        if (condition.getRightExact() == null) {
            return;
        }
        String fieldName = condition.getLeftValuename();
        ValueMetaAndData rhs = condition.getRightExact();
        try {
            // Determine meta and resolve value
            ValueMetaInterface resolvedValueMeta = resolver.getValueMeta(fieldName);
            Object resolvedValue = resolver.getTypedValue(fieldName, rhs.getValueMeta().getType(),
                    rhs.getValueData());

            // We have normally stored object here, adjust value meta accordingly
            if (resolvedValueMeta.getStorageType() != ValueMetaInterface.STORAGE_TYPE_NORMAL) {
                resolvedValueMeta = resolvedValueMeta.clone();
                resolvedValueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            }

            // Set new condition meta and value
            condition.setRightExact(new ValueMetaAndData(resolvedValueMeta, resolvedValue));
        } catch (KettleException e) {
            // Skip conversion of this condition?
        }
    }

    private static void convertListCondition(Condition condition, ValueMetaResolver resolver) {
        String fieldName = condition.getLeftValuename();
        try {
            // Determine meta and resolve values
            ValueMetaInterface resolvedValueMeta = resolver.getValueMeta(fieldName);
            Object[] typedValues = resolver.inListToTypedObjectArray(fieldName, condition.getRightExactString());

            // Encode list values
            String[] typedValueStrings = new String[typedValues.length];
            for (int i = 0; i < typedValues.length; i++) {
                typedValueStrings[i] = resolvedValueMeta.getCompatibleString(typedValues[i]);
            }

            // Set new condition in-list (leave meta as string)
            condition.getRightExact().setValueData(StringUtils.join(typedValueStrings, ';'));
        } catch (KettleException e) {
            // Skip conversion of this condition?
        }
    }

    private void extractConditionParameters(Condition condition, Map<String, String> parameters) {
        if (condition.isAtomic()) {
            if (condition.getFunction() == Condition.FUNC_TRUE) {
                parameters.put(condition.getLeftValuename(), condition.getRightExactString());
                stripFieldNamesFromTrueFunction(condition);
            }
        } else {
            for (Condition sub : condition.getChildren()) {
                extractConditionParameters(sub, parameters);
            }
        }
    }

    /**
     * Strips the "fake" values from a Condition used
     * to pass parameter key/value info in the WHERE clause.
     * E.g. if
     * WHERE PARAMETER('foo') = 'bar'
     * is in the where clause a FUNC_TRUE condition will be
     * created with leftValueName = 'foo' and rightValueName = 'bar'.
     * These need to be stripped out to avoid failing checks
     * for existent field names.
     */
    private void stripFieldNamesFromTrueFunction(Condition condition) {
        assert condition.getFunction() == Condition.FUNC_TRUE;
        condition.setLeftValuename(null);
        condition.setRightValuename(null);
    }

    protected void prepareExecution() throws KettleException {
        // Setup executor with default execution plan
        ImmutableMultimap.Builder<ExecutionPoint, Runnable> builder = ImmutableMultimap.builder();
        builder.putAll(ExecutionPoint.PREPARE, new CopyParameters(parameters, serviceTrans),
                new PrepareExecution(genTrans), new PrepareExecution(serviceTrans));

        builder.putAll(ExecutionPoint.READY, new DefaultTransWiring(this));

        builder.putAll(ExecutionPoint.START, new TransStarter(genTrans), new TransStarter(serviceTrans));

        listenerMap.putAll(builder.build());
    }

    private Map<String, String> getWhereConditionParameters() {
        // Parameters: see which ones are defined in the SQL
        //
        Map<String, String> conditionParameters = new HashMap<>();
        if (sql.getWhereCondition() != null) {
            extractConditionParameters(sql.getWhereCondition().getCondition(), conditionParameters);
        }
        return conditionParameters;
    }

    public static void writeMetadata(DataOutputStream dos, String... metadatas) throws IOException {
        for (String metadata : metadatas) {
            dos.writeUTF(metadata);
        }
    }

    public DataServiceExecutor executeQuery(final DataOutputStream dos) throws IOException {

        writeMetadata(dos, getServiceName(), calculateTransname(getSql(), true),
                getServiceTrans().getContainerObjectId(), calculateTransname(getSql(), false),
                getGenTrans().getContainerObjectId());

        final AtomicBoolean rowMetaWritten = new AtomicBoolean(false);

        // When done, check if no row metadata was written.  The client is still going to expect it...
        // Since we know it, we'll pass it.
        //
        getGenTrans().addTransListener(new TransAdapter() {
            @Override
            public void transFinished(Trans trans) throws KettleException {
                if (rowMetaWritten.compareAndSet(false, true)) {
                    RowMetaInterface stepFields = trans.getTransMeta().getStepFields(getResultStepName());
                    stepFields.writeMeta(dos);
                }
            }
        });

        // Now execute the query transformation(s) and pass the data to the output stream...
        return executeQuery(new RowAdapter() {
            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {

                // On the first row, write the metadata...
                //
                try {
                    if (rowMetaWritten.compareAndSet(false, true)) {
                        rowMeta.writeMeta(dos);
                    }
                    rowMeta.writeData(dos, row);
                } catch (Exception e) {
                    if (!getServiceTrans().isStopped()) {
                        throw new KettleStepException(e);
                    }
                }
            }
        });
    }

    public DataServiceExecutor executeQuery(final RowListener resultRowListener) {
        listenerMap.get(ExecutionPoint.READY).add(new Runnable() {
            @Override
            public void run() {
                // Give back the eventual result rows...
                //
                StepInterface resultStep = genTrans.findRunThread(getResultStepName());
                resultStep.addRowListener(resultRowListener);
            }
        });
        return executeQuery();
    }

    public DataServiceExecutor executeQuery() {
        // Apply Push Down Optimizations
        for (PushDownOptimizationMeta optimizationMeta : service.getPushDownOptimizationMeta()) {
            if (optimizationMeta.isEnabled()) {
                optimizationMeta.activate(this);
            }
        }

        // Run execution plan
        executeListeners(ExecutionPoint.values());

        return this;
    }

    public void executeListeners(ExecutionPoint... stages) {
        for (ExecutionPoint stage : stages) {
            // Copy stage tasks to a new list to prevent accidental concurrent modification
            ImmutableList<Runnable> tasks = ImmutableList.copyOf(listenerMap.get(stage));
            for (Runnable task : tasks) {
                task.run();
            }
            if (!listenerMap.get(stage).equals(tasks)) {
                getGenTrans().getLogChannel().logError(
                        "Listeners were modified while executing {0}. Started with {1} and ended with {2}", stage,
                        tasks, listenerMap.get(stage));
            }
        }
    }

    public RowProducer addRowProducer() throws KettleException {
        return genTrans.addRowProducer(sqlTransGenerator.getInjectorStepName(), 0);
    }

    public void waitUntilFinished() {
        serviceTrans.waitUntilFinished();
        genTrans.waitUntilFinished();
    }

    /**
     * @return the serviceTransMeta
     */
    public TransMeta getServiceTransMeta() {
        return serviceTrans.getTransMeta();
    }

    /**
     * @return the genTransMeta
     */
    public TransMeta getGenTransMeta() {
        return genTrans.getTransMeta();
    }

    public DataServiceMeta getService() {
        return service;
    }

    /**
     * @return the serviceTrans
     */
    public Trans getServiceTrans() {
        return serviceTrans;
    }

    /**
     * @return the genTrans
     */
    public Trans getGenTrans() {
        return genTrans;
    }

    /**
     * @return the serviceName
     */
    public String getServiceName() {
        return sql.getServiceName();
    }

    public Map<String, String> getParameters() {
        return parameters;
    }

    public String getId() {
        return serviceTrans.getContainerObjectId();
    }

    public Boolean hasErrors() {
        return serviceTrans.getErrors() > 0 || genTrans.getErrors() > 0;
    }

    /**
     * Calculate the name of the generated transformation based on the SQL
     *
     * @return the generated name;
     */
    public static String calculateTransname(SQL sql, boolean isService) {
        StringBuilder sbsql = new StringBuilder(sql.getServiceName());
        sbsql.append(" - ");
        if (isService) {
            sbsql.append("Service");
        } else {
            sbsql.append("SQL");
        }
        sbsql.append(" - ");
        sbsql.append(sql.getSqlString().hashCode());

        // Get rid of newlines...
        //
        for (int i = sbsql.length() - 1; i >= 0; i--) {
            if (sbsql.charAt(i) == '\n' || sbsql.charAt(i) == '\r') {
                sbsql.setCharAt(i, ' ');
            }
        }
        return sbsql.toString();
    }

    public void stop() {
        if (serviceTrans.isRunning()) {
            serviceTrans.stopAll();
        }
        if (genTrans.isRunning()) {
            genTrans.stopAll();
        }
    }

    public boolean isStopped() {
        return genTrans.isStopped();
    }

    /**
     * @return the sql
     */
    public SQL getSql() {
        return sql;
    }

    /**
     * @return the resultStepName
     */
    public String getResultStepName() {
        return sqlTransGenerator.getResultStepName();
    }

    public int getRowLimit() {
        return sqlTransGenerator.getRowLimit();
    }

    public int getServiceRowLimit() {
        return sqlTransGenerator.getServiceRowLimit();
    }

    public ListMultimap<ExecutionPoint, Runnable> getListenerMap() {
        return listenerMap;
    }

    /**
     * @author nhudak
     */
    public enum ExecutionPoint {
        PREPARE, OPTIMIZE, READY, START
    }

}