org.pentaho.di.ui.trans.steps.getxmldata.XMLInputFieldsImportProgressDialog.java Source code

Java tutorial

Introduction

Here is the source code for org.pentaho.di.ui.trans.steps.getxmldata.XMLInputFieldsImportProgressDialog.java

Source

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.ui.trans.steps.getxmldata;

import java.io.InputStream;
import java.io.StringReader;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import org.dom4j.Attribute;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.Node;
import org.dom4j.io.SAXReader;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.jface.dialogs.ProgressMonitorDialog;
import org.eclipse.jface.operation.IRunnableWithProgress;
import org.eclipse.swt.widgets.Shell;
import org.pentaho.di.compatibility.Value;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.RowMetaAndData;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.xml.XMLParserFactoryProducer;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.steps.getxmldata.GetXMLDataField;
import org.pentaho.di.trans.steps.getxmldata.GetXMLDataMeta;
import org.pentaho.di.trans.steps.getxmldata.IgnoreDTDEntityResolver;
import org.pentaho.di.ui.core.dialog.ErrorDialog;

/**
 * Takes care of displaying a dialog that will handle the wait while we're finding out loop nodes for an XML file
 * 
 * @author Samatar
 * @since 07-apr-2010
 */
public class XMLInputFieldsImportProgressDialog {
    private static Class<?> PKG = GetXMLDataMeta.class; // for i18n purposes, needed by Translator2!!

    private static String VALUE_NAME = "Name";
    private static String VALUE_PATH = "Path";
    private static String VALUE_ELEMENT = "Element";
    private static String VALUE_RESULT = "result";
    private static String VALUE_TYPE = "Type";
    private static String VALUE_FORMAT = "Format";

    private Shell shell;

    private GetXMLDataMeta meta;

    private String filename;
    private String encoding;

    private int nr;

    private String loopXPath;
    private HashSet<String> list;

    private List<RowMetaAndData> fieldsList;
    private RowMetaAndData[] fields;

    private String xml;
    private String url;

    /**
     * Creates a new dialog that will handle the wait while we're finding out loop nodes for an XML file
     */
    public XMLInputFieldsImportProgressDialog(Shell shell, GetXMLDataMeta meta, String filename, String encoding,
            String loopXPath) {
        this.shell = shell;
        this.meta = meta;
        this.fields = null;
        this.filename = filename;
        this.encoding = encoding;
        this.nr = 0;
        this.loopXPath = loopXPath;
        this.list = new HashSet<String>();
        this.fieldsList = new ArrayList<RowMetaAndData>();
    }

    public XMLInputFieldsImportProgressDialog(Shell shell, GetXMLDataMeta meta, String xmlSource, boolean useUrl,
            String loopXPath) {
        this.shell = shell;
        this.meta = meta;
        this.fields = null;
        this.filename = null;
        this.encoding = null;
        this.nr = 0;
        this.loopXPath = loopXPath;
        this.list = new HashSet<String>();
        this.fieldsList = new ArrayList<RowMetaAndData>();
        if (useUrl) {
            this.xml = null;
            this.url = xmlSource;
        } else {
            this.xml = xmlSource;
            this.url = null;
        }
    }

    public RowMetaAndData[] open() {
        IRunnableWithProgress op = new IRunnableWithProgress() {
            public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
                try {
                    fields = doScan(monitor);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new InvocationTargetException(e,
                            BaseMessages.getString(PKG,
                                    "GetXMLDateLoopNodesImportProgressDialog.Exception.ErrorScanningFile", filename,
                                    e.toString()));
                }
            }
        };

        try {
            ProgressMonitorDialog pmd = new ProgressMonitorDialog(shell);
            pmd.run(true, true, op);
        } catch (InvocationTargetException e) {
            new ErrorDialog(shell,
                    BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.ErrorScanningFile.Title"),
                    BaseMessages.getString(PKG,
                            "GetXMLDateLoopNodesImportProgressDialog.ErrorScanningFile.Message"),
                    e);
        } catch (InterruptedException e) {
            new ErrorDialog(shell,
                    BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.ErrorScanningFile.Title"),
                    BaseMessages.getString(PKG,
                            "GetXMLDateLoopNodesImportProgressDialog.ErrorScanningFile.Message"),
                    e);
        }

        return fields;
    }

    @SuppressWarnings("unchecked")
    private RowMetaAndData[] doScan(IProgressMonitor monitor) throws Exception {
        monitor.beginTask(
                BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.Task.ScanningFile", filename),
                1);

        SAXReader reader = XMLParserFactoryProducer.getSAXReader(null);
        monitor.worked(1);
        if (monitor.isCanceled()) {
            return null;
        }
        // Validate XML against specified schema?
        if (meta.isValidating()) {
            reader.setValidation(true);
            reader.setFeature("http://apache.org/xml/features/validation/schema", true);
        } else {
            // Ignore DTD
            reader.setEntityResolver(new IgnoreDTDEntityResolver());
        }
        monitor.worked(1);
        monitor.beginTask(
                BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.Task.ReadingDocument"), 1);
        if (monitor.isCanceled()) {
            return null;
        }
        InputStream is = null;
        try {

            Document document = null;
            if (!Utils.isEmpty(filename)) {
                is = KettleVFS.getInputStream(filename);
                document = reader.read(is, encoding);
            } else {
                if (!Utils.isEmpty(xml)) {
                    document = reader.read(new StringReader(xml));
                } else {
                    document = reader.read(new URL(url));
                }
            }

            monitor.worked(1);
            monitor.beginTask(
                    BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.Task.DocumentOpened"), 1);
            monitor.worked(1);
            monitor.beginTask(
                    BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.Task.ReadingNode"), 1);

            if (monitor.isCanceled()) {
                return null;
            }
            List<Node> nodes = document.selectNodes(this.loopXPath);
            monitor.worked(1);
            monitor.subTask(BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.Task.FetchNodes"));

            if (monitor.isCanceled()) {
                return null;
            }
            for (Node node : nodes) {
                if (monitor.isCanceled()) {
                    return null;
                }

                nr++;
                monitor.subTask(BaseMessages.getString(PKG,
                        "GetXMLDateLoopNodesImportProgressDialog.Task.FetchNodes", String.valueOf(nr)));
                monitor.subTask(BaseMessages.getString(PKG,
                        "GetXMLDateLoopNodesImportProgressDialog.Task.FetchNodes", node.getPath()));
                setNodeField(node, monitor);
                childNode(node, monitor);

            }
            monitor.worked(1);
        } finally {
            try {
                if (is != null) {
                    is.close();
                }
            } catch (Exception e) { /* Ignore */
            }
        }

        RowMetaAndData[] listFields = fieldsList.toArray(new RowMetaAndData[fieldsList.size()]);

        monitor.setTaskName(
                BaseMessages.getString(PKG, "GetXMLDateLoopNodesImportProgressDialog.Task.NodesReturned"));

        monitor.done();

        return listFields;

    }

    @SuppressWarnings("unchecked")
    private void setNodeField(Node node, IProgressMonitor monitor) {
        Element e = (Element) node;
        // get all attributes
        List<Attribute> lista = e.attributes();
        for (int i = 0; i < lista.size(); i++) {
            setAttributeField(lista.get(i), monitor);
        }

        // Get Node Name
        String nodename = node.getName();
        String nodenametxt = cleanString(node.getPath());

        if (!Utils.isEmpty(nodenametxt) && !list.contains(nodenametxt)) {
            nr++;
            monitor.subTask(BaseMessages.getString(PKG,
                    "GetXMLDataXMLInputFieldsImportProgressDialog.Task.FetchFields", String.valueOf(nr)));
            monitor.subTask(BaseMessages.getString(PKG,
                    "GetXMLDataXMLInputFieldsImportProgressDialog.Task.AddingField", nodename));

            RowMetaAndData row = new RowMetaAndData();
            row.addValue(VALUE_NAME, Value.VALUE_TYPE_STRING, nodename);
            row.addValue(VALUE_PATH, Value.VALUE_TYPE_STRING, nodenametxt);
            row.addValue(VALUE_ELEMENT, Value.VALUE_TYPE_STRING, GetXMLDataField.ElementTypeDesc[0]);
            row.addValue(VALUE_RESULT, Value.VALUE_TYPE_STRING, GetXMLDataField.ResultTypeDesc[0]);

            // Get Node value
            String valueNode = node.getText();

            // Try to get the Type

            if (IsDate(valueNode)) {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "Date");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, "yyyy/MM/dd");
            } else if (IsInteger(valueNode)) {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "Integer");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, null);
            } else if (IsNumber(valueNode)) {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "Number");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, null);
            } else {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "String");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, null);
            }
            fieldsList.add(row);
            list.add(nodenametxt);

        } // end if
    }

    private void setAttributeField(Attribute attribute, IProgressMonitor monitor) {
        // Get Attribute Name
        String attributname = attribute.getName();
        String attributnametxt = cleanString(attribute.getPath());
        if (!Utils.isEmpty(attributnametxt) && !list.contains(attribute.getPath())) {
            nr++;
            monitor.subTask(BaseMessages.getString(PKG,
                    "GetXMLDataXMLInputFieldsImportProgressDialog.Task.FetchFields", String.valueOf(nr)));
            monitor.subTask(BaseMessages.getString(PKG,
                    "GetXMLDataXMLInputFieldsImportProgressDialog.Task.AddingField", attributname));

            RowMetaAndData row = new RowMetaAndData();
            row.addValue(VALUE_NAME, Value.VALUE_TYPE_STRING, attributname);
            row.addValue(VALUE_PATH, Value.VALUE_TYPE_STRING, attributnametxt);
            row.addValue(VALUE_ELEMENT, Value.VALUE_TYPE_STRING, GetXMLDataField.ElementTypeDesc[1]);
            row.addValue(VALUE_RESULT, Value.VALUE_TYPE_STRING, GetXMLDataField.ResultTypeDesc[0]);

            // Get attribute value
            String valueAttr = attribute.getText();

            // Try to get the Type

            if (IsDate(valueAttr)) {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "Date");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, "yyyy/MM/dd");
            } else if (IsInteger(valueAttr)) {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "Integer");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, null);
            } else if (IsNumber(valueAttr)) {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "Number");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, null);
            } else {
                row.addValue(VALUE_TYPE, Value.VALUE_TYPE_STRING, "String");
                row.addValue(VALUE_FORMAT, Value.VALUE_TYPE_STRING, null);
            }
            list.add(attribute.getPath());
        } // end if

    }

    private String cleanString(String inputstring) {
        String retval = inputstring;
        retval = retval.replace(this.loopXPath, "");
        while (retval.startsWith(GetXMLDataMeta.N0DE_SEPARATOR)) {
            retval = retval.substring(1, retval.length());
        }

        return retval;
    }

    private boolean IsDate(String str) {
        // TODO: What about other dates? Maybe something for a CRQ
        try {
            SimpleDateFormat fdate = new SimpleDateFormat("yyyy/MM/dd");
            fdate.setLenient(false);
            fdate.parse(str);
        } catch (Exception e) {
            return false;
        }
        return true;
    }

    private boolean IsInteger(String str) {
        try {
            Integer.parseInt(str);
        } catch (NumberFormatException e) {
            return false;
        }
        return true;
    }

    private boolean IsNumber(String str) {
        try {
            Float.parseFloat(str);
        } catch (Exception e) {
            return false;
        }
        return true;
    }

    private boolean childNode(Node node, IProgressMonitor monitor) {
        boolean rc = false; // true: we found child nodes
        Element ce = (Element) node;
        // List child
        for (int j = 0; j < ce.nodeCount(); j++) {
            Node cnode = ce.node(j);
            if (!Utils.isEmpty(cnode.getName())) {
                Element cce = (Element) cnode;
                if (cce.nodeCount() > 1) {
                    if (childNode(cnode, monitor) == false) {
                        // We do not have child nodes ...
                        setNodeField(cnode, monitor);
                        rc = true;
                    }
                } else {
                    setNodeField(cnode, monitor);
                    rc = true;
                }
            }
        }
        return rc;
    }

}