org.pentaho.metaverse.impl.VfsLineageWriter.java Source code

Java tutorial

Introduction

Here is the source code for org.pentaho.metaverse.impl.VfsLineageWriter.java

Source

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.metaverse.impl;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.regex.Pattern;

import org.apache.commons.vfs2.FileContent;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleFileException;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.metaverse.api.IGraphWriter;
import org.pentaho.metaverse.api.ILineageWriter;
import org.pentaho.metaverse.api.IMetaverseBuilder;
import org.pentaho.metaverse.api.model.IExecutionProfile;
import org.pentaho.metaverse.api.model.LineageHolder;
import org.pentaho.metaverse.graph.GraphMLWriter;
import org.pentaho.metaverse.graph.GraphSONWriter;
import org.pentaho.metaverse.impl.model.ExecutionProfileUtil;
import org.pentaho.metaverse.messages.Messages;
import org.pentaho.metaverse.util.MetaverseUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Created by wseyler on 10/27/15.
 */
public class VfsLineageWriter implements ILineageWriter {

    public static final String DEFAULT_OUTPUT_FOLDER = "tmp://dir";

    private static final Logger log = LoggerFactory.getLogger(VfsLineageWriter.class);

    private IGraphWriter graphWriter = new GraphMLWriter();
    private String outputFolder = DEFAULT_OUTPUT_FOLDER;
    private String outputStrategy = DEFAULT_OUTPUT_STRATEGY;

    protected static SimpleDateFormat dateFolderFormat = new SimpleDateFormat("YYYYMMdd");

    private static enum VFS_Prefixes {
        BZIP2, FILE, FTP, FTPS, GZIP, HDFS, HTTP, HTTPS, JAR, RAM, RES, SFTP, TAR, TEMP, WEBDAV, ZIP
    }

    public VfsLineageWriter() {
    }

    @Override
    public void outputExecutionProfile(LineageHolder holder) throws IOException {
        if (holder != null) {
            IExecutionProfile profile = holder.getExecutionProfile();
            if (profile != null) {
                try (OutputStream fis = getProfileOutputStream(holder)) {
                    if (fis != null) {
                        ExecutionProfileUtil.outputExecutionProfile(fis, profile);
                    } else {
                        log.debug(Messages.getString("DEBUG.noProfileOutputStream"));
                    }
                }
            }
        }
    }

    @Override
    public void outputLineageGraph(LineageHolder holder) throws IOException {
        if (holder != null) {
            IMetaverseBuilder builder = holder.getMetaverseBuilder();
            if (builder != null) {
                // no-op by default, can be used to introduce an artificial delay in the graphml file, for testing purposes
                MetaverseUtil.delay();
                try (OutputStream fos = getGraphOutputStream(holder)) {
                    if (fos != null) {
                        graphWriter.outputGraph(builder.getGraph(), fos);
                    } else {
                        log.debug(Messages.getString("DEBUG.noGraphOutputStream"));
                    }
                }
                MetaverseUtil.delay();
            }
        }
    }

    /**
     * Returns the graph writer associated with this lineage writer object
     *
     * @return an IGraphWriter instance
     */
    public IGraphWriter getGraphWriter() {
        return graphWriter;
    }

    /**
     * Sets the graph writer associated with this lineage writer object
     *
     * @param graphWriter
     *          an IGraphWriter instance
     */
    public void setGraphWriter(IGraphWriter graphWriter) {
        this.graphWriter = graphWriter;
    }

    /**
     * Gets the output folder location for this writer
     *
     * @return a String folder location
     */
    public String getOutputFolder() {
        return outputFolder;
    }

    public static boolean isVFSPrefix(String prefix) {
        for (VFS_Prefixes vfs_prefix : VFS_Prefixes.values()) {
            if (vfs_prefix.name().equalsIgnoreCase(prefix)) {
                return true;
            }
        }
        return false;
    }

    /**
     * Sets the output folder for this writer
     *
     * @param outputFolder
     *          The String output folder to write to
     */
    public void setOutputFolder(String outputFolder) {
        int seperatorIndex = outputFolder.indexOf(":");
        if (seperatorIndex > -1) {
            String prefix = outputFolder.substring(0, outputFolder.indexOf(":"));
            if (isVFSPrefix(prefix)) {
                this.outputFolder = outputFolder;
            } else { // Prefix is not in VFS so try to make a local file from this
                File localFile = new File(outputFolder);
                try {
                    this.outputFolder = "file://" + localFile.getCanonicalPath();
                } catch (IOException e) {
                    log.error(Messages.getString("ERROR.CantUseOutputFile", outputFolder), e);
                }
            }
        } else { // Had no prefix so try to make a local file from this.
            File localFile = new File(outputFolder);
            try {
                this.outputFolder = "file://" + localFile.getCanonicalPath();
            } catch (IOException e) {
                log.error(Messages.getString("ERROR.CantUseOutputFile", outputFolder), e);
            }
        }
    }

    protected OutputStream createOutputStream(LineageHolder holder, String extension) {
        if (holder != null) {
            try {
                IExecutionProfile profile = holder.getExecutionProfile();
                String timestampString = Long.toString(profile.getExecutionData().getStartTime().getTime());
                FileObject destFolder = getOutputDirectoryAsFile(holder);
                String name = Const.NVL(profile.getName(), "unknown");
                FileObject file = destFolder.resolveFile(timestampString + "_" + name + extension);
                FileContent content = file.getContent();
                return content.getOutputStream();
            } catch (Exception e) {
                log.error(Messages.getErrorString("ERROR.CantCreateOutputStream"), e);
                return null;
            }
        } else {
            return null;
        }
    }

    protected FileObject getOutputDirectoryAsFile(LineageHolder holder) {
        try {
            FileObject dateRootFolder = getDateFolder(holder);
            dateRootFolder.createFolder();
            String id = holder.getId() == null ? "unknown_artifact" : holder.getId();
            if (id.startsWith(File.separator)) { // For *nix
                id = id.substring(1);
            } else if (Const.isWindows() && id.charAt(1) == ':') { // For windows
                id = id.replaceFirst(Pattern.quote(":"), "");
            }
            try {
                FileObject folder = dateRootFolder.resolveFile(id);
                folder.createFolder();
                if (folder.isFile()) {
                    // must be a folder
                    throw new IllegalStateException(
                            Messages.getErrorString("ERROR.OutputFolderWrongType", folder.getName().getPath()));
                }
                return folder;
            } catch (Exception e) {
                log.error(Messages.getErrorString("ERROR.CouldNotCreateFile"), e);
                return null;
            }
        } catch (Exception e) {
            log.error(Messages.getErrorString("ERROR.CouldNotCreateFile"), e);
            throw new IllegalStateException(e);
        }
    }

    protected FileObject getDateFolder(LineageHolder holder) throws KettleFileException, FileSystemException {
        String dir = "";
        if (holder != null && holder.getExecutionProfile() != null) {
            IExecutionProfile profile = holder.getExecutionProfile();
            dir += dateFolderFormat.format(profile.getExecutionData().getStartTime());
        } else {
            dir += dateFolderFormat.format(new Date());
        }
        FileObject lineageRootFolder = KettleVFS.getFileObject(getOutputFolder());
        FileObject dateFolder = lineageRootFolder.resolveFile(dir);
        return dateFolder;
    }

    protected OutputStream getProfileOutputStream(LineageHolder holder) {
        return createOutputStream(holder, ".execution.js");
    }

    protected OutputStream getGraphOutputStream(LineageHolder holder) {
        final String ext;
        if (graphWriter instanceof GraphMLWriter) {
            ext = ".graphml";
        } else if (graphWriter instanceof GraphSONWriter) {
            ext = ".graphson";
        } else {
            ext = ".txt";
        }
        return createOutputStream(holder, ext);
    }

    /**
     * Returns the output strategy (all, latest, none, etc.) as a string
     *
     * @return The String name of the output strategy
     */
    @Override
    public String getOutputStrategy() {
        return outputStrategy;
    }

    /**
     * Sets the output strategy (all, latest, none) for this writer
     *
     * @param strategy
     *          The strategy to use when outputting lineage information
     */
    @Override
    public void setOutputStrategy(String strategy) {
        this.outputStrategy = strategy;
    }

    /**
     * Method called on the writer to do any cleanup of the output artifacts, folders, etc.
     */
    @Override
    public void cleanOutput(LineageHolder holder) {
        String folderName = "unknown";
        try {
            FileObject folder = getOutputDirectoryAsFile(holder);
            folderName = folder.getName().getPath();
            folder.deleteAll();
        } catch (IOException ioe) {
            log.error(Messages.getErrorString("ERROR.CouldNotDeleteFile", folderName), ioe);
        }
    }

}