com.panet.imeta.job.entries.deletefiles.JobEntryDeleteFiles.java Source code

Java tutorial

Introduction

Here is the source code for com.panet.imeta.job.entries.deletefiles.JobEntryDeleteFiles.java

Source

/* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/

package com.panet.imeta.job.entries.deletefiles;

import static com.panet.imeta.job.entry.validator.AbstractFileValidator.putVariableSpace;
import static com.panet.imeta.job.entry.validator.AndValidator.putValidators;
import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.andValidator;
import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.fileExistsValidator;
import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.notNullValidator;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSelectInfo;
import org.apache.commons.vfs.FileSelector;
import org.apache.commons.vfs.FileType;
import org.w3c.dom.Node;

import com.panet.imeta.cluster.SlaveServer;
import com.panet.imeta.core.CheckResultInterface;
import com.panet.imeta.core.Const;
import com.panet.imeta.core.Result;
import com.panet.imeta.core.RowMetaAndData;
import com.panet.imeta.core.database.DatabaseMeta;
import com.panet.imeta.core.exception.KettleDatabaseException;
import com.panet.imeta.core.exception.KettleException;
import com.panet.imeta.core.exception.KettleXMLException;
import com.panet.imeta.core.logging.LogWriter;
import com.panet.imeta.core.vfs.KettleVFS;
import com.panet.imeta.core.xml.XMLHandler;
import com.panet.imeta.job.Job;
import com.panet.imeta.job.JobEntryType;
import com.panet.imeta.job.JobMeta;
import com.panet.imeta.job.entry.JobEntryBase;
import com.panet.imeta.job.entry.JobEntryInterface;
import com.panet.imeta.job.entry.validator.ValidatorContext;
import com.panet.imeta.repository.Repository;
import com.panet.imeta.resource.ResourceEntry;
import com.panet.imeta.resource.ResourceReference;
import com.panet.imeta.resource.ResourceEntry.ResourceType;
import com.panet.imeta.shared.SharedObjectInterface;
import com.panet.imeta.trans.step.BaseStepMeta;

/**
 * This defines a 'delete files' job entry.
 * 
 * @author Samatar Hassan
 * @since 06-05-2007
 */
public class JobEntryDeleteFiles extends JobEntryBase implements Cloneable, JobEntryInterface {

    public boolean argFromPrevious;

    public boolean includeSubfolders;

    public String arguments[];

    public String filemasks[];

    public JobEntryDeleteFiles(String n) {
        super(n, ""); //$NON-NLS-1$
        argFromPrevious = false;
        arguments = null;

        includeSubfolders = false;
        setID(-1L);
        setJobEntryType(JobEntryType.DELETE_FILES);
    }

    public JobEntryDeleteFiles() {
        this(""); //$NON-NLS-1$
    }

    public JobEntryDeleteFiles(JobEntryBase jeb) {
        super(jeb);
    }

    public Object clone() {
        JobEntryDeleteFiles je = (JobEntryDeleteFiles) super.clone();
        return je;
    }

    public String getXML() {
        StringBuffer retval = new StringBuffer(300);

        retval.append(super.getXML());
        retval.append("      ").append(XMLHandler.addTagValue("arg_from_previous", argFromPrevious)); //$NON-NLS-1$ //$NON-NLS-2$
        retval.append("      ").append(XMLHandler.addTagValue("include_subfolders", includeSubfolders)); //$NON-NLS-1$ //$NON-NLS-2$

        retval.append("      <fields>").append(Const.CR); //$NON-NLS-1$
        if (arguments != null) {
            for (int i = 0; i < arguments.length; i++) {
                retval.append("        <field>").append(Const.CR); //$NON-NLS-1$
                retval.append("          ").append(XMLHandler.addTagValue("name", arguments[i])); //$NON-NLS-1$ //$NON-NLS-2$
                retval.append("          ").append(XMLHandler.addTagValue("filemask", filemasks[i])); //$NON-NLS-1$ //$NON-NLS-2$
                retval.append("        </field>").append(Const.CR); //$NON-NLS-1$
            }
        }
        retval.append("      </fields>").append(Const.CR); //$NON-NLS-1$

        return retval.toString();
    }

    public void loadXML(Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers,
            Repository rep) throws KettleXMLException {
        try {
            super.loadXML(entrynode, databases, slaveServers);
            argFromPrevious = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "arg_from_previous")); //$NON-NLS-1$ //$NON-NLS-2$
            includeSubfolders = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "include_subfolders")); //$NON-NLS-1$ //$NON-NLS-2$

            Node fields = XMLHandler.getSubNode(entrynode, "fields"); //$NON-NLS-1$

            // How many field arguments?
            int nrFields = XMLHandler.countNodes(fields, "field"); //$NON-NLS-1$
            arguments = new String[nrFields];
            filemasks = new String[nrFields];

            // Read them all...
            for (int i = 0; i < nrFields; i++) {
                Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i); //$NON-NLS-1$

                arguments[i] = XMLHandler.getTagValue(fnode, "name"); //$NON-NLS-1$
                filemasks[i] = XMLHandler.getTagValue(fnode, "filemask"); //$NON-NLS-1$
            }
        } catch (KettleXMLException xe) {
            throw new KettleXMLException(Messages.getString("JobEntryDeleteFiles.UnableToLoadFromXml"), xe); //$NON-NLS-1$
        }
    }

    public void loadRep(Repository rep, long id_jobentry, List<DatabaseMeta> databases,
            List<SlaveServer> slaveServers) throws KettleException {
        try {
            super.loadRep(rep, id_jobentry, databases, slaveServers);
            argFromPrevious = rep.getJobEntryAttributeBoolean(id_jobentry, "arg_from_previous"); //$NON-NLS-1$
            includeSubfolders = rep.getJobEntryAttributeBoolean(id_jobentry, "include_subfolders"); //$NON-NLS-1$

            // How many arguments?
            int argnr = rep.countNrJobEntryAttributes(id_jobentry, "name"); //$NON-NLS-1$
            arguments = new String[argnr];
            filemasks = new String[argnr];

            // Read them all...
            for (int a = 0; a < argnr; a++) {
                arguments[a] = rep.getJobEntryAttributeString(id_jobentry, a, "name"); //$NON-NLS-1$
                filemasks[a] = rep.getJobEntryAttributeString(id_jobentry, a, "filemask"); //$NON-NLS-1$
            }
        } catch (KettleException dbe) {
            throw new KettleException(
                    Messages.getString("JobEntryDeleteFiles.UnableToLoadFromRepo", String.valueOf(id_jobentry)), //$NON-NLS-1$
                    dbe);
        }
    }

    public void saveRep(Repository rep, long id_job) throws KettleException {
        try {
            super.saveRep(rep, id_job);

            rep.saveJobEntryAttribute(id_job, getID(), "arg_from_previous", argFromPrevious); //$NON-NLS-1$
            rep.saveJobEntryAttribute(id_job, getID(), "include_subfolders", includeSubfolders); //$NON-NLS-1$

            // save the arguments...
            if (arguments != null) {
                for (int i = 0; i < arguments.length; i++) {
                    rep.saveJobEntryAttribute(id_job, getID(), i, "name", arguments[i]); //$NON-NLS-1$
                    rep.saveJobEntryAttribute(id_job, getID(), i, "filemask", filemasks[i]); //$NON-NLS-1$
                }
            }
        } catch (KettleDatabaseException dbe) {
            throw new KettleException(
                    Messages.getString("JobEntryDeleteFiles.UnableToSaveToRepo", String.valueOf(id_job)), dbe); //$NON-NLS-1$
        }
    }

    public Result execute(Result result, int nr, Repository rep, Job parentJob) throws KettleException {
        LogWriter log = LogWriter.getInstance();

        List<RowMetaAndData> rows = result.getRows();
        RowMetaAndData resultRow = null;

        int NrErrFiles = 0;

        result.setResult(false);
        result.setNrErrors(1);

        if (argFromPrevious) {
            if (log.isDetailed())
                log.logDetailed(toString(), Messages.getString("JobEntryDeleteFiles.FoundPreviousRows", //$NON-NLS-1$
                        String.valueOf((rows != null ? rows.size() : 0))));
        }

        if (argFromPrevious && rows != null) // Copy the input row to the
        // (command line) arguments
        {
            for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
                resultRow = rows.get(iteration);

                String args_previous = resultRow.getString(0, null);
                String fmasks_previous = resultRow.getString(1, null);

                // ok we can process this file/folder
                if (log.isDetailed())
                    log.logDetailed(toString(), Messages.getString("JobEntryDeleteFiles.ProcessingRow", //$NON-NLS-1$
                            args_previous, fmasks_previous));

                if (!ProcessFile(args_previous, fmasks_previous, parentJob)) {
                    NrErrFiles++;
                }
            }
        } else if (arguments != null) {

            for (int i = 0; i < arguments.length && !parentJob.isStopped(); i++) {

                // ok we can process this file/folder
                if (log.isDetailed())
                    log.logDetailed(toString(),
                            Messages.getString("JobEntryDeleteFiles.ProcessingArg", arguments[i], filemasks[i])); //$NON-NLS-1$
                if (!ProcessFile(arguments[i], filemasks[i], parentJob)) {
                    NrErrFiles++;
                }
            }
        }

        if (NrErrFiles == 0) {
            result.setResult(true);
            result.setNrErrors(0);
        } else {
            result.setNrErrors(NrErrFiles);
            result.setResult(false);
        }

        return result;
    }

    private boolean ProcessFile(String filename, String wildcard, Job parentJob) {
        LogWriter log = LogWriter.getInstance();

        boolean rcode = false;
        FileObject filefolder = null;
        String realFilefoldername = environmentSubstitute(filename);
        String realwildcard = environmentSubstitute(wildcard);

        try {
            filefolder = KettleVFS.getFileObject(realFilefoldername);

            // Here gc() is explicitly called if e.g. createfile is used in the
            // same
            // job for the same file. The problem is that after creating the
            // file the
            // file object is not properly garbaged collected and thus the file
            // cannot
            // be deleted anymore. This is a known problem in the JVM.

            System.gc();

            if (filefolder.exists()) {
                // the file or folder exists
                if (filefolder.getType() == FileType.FOLDER) {
                    // It's a folder
                    if (log.isDetailed())
                        log.logDetailed(toString(),
                                Messages.getString("JobEntryDeleteFiles.ProcessingFolder", realFilefoldername)); //$NON-NLS-1$
                    // Delete Files

                    int Nr = filefolder
                            .delete(new TextFileSelector(filefolder.toString(), realwildcard, parentJob));

                    if (log.isDetailed())
                        log.logDetailed(toString(),
                                Messages.getString("JobEntryDeleteFiles.TotalDeleted", String.valueOf(Nr))); //$NON-NLS-1$
                    rcode = true;
                } else {
                    // It's a file
                    if (log.isDetailed())
                        log.logDetailed(toString(),
                                Messages.getString("JobEntryDeleteFiles.ProcessingFile", realFilefoldername)); //$NON-NLS-1$
                    boolean deleted = filefolder.delete();
                    if (!deleted) {
                        log.logError(toString(),
                                Messages.getString("JobEntryDeleteFiles.CouldNotDeleteFile", realFilefoldername)); //$NON-NLS-1$
                    } else {
                        if (log.isBasic())
                            log.logBasic(toString(),
                                    Messages.getString("JobEntryDeleteFiles.FileDeleted", filename)); //$NON-NLS-1$
                        rcode = true;
                    }
                }
            } else {
                // File already deleted, no reason to try to delete it
                if (log.isBasic())
                    log.logBasic(toString(),
                            Messages.getString("JobEntryDeleteFiles.FileAlreadyDeleted", realFilefoldername)); //$NON-NLS-1$
                rcode = true;
            }
        } catch (IOException e) {
            log.logError(toString(),
                    Messages.getString("JobEntryDeleteFiles.CouldNotProcess", realFilefoldername, e.getMessage())); //$NON-NLS-1$
        } finally {
            if (filefolder != null) {
                try {
                    filefolder.close();
                } catch (IOException ex) {
                }
                ;
            }
        }

        return rcode;
    }

    private class TextFileSelector implements FileSelector {
        LogWriter log = LogWriter.getInstance();
        String file_wildcard = null, source_folder = null;
        Job parentjob;

        public TextFileSelector(String sourcefolderin, String filewildcard, Job parentJob) {

            if (!Const.isEmpty(sourcefolderin)) {
                source_folder = sourcefolderin;
            }

            if (!Const.isEmpty(filewildcard)) {
                file_wildcard = filewildcard;
            }
            parentjob = parentJob;
        }

        public boolean includeFile(FileSelectInfo info) {
            boolean returncode = false;
            FileObject file_name = null;
            try {

                if (!info.getFile().toString().equals(source_folder) && !parentjob.isStopped()) {
                    // Pass over the Base folder itself

                    String short_filename = info.getFile().getName().getBaseName();

                    if (!info.getFile().getParent().equals(info.getBaseFolder())) {

                        // Not in the Base Folder..Only if include sub folders
                        if (includeSubfolders && (info.getFile().getType() == FileType.FILE)
                                && GetFileWildcard(short_filename, file_wildcard)) {
                            if (log.isDetailed())
                                log.logDetailed(toString(), Messages.getString("JobEntryDeleteFiles.DeletingFile", //$NON-NLS-1$
                                        info.getFile().toString()));

                            returncode = true;

                        }
                    } else {
                        // In the Base Folder...

                        if ((info.getFile().getType() == FileType.FILE)
                                && GetFileWildcard(short_filename, file_wildcard)) {
                            if (log.isDetailed())
                                log.logDetailed(toString(), Messages.getString("JobEntryDeleteFiles.DeletingFile", //$NON-NLS-1$
                                        info.getFile().toString()));

                            returncode = true;

                        }

                    }

                }

            } catch (Exception e) {

                log.logError(Messages.getString("JobDeleteFiles.Error.Exception.DeleteProcessError"),
                        Messages.getString("JobDeleteFiles.Error.Exception.DeleteProcess",
                                info.getFile().toString(), e.getMessage()));

                returncode = false;
            } finally {
                if (file_name != null) {
                    try {
                        file_name.close();

                    } catch (IOException ex) {
                    }
                    ;
                }

            }

            return returncode;
        }

        public boolean traverseDescendents(FileSelectInfo info) {
            return true;
        }
    }

    /***************************************************************************
     * 
     * @param selectedfile
     * @param wildcard
     * @return True if the selectedfile matches the wildcard
     **************************************************************************/
    private boolean GetFileWildcard(String selectedfile, String wildcard) {
        Pattern pattern = null;
        boolean getIt = true;

        if (!Const.isEmpty(wildcard)) {
            pattern = Pattern.compile(wildcard);
            // First see if the file matches the regular expression!
            if (pattern != null) {
                Matcher matcher = pattern.matcher(selectedfile);
                getIt = matcher.matches();
            }
        }

        return getIt;
    }

    public void setIncludeSubfolders(boolean includeSubfolders) {
        this.includeSubfolders = includeSubfolders;
    }

    public void setPrevious(boolean argFromPrevious) {
        this.argFromPrevious = argFromPrevious;
    }

    public boolean evaluates() {
        return true;
    }

    public void check(List<CheckResultInterface> remarks, JobMeta jobMeta) {
        boolean res = andValidator().validate(this, "arguments", remarks, putValidators(notNullValidator())); //$NON-NLS-1$

        if (res == false) {
            return;
        }

        ValidatorContext ctx = new ValidatorContext();
        putVariableSpace(ctx, getVariables());
        putValidators(ctx, notNullValidator(), fileExistsValidator());

        for (int i = 0; i < arguments.length; i++) {
            andValidator().validate(this, "arguments[" + i + "]", remarks, ctx); //$NON-NLS-1$ //$NON-NLS-2$
        }
    }

    public List<ResourceReference> getResourceDependencies(JobMeta jobMeta) {
        List<ResourceReference> references = super.getResourceDependencies(jobMeta);
        if (arguments != null) {
            ResourceReference reference = null;
            for (int i = 0; i < arguments.length; i++) {
                String filename = jobMeta.environmentSubstitute(arguments[i]);
                if (reference == null) {
                    reference = new ResourceReference(this);
                    references.add(reference);
                }
                reference.getEntries().add(new ResourceEntry(filename, ResourceType.FILE));
            }
        }
        return references;
    }

    public boolean isArgFromPrevious() {
        return argFromPrevious;
    }

    public String[] getArguments() {
        return arguments;
    }

    public String[] getFilemasks() {
        return filemasks;
    }

    public boolean isIncludeSubfolders() {
        return includeSubfolders;
    }

    @Override
    public void setInfo(Map<String, String[]> p, String id, List<? extends SharedObjectInterface> databases) {
        // TODO Auto-generated method stub
        this.argFromPrevious = BaseStepMeta.parameterToBoolean(p.get(id + ".argFromPrevious"));
        this.includeSubfolders = BaseStepMeta.parameterToBoolean(p.get(id + ".includeSubfolders"));
        String[] fileFolder = p.get(id + "_fileFolderTable.fileFolder");
        String[] wildcard = p.get(id + "_fileFolderTable.wildcard");
        this.arguments = fileFolder;
        this.filemasks = wildcard;
    }

}