org.pentaho.di.job.entries.googledrive.JobEntryGoogleDriveExport.java Source code

Java tutorial

Introduction

Here is the source code for org.pentaho.di.job.entries.googledrive.JobEntryGoogleDriveExport.java

Source

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2016-2016 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.job.entries.googledrive;

import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;

import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.Selectors;
import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.annotations.JobEntry;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleDatabaseException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.Repository;
import org.pentaho.metastore.api.IMetaStore;
import org.w3c.dom.Node;

import com.google.api.services.drive.Drive;
import com.google.api.services.drive.model.File;

@JobEntry(id = "GoogleDriveExport", name = "GoogleDriveExport.Name", description = "GoogleDriveExport.TooltipDesc", image = "org/pentaho/di/job/entries/googledrive/product32.png", categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer", i18nPackageName = "org.pentaho.di.job.entries.googledrive", documentationUrl = "GoogleDriveExport.DocumentationURL", casesUrl = "GoogleDriveExport.CasesURL", forumUrl = "GoogleDriveExport.ForumURL")
public class JobEntryGoogleDriveExport extends JobEntryAbstractGoogleDrive {

    public JobEntryGoogleDriveExport() {
        super();
    }

    protected GoogleDriveExportFileSelection[] fileSelections;
    protected String targetFolder;
    protected boolean createTargetFolder;
    protected boolean addFilesToResult;

    @Override
    public Result execute(Result prev_result, int nr) throws KettleException {
        Result result = prev_result;

        checkFolderExists(KettleVFS.getFileObject(environmentSubstitute(getTargetFolder())), createTargetFolder);

        List<String> activeScope = SCOPE_READONLY;
        for (int i = 0; i < fileSelections.length; i++) {
            if (fileSelections[i].isDeleteSourceFile()) {
                activeScope = SCOPE_READWRITE;
            }
        }
        if (log.isDebug()) {
            if (activeScope == SCOPE_READWRITE) {
                log.logDebug(BaseMessages.getString(PKG, "GoogleDriveExport.Debug.ReadWriteApiPermissions"));
            } else {
                log.logDebug(BaseMessages.getString(PKG, "GoogleDriveExport.Debug.ReadOnlyApiPermissions"));
            }
        }

        Drive driveService = getDriveService(activeScope);

        for (int i = 0; i < fileSelections.length; i++) {
            GoogleDriveExportFileSelection fileSelection = fileSelections[i];
            List<File> driveFiles;
            String queryOptions = fileSelection.getFinalQueryOptions();
            if (log.isDebug()) {
                log.logDebug(BaseMessages.getString(PKG, "GoogleDriveExport.Debug.QueryInfo", queryOptions));
            }
            try {
                driveFiles = searchFiles(driveService, queryOptions);
            } catch (IOException e) {
                log.logError(BaseMessages.getString(PKG, "GoogleDriveExport.Error.BadQuery"), e);
                result.setResult(false);
                result.increaseErrors(1);
                return result;
            }

            if (log.isDetailed()) {
                int fileCount = driveFiles == null ? 0 : driveFiles.size();
                log.logDetailed(BaseMessages.getString(PKG, "GoogleDriveExport.Log.DownloadFileCount", fileCount));
                if (log.isDebug()) {
                    for (File driveFile : driveFiles) {
                        log.logDebug(BaseMessages.getString(PKG, "GoogleDriveExport.Debug.DownloadFileNames",
                                driveFile.getName()));
                    }
                }
            }
            String realTargetFolder = KettleVFS.getFileObject(environmentSubstitute(getTargetFolder()))
                    .getPublicURIString();
            for (File driveFile : driveFiles) {
                if (fileSelection.getExportFormat() != null) {
                    String exportFilename = realTargetFolder + "/" + driveFile.getName() + "."
                            + fileSelection.getExportFormat().getFileExtension();
                    FileObject exportFile = KettleVFS.getFileObject(exportFilename);
                    if (log.isDetailed()) {
                        log.logDetailed(BaseMessages.getString(PKG, "GoogleDriveExport.Log.DownloadingFile",
                                driveFile.getName()));
                    }
                    try {
                        exportFile(driveService, driveFile, exportFile, fileSelection.getExportFormat());
                    } catch (KettleException e) {
                        log.logError(BaseMessages.getString(PKG, "GoogleDriveExport.Error.ExportFile",
                                driveFile.getName()), e);
                        result.setResult(false);
                        result.increaseErrors(1);
                        return result;
                    }

                    if (fileSelection.isDeleteSourceFile()) {
                        if (log.isDetailed()) {
                            log.logDetailed(BaseMessages.getString(PKG, "GoogleDriveExport.Log.DeletingSourceFile",
                                    driveFile.getName()));
                        }
                        try {
                            deleteFile(driveService, driveFile);
                        } catch (IOException e) {
                            log.logError(BaseMessages.getString(PKG, "GoogleDriveExport.Error.DeletingSourceFile",
                                    driveFile.getName()), e);
                            result.setResult(false);
                            result.increaseErrors(1);
                            return result;
                        }
                    }
                }
            }
        }
        return result;
    }

    protected static void deleteFile(Drive driveService, File driveFile) throws IOException {
        driveService.files().delete(driveFile.getId()).execute();
    }

    protected static List<File> searchFiles(Drive driveService, String queryOptions) throws IOException {
        return driveService.files().list().setQ(queryOptions).execute().getFiles();
    }

    protected static void exportFile(Drive driveService, File driveFile, FileObject targetFile,
            GoogleDriveExportFormat exportMapping) throws KettleException {
        Exception savedException = null;
        if (exportMapping != null) {
            FileObject tempFile = KettleVFS.createTempFile(JobEntryGoogleDriveExport.class.getSimpleName(), ".tmp",
                    System.getProperty("java.io.tmpdir"));
            try {
                OutputStream fos = tempFile.getContent().getOutputStream();
                BufferedOutputStream bos = new BufferedOutputStream(fos);
                try {
                    driveService.files().export(driveFile.getId(), exportMapping.getMimeType())
                            .executeMediaAndDownloadTo(bos);
                } catch (IOException e) {
                    // Throw this later, we want to close the output stream first
                    savedException = new KettleException(
                            BaseMessages.getString(PKG, "GoogleDriveExport.Error.ExportingFile"), e);
                }
                try {
                    bos.close();
                } catch (IOException ignore) {
                    // Ignore
                }
                try {
                    fos.close();
                } catch (IOException ignore) {
                    // Ignore
                }
            } catch (IOException e) {
                savedException = new KettleException(
                        BaseMessages.getString(PKG, "GoogleDriveExport.Error.ExportingFile"), e);
            }
            if (tempFile != null) {
                try {
                    targetFile.copyFrom(tempFile, Selectors.SELECT_SELF);
                } catch (FileSystemException e) {
                    savedException = new KettleException(
                            BaseMessages.getString(PKG, "GoogleDriveExport.Error.MovingFileFromTemp"), e);
                }
            }
            if (savedException != null) {
                try {
                    if (targetFile.exists()) {
                        targetFile.delete();
                    }
                } catch (FileSystemException ignore) {
                    // Ignore, couldn't delete a bad output file
                }
                throw new KettleException(savedException);
            }
        }
    }

    @Override
    public String getXML() {
        StringBuilder retval = new StringBuilder();

        retval.append(super.getXML());
        retval.append("      ").append(XMLHandler.addTagValue("addFilesToResult", isAddFilesToResult()));
        retval.append("      ").append(XMLHandler.addTagValue("createTargetFolder", isCreateTargetFolder()));
        retval.append("      ").append(XMLHandler.addTagValue("targetFolder", getTargetFolder()));
        retval.append("      ").append(XMLHandler.openTag("files")).append(Const.CR);
        for (int i = 0; i < fileSelections.length; i++) {
            retval.append("        ").append(XMLHandler.openTag("file")).append(Const.CR);
            retval.append("          ")
                    .append(XMLHandler.addTagValue("queryFileType", fileSelections[i].getFileType().name()));
            retval.append("          ")
                    .append(XMLHandler.addTagValue("exportFileType", fileSelections[i].getExportFormat().name()));
            retval.append("          ")
                    .append(XMLHandler.addTagValue("customQuery", fileSelections[i].getQueryOptions()));
            retval.append("          ")
                    .append(XMLHandler.addTagValue("removeSourceFiles", fileSelections[i].isDeleteSourceFile()));
            retval.append("        ").append(XMLHandler.closeTag("file")).append(Const.CR);
        }
        retval.append("      ").append(XMLHandler.closeTag("files")).append(Const.CR);

        return retval.toString();
    }

    @Override
    public void loadXML(Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers,
            Repository rep, IMetaStore metaStore) throws KettleXMLException {
        try {
            super.loadXML(entrynode, databases, slaveServers, rep, metaStore);
            setAddFilesToResult("Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "addFilesToResult")));
            setCreateTargetFolder("Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "createTargetFolder")));
            setTargetFolder(XMLHandler.getTagValue(entrynode, "targetFolder"));
            Node filesNode = XMLHandler.getSubNode(entrynode, "files");
            if (filesNode != null) {
                int fileCount = XMLHandler.countNodes(filesNode, "file");
                fileSelections = new GoogleDriveExportFileSelection[fileCount];

                for (int i = 0; i < fileCount; i++) {
                    Node fileNode = XMLHandler.getSubNodeByNr(filesNode, "file", i);
                    GoogleDriveFileType fileType = Enum.valueOf(GoogleDriveFileType.class,
                            XMLHandler.getTagValue(fileNode, "queryFileType"));
                    GoogleDriveExportFormat exportFormat = Enum.valueOf(GoogleDriveExportFormat.class,
                            XMLHandler.getTagValue(fileNode, "exportFileType"));
                    String queryOptions = XMLHandler.getTagValue(fileNode, "customQuery");
                    boolean deleteSource = "Y"
                            .equalsIgnoreCase(XMLHandler.getTagValue(fileNode, "removeSourceFiles"));

                    fileSelections[i] = new GoogleDriveExportFileSelection(fileType, exportFormat, queryOptions,
                            deleteSource);
                }
            }
        } catch (Exception e) {
            throw new KettleXMLException(BaseMessages.getString(PKG, "Demo.Error.UnableToLoadFromXML"), e);
        }
    }

    @Override
    public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_job) throws KettleException {
        super.saveRep(rep, metaStore, id_job);
        try {
            rep.saveJobEntryAttribute(id_job, getObjectId(), "addFilesToResult", isAddFilesToResult());
            rep.saveJobEntryAttribute(id_job, getObjectId(), "createTargetFolder", isCreateTargetFolder());
            rep.saveJobEntryAttribute(id_job, getObjectId(), "targetFolder", getTargetFolder());
            rep.saveJobEntryAttribute(id_job, getObjectId(), "fileSelectionCount",
                    fileSelections != null ? fileSelections.length : 0);
            if (fileSelections != null) {
                for (int i = 0; i < fileSelections.length; i++) {
                    rep.saveJobEntryAttribute(id_job, getObjectId(), i, "queryFileType",
                            fileSelections[i].getFileType().name());
                    rep.saveJobEntryAttribute(id_job, getObjectId(), i, "exportFileType",
                            fileSelections[i].getExportFormat().name());
                    rep.saveJobEntryAttribute(id_job, getObjectId(), i, "customQuery",
                            fileSelections[i].getQueryOptions());
                    rep.saveJobEntryAttribute(id_job, getObjectId(), i, "removeSourceFiles",
                            fileSelections[i].isDeleteSourceFile());
                }
            }
        } catch (KettleDatabaseException dbe) {
            throw new KettleException(BaseMessages.getString(PKG, "Demo.Error.UnableToSaveToRepository") + id_job,
                    dbe);
        }
    }

    @Override
    public void loadRep(Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases,
            List<SlaveServer> slaveServers) throws KettleException {
        super.loadRep(rep, metaStore, id_jobentry, databases, slaveServers);
        try {
            setAddFilesToResult(rep.getJobEntryAttributeBoolean(id_jobentry, "addFilesToResult"));
            setCreateTargetFolder(rep.getJobEntryAttributeBoolean(id_jobentry, "createTargetFolder"));
            setTargetFolder(rep.getJobEntryAttributeString(id_jobentry, "targetFolder"));
            long fileCount = rep.getJobEntryAttributeInteger(id_jobentry, "fileSelectionCount");
            fileSelections = new GoogleDriveExportFileSelection[(int) fileCount];
            for (int i = 0; i < fileCount; i++) {
                GoogleDriveFileType fileType = Enum.valueOf(GoogleDriveFileType.class,
                        rep.getJobEntryAttributeString(id_jobentry, i, "queryFileType"));
                GoogleDriveExportFormat exportFormat = Enum.valueOf(GoogleDriveExportFormat.class,
                        rep.getJobEntryAttributeString(id_jobentry, i, "exportFileType"));
                String queryOptions = rep.getJobEntryAttributeString(id_jobentry, i, "customQuery");
                boolean deleteSource = rep.getJobEntryAttributeBoolean(id_jobentry, i, "removeSourceFiles");
                fileSelections[i] = new GoogleDriveExportFileSelection(fileType, exportFormat, queryOptions,
                        deleteSource);
            }
        } catch (KettleDatabaseException dbe) {
            throw new KettleException(
                    BaseMessages.getString(PKG, "Demo.Error.UnableToLoadFromRepository") + id_jobentry, dbe);
        }
    }

    @Override
    public String getDialogClassName() {
        return JobEntryGoogleDriveExportDialog.class.getName();
    }

    @Override
    public boolean evaluates() {
        return true;
    }

    @Override
    public boolean isUnconditional() {
        return false;
    }

    protected void checkFolderExists(FileObject folder, boolean createIfNot) throws KettleException {
        if (folder == null) {
            return;
        }
        try {
            if (!folder.exists() && createIfNot) {
                if (log.isDetailed()) {
                    log.logDetailed(BaseMessages.getString(PKG, "GoogleDriveExport.Log.CreatingTargetFolder"));
                }
                folder.createFolder();
                return;
            } else {
                if (!folder.exists()) {
                    throw new KettleException(BaseMessages.getString(PKG, "GoogleDriveExport.Error.FolderNotExist",
                            folder.getName()));
                } else if (!folder.isFolder()) {
                    throw new KettleException(
                            BaseMessages.getString(PKG, "GoogleDriveExport.Error.NotAFolder", folder.getName()));
                }
            }
        } catch (FileSystemException e) {
            throw new KettleException(e);
        }
    }

    public String getTargetFolder() {
        return targetFolder;
    }

    public void setTargetFolder(String targetFolder) {
        this.targetFolder = targetFolder;
    }

    public boolean isCreateTargetFolder() {
        return createTargetFolder;
    }

    public void setCreateTargetFolder(boolean createTargetFolder) {
        this.createTargetFolder = createTargetFolder;
    }

    public GoogleDriveExportFileSelection[] getFileSelections() {
        return fileSelections;
    }

    public void setFileSelections(GoogleDriveExportFileSelection[] fileSelections) {
        this.fileSelections = fileSelections;
    }

    public boolean isAddFilesToResult() {
        return addFilesToResult;
    }

    public void setAddFilesToResult(boolean addFilesToResult) {
        this.addFilesToResult = addFilesToResult;
    }
}