com.zimbra.cs.datasource.DataSourceManager.java Source code

Java tutorial

Introduction

Here is the source code for com.zimbra.cs.datasource.DataSourceManager.java

Source

/*
 * ***** BEGIN LICENSE BLOCK *****
 * Zimbra Collaboration Suite Server
 * Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 Synacor, Inc.
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software Foundation,
 * version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License along with this program.
 * If not, see <https://www.gnu.org/licenses/>.
 * ***** END LICENSE BLOCK *****
 */
package com.zimbra.cs.datasource;

import static com.zimbra.common.util.TaskUtil.newDaemonThreadFactory;
import static java.util.Collections.newSetFromMap;
import static java.util.concurrent.Executors.newCachedThreadPool;

import java.io.File;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;

import javax.mail.MessagingException;
import javax.mail.Session;
import javax.mail.Transport;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.PostMethod;
import org.json.JSONObject;

import com.sun.mail.smtp.SMTPTransport;
import com.zimbra.common.account.ZAttrProvisioning.AccountStatus;
import com.zimbra.common.localconfig.LC;
import com.zimbra.common.service.ServiceException;
import com.zimbra.common.util.Pair;
import com.zimbra.common.util.StringUtil;
import com.zimbra.common.util.ZimbraHttpConnectionManager;
import com.zimbra.common.util.ZimbraLog;
import com.zimbra.cs.account.Account;
import com.zimbra.cs.account.DataSource;
import com.zimbra.cs.account.DataSource.DataImport;
import com.zimbra.cs.account.DataSourceConfig;
import com.zimbra.cs.account.Provisioning;
import com.zimbra.cs.datasource.imap.ImapSync;
import com.zimbra.cs.db.DbPool;
import com.zimbra.cs.db.DbPool.DbConnection;
import com.zimbra.cs.db.DbScheduledTask;
import com.zimbra.cs.extension.ExtensionUtil;
import com.zimbra.cs.gal.GalImport;
import com.zimbra.cs.ldap.LdapDateUtil;
import com.zimbra.cs.mailbox.Folder;
import com.zimbra.cs.mailbox.Mailbox;
import com.zimbra.cs.mailbox.MailboxManager;
import com.zimbra.cs.mailbox.ScheduledTaskManager;
import com.zimbra.cs.mailclient.smtp.SmtpTransport;
import com.zimbra.cs.util.JMSession;
import com.zimbra.cs.util.Zimbra;
import com.zimbra.soap.admin.type.DataSourceType;

public class DataSourceManager {

    private static DataSourceManager sInstance;

    public static final String CLIENT_ID = "client_id";
    public static final String CLIENT_SECRET = "client_secret";
    public static final String GRANT_TYPE = "grant_type";
    public static final String REFRESH_TOKEN = "refresh_token";
    public static final String ACCESS_TOKEN = "access_token";

    // accountId -> dataSourceId -> ImportStatus
    private static final Map<String, Map<String, ImportStatus>> sImportStatus = new HashMap<String, Map<String, ImportStatus>>();

    // Bug: 40799
    // Methods to keep track of managed data sources so we can easily detect
    // when a data source has been removed while syncing
    private static final Set<Object> sManagedDataSources = newConcurrentHashSet();

    private final DataSourceConfig config;

    private static final ExecutorService executor = newCachedThreadPool(newDaemonThreadFactory("ImportData"));

    private static <E> Set<E> newConcurrentHashSet() {
        return newSetFromMap(new ConcurrentHashMap<E, Boolean>());
    }

    private static Object key(String accountId, String dataSourceId) {
        return new Pair<String, String>(accountId, dataSourceId);
    }

    public static void addManaged(DataSource ds) {
        sManagedDataSources.add(key(ds.getAccountId(), ds.getId()));
    }

    public static void deleteManaged(String accountId, String dataSourceId) {
        sManagedDataSources.remove(key(accountId, dataSourceId));
    }

    public static boolean isManaged(DataSource ds) {
        return sManagedDataSources.contains(key(ds.getAccountId(), ds.getId()));
    }

    public DataSourceManager() {
        this.config = loadConfig();
    }

    private DataSourceConfig loadConfig() {
        try {
            File file = new File(LC.data_source_config.value());
            DataSourceConfig config = DataSourceConfig.read(file);
            ZimbraLog.datasource.debug("Loaded datasource configuration from '%s'", file);

            for (DataSourceConfig.Service service : config.getServices()) {
                ZimbraLog.datasource.debug("Loaded %d folder mappings for service '%s'",
                        service.getFolders().size(), service.getName());
            }
            return config;
        } catch (Exception e) {
            Zimbra.halt("Unable to load datasource config", e);
            return null;
        }
    }

    /**
     * @param ds not used
     * @param folder not used
     */
    public boolean isSyncCapable(DataSource ds, Folder folder) {
        return true;
    }

    /**
     * @param ds not used
     * @param folder not used
     */
    public boolean isSyncEnabled(DataSource ds, Folder folder) {
        return true;
    }

    public synchronized static DataSourceManager getInstance() {
        if (sInstance == null) {
            String className = LC.zimbra_class_datasourcemanager.value();
            if (!StringUtil.isNullOrEmpty(className)) {
                try {
                    try {
                        sInstance = (DataSourceManager) Class.forName(className).newInstance();
                    } catch (ClassNotFoundException cnfe) {
                        // ignore and look in extensions
                        sInstance = (DataSourceManager) ExtensionUtil.findClass(className).newInstance();
                    }
                } catch (Exception e) {
                    ZimbraLog.system.error("Unable to initialize %s.", className, e);
                }
            }
            if (sInstance == null) {
                sInstance = new DataSourceManager();
                ZimbraLog.datasource.info("Initialized %s.", sInstance.getClass().getName());
            }
        }

        return sInstance;
    }

    public static DataSourceConfig getConfig() {
        return getInstance().config;
    }

    public Mailbox getMailbox(DataSource ds) throws ServiceException {
        return MailboxManager.getInstance().getMailboxByAccount(ds.getAccount());
    }

    public DataImport getDataImport(DataSource ds) throws ServiceException {
        return getDataImport(ds, false);
    }

    public DataImport getDataImport(DataSource ds, boolean test) throws ServiceException {
        switch (ds.getType()) {
        case pop3:
            return new Pop3Sync(ds);
        case imap:
            return new ImapSync(ds, test);
        case caldav:
            return new CalDavDataImport(ds);
        case rss:
        case cal:
            return new RssImport(ds);
        case gal:
            return new GalImport(ds);
        case xsync:
            try {
                String className = LC.data_source_xsync_class.value();
                if (className != null && className.length() > 0) {
                    Class<?> cmdClass;
                    try {
                        cmdClass = Class.forName(className);
                    } catch (ClassNotFoundException x) {
                        cmdClass = ExtensionUtil.findClass(className);
                    }
                    Constructor<?> constructor = cmdClass.getConstructor(new Class[] { DataSource.class });
                    return (DataImport) constructor.newInstance(ds);
                }
            } catch (Exception x) {
                ZimbraLog.datasource.warn("Failed instantiating xsync class: %s", ds, x);
            }
        default:
            // yab is handled by OfflineDataSourceManager
            throw new IllegalArgumentException("Unknown data import type: " + ds.getType());
        }
    }

    public static String getDefaultImportClass(DataSourceType ds) {
        switch (ds) {
        case caldav:
            return CalDavDataImport.class.getName();
        case gal:
            return GalImport.class.getName();
        }
        return null;
    }

    /*
     * Tests connecting to a data source.  Do not actually create the
     * data source.
     */
    public static void test(DataSource ds) throws ServiceException {
        ZimbraLog.datasource.info("Testing: %s", ds);
        try {
            DataImport di = getInstance().getDataImport(ds, true);
            di.test();

            if (ds.isSmtpEnabled()) {
                Session session = JMSession.getSession(ds);
                Transport smtp = session.getTransport();
                String emailAddress = ds.getEmailAddress();
                if (smtp instanceof SmtpTransport) {
                    test((SmtpTransport) smtp, emailAddress);
                } else {
                    test((SMTPTransport) smtp, emailAddress);
                }
            }
            ZimbraLog.datasource.info("Test succeeded: %s", ds);
        } catch (ServiceException x) {
            ZimbraLog.datasource.info("Test failed: %s", ds, x);
            throw x;
        } catch (Exception e) {
            ZimbraLog.datasource.info("Test failed: %s", ds, e);
            throw ServiceException.INVALID_REQUEST("Datasource test failed", e);
        }
    }

    private static void test(SMTPTransport smtp, String mailfrom) throws MessagingException {
        smtp.connect();
        smtp.issueCommand("MAIL FROM:<" + mailfrom + ">", 250);
        smtp.issueCommand("RSET", 250);
        smtp.close();
    }

    private static void test(SmtpTransport smtp, String mailfrom) throws MessagingException {
        smtp.connect();
        smtp.mail(mailfrom);
        smtp.rset();
        smtp.close();
    }

    public static List<ImportStatus> getImportStatus(Account account) throws ServiceException {
        List<DataSource> dsList = Provisioning.getInstance().getAllDataSources(account);
        List<ImportStatus> allStatus = new ArrayList<ImportStatus>();
        for (DataSource ds : dsList) {
            allStatus.add(getImportStatus(account, ds));
        }
        return allStatus;
    }

    public static ImportStatus getImportStatus(Account account, DataSource ds) {
        ImportStatus importStatus;

        synchronized (sImportStatus) {
            Map<String, ImportStatus> isMap = sImportStatus.get(account.getId());
            if (isMap == null) {
                isMap = new HashMap<String, ImportStatus>();
                sImportStatus.put(account.getId(), isMap);
            }
            importStatus = isMap.get(ds.getId());
            if (importStatus == null) {
                importStatus = new ImportStatus(ds.getId());
                isMap.put(ds.getId(), importStatus);
            }
        }

        return importStatus;
    }

    public static void asyncImportData(final DataSource ds) {
        ZimbraLog.datasource.debug("Requesting async import for DataSource %s", ds.getId());

        executor.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    // todo exploit comonality with DataSourceTask
                    ZimbraLog.clearContext();
                    ZimbraLog.addMboxToContext(ds.getMailbox().getId());
                    ZimbraLog.addAccountNameToContext(ds.getAccount().getName());
                    ZimbraLog.addDataSourceNameToContext(ds.getName());
                    ZimbraLog.datasource.debug("Running on-demand import for DataSource %s", ds.getId());

                    DataSourceManager.importData(ds);

                } catch (Exception e) {
                    ZimbraLog.datasource.warn("On-demand DataSource import failed.", e);
                } finally {
                    ZimbraLog.clearContext();
                }
            }
        });
    }

    public static void importData(DataSource ds) throws ServiceException {
        importData(ds, null, true);
    }

    public static void importData(DataSource fs, boolean fullSync) throws ServiceException {
        importData(fs, null, fullSync);
    }

    /**
     * Executes the data source's {@link MailItemImport} implementation to import data in the current thread.
     */
    public static void importData(DataSource ds, List<Integer> folderIds, boolean fullSync)
            throws ServiceException {

        ZimbraLog.datasource.info("Requested import.");
        AccountStatus status = ds.getAccount().getAccountStatus();
        if (!(status.isActive() || status.isLocked() || status.isLockout())) {
            ZimbraLog.datasource.info("Account is not active. Skipping import.");
            return;
        }
        if (DataSourceManager.getInstance().getMailbox(ds).getMaintenance() != null) {
            ZimbraLog.datasource.info("Mailbox is in maintenance mode. Skipping import.");
            return;
        }
        ImportStatus importStatus = getImportStatus(ds.getAccount(), ds);
        synchronized (importStatus) {
            if (importStatus.isRunning()) {
                ZimbraLog.datasource.info("Attempted to start import while "
                        + " an import process was already running.  Ignoring the second request.");
                return;
            }
            importStatus.mHasRun = true;
            importStatus.mIsRunning = true;
            importStatus.mSuccess = false;
            importStatus.mError = null;
        }

        boolean success = false;
        String error = null;

        addManaged(ds);

        try {
            ZimbraLog.datasource.info("Importing data for data source '%s'", ds.getName());
            getInstance().getDataImport(ds).importData(folderIds, fullSync);
            success = true;
            resetErrorStatus(ds);
        } catch (ServiceException x) {
            error = generateErrorMessage(x);
            setErrorStatus(ds, error);
            throw x;
        } finally {
            ZimbraLog.datasource.info("Import completed for data source '%s'", ds.getName());
            synchronized (importStatus) {
                importStatus.mSuccess = success;
                importStatus.mError = error;
                importStatus.mIsRunning = false;
            }
        }
    }

    public static void resetErrorStatus(DataSource ds) {
        if (ds.getAttr(Provisioning.A_zimbraDataSourceFailingSince) != null
                || ds.getAttr(Provisioning.A_zimbraDataSourceLastError) != null) {
            Map<String, Object> attrs = new HashMap<String, Object>();
            attrs.put(Provisioning.A_zimbraDataSourceFailingSince, null);
            attrs.put(Provisioning.A_zimbraDataSourceLastError, null);
            try {
                Provisioning.getInstance().modifyAttrs(ds, attrs);
            } catch (ServiceException e) {
                ZimbraLog.datasource.warn("Unable to reset error status for data source %s.", ds.getName());
            }
        }
    }

    private static void setErrorStatus(DataSource ds, String error) {
        Map<String, Object> attrs = new HashMap<String, Object>();
        attrs.put(Provisioning.A_zimbraDataSourceLastError, error);
        if (ds.getAttr(Provisioning.A_zimbraDataSourceFailingSince) == null) {
            attrs.put(Provisioning.A_zimbraDataSourceFailingSince, LdapDateUtil.toGeneralizedTime(new Date()));
        }
        try {
            Provisioning.getInstance().modifyDataSource(ds.getAccount(), ds.getId(), attrs);
        } catch (ServiceException e) {
            ZimbraLog.datasource.warn("Unable to set error status for data source %s.", ds.getName());
        }
    }

    private static String generateErrorMessage(Throwable t) {
        StringBuilder buf = new StringBuilder();
        while (t != null) {
            // HACK: go with JavaMail error message
            if (t.getClass().getName().startsWith("javax.mail.")) {
                String msg = t.getMessage();
                return msg != null ? msg : t.toString();
            }
            if (buf.length() > 0) {
                buf.append(", ");
            }
            String msg = t.getMessage();
            buf.append(msg != null ? msg : t.toString());
            t = t.getCause();
        }
        return buf.toString();
    }

    static void cancelTask(Mailbox mbox, String dsId) throws ServiceException {
        ScheduledTaskManager.cancel(DataSourceTask.class.getName(), dsId, mbox.getId(), false);
        DbScheduledTask.deleteTask(DataSourceTask.class.getName(), dsId);
    }

    public static DataSourceTask getTask(Mailbox mbox, String dsId) {
        return (DataSourceTask) ScheduledTaskManager.getTask(DataSourceTask.class.getName(), dsId, mbox.getId());
    }

    /**
     * Cancels scheduling for this <tt>DataSource</tt>
     *
     * @param account
     * @param dsId
     * @throws ServiceException
     */
    public static void cancelSchedule(Account account, String dsId) throws ServiceException {
        updateSchedule(account, null, dsId, true);
    }

    /**
     * Cancels scheduling for this <tt>DataSource</tt>
     *
     * @param account Account for the DataSource, cannot be null
     * @param ds The DataSource, cannot be null
     * @throws ServiceException
     */
    public static void updateSchedule(Account account, DataSource ds) throws ServiceException {
        updateSchedule(account, ds, ds.getId(), false);
    }

    /**
     *
     * Updates scheduling data for this <tt>DataSource</tt> both in memory and in the
     * <tt>data_source_task</tt> database table.
     *
     * @param account Account for the DataSource, cannot be null.
     * @param ds The DataSource.  Ignored if cancelSchedule is true.
     * @param dsId zimbraId of the DataSource.
     * @param cancelSchedule cancel scheduling for the DataSource.
     * @throws ServiceException
     */
    private static void updateSchedule(Account account, DataSource ds, String dsId, boolean cancelSchedule)
            throws ServiceException {
        if (!LC.data_source_scheduling_enabled.booleanValue()) {
            return;
        }
        String accountId = account.getId();
        ZimbraLog.datasource.debug("Updating schedule for account %s, data source %s", accountId, dsId);

        int mboxId = MailboxManager.getInstance().lookupMailboxId(account.getId());
        if (mboxId == -1)
            return;

        if (cancelSchedule) {
            ZimbraLog.datasource.info("Data source %s was deleted.  Deleting scheduled task.", dsId);
            ScheduledTaskManager.cancel(DataSourceTask.class.getName(), dsId, mboxId, false);
            DbScheduledTask.deleteTask(DataSourceTask.class.getName(), dsId);
            deleteManaged(accountId, dsId);
            return;
        }
        if (!ds.isEnabled()) {
            ZimbraLog.datasource.info("Data source %s is disabled.  Deleting scheduled task.", dsId);
            ScheduledTaskManager.cancel(DataSourceTask.class.getName(), dsId, mboxId, false);
            DbScheduledTask.deleteTask(DataSourceTask.class.getName(), dsId);
            return;
        }

        ZimbraLog.datasource.info("Updating schedule for data source %s", ds.getName());
        DbConnection conn = null;
        try {
            conn = DbPool.getConnection();
            ScheduledTaskManager.cancel(conn, DataSourceTask.class.getName(), ds.getId(), mboxId, false);
            if (ds.isScheduled()) {
                DataSourceTask task = new DataSourceTask(mboxId, accountId, dsId, ds.getPollingInterval());
                ZimbraLog.datasource.debug("Scheduling %s", task);
                ScheduledTaskManager.schedule(conn, task);
            }
            conn.commit();
        } catch (ServiceException e) {
            ZimbraLog.datasource.warn("Unable to schedule data source %s", ds.getName(), e);
            DbPool.quietRollback(conn);
        } finally {
            DbPool.quietClose(conn);
        }
    }

    public static void refreshOAuthToken(DataSource ds) {
        PostMethod postMethod = null;
        try {
            postMethod = new PostMethod(ds.getOauthRefreshTokenUrl());
            postMethod.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");
            postMethod.addParameter(CLIENT_ID, ds.getOauthClientId());
            postMethod.addParameter(CLIENT_SECRET, ds.getDecryptedOAuthClientSecret());
            postMethod.addParameter(REFRESH_TOKEN, ds.getOauthRefreshToken());
            postMethod.addParameter(GRANT_TYPE, REFRESH_TOKEN);

            HttpClient httpClient = ZimbraHttpConnectionManager.getExternalHttpConnMgr().getDefaultHttpClient();
            int status = httpClient.executeMethod(postMethod);
            if (status == HttpStatus.SC_OK) {
                ZimbraLog.datasource.info("Refreshed oauth token status=%d", status);
                JSONObject response = new JSONObject(postMethod.getResponseBodyAsString());
                String oauthToken = response.getString(ACCESS_TOKEN);
                Map<String, Object> attrs = new HashMap<String, Object>();
                attrs.put(Provisioning.A_zimbraDataSourceOAuthToken,
                        DataSource.encryptData(ds.getId(), oauthToken));
                Provisioning provisioning = Provisioning.getInstance();
                provisioning.modifyAttrs(ds, attrs);
            } else {
                ZimbraLog.datasource.info("Could not refresh oauth token status=%d", status);
            }
        } catch (Exception e) {
            ZimbraLog.datasource.warn("Exception while refreshing oauth token", e);
        } finally {
            if (postMethod != null) {
                postMethod.releaseConnection();
            }
        }
    }
}