org.kuali.kra.external.Cfda.service.impl.CfdaServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for org.kuali.kra.external.Cfda.service.impl.CfdaServiceImpl.java

Source

/*
 * Kuali Coeus, a comprehensive research administration system for higher education.
 * 
 * Copyright 2005-2015 Kuali, Inc.
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.kuali.kra.external.Cfda.service.impl;

import au.com.bytecode.opencsv.CSVReader;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPReply;
import org.kuali.kra.award.home.CFDA;
import org.kuali.kra.external.Cfda.CfdaService;
import org.kuali.kra.external.Cfda.CfdaUpdateResults;
import org.kuali.kra.infrastructure.Constants;
import org.kuali.rice.core.api.datetime.DateTimeService;
import org.kuali.rice.core.framework.persistence.jdbc.sql.SQLUtils;
import org.kuali.rice.coreservice.framework.parameter.ParameterService;
import org.kuali.rice.krad.service.BusinessObjectService;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;

/**
 * This class is an implementation of the CfdaService. 
 * Updates the CFDA table with values from cfda.gov
 */
public class CfdaServiceImpl implements CfdaService {

    private ParameterService parameterService;
    private BusinessObjectService businessObjectService;
    private DateTimeService dateTimeService;
    private static final String FTP_PREFIX = "ftp://";
    private String cfdaFileName;
    private String govURL;

    private static final Log LOG = LogFactory.getLog(CfdaServiceImpl.class);
    protected static Comparator cfdaComparator;

    static {
        cfdaComparator = new Comparator() {
            public int compare(Object o1, Object o2) {
                String lhs = (String) o1;
                String rhs = (String) o2;
                return lhs.compareTo(rhs);
            }
        };
    }

    /**
     * This method retrieves cfda codes from the government site
     * @return
     * @throws IOException
     */
    public SortedMap<String, CFDA> retrieveGovCodes() throws IOException {

        SortedMap<String, CFDA> govMap = new TreeMap<String, CFDA>();
        createGovURL();
        LOG.info("Getting government file: " + cfdaFileName + " from URL " + govURL + " for update");

        InputStream inputStream = null;
        FTPClient ftp = connect(getGovURL());
        try {
            inputStream = ftp.retrieveFileStream(cfdaFileName);
            if (inputStream != null) {
                LOG.info("reading input stream");
                InputStreamReader screenReader = new InputStreamReader(inputStream);
                CSVReader csvReader = new CSVReader(screenReader, ',', '"', 1);
                List<String[]> lines = csvReader.readAll();
                for (String[] line : lines) {
                    String title = line[0];
                    String number = line[1];
                    CFDA cfda = new CFDA();
                    cfda.setCfdaNumber(number);
                    cfda.setCfdaProgramTitleName(title);
                    cfda.setCfdaMaintenanceTypeId(Constants.CFDA_MAINT_TYP_ID_AUTOMATIC);
                    govMap.put(number, cfda);
                }
            } else {
                // If file name is incorrect
                throw new IOException("Input stream is null. The file " + cfdaFileName
                        + " could not be retrieved from " + govURL);
            }
        } finally {
            disconnect(ftp);
        }
        return govMap;
    }

    /**
     * This method connects to the FTP server.
     * @param url
     * @return ftp
     */
    public FTPClient connect(String url) {
        FTPClient ftp = new FTPClient();
        try {
            ftp.connect(getGovURL());
            // Entering passive mode to prevent firewall issues. The client will establish a  data transfer
            // connection.
            ftp.enterLocalPassiveMode();
            int reply = ftp.getReplyCode();
            if (!FTPReply.isPositiveCompletion(reply)) {
                LOG.error("FTP connection to server not established.");
                throw new IOException("FTP connection to server not established.");
            }

            boolean loggedIn = ftp.login(Constants.CFDA_GOV_LOGIN_USERNAME, "");
            LOG.info("Logged in as " + Constants.CFDA_GOV_LOGIN_USERNAME);
            if (!loggedIn) {
                LOG.error("Could not login as anonymous.");
                throw new IOException("Could not login as anonymous.");
            }

        } catch (IOException io) {
            LOG.error(io.getMessage());
        }
        return ftp;
    }

    /**
     * This method disconnects from server.
     * @param ftp
     * @throws IOException
     */
    public void disconnect(FTPClient ftp) throws IOException {
        ftp.logout();
        if (ftp.isConnected()) {
            ftp.disconnect();
        }
    }

    /**
     * This method gets the url from the parameter and creates the fileName and 
     * the actual URL used to FTP.
     */
    protected void createGovURL() {
        // Example url ftp://ftp.cfda.gov/programs09187.csv
        String url = getParameterService().getParameterValueAsString(Constants.MODULE_NAMESPACE_AWARD,
                Constants.PARAMETER_COMPONENT_DOCUMENT, Constants.CFDA_GOV_URL_PARAMETER);
        String fileName = StringUtils.substringAfterLast(url, "/");
        url = StringUtils.substringBeforeLast(url, "/");

        if (StringUtils.contains(url, FTP_PREFIX)) {
            url = StringUtils.remove(url, FTP_PREFIX);
        }

        Calendar calendar = dateTimeService.getCurrentCalendar();
        // need to pull off the '20' in 2009
        String year = "" + calendar.get(Calendar.YEAR);
        year = year.substring(2, 4);
        fileName = fileName + year;

        // the last 3 numbers in the file name are the day of the year, but the files are from "yesterday"
        fileName = fileName + String.format("%03d", calendar.get(Calendar.DAY_OF_YEAR) - 1);
        fileName = fileName + ".csv";

        setGovURL(url);
        setCfdaFileName(fileName);
    }

    protected String getGovURL() {
        return govURL;
    }

    protected String getCfdaFileName() {
        return cfdaFileName;
    }

    protected void setGovURL(String url) {
        govURL = url;
    }

    protected void setCfdaFileName(String fileName) {
        cfdaFileName = fileName;
    }

    /**
     * This method updates the CFDA table with the values received from the 
     * gov site.
     * @see org.kuali.kra.external.Cfda.CfdaService#updateCfda()
     */
    public CfdaUpdateResults updateCfda() {
        CfdaUpdateResults updateResults = new CfdaUpdateResults();
        StringBuilder message = new StringBuilder();
        Map<String, CFDA> govCfdaMap;

        try {
            govCfdaMap = retrieveGovCodes();
        } catch (IOException ioe) {
            message.append("Problem encountered while retrieving cfda numbers, " + "the database was not updated."
                    + ioe.getMessage());
            updateResults.setMessage(message.toString());
            return updateResults;
        }

        SortedMap<String, CFDA> kcMap = getCfdaValuesInDatabase();
        updateResults.setNumberOfRecordsInKcDatabase(kcMap.size());
        updateResults.setNumberOfRecordsRetrievedFromWebSite(govCfdaMap.size());

        for (String key : kcMap.keySet()) {
            CFDA kcCfda = kcMap.get(key);
            CFDA govCfda = govCfdaMap.get(key);

            if (kcCfda.getCfdaMaintenanceTypeId().equalsIgnoreCase(Constants.CFDA_MAINT_TYP_ID_MANUAL)) {
                // Leave it alone. It's maintained manually.
                updateResults.setNumberOfRecordsNotUpdatedBecauseManual(
                        1 + updateResults.getNumberOfRecordsNotUpdatedBecauseManual());
            } else if (kcCfda.getCfdaMaintenanceTypeId().equalsIgnoreCase(Constants.CFDA_MAINT_TYP_ID_AUTOMATIC)) {

                if (govCfda == null) {
                    if (kcCfda.getActive()) {
                        kcCfda.setActive(false);
                        businessObjectService.save(kcCfda);
                        updateResults.setNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite(
                                updateResults.getNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite() + 1);
                    } else {
                        // Leave it alone for historical purposes
                        updateResults.setNumberOfRecordsNotUpdatedForHistoricalPurposes(
                                updateResults.getNumberOfRecordsNotUpdatedForHistoricalPurposes() + 1);
                    }
                } else {
                    if (kcCfda.getActive()) {
                        /*if (!kcCfda.getCfdaProgramTitleName().equalsIgnoreCase(govCfda.getCfdaProgramTitleName())) {
                        message.append("The program title for CFDA " + kcCfda.getCfdaNumber() + " changed from " 
                                        + kcCfda.getCfdaProgramTitleName() + " to " + govCfda.getCfdaProgramTitleName() + ".<BR>");
                         }*/
                        updateResults.setNumberOfRecordsUpdatedBecauseAutomatic(
                                updateResults.getNumberOfRecordsUpdatedBecauseAutomatic() + 1);
                    } else {
                        kcCfda.setActive(true);
                        updateResults
                                .setNumberOfRecordsReActivated(updateResults.getNumberOfRecordsReActivated() + 1);
                    }

                    kcCfda.setCfdaProgramTitleName(govCfda.getCfdaProgramTitleName());
                    businessObjectService.save(kcCfda);
                }
            }

            // Remove it from the govMap so the remaining codes are new 
            govCfdaMap.remove(key);
        }
        // New CFDA number from govt, added to the db
        updateResults.setMessage(message.toString());
        addNew(govCfdaMap);
        updateResults.setNumberOfRecordsNewlyAddedFromWebSite(govCfdaMap.size() + 1);
        return updateResults;
    }

    /**
     * This method gets the current CFDA values in the table.
     * @return
     */
    protected SortedMap<String, CFDA> getCfdaValuesInDatabase() {
        Collection<CFDA> cfdaValues = getBusinessObjectService().findAll(CFDA.class);

        SortedMap<String, CFDA> mapDatabaseCfda = new TreeMap<String, CFDA>(cfdaComparator);
        for (CFDA o : cfdaValues) {
            mapDatabaseCfda.put(o.getCfdaNumber(), o);
        }
        return mapDatabaseCfda;
    }

    /**
     * This method adds new cfda numbers to the cfda table
     * @param newCfdas
     */
    protected void addNew(Map<String, CFDA> newCfdas) {
        for (String cfdaKey : newCfdas.keySet()) {
            CFDA cfda = newCfdas.get(cfdaKey);

            String cfdaProgramTitleName = trimProgramTitleName(cfda.getCfdaProgramTitleName());
            // all new cfda numbers are set to automatic and active
            cfda.setCfdaProgramTitleName(SQLUtils.cleanString(cfdaProgramTitleName));
            cfda.setActive(true);
            cfda.setCfdaMaintenanceTypeId(Constants.CFDA_MAINT_TYP_ID_AUTOMATIC);
            getBusinessObjectService().save(cfda);
        }
    }

    /**
     * This method trims the program title name so it cannot be longer 
     * than the specified length.
     * @param programTitleName
     * @return
     */
    protected String trimProgramTitleName(String programTitleName) {
        if (programTitleName != null && programTitleName.length() > Constants.MAX_ALLOWABLE_CFDA_PGM_TITLE_NAME) {
            programTitleName = programTitleName.substring(0, Constants.MAX_ALLOWABLE_CFDA_PGM_TITLE_NAME);
        }
        return programTitleName;
    }

    public BusinessObjectService getBusinessObjectService() {
        return businessObjectService;
    }

    /**
     * Sets the businessObjectService. Injected by spring.
     * @return
     */
    public void setBusinessObjectService(BusinessObjectService businessObjectService) {
        this.businessObjectService = businessObjectService;
    }

    /**
     * Sets the dateTimeService. Injected by spring.
     * @return
     */
    public void setDateTimeService(DateTimeService dateTimeService) {
        this.dateTimeService = dateTimeService;
    }

    /*
     * Sets the parameterService attribute value. Injected by Spring.
     * @param parameterService The parameterService to set.
     */
    public void setParameterService(ParameterService parameterService) {
        this.parameterService = parameterService;
    }

    /**
     * Sets the parameterService. Injected by spring.
     * @return
     */
    public ParameterService getParameterService() {
        return parameterService;
    }
}