net.audumla.climate.bom.BOMDataLoader.java Source code

Java tutorial

Introduction

Here is the source code for net.audumla.climate.bom.BOMDataLoader.java

Source

package net.audumla.climate.bom;

/*
 * *********************************************************************
 *  ORGANIZATION : audumla.net
 *  More information about this project can be found at the following locations:
 *  http://www.audumla.net/
 *  http://audumla.googlecode.com/
 * *********************************************************************
 *  Copyright (C) 2012 - 2013 Audumla.net
 *  Licensed under the Creative Commons Attribution-NonCommercial-NoDerivs 3.0 Unported License.
 *  You may not use this file except in compliance with the License located at http://creativecommons.org/licenses/by-nc-nd/3.0/
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
 *  "AS I BASIS", WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and limitations under the License.
 */

import net.audumla.collections.ExpiringMap;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.IOUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPClientConfig;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.log4j.Logger;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;

public class BOMDataLoader {
    private static final Logger LOG = Logger.getLogger(BOMDataLoader.class);
    public static String BOMHTTP = "www.bom.gov.au";
    public static String HTTP = "http://";
    public static String FTP = "ftp://";
    public static String BOMFTP = "ftp2.bom.gov.au";
    public static String BOMBaseFTPDir = "/anon/gen/";
    public static String FWO = "fwo/";
    public static String CLIMATE = "clim_data/";
    private static String localBaseDir = "audumla.net/";
    private static BOMDataLoader instance = new BOMDataLoader();
    //   private FTPClient ftpClient;
    private Map<String, FTPClient> ftpClients = new HashMap<String, FTPClient>();
    private ExpiringMap cache = new ExpiringMap();

    private BOMDataLoader() {
        try {
            PropertiesConfiguration config = new PropertiesConfiguration("bomdatacache.properties");
            cache.loadProperties(config);
        } catch (Exception e) {
            LOG.error("Cannot load BOM data loader config", e);
        }

    }

    public static BOMDataLoader instance() {
        return instance;
    }

    public boolean hasDataExpired(String conntype, String host, String location) {
        return cache.hasDataExpired(generateLocalFileName(conntype, host + "_" + location));
    }

    private String generateLocalFileName(String conntype, String location) {
        return (localBaseDir + conntype + location).replaceAll("/|\\&|\\?|\\:", "_");
    }

    synchronized public BufferedReader getData(String conntype, String host, String location) {
        String tmpFileName = generateLocalFileName(conntype, host + "_" + location);
        boolean loadedRemote = false;
        if (cache.hasDataExpired(tmpFileName)) {
            try {
                File file = new File(new File(System.getProperty("java.io.tmpdir")), tmpFileName);

                if (conntype.equals(FTP)) {
                    storeFTPFile(host, location, file);
                } else {
                    storeHTTPFile(host, location, file);
                }
                cache.add(tmpFileName);
                loadedRemote = true;
            } catch (Exception ex) {
                throw new UnsupportedOperationException("Cannot load data from - " + location, ex);
            }
        }
        try {
            File file = new File(new File(System.getProperty("java.io.tmpdir")), tmpFileName);
            file.setReadOnly();
            FileReader fr = new FileReader(file);
            if (!loadedRemote) {
                LOG.info("Retrieved cached content - " + tmpFileName);
            }
            return new BufferedReader(fr);
        } catch (FileNotFoundException e) {
            throw new UnsupportedOperationException("Cannot load file - " + tmpFileName);
        }
    }

    private boolean storeHTTPFile(String host, String location, File of) throws MalformedURLException, IOException {
        try {
            InputStream is = new URL(HTTP + host + "/" + location).openStream();
            if (of.exists()) {
                of.delete();
            }
            FileOutputStream fos = new FileOutputStream(of);
            of.createNewFile();
            IOUtils.copy(new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))), fos);
            LOG.info("Retrieved remote HTTP content - " + host + "/" + location);
        } catch (FileNotFoundException ex) {
            return false;
        }
        return true;
    }

    public FTPFile getFTPFile(String host, String location) throws IOException {
        FTPClient ftp = getFTPClient(host);
        try {
            synchronized (ftp) {
                FTPFile[] files = ftp.listFiles(location);
                if (files.length > 0) {
                    return files[0];
                } else {
                    return null;
                }
            }
        } catch (IOException e) {
            try {
                ftp.logout();
                ftp.disconnect();
            } catch (Exception ex) {
                LOG.error("Failure to close connection", ex);
            }
            throw new UnsupportedOperationException(
                    "Error locating File " + host + location + " FTP Code -> " + ftp.getReplyCode(), e);
        } finally {
            /*            if (!ftp.completePendingCommand()) {
            ftp.logout();
            ftp.disconnect();
                        }
                        ftp.disconnect();
            */
        }
    }

    private synchronized FTPClient getFTPClient(String host) {
        FTPClient ftp = ftpClients.get(host);
        if (ftp == null || !ftp.isAvailable() || !ftp.isConnected()) {
            ftp = new FTPClient();
            FTPClientConfig config = new FTPClientConfig();
            ftp.configure(config);
            try {
                ftp.setControlKeepAliveTimeout(30);
                ftp.setControlKeepAliveReplyTimeout(5);
                ftp.setDataTimeout(3000);
                ftp.setDefaultTimeout(1000);
                int reply;
                ftp.connect(host);
                LOG.debug("Connected to " + host);
                reply = ftp.getReplyCode();
                if (!FTPReply.isPositiveCompletion(reply)) {
                    ftp.disconnect();
                    LOG.error("FTP server '" + host + "' refused connection.");
                } else {
                    if (!ftp.login("anonymous", "guest")) {
                        LOG.error("Unable to login to server " + host);
                    }
                    ftp.setSoTimeout(60000);
                    ftp.enterLocalPassiveMode();
                    ftp.setFileType(FTPClient.BINARY_FILE_TYPE);

                }
            } catch (IOException e) {
                LOG.error("Unable to connect to " + host, e);
            }
            ftpClients.put(host, ftp);
        }
        if (!ftp.isConnected() || !ftp.isAvailable()) {
            throw new UnsupportedOperationException("Cannot connect to " + host);
        }
        return ftp;
    }

    private boolean storeFTPFile(String host, String location, File of) throws Exception {
        if (of.exists()) {
            of.delete();
        }
        FileOutputStream fos = new FileOutputStream(of);
        of.createNewFile();
        if (!storeFTPFile(host, location, fos)) {
            of.delete();
            return false;
        }
        return true;
    }

    private boolean storeFTPFile(String host, String location, OutputStream os) {
        FTPClient ftp = getFTPClient(host);
        try {
            if (getFTPFile(host, location) != null) {
                synchronized (ftp) {
                    IOUtils.copy(ftp.retrieveFileStream(location), os);
                }
            } else {
                return false;
            }
        } catch (IOException ex) {
            LOG.warn("Unable to load file " + location + " FTP Reply -> " + ftp.getReplyCode(), ex);
            return false;
        } finally {
            try {
                os.close();
            } catch (Exception ex) {
                LOG.error("Unknown error encountered storing file " + location, ex);
            }
        }
        LOG.info("Retrieved remote FTP content - " + "ftp://" + host + location);
        try {
            if (!ftp.completePendingCommand()) {
                ftp.logout();
                ftp.disconnect();
            }
        } catch (Exception ex) {
            LOG.error("Unknown error encountered storing file " + location, ex);
        }
        return true;
    }
}