org.ecoinformatics.datamanager.download.DownloadHandler.java Source code

Java tutorial

Introduction

Here is the source code for org.ecoinformatics.datamanager.download.DownloadHandler.java

Source

/**
 *    '$RCSfile: DownloadHandler.java,v $'
 *
 *     '$Author: leinfelder $'
 *       '$Date: 2008-08-08 21:40:51 $'
 *   '$Revision: 1.28 $'
 *
 *  For Details: http://kepler.ecoinformatics.org
 *
 * Copyright (c) 2003 The Regents of the University of California.
 * All rights reserved.
 *
 * Permission is hereby granted, without written agreement and without
 * license or royalty fees, to use, copy, modify, and distribute this
 * software and its documentation for any purpose, provided that the
 * above copyright notice and the following two paragraphs appear in
 * all copies of this software.
 *
 * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY
 * FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
 * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
 * IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY
 * OF SUCH DAMAGE.
 *
 * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE
 * PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY
 * OF CALIFORNIA HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT,
 * UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
 */
package org.ecoinformatics.datamanager.download;

import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Hashtable;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPClient;

import org.ecoinformatics.datamanager.database.DatabaseLoader;
import org.ecoinformatics.datamanager.parser.Entity;
import org.ecoinformatics.datamanager.quality.QualityCheck;
import org.ecoinformatics.datamanager.quality.QualityReport;
import org.ecoinformatics.datamanager.quality.QualityCheck.Status;
import org.ecoinformatics.ecogrid.authenticatedqueryservice.AuthenticatedQueryServiceGetToStreamClient;
import org.ecoinformatics.ecogrid.queryservice.QueryServiceGetToStreamClient;

/**
 * This class will read a input stream from remote entity for given URL and
 * write data into given local storage systems. This is the main class of download
 * component. The class implements Runnable interface, so the download process
 * will be run in another thread.
 * 
 * @author tao
 */
public class DownloadHandler implements Runnable {
    /*
     * Class fields
     */

    private static String anonymousFtpPasswd = "anonymous@domain.org";
    public static Log log = LogFactory.getLog(DownloadHandler.class);
    // Used in quality reporting
    protected final static String ONLINE_URLS_EXCEPTION_MESSAGE = "Error reading from the data source.";

    /*
     * Constants
     */
    private static final String ANONYMOUS = "anonymous";
    private static final String SRBUSERNAME = "testuser.sdsc";
    private static final String SRBPASSWD = "TESTUSER";
    private static final int SLEEPTIME = 100;
    private static final int MAXLOOPNUMBER = 200000;

    protected static Hashtable<String, DownloadHandler> handlerList = new Hashtable<String, DownloadHandler>();
    private static String SRBENDPOINT = "http://srbbrick8.sdsc.edu:8080/SRBImpl/services/SRBQueryService";
    private static String SRBMACHINE = "srb-mcat.sdsc.edu";

    /*
     * Instance fields
     */

    //protected DownloadHandler handler = null;
    //private String identifier = null;
    private String url = null;
    private DataStorageInterface[] dataStorageClassList = null;
    private String[] errorMessages = null;
    protected boolean completed = false;
    protected boolean success = false;
    protected boolean busy = false;
    private Exception exception = null;

    protected String ecogridEndPoint = "http://ecogrid.ecoinformatics.org/knb/services/QueryService";
    protected Entity entity = null;
    protected String sessionId = null;

    /*
     * Constructors
     */

    /**
     * This version of the constructor stores the entity object for which the download is 
     * being performed. This is to support quality reporting, where information about the
     * associated entity is needed as part of the quality information being reported on.
     * 
     * @param entity  the Entity object for which this DownloadHandler is downloading data
     * @param url     the url (or identifier) of entity need be downloaded
     */
    protected DownloadHandler(Entity entity, String url, EcogridEndPointInterface endPoint) {
        this(url, endPoint);
        this.entity = entity;
    }

    /**
     * Constructor of this class
     * @param url  the url (or identifier) of entity need be downloaded
     */
    protected DownloadHandler(String url, EcogridEndPointInterface endPoint) {
        this.url = url;
        if (endPoint != null) {
            ecogridEndPoint = endPoint.getMetacatEcogridEndPoint();
            SRBENDPOINT = endPoint.getSRBEcogridEndPoint();
            SRBMACHINE = endPoint.getSRBMachineName();

            //do we have authenticated version?
            if (endPoint instanceof AuthenticatedEcogridEndPointInterface) {
                sessionId = ((AuthenticatedEcogridEndPointInterface) endPoint).getSessionId();
                //can we actually use it?
                if (sessionId != null) {
                    ecogridEndPoint = ((AuthenticatedEcogridEndPointInterface) endPoint)
                            .getMetacatAuthenticatedEcogridEndPoint();
                }
            }
        }
        //loadOptions();
        //this.identifier = identifier;
        //this.dataStorageClassList = dataStorageClassList;
    }

    /*
     * Class methods
     */

    /**
     * Gets a downloadHandler with specified url from the hash.
     * Return null if no handler found for this source.
     * 
     * @param source  the source URL to which the returned download handler is
     *                associated. The source URL is the key, the download
     *                handler object is the associated value.
     * @return  the DownloadHandler value associated with the source, or null
     *          if DownloadHandler object is associated with this source.
     */
    protected static synchronized DownloadHandler getHandlerFromHash(String source) {
        DownloadHandler handler = null;

        if (source != null) {
            handler = handlerList.get(source);
            // assign download handler to one in List      
        }

        return handler;
    }

    /**
     * Gets an instance of the DownloadHandler Object for this URL.
      * 
     * @param url The url (or identifier) of entity to be downloaded
     * @param endPoint the object which provides ecogrid endpoint information
     * @return  DownloadHandler object associated with this URL
     * 
     */
    public static DownloadHandler getInstance(String url, EcogridEndPointInterface endPoint) {
        DownloadHandler handler = getHandlerFromHash(url);

        if (handler == null) {
            log.debug("Constructing DownloadHandler for URL: " + url);
            handler = new DownloadHandler(url, endPoint);
        }

        return handler;
    }

    /**
     * Gets an instance of the DownloadHandler Object for this URL. This version
     * of the method passes the associated entity object as a parameter, and
     * also calls the constructor that accepts the entity object. This is
     * to support quality reporting. When creating a qualityCheck object relating
     * to downloading, the DownloadHandler will need some information from the
     * entity such as the packageId and entity name.
     * 
     * @param entity   The entity object for which the download is being performed
     * @param url The url (or identifier) of entity to be downloaded
     * @param endPoint the object which provides ecogrid endpoint information
     * @return  DownloadHandler object associated with this URL
     * 
     */
    public static DownloadHandler getInstance(Entity entity, String url, EcogridEndPointInterface endPoint) {
        DownloadHandler handler = getHandlerFromHash(url);

        if (handler == null) {
            log.debug("Constructing DownloadHandler for URL: " + url);
            handler = new DownloadHandler(entity, url, endPoint);
        }

        return handler;
    }

    /**
     * Sets the DownloadHandler object into the hash table. This will be called
     * at the start of download process. So we can keep track which handler is 
     * doing the download job now. Since it will access a static variable 
     * handlerList in different thread, it should be static and synchronized
     * 
     * @param  downloadHandler  the DownloadHandler object to be stored in the
     *                          hash
     *
     */
    private static synchronized void putDownloadHandlerIntoHash(DownloadHandler downloadHandler) {
        if (downloadHandler != null) {
            String source = downloadHandler.getUrl();
            if (source != null) {
                //System.out.println("add the source "+source);
                handlerList.put(source, downloadHandler);
            }
        }
    }

    /**
     * Removes the downloadHandler obj from the hash table. This method will be
     * called at the end of download process. Since it will access a static 
     * variable handlerList in different thread, it should be static and 
     * synchronized;
     * 
     * @param  downloadHandler  the DownloadHandler object to be removed
     *                          from the hash
     */
    private static synchronized void removeDownloadHandlerFromHash(DownloadHandler downloadHandler) {
        if (downloadHandler != null) {
            String source = downloadHandler.getUrl();

            if (source != null) {
                //System.out.println("remove the source "+source);
                handlerList.remove(source);
            }
        }
    }

    /**
     * Setter method for the anonymousFtpPasswd class variable.
     * 
     * @param passwd  The anonymous FTP password setting. FTP servers
     *                usually just check that the value parses as a
     *                syntactically valid email address such as
     *                'anonymous@domain.org' (the default setting).
     */
    public static void setAnonymousFtpPasswd(String passwd) {
        anonymousFtpPasswd = passwd;
    }

    /*
     * Instance methods
     */

    /**
     * This method will download data for the given url in a new thread.
      * It implements from Runnable Interface.
     */
    public void run() {
        DownloadHandler handler = getHandlerFromHash(url);

        if (handler != null) {
            /*
               * A handler which points to the same URL is busy in downloading 
               * process, so do nothing, just wait for the handler to finish the 
               * download.
               */
            int index = 0;

            while (handler.isBusy() && index < MAXLOOPNUMBER) {
                try {
                    Thread.sleep(SLEEPTIME);
                } catch (Exception e) {
                    break;
                }
                index++;
            }

            success = handler.isSuccess();
            //System.out.println("after setting success "+success);
            busy = false;
            completed = true;
            //System.out.println(" don't need download");
            return;
        } else {
            // if no handler which points same url, put the handler into hash table for tracking
            //System.out.println("need download");
            putDownloadHandlerIntoHash(this);
        }

        busy = true;
        completed = false;
        //System.out.println("start get source"+url);

        try {
            success = getContentFromSource(url);
        } catch (Exception e) {
            log.error("Error in DownloadHandler run method " + e.getMessage());
        }

        //System.out.println("after get source"+url);
        // waiting DataStorageInterface to finished serialize( some DataStorageInterface will
        // span another thread
        waitingStorageInterfaceSerialize();

        if (dataStorageClassList != null) {
            int length = dataStorageClassList.length;
            for (int i = 0; i < length; i++) {
                DataStorageInterface storage = dataStorageClassList[i];
                if (storage != null) {
                    success = success && storage.isSuccess(url);
                    if (storage instanceof DatabaseLoader) {
                        exception = storage.getException();
                    }
                }
            }

        }

        // downloading is done, remove the handler from hash.
        removeDownloadHandlerFromHash(this);
        //this.notifyAll();
        busy = false;
        completed = true;
    }

    /*
     * Waits until DataStorageClass finishes serializing.
     * Sometimes the outputstream which DownloadHandler gets from
     * DataStorageClass is not in same thread as the main one there.
     * A good example of this is the DatabaseLoader class.
     */
    private void waitingStorageInterfaceSerialize() {
        if (dataStorageClassList != null) {
            int length = dataStorageClassList.length;
            boolean completedInDataStorageClassList = true;

            for (int i = 0; i < length; i++) {
                DataStorageInterface storage = dataStorageClassList[i];
                if (storage != null) {
                    if (storage.doesDataExist(url)) {

                    } else {
                        completedInDataStorageClassList = completedInDataStorageClassList
                                && storage.isCompleted(url);
                    }
                }
            }

            while (!completedInDataStorageClassList) {
                completedInDataStorageClassList = true;

                for (int i = 0; i < length; i++) {
                    DataStorageInterface storage = dataStorageClassList[i];

                    if (storage != null) {
                        if (storage.doesDataExist(url)) {

                        } else {
                            completedInDataStorageClassList = completedInDataStorageClassList
                                    && storage.isCompleted(url);
                        }
                    }
                }
            }
        }
    }

    /**
     * Downloads data into the given list of DataStorageInterface objects. 
     * This method will create, start and wait for another thread to download 
     * data.
     * 
     * @param  dataStorages  The list of destinations for the downloaded data
     * @return true if successful, else false
     */
    public boolean download(DataStorageInterface[] dataStorages) throws DataSourceNotFoundException, Exception {
        this.setDataStorageClassList(dataStorages);
        Thread loadData = new Thread(this);
        loadData.start();
        int index = 0;

        while (!this.isCompleted() && index < MAXLOOPNUMBER) {
            if (exception != null) {
                throw exception;
            }
            Thread.sleep(SLEEPTIME);
            index++;
        }

        success = this.isSuccess();

        if (exception != null) {
            throw exception;
        }

        return success;
    }

    /**
     * Returns the thread status - busy or not
     *  
     * @return boolean variable busy - if this thread is busy on downloading
     */
    public boolean isBusy() {
        return busy;
    }

    /**
     * Returns the status of downloading, completed or not
     * 
     * @return value of completetion
     */
    public boolean isCompleted() {
        return completed;
    }

    /**
     * Returns the status of downloading, success or not
     * 
     * @return The value of success
     */
    public boolean isSuccess() {
        return success;
    }

    /**
     * Retruns the objects of DataStorageInterface associated with this class.
     * 
     * @return The array of DataStorageInterface object
     */
    public DataStorageInterface[] getDataStorageClassList() {
        return dataStorageClassList;
    }

    /**
     * Sets the objects of DataStorageInterface associated with this class.
     * 
     * @param dataStorageClassList  The array of DataStorageInterface which 
     *                              will be associcated to this class
     */
    public void setDataStorageClassList(DataStorageInterface[] dataStorageClassList) {
        this.dataStorageClassList = dataStorageClassList;
    }

    /**
     * Gets content from given source and writes it to DataStorageInterface 
     * to store them. This method will be called by run()
     * 
     * @param resourceName  the URL to the source data to be retrieved
     */
    protected boolean getContentFromSource(String resourceName) {
        boolean successFlag = false;
        QualityCheck onlineURLsQualityCheck = null;
        boolean onlineURLsException = false; // used to determine status of onlineURLs quality check

        if (resourceName != null) {
            resourceName = resourceName.trim();
        }

        if (resourceName != null && (resourceName.startsWith("http://") || resourceName.startsWith("https://")
                || resourceName.startsWith("file://") || resourceName.startsWith("ftp://"))) {
            // get the data from a URL
            int responseCode = 0;
            String responseMessage = null;

            try {
                URL url = new URL(resourceName);
                boolean isFTP = false;

                if (entity != null) {
                    String contentType = null;

                    // Find the right MIME type and set it as content type
                    if (resourceName.startsWith("http")) {
                        HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
                        httpURLConnection.setRequestMethod("HEAD");
                        httpURLConnection.connect();
                        contentType = httpURLConnection.getContentType();
                        responseCode = httpURLConnection.getResponseCode();
                        responseMessage = httpURLConnection.getResponseMessage();
                    } else if (resourceName.startsWith("file")) {
                        URLConnection urlConnection = url.openConnection();
                        urlConnection.connect();
                        contentType = urlConnection.getContentType();
                    } else { // FTP
                        isFTP = true;
                        contentType = "application/octet-stream";
                    }

                    entity.setUrlContentType(contentType);
                }

                if (!isFTP) { // HTTP(S) or FILE
                    InputStream filestream = url.openStream();

                    try {
                        successFlag = this.writeRemoteInputStreamIntoDataStorage(filestream);
                    } catch (IOException e) {
                        exception = e;
                        String errorMessage = e.getMessage();
                        if (errorMessage.startsWith(ONLINE_URLS_EXCEPTION_MESSAGE)) {
                            onlineURLsException = true;
                        }
                    } finally {
                        filestream.close();
                    }
                } else { // FTP
                    String[] urlParts = resourceName.split("/");
                    String address = urlParts[2];
                    String dir = "/";
                    for (int i = 3; i < urlParts.length - 1; i++) {
                        dir += urlParts[i] + "/";
                    }
                    String fileName = urlParts[urlParts.length - 1];
                    FTPClient ftpClient = new FTPClient();
                    ftpClient.connect(address);
                    ftpClient.login(ANONYMOUS, anonymousFtpPasswd);
                    ftpClient.changeWorkingDirectory(dir);
                    ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
                    ftpClient.enterLocalPassiveMode(); // necessary to avoid firewall blocking
                    InputStream filestream = ftpClient.retrieveFileStream(fileName);
                    try {
                        successFlag = this.writeRemoteInputStreamIntoDataStorage(filestream);
                    } catch (IOException e) {
                        exception = e;
                        String errorMessage = e.getMessage();
                        if (errorMessage.startsWith(ONLINE_URLS_EXCEPTION_MESSAGE)) {
                            onlineURLsException = true;
                        }
                    } finally {
                        try {
                            filestream.close();
                        } catch (IOException e) {
                            exception = new DataSourceNotFoundException(String
                                    .format("Error closing local file '%s': %s", resourceName, e.getMessage()));
                            onlineURLsException = true;
                        }
                    }

                    // logout and disconnect if FTP session
                    if (resourceName.startsWith("ftp") && ftpClient != null) {
                        try {
                            ftpClient.enterLocalActiveMode();
                            ftpClient.logout();
                            ftpClient.disconnect();
                        } catch (IOException e) {
                            exception = new DataSourceNotFoundException(
                                    String.format("Error disconnecting from FTP with resource '%s': %s",
                                            resourceName, e.getMessage()));
                            onlineURLsException = true;
                        }
                    }
                }
            } catch (MalformedURLException e) {
                String eClassName = e.getClass().getName();
                String eMessage = String.format("%s: %s", eClassName, e.getMessage());
                exception = new DataSourceNotFoundException(
                        String.format("The URL '%s' is a malformed URL: %s", resourceName, eMessage));
            } catch (IOException e) {
                String eClassName = e.getClass().getName();
                String eMessage = String.format("%s: %s", eClassName, e.getMessage());
                if (responseCode > 0) {
                    eMessage = String.format("Response Code: %d %s; %s", responseCode, responseMessage, eMessage);
                }
                exception = new DataSourceNotFoundException(
                        String.format("The URL '%s' is not reachable: %s", resourceName, eMessage));
            }

            // Initialize the "Online URLs are live" quality check
            String qualityCheckIdentifier = "onlineURLs";
            QualityCheck qualityCheckTemplate = QualityReport.getQualityCheckTemplate(qualityCheckIdentifier);
            onlineURLsQualityCheck = new QualityCheck(qualityCheckIdentifier, qualityCheckTemplate);

            if (QualityCheck.shouldRunQualityCheck(entity, onlineURLsQualityCheck)) {
                String resourceNameEscaped = embedInCDATA(resourceName);

                if (!onlineURLsException) {
                    onlineURLsQualityCheck.setStatus(Status.valid);
                    onlineURLsQualityCheck.setFound("true");
                    onlineURLsQualityCheck.setExplanation("Succeeded in accessing URL: " + resourceNameEscaped);
                } else {
                    onlineURLsQualityCheck.setFailedStatus();
                    onlineURLsQualityCheck.setFound("false");
                    String explanation = "Failed to access URL: " + resourceNameEscaped;
                    explanation = explanation + "; " + embedInCDATA(exception.getMessage());
                    onlineURLsQualityCheck.setExplanation(explanation);
                }

                entity.addQualityCheck(onlineURLsQualityCheck);
            }

            return successFlag;
        } else if (resourceName != null && resourceName.startsWith("ecogrid://")) {
            // get the docid from url
            int start = resourceName.indexOf("/", 11) + 1;
            //log.debug("start: " + start);
            int end = resourceName.indexOf("/", start);

            if (end == -1) {
                end = resourceName.length();
            }

            //log.debug("end: " + end);
            String ecogridIdentifier = resourceName.substring(start, end);
            // pass this docid and get data item
            //System.out.println("the endpoint is "+ECOGRIDENDPOINT);
            //System.out.println("The identifier is "+ecogridIdentifier);
            //return false;
            return getContentFromEcoGridSource(ecogridEndPoint, ecogridIdentifier);
        } else if (resourceName != null && resourceName.startsWith("srb://")) {
            // get srb docid from the url
            String srbIdentifier = transformSRBurlToDocid(resourceName);
            // reset endpoint for srb (This is hack we need to figure ou
            // elegent way to do this
            //mEndPoint = Config.getValue("//ecogridService/srb/endPoint");
            // pass this docid and get data item
            //log.debug("before get srb data");
            return getContentFromEcoGridSource(SRBENDPOINT, srbIdentifier);
        } else {
            successFlag = false;
            return successFlag;
        }
    }

    /*
     * Embeds a string value inside CDATA opening and closing tags.
     */
    private String embedInCDATA(String value) {
        return "<![CDATA[" + value + "]]>";
    }

    /**
     *  Get data from ecogrid server base on given end point and identifier.
     *  This method will handle the distribution url for ecogrid or srb protocol
     *  This method will be called by getContentFromSource().
     *  
     *  @param  endPoint    the end point of ecogrid service
     *  @param  identifier  the entity identifier in ecogrid service
     */
    protected boolean getContentFromEcoGridSource(String endPoint, String identifier) {
        // create a ecogrid client object and get the full record from the
        // client
        //System.out.println("=================the endpoint is "+endPoint);
        //System.out.println("=================the identifier is "+identifier);
        boolean successFlag = false;

        if (endPoint != null && identifier != null) {
            //log.debug("Get " + identifier + " from " + endPoint);

            try {
                //fatory
                //log.debug("This is instance pattern");

                //log.debug("Get from EcoGrid: " + identifier);
                NeededOutputStream[] outputStreamList = getOutputStreamList();

                if (outputStreamList != null) {
                    boolean oneLoopSuccess = true;

                    for (int i = 0; i < outputStreamList.length; i++) {
                        NeededOutputStream stream = outputStreamList[i];

                        if (stream != null && stream.getNeeded()) {
                            BufferedOutputStream bos = new BufferedOutputStream(stream.getOutputStream());

                            //use the appropriate client
                            URL endPointURL = new URL(endPoint);
                            if (sessionId != null) {
                                AuthenticatedQueryServiceGetToStreamClient authenticatedEcogridClient = new AuthenticatedQueryServiceGetToStreamClient(
                                        endPointURL);
                                authenticatedEcogridClient.get(identifier, sessionId, bos);
                            } else {
                                QueryServiceGetToStreamClient ecogridClient = new QueryServiceGetToStreamClient(
                                        endPointURL);
                                ecogridClient.get(identifier, bos);
                            }
                            bos.flush();
                            bos.close();

                            if (oneLoopSuccess) {
                                successFlag = true;
                            }
                        } else if (stream != null) {
                            if (oneLoopSuccess) {
                                successFlag = true;
                            }
                        } else {
                            oneLoopSuccess = false;
                            successFlag = false;
                        }
                    }
                } else {
                    successFlag = false;
                }

                return successFlag;

            } catch (Exception ee) {
                log.error("DownloadHandler - error getting content from Ecogrid ", ee);
                ee.printStackTrace();
                successFlag = false;
                return successFlag;
            }
        } else {
            //System.out.println("in else path of get data from other source");
            // this is not ecogrid source, we need download by other protocol
            //return getContentFromSource(identifier);
            return false;
        }

    }

    /*
     * This method will transfer a srb url to srb docid in ecogrid.
     * srb id should look like: 
     *   srb://seek:/home/beam.seek/IPCC_climate/Present/ccld6190.dat
     * and correspond docid looks like:
     *   srb://testuser:TESTUSER@orion.sdsc.edu/home/beam.seek/IPCC_climate/Present/ccld6190.dat
     */
    private String transformSRBurlToDocid(String srbURL) {
        String docid = null;

        if (srbURL == null) {
            return docid;
        }

        String regex = "seek:";
        srbURL = srbURL.trim();
        //log.debug("The srb url is "+srbURL);
        // get user name , passwd and machine namefrom configure file
        String user = SRBUSERNAME;//Config.getValue("//ecogridService/srb/user");
        String passwd = SRBPASSWD;//Config.getValue("//ecogridService/srb/passwd");
        String machineName = SRBMACHINE;//Config.getValue("//ecogridService/srb/machineName");
        String replacement = user + ":" + passwd + "@" + machineName;
        docid = srbURL.replaceFirst(regex, replacement);
        //log.debug("The srb id is " + docid);

        return docid;
    }

    /*
     * Method to get an array of outputstream for datastorage class.
     * If dataStorageClassList is null, null will return.
     * If some dataStorage object couldn't get an outputstream or
     * identifier is already in the datastorage object, null will associate
     * with this object.
     * 
     */
    private NeededOutputStream[] getOutputStreamList() {
        NeededOutputStream[] list = null;

        if (dataStorageClassList != null) {
            list = new NeededOutputStream[dataStorageClassList.length];

            for (int i = 0; i < dataStorageClassList.length; i++) {
                DataStorageInterface dataStorge = dataStorageClassList[i];

                if (dataStorge != null && !dataStorge.doesDataExist(url)) {
                    log.debug("DownloadHandler.startSerialize()");
                    OutputStream osw = dataStorge.startSerialize(url);
                    NeededOutputStream stream = new NeededOutputStream(osw, true);
                    list[i] = stream;
                } else if (dataStorge != null) {
                    OutputStream osw = null;
                    NeededOutputStream stream = new NeededOutputStream(osw, false);
                    list[i] = stream;
                } else {
                    list[i] = null;
                }
            }
        }

        return list;
    }

    /*
     * Method to close a OutputStream array. Calls the finishSerialize() method
     * for each of the DataStorageInterface objects in the
     * dataStorageClassList.
     */
    private void finishSerialize(String error) throws IOException {
        if (dataStorageClassList != null) {
            int length = dataStorageClassList.length;
            //boolean completedInDataStorageClassList = true;

            for (int i = 0; i < length; i++) {
                DataStorageInterface storage = dataStorageClassList[i];

                //if (storage != null && storage.isCompleted(url))
                if (storage != null) {
                    storage.finishSerialize(url, error);
                }
            }
        }
        log.debug("DownloadHandler.finishSerialize()");
    }

    /**
     * Gets Error messages for downloading process.
     * 
     * @return The array of errory messages
     */
    public String[] getErrorMessages() {
        return errorMessages;
    }

    /*
     * This class add a new flag in OutputStream class - need to write the
     * remote input stream to this output stream or not.
     * If data storage insterface already download this identifier,
     * this OuptStream will be marked as NonNeeded. So download would NOT
     * happen again
     * 
     * @author tao
     *
     */
    private class NeededOutputStream {
        /*
         * Instance fields
         */

        private OutputStream stream = null;
        private boolean needed = true;

        /**
         * Constructor
         * @param stream  OutputStream in this object
         * @param needed  Need of writting inputstream to this output stream
         */
        public NeededOutputStream(OutputStream stream, boolean needed) {
            this.stream = stream;
            this.needed = needed;
        }

        /*
         * Instance methods
         */

        /**
         * Gets the OuptStream associated with this object
         * @return The OutputStream  object
         */
        public OutputStream getOutputStream() {
            return stream;
        }

        /**
         * Gets the value if need write the inpustream to this output stream
         * @return The boolean value of need to write
         */
        public boolean getNeeded() {
            return needed;
        }
    }

    /**
     * Gets the URL that this DownloadHandler object is associated with. 
     */
    public String getUrl() {
        return url;
    }

    /*
     * This method will read from remote inputsream and write it
     * to StorageSystem. It only handles http or ftp protocals. It does not
     * handle ecogrid protocol.
     */
    protected boolean writeRemoteInputStreamIntoDataStorage(InputStream inputStream) throws IOException {
        boolean successFlag = false;

        try {
            NeededOutputStream[] outputStreamList = getOutputStreamList();

            if (outputStreamList != null) {
                // start reading, to write to the ouputstreams
                byte[] b = new byte[1024];
                int bytesRead = inputStream.read(b, 0, 1024);

                if (bytesRead < 1) {
                    throw new IOException(
                            String.format("%s %s", ONLINE_URLS_EXCEPTION_MESSAGE, "0 bytes were read."));
                }

                int kilobytes = 1;
                //System.err.printf("  Kilobytes read: .\n");

                /*
                 * Store the first kilobyte of data in the entity for
                 * subsequent use in quality reporting
                 */
                if (entity != null && entity.getFirstKilobyte() == null) {
                    String firstKilobyte = new String(b);
                    entity.setFirstKilobyte(firstKilobyte);
                }

                NeededOutputStream stream = null;
                OutputStream os = null;
                while (bytesRead > -1) {
                    // write to each outputstreams
                    for (int i = 0; i < outputStreamList.length; i++) {
                        stream = outputStreamList[i];
                        if (stream != null && stream.getNeeded()) {
                            os = stream.getOutputStream();
                            os.write(b, 0, bytesRead);
                        }
                    }
                    // get the next bytes
                    bytesRead = inputStream.read(b, 0, 1024);
                    kilobytes++;
                    /*if (kilobytes < 1000) {
                    System.err.printf(".");
                    if (kilobytes % 100 == 0) {
                    System.err.printf("\n");
                    }
                    }               
                    else if (kilobytes % 1000 == 0) {
                    System.err.printf("\n  Kilobytes read: %d", kilobytes);
                    }*/
                }
                //System.err.printf("\n");

                // done writing to the streams
                for (int i = 0; i < outputStreamList.length; i++) {
                    stream = outputStreamList[i];
                    if (stream != null && stream.getNeeded()) {
                        os.flush();
                        os.close();
                    }
                }
                successFlag = true;
                //String error = null;
                //finishSerialize(error);
                log.info(String.format("  Total Kilobytes Read: %d\n", kilobytes));
                log.debug("DownloadHandler.finishSerialize()");
            } else {
                successFlag = false;
            }

            return successFlag;

        } catch (IOException e) {
            successFlag = false;
            e.printStackTrace();
            throw (e);
        } catch (Exception ee) {
            successFlag = false;
            return successFlag;
        }

    }

}