Java tutorial
/** * '$RCSfile$' * Copyright: 2000 Regents of the University of California and the * National Center for Ecological Analysis and Synthesis * Purpose: To test the ReplicationServerList class by JUnit * Authors: Jing Tao * * '$Author: daigle $' * '$Date: 2009-11-07 08:46:33 +1000 (Sat, 07 Nov 2009) $' * '$Revision: 5110 $' * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package edu.ucsb.nceas.metacattest; import edu.ucsb.nceas.metacat.*; import edu.ucsb.nceas.metacat.client.Metacat; import edu.ucsb.nceas.metacat.client.MetacatFactory; import edu.ucsb.nceas.metacat.replication.*; import edu.ucsb.nceas.metacat.properties.PropertyService; import edu.ucsb.nceas.metacat.util.MetacatUtil; import edu.ucsb.nceas.utilities.IOUtil; import edu.ucsb.nceas.utilities.XMLUtilities; //import edu.ucsb.nceas.morpho.framework.*; import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestResult; import junit.framework.TestSuite; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import java.io.*; import java.net.*; import java.text.SimpleDateFormat; import java.util.*; /** * This class is used to change the data file location for IPCC eml documents. * Currently IPCC eml documents point data file ti SRB server. However, the srb * earthgrid is not very stable. We decided to change the online URL from srb to knb. * So this class will handle this case. * Before running this program, it needs: * 1. Downloaded data files from SRB * 2. A list of IPCC docid(with revision number) text file. If the text file is not available, it need * a metacat query file to search metacat to get the doicd list. * What the class will do: * 1. It will read the eml from Metacat. * 2. Get online URL information from eml document by DOM parser. * 3. Base on the URL information, this program will find the data file in * the direcotry which contains the srb data file. * 4. It will generate docid for the data file. * 5. Upload the download srb data file to Metacat with assigned docid. * 6. Modify the eml document with the new URL information (pointing to * knb) and new version number in eml * 7. Update it to a new version in Metacat. * 8 . Go through above 7 steps for every eml document in the list. * */ public class UploadIPCCDataTest extends TestCase { /* Initialize properties*/ static { try { PropertyService.getInstance(); } catch (Exception e) { System.err.println("Exception in initialize option in MetacatServletNetTest " + e.getMessage()); } } /**Constants*/ private static String SRBDATAFILEDIR = "/home/tao/data-file/IPCC"; // Dir for storing srb data file private static String DOCLISTFILE = "docidList"; // File name which stores IPCC document id private static String METACATURL = "http://chico.dyndns.org/knb/metacat"; private static String USERNAME = "uid=dpennington,o=LTER,dc=ecoinformatics,dc=org"; private static String PASSWORD = "password"; private static String TABLEONLINEURL = "/eml:eml/dataset/dataTable/physical/distribution/online/url"; private static String SPATIALONLINEURL = "/eml:eml/dataset/spatialRaster/physical/distribution/online/url"; private static String PACKAGEID = "/eml:eml/@packageId"; private static String SRB = "srb://"; private static String KNB = "ecogrid://knb/"; private static String DATAIDPREFIX = "IPCC"; private static String DOT = "."; private static String SUCCESSLOG = "update.log"; private static String ERRORLOG = "error.log"; private static String CURRENT_CORRECTFILENAME = "correct_filename.csv"; private File log = new File(SUCCESSLOG); private File error = new File(ERRORLOG); /** * Constructor to build the test * * @param name the name of the test method */ public UploadIPCCDataTest(String name) { super(name); } /** * Create a suite of tests to be run together */ public static Test suite() { TestSuite suite = new TestSuite(); //suite.addTest(new UploadIPCCDataTest("modifyEMLDocsWithCorrectDataFileName")); //suite.addTest(new UploadIPCCDataTest("modifyEMLDocsWithIncorrectDataFileName")); return suite; } /** * Modify EML Docs' data url online from SRB to ecogrid. *Those eml docs pointe valide srb file names. */ public void modifyEMLDocsWithCorrectDataFileName() { boolean originalDataFileIncorrect = false; updateEML(originalDataFileIncorrect); } /** * Modify EML Docs' data url online from SRB to ecogrid. *Those eml docs pointe invalide srb file names. */ public void modifyEMLDocsWithIncorrectDataFileName() { boolean originalDataFileIncorrect = true; updateEML(originalDataFileIncorrect); } /* * Upload the data file to Metacat and modify the eml documents * @return * @throws Exception */ private void updateEML(boolean originalDataFileIncorrect) { // Get eml document first Vector list = getDocumentList(); //If list is not empty, goes through every document by handleSingleEML method - //1. It will read the eml from Metacat. // 2. Get online URL information from eml document by DOM parser. // 3. Base on the URL information, this program will find the data file in // the direcotry which contains the srb data file. // 4. It will generate docid for the data file // 5. At last upload the download srb data file to Metacat with assigned docid. // 6. Modify the eml document with the new URL information (pointing to // knb) and new version number in eml. // 7.Update it to a new version in Metacat. if (list != null && !list.isEmpty()) { int size = list.size(); for (int i = 0; i < size; i++) { String docid = null; try { docid = (String) list.elementAt(i); String dataId = handleSingleEML(docid, originalDataFileIncorrect); String message = "Successfully update eml " + docid + " with data id " + dataId; writeLog(log, message); } catch (Exception e) { System.err.println("Failed to handle eml document " + docid + " since " + e.getMessage()); String message = "failed to update eml " + docid + "\n " + e.getMessage(); writeLog(error, message); } } } else { System.err.println("There is no EML document to handle"); } } /* * Does actually job to upload data file and modify eml document for a given id. * Here are its tasks: * 1. It will read the eml from Metacat. * 2. Get online URL information from eml document by DOM parser. * 3. Base on the URL information, this program will find the data file in * the direcotry which contains the srb data file. * 4. It will generate docid for the data file. * 5. Upload the download srb data file to Metacat with assigned docid. * 6. Modify the eml document with the new URL information (pointing to * knb) and new version number in eml * 7. Update it to a new version in Metacat. * */ private String handleSingleEML(String docid, boolean originalDataFileIncorrect) throws Exception { Metacat metacat = MetacatFactory.createMetacatConnection(METACATURL); // login metacat String response = metacat.login(USERNAME, PASSWORD); if (response.indexOf("<login>") == -1) { throw new Exception("login failed " + response); } // 1. Reads eml document from metacat Reader r = new InputStreamReader(metacat.read(docid)); Document DOMdoc = XMLUtilities.getXMLReaderAsDOMDocument(r); Node rootNode = (Node) DOMdoc.getDocumentElement(); //2. Gets online url information. If onlineUrl is not SRB, through an exception String onlineUrl = getOnLineURL(rootNode); //System.out.println("=================The url is "+onlineUrl); //3. Find the srb data file String dataFileName = getDataFileNameFromURL(onlineUrl); //System.out.println("the data fileName in eml "+dataFileName); //If the dataFileName in original eml is wrong, we need to look up the // the correct name first if (originalDataFileIncorrect) { Hashtable correctName = getCurrent_CorrectFileNamesPair(); dataFileName = (String) correctName.get(dataFileName); } //System.out.println("=================The data file is "+dataFileName); File dataFile = null; dataFile = new File(SRBDATAFILEDIR, dataFileName); if (!dataFile.exists()) { throw new Exception("Couldn't find the data file in srb data directory " + dataFile); } //4. Generate docid for data file String dataId = generateId(); //System.out.println("=======The docid for data file will be "+dataId); //5. upload data file to Metacat response = metacat.upload(dataId, dataFile); if (response.indexOf("<success>") == -1) { throw new Exception("Couldn't upload data file " + dataFileName + " with id " + dataId + " into Metacat since " + response); } //6. Updates eml online url and package id in DOM String newId = updateEMLDoc(rootNode, docid, dataId); //System.out.println("The new docid is ========"+newId); //Put EML DOM with the new packagId and oneline url into a StringWriter and store it to String StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter); XMLUtilities.print(rootNode, printWriter); String xml = stringWriter.toString(); //System.out.println("the xml is "+xml); //7.insert new (update) EML document into Metacat StringReader xmlReader = new StringReader(xml); response = metacat.update(newId, xmlReader, null); if (response.indexOf("<success>") == -1) { throw new Exception("Upload data file " + dataFileName + " with id " + dataId + " successfully but update eml " + newId + " failed since " + response); } metacat.logout(); return dataId; } /* * Gets onlineUrl value from a given eml DOM document. * The online url xpath can be "/eml/dataset/dataTable/physical/distribution/online/url" * or "/eml/dataset/spatialRaster/physical/distribution/online/url" */ private String getOnLineURL(Node root) throws Exception { String url = null; if (root == null) { throw new Exception("root node for this EML is null and couldn't get online url from it"); } Node urlNode = XMLUtilities.getTextNodeWithXPath(root, TABLEONLINEURL); // in table online url does exist, we will try to use another xpath - SPATIALONLEURL if (urlNode == null) { urlNode = XMLUtilities.getTextNodeWithXPath(root, SPATIALONLINEURL); } // Couldn't find any matche element, throw exception if (urlNode == null) { throw new Exception("Couldn't find any onlie url information in eml document"); } //Gets text node value and if the url doesn't contain "srb;//", it will throw a exception url = urlNode.getNodeValue(); if (url == null || url.indexOf(SRB) == -1) { throw new Exception("The online url doesn't have srb protocol and we don't need to handle"); } return url; } /* * Automatically to generate a unique id for ddata file. * This id will be looked like - DATAIDPREFIX.numberBaseonTime.1, e.g * IPCC.20072321.1 */ private String generateId() { int version = 1; StringBuffer docid = new StringBuffer(DATAIDPREFIX); docid.append(DOT); // Create a calendar to get the date formatted properly String[] ids = TimeZone.getAvailableIDs(-8 * 60 * 60 * 1000); SimpleTimeZone pdt = new SimpleTimeZone(-8 * 60 * 60 * 1000, ids[0]); pdt.setStartRule(Calendar.APRIL, 1, Calendar.SUNDAY, 2 * 60 * 60 * 1000); pdt.setEndRule(Calendar.OCTOBER, -1, Calendar.SUNDAY, 2 * 60 * 60 * 1000); Calendar calendar = new GregorianCalendar(pdt); Date trialTime = new Date(); calendar.setTime(trialTime); int time = 0; docid.append(calendar.get(Calendar.YEAR)); time = calendar.get(Calendar.DAY_OF_YEAR); if (time < 10) { docid.append("0"); docid.append("0"); docid.append(time); } else if (time < 100) { docid.append("0"); docid.append(time); } else { docid.append(time); } time = calendar.get(Calendar.HOUR_OF_DAY); if (time < 10) { docid.append("0"); docid.append(time); } else { docid.append(time); } time = calendar.get(Calendar.MINUTE); if (time < 10) { docid.append("0"); docid.append(time); } else { docid.append(time); } time = calendar.get(Calendar.SECOND); if (time < 10) { docid.append("0"); docid.append(time); } else { docid.append(time); } //sometimes this number is not unique, so we append a random number int random = (new Double(Math.random() * 100)).intValue(); docid.append(random); docid.append(DOT); docid.append(version); return docid.toString(); } /* * Get data file name from online url. SRB oneline url will looks like - * srb://seek:/home/beam.seek/IPCC_climate/Present/ccld6190.dat. * The last part - ccld6190.dat is the file name. This method will get * the file name from the give url */ private String getDataFileNameFromURL(String onlineUrl) throws Exception { String dataFile = null; String slash = "/"; if (onlineUrl != null) { int index = onlineUrl.lastIndexOf(slash); try { dataFile = onlineUrl.substring(index + 1); } catch (Exception e) { throw new Exception( "Couldn't get data file name from the given url " + onlineUrl + " since " + e.getMessage()); } } return dataFile; } /* * Gets eml document list from text file. The text file format should be: * tao.1.1 * tao.2.1 */ private Vector getDocumentListFromFile() throws Exception { Vector docList = new Vector(); File docListFile = new File(SRBDATAFILEDIR, DOCLISTFILE); FileReader docListFileReader = new FileReader(docListFile); BufferedReader readDocList = new BufferedReader(docListFileReader); // Read every line from the text file and put it into a vector String docid = readDocList.readLine(); while (docid != null) { // If the docid string is not empty, put it into vector if (!docid.trim().equals("")) { docList.add(docid.trim()); } docid = readDocList.readLine(); } return docList; } /* * Update the given eml document (in DOM). There are two places to be updated * The package id will be increased 1, i.e., from 1 to 2. The distribution online url will * point to the new ecogrid id, i.e. , ecogrid://knb/IPCC.2007.1 */ private String updateEMLDoc(Node root, String docid, String dataId) throws Exception { // update package id docid = getIncreasedNewDocid(docid); XMLUtilities.addAttributeNodeToDOMTree(root, PACKAGEID, docid); // update online url. oneline url should either in spatialRaster or dataTable. // First try to see if spatialRaster exist or not. If not try data table String newUrl = KNB + dataId; //new url looks like ecogrid://knb/IPCC.2007.1 boolean isSpatialRaster = true; boolean isDataTable = false; Node urlNode = XMLUtilities.getTextNodeWithXPath(root, SPATIALONLINEURL); if (urlNode == null) { // has no spatialRaster isSpatialRaster = false; } else { // has spatialRaster isSpatialRaster = true; } // determin if has datable or not urlNode = XMLUtilities.getTextNodeWithXPath(root, TABLEONLINEURL); if (urlNode != null) { isDataTable = true; } if (isSpatialRaster && !isDataTable) { //only has spatialRaster and no dataTable, update spatialRaster online url XMLUtilities.addTextNodeToDOMTree(root, SPATIALONLINEURL, newUrl); } else if (!isSpatialRaster && isDataTable) { //only has dataTable and no spatialRaster, update dataTable online url XMLUtilities.addTextNodeToDOMTree(root, TABLEONLINEURL, newUrl); } else { //some strange things happen throw new Exception("The eml either has both dataTable or spatialRaster OR doesn't has any entity"); } return docid; } /* * Gets new docid with increased version. Docid looks like tao.1.1. The new docid will be * tao.1.2. */ private String getIncreasedNewDocid(String docid) throws Exception { int rev = 1; String revision = null; String prefix = null; String newId = null; if (docid != null) { int index = docid.lastIndexOf(DOT); try { // Get revsion part(1) revision = docid.substring(index + 1); // Get prefix part (tao.1.) prefix = docid.substring(0, index + 1); // increase version from 1 to 2 rev = (new Integer(revision)).intValue(); rev++; // combines the prefix tao.1. and new revision2 to get tao.1.2 newId = prefix + rev; } catch (Exception e) { throw new Exception("Couldn't increase revsion number from the given docid " + docid + " since " + e.getMessage()); } } return newId; } /* * Gets eml document list from searching Metacat * TO-DO: This method need to be implemented */ private Vector getDocumentListFromMetacat() { Vector docList = new Vector(); return docList; } /* * Get eml document list. First this method will try * to get the eml document list form text file. If the result is empty or * it caught an exception it will try to get eml document list from metacat. */ private Vector getDocumentList() { Vector list = null; try { //First, try to get eml doc list from text file list = getDocumentListFromFile(); if (list == null || list.isEmpty()) { throw new Exception("The eml doclist is empty in text file"); } } catch (Exception e) { System.err.println("Couldn't get eml document list from text file: " + e.getMessage()); // If an exception happened, try to get eml doc list from metacat list = getDocumentListFromMetacat(); } if (list != null) { System.out.println("the list is " + list); } return list; } /* * Writes error message into log file. */ private void writeLog(File file, String message) { try { FileOutputStream fos = new FileOutputStream(file, true); PrintWriter pw = new PrintWriter(fos); SimpleDateFormat formatter = new SimpleDateFormat("yy-MM-dd HH:mm:ss"); java.util.Date localtime = new java.util.Date(); String dateString = formatter.format(localtime); dateString += " :: " + message; //time stamp each entry pw.println(dateString); pw.flush(); pw.close(); fos.close(); } catch (Exception e) { System.out.println( "error writing to replication log from " + "MetacatReplication.replLog: " + e.getMessage()); //e.printStackTrace(System.out); } } /* * Read a csv file which contains current data file name and correct data file name. * The format of csv file is: * currentname1,correctname1 * currentname2,correctname2 * ........ * The return value is hash table, the current data file name is key and correct file name is * value. */ private Hashtable getCurrent_CorrectFileNamesPair() throws Exception { Hashtable fileNamesHash = new Hashtable(); File current_correctFileNames = new File(CURRENT_CORRECTFILENAME); FileReader fileReader = new FileReader(current_correctFileNames); BufferedReader readDocList = new BufferedReader(fileReader); // Read every line from the text file, this line will look like: // currentname1,correctname1 String lineString = readDocList.readLine(); while (lineString != null) { //Get the comma index number int commaIndex = lineString.indexOf(","); if (commaIndex != -1) { //Get the current file name part String currentName = lineString.substring(0, commaIndex); //Get the correct file name part String correctName = lineString.substring(commaIndex + 1, lineString.length()); if (currentName != null && correctName != null) { fileNamesHash.put(currentName.trim(), correctName.trim()); } } lineString = readDocList.readLine(); } return fileNamesHash; } }