Java tutorial
package com.alcatel_lucent.nz.wnmsextract.reader; /* * This file is part of wnmsextract. * * wnmsextract is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * wnmsextract is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** * @author Ryan Scott */ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Enumeration; import java.util.List; import java.util.zip.Adler32; import java.util.zip.CheckedInputStream; import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import com.alcatel_lucent.nz.wnmsextract.document.ALUFileFilter; import com.alcatel_lucent.nz.wnmsextract.document.DocumentType; /** * This is the old method for unzipping files. it was a bit clunky so has * been replaced with FileSelector. Some scripts may still rely on it * however, like Ryan's excel conversion stuff */ @Deprecated //public class FileUtilities implements FileSelector public class FileUtilities extends FileSelector { private static final String CDIR = File.separator + "~COMPRESSED" + File.separator; private static final String RDIR = File.separator + "RAW" + File.separator; //private static boolean debug = false; private static Logger jlog = Logger.getLogger("com.alcatel_lucent.nz.wnmsextract.schedule.FileUtilities"); private static FileUtilities fileutility; private List<File> xmlFiles = new ArrayList<File>(); private Calendar cal; private File sourcepath; private File temppath; //Singleton class private FileUtilities() { } public static synchronized FileUtilities getInstance() { if (fileutility == null) { fileutility = new FileUtilities(); } return fileutility; } // Returns a list of files of XMLs that need to be processed @Override public void extract() { File source_raw = new File(getSourcePath().getAbsolutePath() + RDIR); File source_compressed = getSourcePath();//+CDIR); File dest_local = getTempPath(); File dest_localdate = new File(getTempPath().getAbsolutePath() + calendarToString(cal)); // Delete the previous days data deletePreviousDay(cal, getTempPath().getAbsolutePath()); // Set up the filefilter //ALUFileFilter filefilter = (ALUFileFilter) doctype.getFileFilter(); //filefilter.setAcceptDate(cal); jlog.info("Extract [RAW:" + source_raw + ", CMP:" + source_compressed + "]"); // Flag for if we have found data for the specified date boolean dataFound = false; // Used to store the filtered XML files //--------------------------------------------------------------------------- // Check cwd for current zip boolean zipPresentCwd = false; for (File f1 : dest_local.listFiles()) { // Check if we have the folder if (f1.getName().compareTo(calendarToString(cal)) == 0) { zipPresentCwd = true; dataFound = true; break; } } jlog.debug("Zip Present CWD " + zipPresentCwd); //--------------------------------------------------------------------------- // If data isn't present in cwd we need to get it if (zipPresentCwd == false) { boolean zipPresentCompressed = false; // Check for the input zip in the compressed folder for (File file_in_compressed : source_compressed.listFiles()) { // Check if we have the .zip file if (file_in_compressed.getName().compareTo(calendarToString(cal) + ".zip") == 0) { zipPresentCompressed = true; dataFound = true; break; } } // If its in the compressed folder we need to decompress it, jlog.debug("Zip Present ~COMPRESSED " + zipPresentCompressed); if (zipPresentCompressed) { decompressCompressed(source_compressed, cal, dest_local); } // Else we check if its in the RAW folder else { // Assuming if there is files in the RAW directory the data is there if ((new File(getSourcePath().getAbsolutePath() + CDIR)).listFiles().length != 0) { decompressRaw(source_raw, cal, dest_localdate); dataFound = true; } } } //--------------------------------------------------------------------------- // If we have found the data and copied to temppath jlog.debug("Data Found " + dataFound); if (dataFound) { // Take a list of all the gzips List<File> gzips = Arrays.asList((new File(temppath + calendarToString(cal))).listFiles()); // Decompress all the files from the temppath to XML if its not already in the cwd if (zipPresentCwd == false) { decompressGZip(gzips); } // Take a list of all the files after uncompressing List<File> all = Arrays.asList((new File(temppath + calendarToString(cal))).listFiles()); // Loop through all the files and create a new file list of all the XML files in the temppath for (File f1 : all) { // Check if it is accepted by the file filter //if(filefilter.accept(f1)) // Check to see if the size is zero then do what? xmlFiles.add(f1); } } // We haven't found the data anywhere, we have a problem else { System.out.println("Data not found"); } //System.out.println("if we've made it to here extract hasn't crashed (though it may not have worked)"); jlog.debug("XMLFILES for ?<<<"); for (File xf : xmlFiles) { jlog.debug(xf); } jlog.debug(">>>"); } // Returns a list of files that conform to the filter @SuppressWarnings("unused") private List<File> getFileType(DocumentType doctype, Calendar cal, File temppath) { //Set the date to filter on ALUFileFilter ff = (ALUFileFilter) doctype.getFileFilter(); ff.setAcceptDate(cal); // Loop through all the files in the temppath List<File> filteredFiles = new ArrayList<File>(); for (File f3 : (temppath).listFiles()) { // If the filter is accepted add the files to the new list if (ff.accept(f3)) { filteredFiles.add(f3); } } return filteredFiles; } // Copies all the files from the source directory to the dest directory public void decompressRaw(File source, Calendar cal, File dest) { boolean foundData = false; //File sourceFolder = null; // Try find the directory for (File f : source.listFiles()) { if ((f.getName().compareTo(calendarToString(cal)) == 0) && f.isDirectory()) { foundData = true; copyFile(f, dest, true); break; } } if (!foundData) { jlog.fatal("Couldn't create directory : " + dest.getAbsolutePath() + File.separator); } return; } // Decompresses the zip from the compressed CPV to the destination public void decompressCompressed(File sourcepath, Calendar cal, File destpath) { File sourcezip = null; File destzip = null; // Try get the input zip, should be path + %date%.zip try { sourcezip = new File(sourcepath.getAbsolutePath() + File.separator + calendarToString(cal) + ".zip"); } catch (NullPointerException e) { jlog.fatal("Can't find zip file" + calendarToString(cal) + ".zip " + e); // Kill thread? } // Try open the output zip, want it to be in temppath\%date%.zip try { destzip = new File(destpath.getAbsolutePath() + File.separator + calendarToString(cal) + ".zip"); } catch (NullPointerException e) { jlog.fatal("Can't create output zip file " + e); } // Try copy the file to the temppath if (!copyFile(sourcezip, destzip, false)) { jlog.info("Failed copying source zip " + sourcezip.getAbsolutePath()); if (!copyFile(sourcezip, destzip, false)) { jlog.info("Second attempt failed copying source zip " + sourcezip.getAbsolutePath()); // kill thread return; } } //Recursively decompress the zip from temppath -> temppath\%date%\ File decompressPath = new File( destpath.getAbsolutePath() + File.separator + calendarToString(cal) + File.separator); decompressZip(destzip, decompressPath); // Delete source file deleteFile(destzip); } // Get the checksum of a file public long checksum(File input) { long checksum = -1; try { FileInputStream fis = null; CheckedInputStream cis = null; Adler32 adler = null; fis = new FileInputStream(input); adler = new Adler32(); cis = new CheckedInputStream(fis, adler); byte[] buffer = new byte[1024]; while (cis.read(buffer) >= 0) { checksum = cis.getChecksum().getValue(); } } catch (IOException e) { jlog.fatal("IO Exception on " + input.getAbsolutePath() + e); } return checksum; } // Recursively decompress a Zip, will decompress to the temppath public void decompressZip(File inputZipPath, File zipPath) { int BUFFER = 2048; List<File> zipFiles = new ArrayList<File>(); try { zipPath.mkdir(); } catch (SecurityException e) { jlog.fatal("Security exception when creating " + zipPath.getName()); } ZipFile zipFile = null; boolean isZip = true; // Open Zip file for reading (should be in temppath) try { zipFile = new ZipFile(inputZipPath, ZipFile.OPEN_READ); } catch (IOException e) { jlog.fatal("IO exception in " + inputZipPath.getName()); } // Create an enumeration of the entries in the zip file Enumeration<? extends ZipEntry> zipFileEntries = zipFile.entries(); if (isZip) { // Process each entry while (zipFileEntries.hasMoreElements()) { // Get a zip file entry ZipEntry entry = zipFileEntries.nextElement(); String currentEntry = entry.getName(); File destFile = null; // destFile should be pointing to temppath\%date%\ try { destFile = new File(zipPath.getAbsolutePath(), currentEntry); destFile = new File(zipPath.getAbsolutePath(), destFile.getName()); } catch (NullPointerException e) { jlog.fatal("File not found" + destFile.getName()); } // If the entry is a .zip add it to the list so that it can be extracted if (currentEntry.endsWith(".zip")) { zipFiles.add(destFile); } try { // Extract file if not a directory if (!entry.isDirectory()) { // Stream the zip entry BufferedInputStream is = new BufferedInputStream(zipFile.getInputStream(entry)); int currentByte; // establish buffer for writing file byte data[] = new byte[BUFFER]; FileOutputStream fos = null; // Write the current file to disk try { fos = new FileOutputStream(destFile); } catch (FileNotFoundException e) { jlog.fatal("File not found " + destFile.getName()); } catch (SecurityException e) { jlog.fatal("Access denied to " + destFile.getName()); } BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER); // read and write until last byte is encountered while ((currentByte = is.read(data, 0, BUFFER)) != -1) { dest.write(data, 0, currentByte); } dest.flush(); dest.close(); is.close(); } } catch (IOException ioe) { jlog.fatal("IO exception in " + zipFile.getName()); } } try { zipFile.close(); } catch (IOException e) { jlog.fatal("IO exception when closing " + zipFile.getName()); } } // Recursively decompress the list of zip files for (File f : zipFiles) { decompressZip(f, zipPath); } return; } // Decompress a list of Gzips to the same folder they came from!!! public void decompressGZip(List<File> fileList) { List<File> failedFiles = new ArrayList<File>(); for (File f1 : fileList) { if (f1.getName().endsWith(".gz")) { // if it fails if (!decompressGZipFile(f1)) { failedFiles.add(f1); } } } // Check for failed files if (failedFiles.size() > 0) { for (File f1 : failedFiles) { if (f1.getName().endsWith(".gz")) { // if it fails again the file is probably a dud if (!decompressGZipFile(f1)) { jlog.fatal(".gz file not extracted: " + f1.getName()); deleteFile(f1); } } // Should never get here else { jlog.fatal("Failed extraction, file being deleted: " + f1.getName()); deleteFile(f1); } } } return; } // Decompresses a GZIP file public boolean decompressGZipFile(File input) { // Open the compressed file GZIPInputStream in = null; // If the file is of zero length delete it if (input.length() == 0) { deleteFile(input); return true; } try { in = new GZIPInputStream(new FileInputStream(input)); } catch (FileNotFoundException e) { jlog.fatal(".gz file not found: " + input); e.printStackTrace(); return false; } catch (IOException e) { jlog.fatal("IO exception on: " + input); e.printStackTrace(); return false; } // Open the output file String target = input.getAbsolutePath().replace(".gz", ".xml"); OutputStream out = null; try { out = new FileOutputStream(target); } catch (FileNotFoundException e) { jlog.fatal("File not found: " + target); e.printStackTrace(); return false; } // Transfer bytes from the compressed file to the output file byte[] buf = new byte[1024]; int len; try { while ((len = in.read(buf)) > 0) { out.write(buf, 0, len); } } catch (IOException e) { jlog.fatal("IO exception when writing to " + target); e.printStackTrace(); return false; } finally { // Close the file and stream try { in.close(); } catch (IOException e) { jlog.fatal("Can't close input stream"); e.printStackTrace(); return false; } try { out.close(); } catch (IOException e) { jlog.fatal("Can't close output stream"); e.printStackTrace(); return false; } } // Check if the gzip file size == 0 File outputFile = new File(target); // If the file is of zero length something bad happened if (outputFile.length() == 0) { return false; } // Delete the file deleteFile(input); return true; } // Delete a file, input a string of the path @SuppressWarnings("unused") private void deleteFile(String filename) { deleteFile(new File(filename)); } // Delete a file public void deleteFile(File file) { file.delete(); return; } // Copy a file to another public boolean copyFile(File in, File out, boolean filesOnly) { if (in.isDirectory()) { try { if (filesOnly) { String[] params = { "zip", "gz" }; for (Object obj : FileUtils.listFiles(in, params, true)) { File x = (File) obj; if (x.getName().endsWith(".zip")) { decompressZip(x, out); } FileUtils.copyFileToDirectory(x, out); } } else { FileUtils.copyDirectory(in, out); } } catch (IOException e) { jlog.fatal("Failed to copy directory: " + in.getAbsolutePath()); return false; } } else { try { FileUtils.copyFile(in, out); } catch (IOException e) { jlog.fatal("Failed to copy file: " + in.getAbsolutePath()); return false; } } return true; } // Returns a string of the calendar in the format yyyyMMdd public static String calendarToString(Calendar cal) { DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); return (dateFormat.format(cal.getTime())); } /** * Deletes any files downloaded from the previous day * @param cal. The day being processed NOT the day being deleted */ public static boolean deletePreviousDay(Calendar today, String temppath) { Calendar yesterday = (Calendar) today.clone(); yesterday.add(Calendar.DATE, -2); File to_delete = new File(temppath + calendarToString(yesterday)); return to_delete.delete(); } //Autogenerated/replaced interface matching methods //@Override @Override public void setCalendar(Calendar cal) { this.cal = cal; } /** * fetchFileList. Returns a list of files matching the requested doctype in the current extract directory * @param doctype * @return */ //@Override public List<File> getFileList(DocumentType doctype) { //Set the date to filter on ALUFileFilter ff = (ALUFileFilter) doctype.getFileFilter(); ff.setAcceptDate(cal); // Loop through all the files in the temppath List<File> filteredFiles = new ArrayList<File>(); for (File f3 : (getTempPath()).listFiles()) { // If the filter is accepted add the files to the new list if (ff.accept(f3)) { filteredFiles.add(f3); } } return filteredFiles; } //autogen/replaced interface methods //@Override @Override public void setSourcePath(File sourcepath) { this.sourcepath = sourcepath; } @Override public File getSourcePath() { return this.sourcepath; } //@Override @Override public void setTempPath(File temppath) { this.temppath = temppath; } @Override public File getTempPath() { return this.temppath; } @Override public String docPath() { // TODO Auto-generated method stub return null; } }