List of usage examples for java.util.zip CheckedInputStream CheckedInputStream
public CheckedInputStream(InputStream in, Checksum cksum)
From source file:com.blockwithme.longdb.tools.Utils.java
/** Get checksum CRC32 for the file content. * // w w w .j ava 2 s . com * @param theFile * the file * @param isCommpressed * true if commpressed * @return the checksum (CRC32) of the file. * @throws Exception */ @SuppressWarnings("resource") public static long getCRC32(final File theFile, final boolean isCommpressed) throws Exception { CheckedInputStream cis = null; try { final CRC32 checksum = new CRC32(); // TODO: Would a buffered input stream make this faster? cis = new CheckedInputStream((isCommpressed ? new GZIPInputStream(new FileInputStream(theFile)) : new FileInputStream(theFile)), checksum); final byte[] tempBuf = new byte[ONE_K]; while (cis.read(tempBuf) >= 0) ; // just read the full stream. // $codepro.audit.disable return checksum.getValue(); } finally { if (cis != null) cis.close(); } }
From source file:com.alexholmes.hdfsslurper.WorkerThread.java
private void process(FileStatus srcFileStatus) throws IOException, InterruptedException { Path stagingFile = null;//from ww w.j av a 2 s . c om FileSystem destFs = null; String filenameBatchidDelimiter = config.getFileNameBatchIdDelimiter(); try { FileSystem srcFs = srcFileStatus.getPath().getFileSystem(config.getConfig()); // run a script which can change the name of the file as well as // write out a new version of the file // if (config.getWorkScript() != null) { Path newSrcFile = stageSource(srcFileStatus); srcFileStatus = srcFileStatus.getPath().getFileSystem(config.getConfig()).getFileStatus(newSrcFile); } Path srcFile = srcFileStatus.getPath(); // get the target HDFS file // Path destFile = getHdfsTargetPath(srcFileStatus); if (config.getCodec() != null) { String ext = config.getCodec().getDefaultExtension(); if (!destFile.getName().endsWith(ext)) { destFile = new Path(destFile.toString() + ext); } } destFs = destFile.getFileSystem(config.getConfig()); // get the staging HDFS file // stagingFile = fileSystemManager.getStagingFile(srcFileStatus, destFile); String batchId = srcFile.toString().substring( srcFile.toString().lastIndexOf(filenameBatchidDelimiter) + 1, srcFile.toString().length()); log.info("event#Copying source file '" + srcFile + "' to staging destination '" + stagingFile + "'" + "$batchId#" + batchId); // if the directory of the target file doesn't exist, attempt to // create it // Path destParentDir = destFile.getParent(); if (!destFs.exists(destParentDir)) { log.info("event#Attempting creation of target directory: " + destParentDir.toUri()); if (!destFs.mkdirs(destParentDir)) { throw new IOException("event#Failed to create target directory: " + destParentDir.toUri()); } } // if the staging directory doesn't exist, attempt to create it // Path destStagingParentDir = stagingFile.getParent(); if (!destFs.exists(destStagingParentDir)) { log.info("event#Attempting creation of staging directory: " + destStagingParentDir.toUri()); if (!destFs.mkdirs(destStagingParentDir)) { throw new IOException("event#Failed to create staging directory: " + destParentDir.toUri()); } } // copy the file // InputStream is = null; OutputStream os = null; CRC32 crc = new CRC32(); try { is = new BufferedInputStream(srcFs.open(srcFile)); if (config.isVerify()) { is = new CheckedInputStream(is, crc); } os = destFs.create(stagingFile); if (config.getCodec() != null) { os = config.getCodec().createOutputStream(os); } IOUtils.copyBytes(is, os, 4096, false); } finally { IOUtils.closeStream(is); IOUtils.closeStream(os); } long srcFileSize = srcFs.getFileStatus(srcFile).getLen(); long destFileSize = destFs.getFileStatus(stagingFile).getLen(); if (config.getCodec() == null && srcFileSize != destFileSize) { throw new IOException( "event#File sizes don't match, source = " + srcFileSize + ", dest = " + destFileSize); } log.info("event#Local file size = " + srcFileSize + ", HDFS file size = " + destFileSize + "$batchId#" + batchId); if (config.isVerify()) { verify(stagingFile, crc.getValue()); } if (destFs.exists(destFile)) { destFs.delete(destFile, false); } log.info("event#Moving staging file '" + stagingFile + "' to destination '" + destFile + "'" + "$batchId#" + batchId); if (!destFs.rename(stagingFile, destFile)) { throw new IOException("event#Failed to rename file"); } if (config.isCreateLzopIndex() && destFile.getName().endsWith(lzopExt)) { Path lzoIndexPath = new Path(destFile.toString() + LzoIndex.LZO_INDEX_SUFFIX); if (destFs.exists(lzoIndexPath)) { log.info("event#Deleting index file as it already exists"); destFs.delete(lzoIndexPath, false); } indexer.index(destFile); } fileSystemManager.fileCopyComplete(srcFileStatus); } catch (Throwable t) { log.error("event#Caught exception working on file " + srcFileStatus.getPath(), t); // delete the staging file if it still exists // try { if (destFs != null && destFs.exists(stagingFile)) { destFs.delete(stagingFile, false); } } catch (Throwable t2) { log.error("event#Failed to delete staging file " + stagingFile, t2); } fileSystemManager.fileCopyError(srcFileStatus); } }
From source file:com.panet.imeta.core.row.ValueDataUtil.java
public static Long ChecksumCRC32(ValueMetaInterface metaA, Object dataA) { long checksum = 0; FileObject file = null;/*from w w w .ja v a 2 s .c o m*/ try { file = KettleVFS.getFileObject(dataA.toString()); CheckedInputStream cis = null; // Computer CRC32 checksum cis = new CheckedInputStream((FileInputStream) ((LocalFile) file).getInputStream(), new CRC32()); byte[] buf = new byte[128]; while (cis.read(buf) >= 0) { } checksum = cis.getChecksum().getValue(); } catch (Exception e) { } finally { if (file != null) try { file.close(); } catch (Exception e) { } ; } return checksum; }
From source file:net.sourceforge.jaulp.file.checksum.ChecksumUtils.java
/** * Gets the checksum from the given file. If the flag crc is true than the CheckedInputStream is * constructed with an instance of <code>java.util.zip.CRC32</code> otherwise with an instance * of <code>java.util.zip.Adler32</code>. * * @param file// w ww .j a va2 s. c o m * The file The file from what to get the checksum. * @param crc * The crc If the flag crc is true than the CheckedInputStream is constructed with an * instance of {@link java.util.zip.CRC32} object otherwise it is constructed with an * instance of * @return The checksum from the given file as long. * @throws FileNotFoundException * Is thrown if the file is not found. * @throws IOException * Signals that an I/O exception has occurred. {@link java.util.zip.CRC32} object * otherwise it is constructed with an instance of {@link java.util.zip.Adler32} * object. {@link java.util.zip.Adler32} object. */ public static long getChecksum(File file, boolean crc) throws FileNotFoundException, IOException { CheckedInputStream cis = null; if (crc) { cis = new CheckedInputStream(new FileInputStream(file), new CRC32()); } else { cis = new CheckedInputStream(new FileInputStream(file), new Adler32()); } int length = (int) file.length(); byte[] buffer = new byte[length]; long checksum = 0; while (cis.read(buffer) >= 0) { checksum = cis.getChecksum().getValue(); } checksum = cis.getChecksum().getValue(); StreamUtils.closeInputStream(cis); return checksum; }
From source file:com.panet.imeta.core.row.ValueDataUtil.java
public static Long ChecksumAdler32(ValueMetaInterface metaA, Object dataA) { long checksum = 0; FileObject file = null;//from w w w .ja va 2 s .co m try { file = KettleVFS.getFileObject(dataA.toString()); CheckedInputStream cis = null; // Computer Adler-32 checksum cis = new CheckedInputStream((FileInputStream) ((LocalFile) file).getInputStream(), new Adler32()); byte[] buf = new byte[128]; while (cis.read(buf) >= 0) { } checksum = cis.getChecksum().getValue(); } catch (Exception e) { //throw new Exception(e); } finally { if (file != null) try { file.close(); } catch (Exception e) { } ; } return checksum; }
From source file:de.alpharogroup.file.checksum.ChecksumUtils.java
/** * Gets the checksum from the given file. If the flag crc is true than the CheckedInputStream is * constructed with an instance of <code>java.util.zip.CRC32</code> otherwise with an instance * of <code>java.util.zip.Adler32</code>. * * @param file/* w ww . j a v a 2 s .co m*/ * The file The file from what to get the checksum. * @param crc * The crc If the flag crc is true than the CheckedInputStream is constructed with an * instance of {@link java.util.zip.CRC32} object otherwise it is constructed with an * instance of * @return The checksum from the given file as long. * @throws FileNotFoundException * Is thrown if the file is not found. * @throws IOException * Signals that an I/O exception has occurred. {@link java.util.zip.CRC32} object * otherwise it is constructed with an instance of {@link java.util.zip.Adler32} * object. {@link java.util.zip.Adler32} object. */ public static long getChecksum(final File file, final boolean crc) throws FileNotFoundException, IOException { CheckedInputStream cis = null; if (crc) { cis = new CheckedInputStream(new FileInputStream(file), new CRC32()); } else { cis = new CheckedInputStream(new FileInputStream(file), new Adler32()); } final int length = (int) file.length(); final byte[] buffer = new byte[length]; long checksum = 0; while (cis.read(buffer) >= 0) { checksum = cis.getChecksum().getValue(); } checksum = cis.getChecksum().getValue(); StreamUtils.closeInputStream(cis); return checksum; }
From source file:de.alpharogroup.file.checksum.ChecksumExtensions.java
/** * Gets the checksum from the given file. If the flag crc is true than the CheckedInputStream is * constructed with an instance of <code>java.util.zip.CRC32</code> otherwise with an instance * of <code>java.util.zip.Adler32</code>. * * @param file//from ww w . j a va2 s . c o m * The file The file from what to get the checksum. * @param crc * The crc If the flag crc is true than the CheckedInputStream is constructed with an * instance of {@link java.util.zip.CRC32} object otherwise it is constructed with an * instance of * @return The checksum from the given file as long. * @throws FileNotFoundException * Is thrown if the file is not found. * @throws IOException * Signals that an I/O exception has occurred. {@link java.util.zip.CRC32} object * otherwise it is constructed with an instance of {@link java.util.zip.Adler32} * object. {@link java.util.zip.Adler32} object. */ public static long getChecksum(final File file, final boolean crc) throws FileNotFoundException, IOException { CheckedInputStream cis = null; if (crc) { cis = new CheckedInputStream(new FileInputStream(file), new CRC32()); } else { cis = new CheckedInputStream(new FileInputStream(file), new Adler32()); } final int length = (int) file.length(); final byte[] buffer = new byte[length]; long checksum = 0; while (cis.read(buffer) >= 0) { checksum = cis.getChecksum().getValue(); } checksum = cis.getChecksum().getValue(); StreamExtensions.closeInputStream(cis); return checksum; }
From source file:com.alexholmes.hdfsslurper.WorkerThread.java
private long hdfsFileCRC32(Path path) throws IOException { InputStream in = null;/*from w w w. jav a 2s . c om*/ CRC32 crc = new CRC32(); try { InputStream is = new BufferedInputStream(path.getFileSystem(config.getConfig()).open(path)); if (config.getCodec() != null) { is = config.getCodec().createInputStream(is); } in = new CheckedInputStream(is, crc); org.apache.commons.io.IOUtils.copy(in, new NullOutputStream()); } finally { org.apache.commons.io.IOUtils.closeQuietly(in); } return crc.getValue(); }
From source file:com.alcatel_lucent.nz.wnmsextract.reader.FileUtilities.java
public long checksum(File input) { long checksum = -1; try {//from w w w.j a va2 s . c o m FileInputStream fis = null; CheckedInputStream cis = null; Adler32 adler = null; fis = new FileInputStream(input); adler = new Adler32(); cis = new CheckedInputStream(fis, adler); byte[] buffer = new byte[1024]; while (cis.read(buffer) >= 0) { checksum = cis.getChecksum().getValue(); } } catch (IOException e) { jlog.fatal("IO Exception on " + input.getAbsolutePath() + e); } return checksum; }
From source file:com.adito.core.CoreUtil.java
/** * @param f//from ww w. j a v a2 s . co m * @return long * @throws IOException */ public static long generateChecksum(File f) throws IOException { Adler32 alder = new Adler32(); FileInputStream fin = new FileInputStream(f); CheckedInputStream in = new CheckedInputStream(fin, alder); byte[] buf = new byte[32768]; Util.readFullyIntoBuffer(in, buf); alder = (Adler32) in.getChecksum(); try { in.close(); } catch (IOException ex) { } try { fin.close(); } catch (IOException ex1) { } return alder.getValue(); }