List of usage examples for java.util.zip CRC32 CRC32
public CRC32()
From source file:com.nridge.core.base.field.data.DataBag.java
/** * Convenience method will calculate a unique type id property for * the bag based on each field name using a CRC32 algorithm. *//*from ww w .j av a 2 s . co m*/ public void setTypeIdByNames() { CRC32 crc32 = new CRC32(); crc32.reset(); if (StringUtils.isNotEmpty(mName)) crc32.update(mName.getBytes()); else { for (DataField dataField : mFields) crc32.update(dataField.getName().getBytes()); } setTypeId(crc32.getValue()); }
From source file:com.bigdata.dastor.db.commitlog.CommitLog.java
public static void recover(File[] clogs) throws IOException { Set<Table> tablesRecovered = new HashSet<Table>(); List<Future<?>> futures = new ArrayList<Future<?>>(); for (File file : clogs) { int bufferSize = (int) Math.min(file.length(), 32 * 1024 * 1024); BufferedRandomAccessFile reader = new BufferedRandomAccessFile(file.getAbsolutePath(), "r", bufferSize); try {/*from www .ja va 2 s .c om*/ final CommitLogHeader clHeader; try { clHeader = CommitLogHeader.readCommitLogHeader(reader); } catch (EOFException eofe) { logger.info( "Attempted to recover an incomplete CommitLogHeader. Everything is ok, don't panic."); continue; } /* seek to the lowest position where any CF has non-flushed data */ int lowPos = CommitLogHeader.getLowestPosition(clHeader); if (lowPos == 0) continue; reader.seek(lowPos); if (logger.isDebugEnabled()) logger.debug("Replaying " + file + " starting at " + lowPos); /* read the logs populate RowMutation and apply */ while (!reader.isEOF()) { if (logger.isDebugEnabled()) logger.debug("Reading mutation at " + reader.getFilePointer()); long claimedCRC32; byte[] bytes; try { bytes = new byte[(int) reader.readLong()]; // readlong can throw EOFException too reader.readFully(bytes); claimedCRC32 = reader.readLong(); } catch (EOFException e) { // last CL entry didn't get completely written. that's ok. break; } ByteArrayInputStream bufIn = new ByteArrayInputStream(bytes); Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); if (claimedCRC32 != checksum.getValue()) { // this part of the log must not have been fsynced. probably the rest is bad too, // but just in case there is no harm in trying them. continue; } /* deserialize the commit log entry */ final RowMutation rm = RowMutation.serializer().deserialize(new DataInputStream(bufIn)); if (logger.isDebugEnabled()) logger.debug(String.format("replaying mutation for %s.%s: %s", rm.getTable(), rm.key(), "{" + StringUtils.join(rm.getColumnFamilies(), ", ") + "}")); final Table table = Table.open(rm.getTable()); tablesRecovered.add(table); final Collection<ColumnFamily> columnFamilies = new ArrayList<ColumnFamily>( rm.getColumnFamilies()); final long entryLocation = reader.getFilePointer(); Runnable runnable = new WrappedRunnable() { public void runMayThrow() throws IOException { /* remove column families that have already been flushed before applying the rest */ for (ColumnFamily columnFamily : columnFamilies) { int id = table.getColumnFamilyId(columnFamily.name()); if (!clHeader.isDirty(id) || entryLocation <= clHeader.getPosition(id)) { rm.removeColumnFamily(columnFamily); } } if (!rm.isEmpty()) { Table.open(rm.getTable()).apply(rm, null, false); } } }; futures.add(StageManager.getStage(StageManager.MUTATION_STAGE).submit(runnable)); if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) { FBUtilities.waitOnFutures(futures); futures.clear(); } } } finally { reader.close(); logger.info("Finished reading " + file); } } // wait for all the writes to finish on the mutation stage FBUtilities.waitOnFutures(futures); logger.debug("Finished waiting on mutations from recovery"); // flush replayed tables futures.clear(); for (Table table : tablesRecovered) futures.addAll(table.flush()); FBUtilities.waitOnFutures(futures); }
From source file:guru.benson.pinch.Pinch.java
/** * Extract all ZipEntries from the ZIP central directory. * * @param buf//from w ww . j a v a2 s .co m * The byte buffer containing the ZIP central directory. * * @return A list with all ZipEntries. */ private static ArrayList<ExtendedZipEntry> parseHeaders(ByteBuffer buf) { ArrayList<ExtendedZipEntry> zeList = new ArrayList<ExtendedZipEntry>(); buf.order(ByteOrder.LITTLE_ENDIAN); int offset = 0; while (offset < buf.limit() - ZipConstants.CENHDR) { short fileNameLen = buf.getShort(offset + ZipConstants.CENNAM); short extraFieldLen = buf.getShort(offset + ZipConstants.CENEXT); short fileCommentLen = buf.getShort(offset + ZipConstants.CENCOM); String fileName = new String(buf.array(), offset + ZipConstants.CENHDR, fileNameLen); ExtendedZipEntry zeGermans = new ExtendedZipEntry(fileName); zeGermans.setMethod(buf.getShort(offset + ZipConstants.CENHOW)); CRC32 crc = new CRC32(); crc.update(buf.getInt(offset + ZipConstants.CENCRC)); zeGermans.setCrc(crc.getValue()); zeGermans.setCompressedSize(buf.getInt(offset + ZipConstants.CENSIZ)); zeGermans.setSize(buf.getInt(offset + ZipConstants.CENLEN)); zeGermans.setInternalAttr(buf.getShort(offset + ZipConstants.CENATT)); zeGermans.setExternalAttr(buf.getShort(offset + ZipConstants.CENATX)); zeGermans.setOffset((long) buf.getInt(offset + ZipConstants.CENOFF)); zeGermans.setExtraLength(extraFieldLen); zeList.add(zeGermans); offset += ZipConstants.CENHDR + fileNameLen + extraFieldLen + fileCommentLen; } return zeList; }
From source file:com.splout.db.dnode.HttpFileExchanger.java
@Override public void handle(HttpExchange exchange) throws IOException { DataInputStream iS = null;/* w ww .j a va2 s . c o m*/ FileOutputStream writer = null; File dest = null; String tablespace = null; Integer partition = null; Long version = null; try { iS = new DataInputStream(new GZIPInputStream(exchange.getRequestBody())); String fileName = exchange.getRequestHeaders().getFirst("filename"); tablespace = exchange.getRequestHeaders().getFirst("tablespace"); partition = Integer.valueOf(exchange.getRequestHeaders().getFirst("partition")); version = Long.valueOf(exchange.getRequestHeaders().getFirst("version")); dest = new File( new File(tempDir, DNodeHandler.getLocalStoragePartitionRelativePath(tablespace, partition, version)), fileName); // just in case, avoid copying the same file concurrently // (but we also shouldn't avoid this in other levels of the app) synchronized (currentTransfersMonitor) { if (currentTransfers.containsKey(dest.toString())) { throw new IOException("Incoming file already being transferred - " + dest); } currentTransfers.put(dest.toString(), new Object()); } if (!dest.getParentFile().exists()) { dest.getParentFile().mkdirs(); } if (dest.exists()) { dest.delete(); } writer = new FileOutputStream(dest); byte[] buffer = new byte[config.getInt(FetcherProperties.DOWNLOAD_BUFFER)]; Checksum checkSum = new CRC32(); // 1- Read file size long fileSize = iS.readLong(); log.debug("Going to read file [" + fileName + "] of size: " + fileSize); // 2- Read file contents long readSoFar = 0; do { long missingBytes = fileSize - readSoFar; int bytesToRead = (int) Math.min(missingBytes, buffer.length); int read = iS.read(buffer, 0, bytesToRead); checkSum.update(buffer, 0, read); writer.write(buffer, 0, read); readSoFar += read; callback.onProgress(tablespace, partition, version, dest, fileSize, readSoFar); } while (readSoFar < fileSize); // 3- Read CRC long expectedCrc = iS.readLong(); if (expectedCrc == checkSum.getValue()) { log.info("File [" + dest.getAbsolutePath() + "] received -> Checksum -- " + checkSum.getValue() + " matches expected CRC [OK]"); callback.onFileReceived(tablespace, partition, version, dest); } else { log.error("File received [" + dest.getAbsolutePath() + "] -> Checksum -- " + checkSum.getValue() + " doesn't match expected CRC: " + expectedCrc); callback.onBadCRC(tablespace, partition, version, dest); dest.delete(); } } catch (Throwable t) { log.error(t); callback.onError(t, tablespace, partition, version, dest); if (dest != null && dest.exists() && !t.getMessage().contains("Incoming file already being transferred")) { dest.delete(); } } finally { if (writer != null) { writer.close(); } if (iS != null) { iS.close(); } if (dest != null) { currentTransfers.remove(dest.toString()); } } }
From source file:org.apache.lucene.store.FSDirectory.java
public synchronized String getCacheKey(String[] filelist) { CacheKeyBuffer kkk = new CacheKeyBuffer(filelist, this.dir_uuid, System.currentTimeMillis() / 600000l, this.getP()); String rtn = CACHE_BUFFER.get(kkk); if (rtn == null) { StringBuffer buff = new StringBuffer(); buff.append(this.getClass().getName()).append("_"); buff.append(this.directory.getAbsolutePath()).append("_"); CRC32 crc32 = new CRC32(); crc32.update(0);/*from ww w .java 2 s . c om*/ long filesize = 0; long filemodify = 0; if (filelist != null) { buff.append(filelist.length).append("_"); for (String s : filelist) { crc32.update(new String(s).getBytes()); try { filesize += this.fileLength(s); } catch (Throwable e) { logger.error("filelength", e); } try { filemodify = Math.max(filemodify, this.fileModified(s)); } catch (Throwable e) { logger.error("filelength", e); } } } long crcvalue = crc32.getValue(); buff.append(crcvalue).append("_"); buff.append(filesize).append("_"); buff.append(filemodify).append("_"); buff.append(format.format(new Date(filemodify))); rtn = buff.toString(); CACHE_BUFFER.put(kkk, rtn); } return rtn; }
From source file:com.jhash.oimadmin.Utils.java
public static void createJarFileFromContent(Map<String, byte[]> content, String[] fileSequence, String jarFileName) {/*from w w w . j a v a2 s.c o m*/ logger.trace("createJarFileFromContent({},{})", content, jarFileName); logger.trace("Trying to create a new jar file"); try (ZipOutputStream jarFileOutputStream = new ZipOutputStream(new FileOutputStream(jarFileName))) { jarFileOutputStream.setMethod(ZipOutputStream.STORED); for (String jarItem : fileSequence) { logger.trace("Processing item {}", jarItem); byte[] fileContent = content.get(jarItem); if (fileContent == null) throw new NullPointerException("Failed to locate content for file " + jarItem); JarEntry pluginXMLFileEntry = new JarEntry(jarItem); pluginXMLFileEntry.setTime(System.currentTimeMillis()); pluginXMLFileEntry.setSize(fileContent.length); pluginXMLFileEntry.setCompressedSize(fileContent.length); CRC32 crc = new CRC32(); crc.update(fileContent); pluginXMLFileEntry.setCrc(crc.getValue()); jarFileOutputStream.putNextEntry(pluginXMLFileEntry); jarFileOutputStream.write(fileContent); jarFileOutputStream.closeEntry(); } } catch (Exception exception) { throw new OIMAdminException("Failed to create the Jar file " + jarFileName, exception); } }
From source file:com.occamlab.te.parsers.ImageParser.java
private static void processBufferedImage(BufferedImage buffimage, String formatName, NodeList nodes) throws Exception { HashMap<Object, Object> bandMap = new HashMap<Object, Object>(); for (int i = 0; i < nodes.getLength(); i++) { Node node = nodes.item(i); if (node.getNodeType() == Node.ELEMENT_NODE) { if (node.getLocalName().equals("subimage")) { Element e = (Element) node; int x = Integer.parseInt(e.getAttribute("x")); int y = Integer.parseInt(e.getAttribute("y")); int w = Integer.parseInt(e.getAttribute("width")); int h = Integer.parseInt(e.getAttribute("height")); processBufferedImage(buffimage.getSubimage(x, y, w, h), formatName, e.getChildNodes()); } else if (node.getLocalName().equals("checksum")) { CRC32 checksum = new CRC32(); Raster raster = buffimage.getRaster(); DataBufferByte buffer; if (node.getParentNode().getLocalName().equals("subimage")) { WritableRaster outRaster = raster.createCompatibleWritableRaster(); buffimage.copyData(outRaster); buffer = (DataBufferByte) outRaster.getDataBuffer(); } else { buffer = (DataBufferByte) raster.getDataBuffer(); }//from w ww . j av a 2 s.c o m int numbanks = buffer.getNumBanks(); for (int j = 0; j < numbanks; j++) { checksum.update(buffer.getData(j)); } Document doc = node.getOwnerDocument(); node.appendChild(doc.createTextNode(Long.toString(checksum.getValue()))); } else if (node.getLocalName().equals("count")) { String band = ((Element) node).getAttribute("bands"); String sample = ((Element) node).getAttribute("sample"); if (sample.equals("all")) { bandMap.put(band, null); } else { HashMap<Object, Object> sampleMap = (HashMap<Object, Object>) bandMap.get(band); if (sampleMap == null) { if (!bandMap.containsKey(band)) { sampleMap = new HashMap<Object, Object>(); bandMap.put(band, sampleMap); } } sampleMap.put(Integer.decode(sample), new Integer(0)); } } else if (node.getLocalName().equals("transparentNodata")) { // 2011-08-24 // PwD String transparentNodata = checkTransparentNodata(buffimage, node); node.setTextContent(transparentNodata); } } } Iterator bandIt = bandMap.keySet().iterator(); while (bandIt.hasNext()) { String band_str = (String) bandIt.next(); int band_indexes[]; if (buffimage.getType() == BufferedImage.TYPE_BYTE_BINARY || buffimage.getType() == BufferedImage.TYPE_BYTE_GRAY) { band_indexes = new int[1]; band_indexes[0] = 0; } else { band_indexes = new int[band_str.length()]; for (int i = 0; i < band_str.length(); i++) { if (band_str.charAt(i) == 'A') band_indexes[i] = 3; if (band_str.charAt(i) == 'B') band_indexes[i] = 2; if (band_str.charAt(i) == 'G') band_indexes[i] = 1; if (band_str.charAt(i) == 'R') band_indexes[i] = 0; } } Raster raster = buffimage.getRaster(); java.util.HashMap sampleMap = (java.util.HashMap) bandMap.get(band_str); boolean addall = (sampleMap == null); if (sampleMap == null) { sampleMap = new java.util.HashMap(); bandMap.put(band_str, sampleMap); } int minx = raster.getMinX(); int maxx = minx + raster.getWidth(); int miny = raster.getMinY(); int maxy = miny + raster.getHeight(); int bands[][] = new int[band_indexes.length][raster.getWidth()]; for (int y = miny; y < maxy; y++) { for (int i = 0; i < band_indexes.length; i++) { raster.getSamples(minx, y, maxx, 1, band_indexes[i], bands[i]); } for (int x = minx; x < maxx; x++) { int sample = 0; for (int i = 0; i < band_indexes.length; i++) { sample |= bands[i][x] << ((band_indexes.length - i - 1) * 8); } Integer sampleObj = new Integer(sample); boolean add = addall; if (!addall) { add = sampleMap.containsKey(sampleObj); } if (add) { Integer count = (Integer) sampleMap.get(sampleObj); if (count == null) { count = new Integer(0); } count = new Integer(count.intValue() + 1); sampleMap.put(sampleObj, count); } } } } Node node = nodes.item(0); while (node != null) { if (node.getNodeType() == Node.ELEMENT_NODE) { if (node.getLocalName().equals("count")) { String band = ((Element) node).getAttribute("bands"); String sample = ((Element) node).getAttribute("sample"); HashMap sampleMap = (HashMap) bandMap.get(band); Document doc = node.getOwnerDocument(); if (sample.equals("all")) { Node parent = node.getParentNode(); Node prevSibling = node.getPreviousSibling(); Iterator sampleIt = sampleMap.keySet().iterator(); Element countnode = null; int digits; String prefix; switch (buffimage.getType()) { case BufferedImage.TYPE_BYTE_BINARY: digits = 1; prefix = ""; break; case BufferedImage.TYPE_BYTE_GRAY: digits = 2; prefix = "0x"; break; default: prefix = "0x"; digits = band.length() * 2; } while (sampleIt.hasNext()) { countnode = doc.createElementNS(node.getNamespaceURI(), "count"); Integer sampleInt = (Integer) sampleIt.next(); Integer count = (Integer) sampleMap.get(sampleInt); if (band.length() > 0) { countnode.setAttribute("bands", band); } countnode.setAttribute("sample", prefix + HexString(sampleInt.intValue(), digits)); Node textnode = doc.createTextNode(count.toString()); countnode.appendChild(textnode); parent.insertBefore(countnode, node); if (sampleIt.hasNext()) { if (prevSibling != null && prevSibling.getNodeType() == Node.TEXT_NODE) { parent.insertBefore(prevSibling.cloneNode(false), node); } } } parent.removeChild(node); node = countnode; } else { Integer count = (Integer) sampleMap.get(Integer.decode(sample)); if (count == null) count = new Integer(0); Node textnode = doc.createTextNode(count.toString()); node.appendChild(textnode); } } } node = node.getNextSibling(); } }
From source file:bobs.is.compress.sevenzip.SevenZFile.java
private Archive readHeaders(final byte[] password) throws IOException { final byte[] signature = new byte[6]; file.readFully(signature);// www. ja v a 2 s . co m if (!Arrays.equals(signature, sevenZSignature)) { throw new IOException("Bad 7z signature"); } // 7zFormat.txt has it wrong - it's first major then minor final byte archiveVersionMajor = file.readByte(); final byte archiveVersionMinor = file.readByte(); if (archiveVersionMajor != 0) { throw new IOException( String.format("Unsupported 7z version (%d,%d)", archiveVersionMajor, archiveVersionMinor)); } final long startHeaderCrc = 0xffffFFFFL & Integer.reverseBytes(file.readInt()); final StartHeader startHeader = readStartHeader(startHeaderCrc); final int nextHeaderSizeInt = (int) startHeader.nextHeaderSize; if (nextHeaderSizeInt != startHeader.nextHeaderSize) { throw new IOException("cannot handle nextHeaderSize " + startHeader.nextHeaderSize); } file.seek(SIGNATURE_HEADER_SIZE + startHeader.nextHeaderOffset); final byte[] nextHeader = new byte[nextHeaderSizeInt]; file.readFully(nextHeader); final CRC32 crc = new CRC32(); crc.update(nextHeader); if (startHeader.nextHeaderCrc != crc.getValue()) { throw new IOException("NextHeader CRC mismatch"); } final ByteArrayInputStream byteStream = new ByteArrayInputStream(nextHeader); DataInputStream nextHeaderInputStream = new DataInputStream(byteStream); Archive archive = new Archive(); int nid = nextHeaderInputStream.readUnsignedByte(); if (nid == NID.kEncodedHeader) { nextHeaderInputStream = readEncodedHeader(nextHeaderInputStream, archive, password); // Archive gets rebuilt with the new header archive = new Archive(); nid = nextHeaderInputStream.readUnsignedByte(); } if (nid == NID.kHeader) { readHeader(nextHeaderInputStream, archive); nextHeaderInputStream.close(); } else { throw new IOException("Broken or unsupported archive: no Header"); } return archive; }
From source file:org.apache.hadoop.raid.TestRaidDfs.java
private void validateFile(FileSystem fileSys, Path name1, Path name2, long crc) throws IOException { FileStatus stat1 = fileSys.getFileStatus(name1); FileStatus stat2 = fileSys.getFileStatus(name2); assertTrue(" Length of file " + name1 + " is " + stat1.getLen() + " is different from length of file " + name1 + " " + stat2.getLen(), stat1.getLen() == stat2.getLen()); CRC32 newcrc = new CRC32(); FSDataInputStream stm = fileSys.open(name2); final byte[] b = new byte[4192]; int num = 0;/*from ww w. ja v a 2 s. c o m*/ while (num >= 0) { num = stm.read(b); if (num < 0) { break; } newcrc.update(b, 0, num); } stm.close(); LOG.info(" Newcrc " + newcrc.getValue() + " old crc " + crc); if (newcrc.getValue() != crc) { fail("CRC mismatch of files " + name1 + " with file " + name2); } }
From source file:org.phpmaven.phar.PharJavaPackager.java
private void packFile(final ByteArrayOutputStream fileEntriesBaos, final ByteArrayOutputStream compressedFilesBaos, final File fileToPack, String filePath) throws IOException { if (DEBUG) {/* w ww. j a v a 2 s. c o m*/ System.out.println("Packing file " + fileToPack + " with " + fileToPack.length() + " bytes."); } final byte[] fileBytes = filePath.getBytes("UTF-8"); writeIntLE(fileEntriesBaos, fileBytes.length); fileEntriesBaos.write(fileBytes); // TODO Complain with files larger than 4 bytes file length writeIntLE(fileEntriesBaos, (int) fileToPack.length()); writeIntLE(fileEntriesBaos, (int) (fileToPack.lastModified() / 1000)); final byte[] uncompressed = FileUtils.readFileToByteArray(fileToPack); if (DEBUG) { System.out.println("read " + uncompressed.length + " bytes from file."); } final ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); // final GZIPOutputStream gzipStream = new GZIPOutputStream(compressedStream); // gzipStream.write(uncompressed); // gzipStream.flush(); final CRC32 checksum = new CRC32(); checksum.update(uncompressed); final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); deflater.setInput(uncompressed); deflater.finish(); final byte[] buf = new byte[Short.MAX_VALUE]; while (!deflater.needsInput()) { final int bytesRead = deflater.deflate(buf); compressedStream.write(buf, 0, bytesRead); } final byte[] compressed = compressedStream.toByteArray(); if (DEBUG) { System.out.println("compressed to " + compressed.length + " bytes."); } // final Inflater decompresser = new Inflater(); // decompresser.setInput(compressed); // byte[] result = new byte[5000]; // try { // int resultLength = decompresser.inflate(result); // final String str = new String(result, 0, resultLength); // int i = 42; // } catch (DataFormatException e) { // // TODO Auto-generated catch block // e.printStackTrace(); // } // decompresser.end(); compressedFilesBaos.write(compressed); writeIntLE(fileEntriesBaos, compressed.length); writeIntLE(fileEntriesBaos, checksum.getValue()); // bits: 0x00001000, gzip fileEntriesBaos.write(0); fileEntriesBaos.write(0x10); fileEntriesBaos.write(0); fileEntriesBaos.write(0); // 0 bytes manifest writeIntLE(fileEntriesBaos, 0); }