List of usage examples for java.util.zip Checksum update
public void update(byte[] b, int off, int len);
From source file:com.bigdata.dastor.db.commitlog.CommitLog.java
public static void recover(File[] clogs) throws IOException { Set<Table> tablesRecovered = new HashSet<Table>(); List<Future<?>> futures = new ArrayList<Future<?>>(); for (File file : clogs) { int bufferSize = (int) Math.min(file.length(), 32 * 1024 * 1024); BufferedRandomAccessFile reader = new BufferedRandomAccessFile(file.getAbsolutePath(), "r", bufferSize); try {//from ww w. j a v a 2 s .c o m final CommitLogHeader clHeader; try { clHeader = CommitLogHeader.readCommitLogHeader(reader); } catch (EOFException eofe) { logger.info( "Attempted to recover an incomplete CommitLogHeader. Everything is ok, don't panic."); continue; } /* seek to the lowest position where any CF has non-flushed data */ int lowPos = CommitLogHeader.getLowestPosition(clHeader); if (lowPos == 0) continue; reader.seek(lowPos); if (logger.isDebugEnabled()) logger.debug("Replaying " + file + " starting at " + lowPos); /* read the logs populate RowMutation and apply */ while (!reader.isEOF()) { if (logger.isDebugEnabled()) logger.debug("Reading mutation at " + reader.getFilePointer()); long claimedCRC32; byte[] bytes; try { bytes = new byte[(int) reader.readLong()]; // readlong can throw EOFException too reader.readFully(bytes); claimedCRC32 = reader.readLong(); } catch (EOFException e) { // last CL entry didn't get completely written. that's ok. break; } ByteArrayInputStream bufIn = new ByteArrayInputStream(bytes); Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); if (claimedCRC32 != checksum.getValue()) { // this part of the log must not have been fsynced. probably the rest is bad too, // but just in case there is no harm in trying them. continue; } /* deserialize the commit log entry */ final RowMutation rm = RowMutation.serializer().deserialize(new DataInputStream(bufIn)); if (logger.isDebugEnabled()) logger.debug(String.format("replaying mutation for %s.%s: %s", rm.getTable(), rm.key(), "{" + StringUtils.join(rm.getColumnFamilies(), ", ") + "}")); final Table table = Table.open(rm.getTable()); tablesRecovered.add(table); final Collection<ColumnFamily> columnFamilies = new ArrayList<ColumnFamily>( rm.getColumnFamilies()); final long entryLocation = reader.getFilePointer(); Runnable runnable = new WrappedRunnable() { public void runMayThrow() throws IOException { /* remove column families that have already been flushed before applying the rest */ for (ColumnFamily columnFamily : columnFamilies) { int id = table.getColumnFamilyId(columnFamily.name()); if (!clHeader.isDirty(id) || entryLocation <= clHeader.getPosition(id)) { rm.removeColumnFamily(columnFamily); } } if (!rm.isEmpty()) { Table.open(rm.getTable()).apply(rm, null, false); } } }; futures.add(StageManager.getStage(StageManager.MUTATION_STAGE).submit(runnable)); if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) { FBUtilities.waitOnFutures(futures); futures.clear(); } } } finally { reader.close(); logger.info("Finished reading " + file); } } // wait for all the writes to finish on the mutation stage FBUtilities.waitOnFutures(futures); logger.debug("Finished waiting on mutations from recovery"); // flush replayed tables futures.clear(); for (Table table : tablesRecovered) futures.addAll(table.flush()); FBUtilities.waitOnFutures(futures); }
From source file:com.jkoolcloud.tnt4j.streams.configure.state.AbstractFileStreamStateHandler.java
/** * Check the line CRC./*from w w w.j a v a2 s .c o m*/ * * @param line * line to check * @param crc * CRC to match * * @return {@code true} if matched */ private static boolean checkCrc(String line, Long crc) { if (line == null || crc == null) { return false; } Checksum crcLine = new CRC32(); final byte[] bytes4Line = line.getBytes(); crcLine.update(bytes4Line, 0, bytes4Line.length); final long lineCRC = crcLine.getValue(); return lineCRC == crc; }
From source file:com.autoupdateapk.AutoUpdateApk.java
private static int crc32(String str) { byte bytes[] = str.getBytes(); Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); return (int) checksum.getValue(); }
From source file:com.jkoolcloud.tnt4j.streams.configure.state.AbstractFileStreamStateHandler.java
/** * Calculates CRC value for bytes read from provided input stream. * * @param in//from w w w. ja va 2 s.c o m * reader to read bytes for CRC calculation * * @return calculated CRC value * * @throws IOException * if input can't be read. */ protected static Long getInputCrc(Reader in) throws IOException { char[] buff = new char[BYTES_TO_COUNT_CRC]; Checksum crc = new CRC32(); int readLen = in.read(buff); if (readLen > 0) { String str = new String(buff, 0, readLen); final byte[] bytes = str.getBytes(Utils.UTF8); crc.update(bytes, 0, bytes.length); } return crc.getValue(); }
From source file:org.codice.ddf.checksum.impl.CRC32ChecksumProvider.java
@Override public String calculateChecksum(InputStream inputStream) throws IOException { if (inputStream == null) { throw new IllegalArgumentException("InputStream cannot be null"); }//from w w w . ja v a2s . c o m byte[] bytes = IOUtils.toByteArray(inputStream); Checksum checksum = new CRC32(); checksum.update(bytes, 0, bytes.length); long checkSumValue = checksum.getValue(); return Long.toHexString(checkSumValue); }
From source file:de.berlin.magun.nfcmime.core.CrcGenerator.java
/** * Generates a CRC32 checksum for an array of NdefRecords. * @param records/*from ww w.j a va2 s . c om*/ * @return CRC32 checksum represented as long. */ public long getChecksum(NdefRecord[] records) { byte[] overallBytes = getByteArray(records); Checksum cs = new CRC32(); cs.update(overallBytes, 0, overallBytes.length); return cs.getValue(); }
From source file:com.threewks.thundr.util.Encoder.java
public Encoder crc32() { Checksum checksum = new CRC32(); checksum.update(data, 0, data.length); long checksumValue = checksum.getValue(); data = Long.toHexString(checksumValue).getBytes(); return unhex(); }
From source file:CRC32HashBuilder.java
/** * {@inheritDoc}/*from www .j a v a 2 s. c o m*/ */ public String getHash(final InputStream input) throws IOException { if (input == null) { throw new IllegalArgumentException("Content cannot be null!"); } final Checksum checksum = new CRC32(); final byte[] bytes = new byte[1024]; int len = 0; while ((len = input.read(bytes)) >= 0) { checksum.update(bytes, 0, len); } final String hash = new BigInteger(Long.toString(checksum.getValue())).toString(16); return hash; }
From source file:kr.ac.cau.mecs.cass.signal.payload.JSONObjectPayload.java
@Override public void serializeRawData(Payload payload) { String data = this.data.toString(); Checksum crc32 = new CRC32(); Base64 base64 = new Base64(); byte[] bytedata = data.getBytes(Charset.forName("UTF-8")); crc32.reset();//from ww w .j a va 2s.c o m crc32.update(bytedata, 0, bytedata.length); String encoded = base64.encode(bytedata); payload.setCrc(crc32.getValue()); payload.setLength(encoded.length()); payload.setRawdata(encoded); payload.setType(0x11); }
From source file:com.jkoolcloud.tnt4j.streams.configure.state.FileStreamStateHandlerTest.java
@Test public void findStreamingFile() throws Exception { FileStreamStateHandler rwd = new FileStreamStateHandler(); File testFilesDir = new File(samplesDir, "/multiple-logs/"); File[] testFiles = testFilesDir.listFiles((FilenameFilter) new WildcardFileFilter("orders*")); // NON-NLS FileAccessState newFAS = new FileAccessState(); int count = 0; File fileToSearchFor = null;//from w ww . j a v a 2 s . co m int lineLastRead = 0; File fileWritten = null; for (File testFile : testFiles) { count++; FileReader in; LineNumberReader reader; Long fileCRC = rwd.getFileCrc(testFile); if (count == 2) { newFAS.currentFileCrc = fileCRC; fileToSearchFor = testFile; } in = new FileReader(testFile); reader = new LineNumberReader(in); reader.setLineNumber(0); String line = reader.readLine(); int count2 = 0; while (line != null) { count2++; Checksum crcLine = new CRC32(); final byte[] bytes4Line = line.getBytes(); crcLine.update(bytes4Line, 0, bytes4Line.length); final long lineCRC = crcLine.getValue(); final int lineNumber = reader.getLineNumber(); System.out.println("for " + lineNumber + " line CRC is " + lineCRC); // NON-NLS if (count2 == 3) { newFAS.currentLineCrc = lineCRC; newFAS.currentLineNumber = lineNumber; newFAS.lastReadTime = System.currentTimeMillis(); lineLastRead = lineNumber; } line = reader.readLine(); } fileWritten = AbstractFileStreamStateHandler.writeState(newFAS, testFilesDir, "TestStream"); // NON-NLS Utils.close(reader); } final File findLastProcessed = rwd.findStreamingFile(newFAS, testFiles); assertEquals(fileToSearchFor, findLastProcessed); final int lineLastReadRecorded = rwd.checkLine(findLastProcessed, newFAS); assertEquals(lineLastRead, lineLastReadRecorded); fileWritten.delete(); }