List of usage examples for java.util.zip CRC32 CRC32
public CRC32()
From source file:com.neusoft.clw.infomanage.ridingplan.action.RidingPlanAction.java
private String doChecksum(String fileName) { long checksum = 0; try {//from www . j a va 2 s . c om CheckedInputStream cis = null; try { // Computer CRC32 checksum cis = new CheckedInputStream(new FileInputStream(fileName), new CRC32()); } catch (FileNotFoundException e) { LOG.error("File not found."); } byte[] buf = new byte[128]; while (cis.read(buf) >= 0) { } checksum = cis.getChecksum().getValue(); } catch (IOException e) { e.printStackTrace(); } return String.valueOf(checksum); }
From source file:org.apache.nifi.processors.standard.TailFile.java
/** * Finds any files that have rolled over and have not yet been ingested by * this Processor. Each of these files that is found will be ingested as its * own FlowFile. If a file is found that has been partially ingested, the * rest of the file will be ingested as a single FlowFile but the data that * already has been ingested will not be ingested again. * * @param context the ProcessContext to use in order to obtain Processor * configuration./*w w w . j a v a 2 s. c o m*/ * @param session the ProcessSession to use in order to interact with * FlowFile creation and content. * @param expectedChecksum the checksum value that is expected for the * oldest file from offset 0 through <position>. * @param timestamp the latest Last Modfiied Timestamp that has been * consumed. Any data that was written before this data will not be * ingested. * @param position the byte offset in the file being tailed, where tailing * last left off. * * @return <code>true</code> if the file being tailed has rolled over, false * otherwise */ private boolean recoverRolledFiles(final ProcessContext context, final ProcessSession session, final String tailFile, final List<File> rolledOffFiles, final Long expectedChecksum, final long timestamp, final long position) { try { getLogger().debug("Recovering Rolled Off Files; total number of files rolled off = {}", new Object[] { rolledOffFiles.size() }); TailFileObject tfo = states.get(tailFile); // For first file that we find, it may or may not be the file that we were last reading from. // As a result, we have to read up to the position we stored, while calculating the checksum. If the checksums match, // then we know we've already processed this file. If the checksums do not match, then we have not // processed this file and we need to seek back to position 0 and ingest the entire file. // For all other files that have been rolled over, we need to just ingest the entire file. boolean rolloverOccurred = !rolledOffFiles.isEmpty(); if (rolloverOccurred && expectedChecksum != null && rolledOffFiles.get(0).length() >= position) { final File firstFile = rolledOffFiles.get(0); final long startNanos = System.nanoTime(); if (position > 0) { try (final InputStream fis = new FileInputStream(firstFile); final CheckedInputStream in = new CheckedInputStream(fis, new CRC32())) { StreamUtils.copy(in, new NullOutputStream(), position); final long checksumResult = in.getChecksum().getValue(); if (checksumResult == expectedChecksum) { getLogger().debug("Checksum for {} matched expected checksum. Will skip first {} bytes", new Object[] { firstFile, position }); // This is the same file that we were reading when we shutdown. Start reading from this point on. rolledOffFiles.remove(0); FlowFile flowFile = session.create(); flowFile = session.importFrom(in, flowFile); if (flowFile.getSize() == 0L) { session.remove(flowFile); // use a timestamp of lastModified() + 1 so that we do not ingest this file again. cleanup(); tfo.setState( new TailFileState(tailFile, null, null, 0L, firstFile.lastModified() + 1L, firstFile.length(), null, tfo.getState().getBuffer())); } else { final Map<String, String> attributes = new HashMap<>(3); attributes.put(CoreAttributes.FILENAME.key(), firstFile.getName()); attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain"); attributes.put("tailfile.original.path", tailFile); flowFile = session.putAllAttributes(flowFile, attributes); session.getProvenanceReporter().receive(flowFile, firstFile.toURI().toString(), "FlowFile contains bytes 0 through " + position + " of source file", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos)); session.transfer(flowFile, REL_SUCCESS); getLogger().debug("Created {} from rolled over file {} and routed to success", new Object[] { flowFile, firstFile }); // use a timestamp of lastModified() + 1 so that we do not ingest this file again. cleanup(); tfo.setState( new TailFileState(tailFile, null, null, 0L, firstFile.lastModified() + 1L, firstFile.length(), null, tfo.getState().getBuffer())); // must ensure that we do session.commit() before persisting state in order to avoid data loss. session.commit(); persistState(tfo, context); } } else { getLogger().debug( "Checksum for {} did not match expected checksum. Checksum for file was {} but expected {}. Will consume entire file", new Object[] { firstFile, checksumResult, expectedChecksum }); } } } } // For each file that we found that matches our Rollover Pattern, and has a last modified date later than the timestamp // that we recovered from the state file, we need to consume the entire file. The only exception to this is the file that // we were reading when we last stopped, as it may already have been partially consumed. That is taken care of in the // above block of code. for (final File file : rolledOffFiles) { tfo.setState(consumeFileFully(file, context, session, tfo)); } return rolloverOccurred; } catch (final IOException e) { getLogger().error("Failed to recover files that have rolled over due to {}", new Object[] { e }); return false; } }
From source file:org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.java
/** * Calculate the CRC of a file to use to determine if it has changed. * * @param filePath File to get the CRC for * * @return String containing the CRC//from w w w . j av a 2 s . co m * * @throws SharedConfigurationException */ private static String calculateCRC(String filePath) throws SharedConfigurationException { File file = new File(filePath); try { FileInputStream fileStream = new FileInputStream(file); CRC32 crc = new CRC32(); byte[] buffer = new byte[65536]; int bytesRead = fileStream.read(buffer); while (-1 != bytesRead) { crc.update(buffer, 0, bytesRead); bytesRead = fileStream.read(buffer); } return String.valueOf(crc.getValue()); } catch (IOException ex) { throw new SharedConfigurationException( String.format("Failed to calculate CRC for %s", file.getAbsolutePath()), ex); } }
From source file:cn.com.sinosoft.util.io.FileUtils.java
/** * Computes the checksum of a file using the CRC32 checksum routine. * The value of the checksum is returned. * * @param file the file to checksum, must not be <code>null</code> * @return the checksum value/*from w w w.j a va 2 s. c o m*/ * @throws NullPointerException if the file or checksum is <code>null</code> * @throws IllegalArgumentException if the file is a directory * @throws IOException if an IO error occurs reading the file * @since Commons IO 1.3 */ public static long checksumCRC32(File file) throws IOException { CRC32 crc = new CRC32(); checksum(file, crc); return crc.getValue(); }
From source file:com.cardio3g.MainActivity.java
/** * Go through each of the Expansion APK files and open each as a zip file. * Calculate the CRC for each file and return false if any fail to match. * * @return true if XAPKZipFile is successful *//* www. j a va 2 s.c o m*/ void validateXAPKZipFiles() { AsyncTask<Object, DownloadProgressInfo, Boolean> validationTask = new AsyncTask<Object, DownloadProgressInfo, Boolean>() { @Override protected void onPreExecute() { mDownloadViewGroup.setVisibility(View.VISIBLE); super.onPreExecute(); } @Override protected Boolean doInBackground(Object... params) { for (XAPKFile xf : xAPKS) { String fileName = Helpers.getExpansionAPKFileName(MainActivity.this, xf.mIsMain, xf.mFileVersion); if (!Helpers.doesFileExist(MainActivity.this, fileName, xf.mFileSize, false)) return false; fileName = Helpers.generateSaveFileName(MainActivity.this, fileName); ZipResourceFile zrf; byte[] buf = new byte[1024 * 256]; try { zrf = new ZipResourceFile(fileName); ZipResourceFile.ZipEntryRO[] entries = zrf.getAllEntries(); /** * First calculate the total compressed length */ long totalCompressedLength = 0; for (ZipResourceFile.ZipEntryRO entry : entries) { totalCompressedLength += entry.mCompressedLength; } float averageVerifySpeed = 0; long totalBytesRemaining = totalCompressedLength; long timeRemaining; /** * Then calculate a CRC for every file in the Zip file, * comparing it to what is stored in the Zip directory. * Note that for compressed Zip files we must extract * the contents to do this comparison. */ for (ZipResourceFile.ZipEntryRO entry : entries) { if (-1 != entry.mCRC32) { long length = entry.mUncompressedLength; CRC32 crc = new CRC32(); DataInputStream dis = null; try { dis = new DataInputStream(zrf.getInputStream(entry.mFileName)); long startTime = SystemClock.uptimeMillis(); while (length > 0) { int seek = (int) (length > buf.length ? buf.length : length); dis.readFully(buf, 0, seek); crc.update(buf, 0, seek); length -= seek; long currentTime = SystemClock.uptimeMillis(); long timePassed = currentTime - startTime; if (timePassed > 0) { float currentSpeedSample = (float) seek / (float) timePassed; if (0 != averageVerifySpeed) { averageVerifySpeed = SMOOTHING_FACTOR * currentSpeedSample + (1 - SMOOTHING_FACTOR) * averageVerifySpeed; } else { averageVerifySpeed = currentSpeedSample; } totalBytesRemaining -= seek; timeRemaining = (long) (totalBytesRemaining / averageVerifySpeed); this.publishProgress(new DownloadProgressInfo(totalCompressedLength, totalCompressedLength - totalBytesRemaining, timeRemaining, averageVerifySpeed)); } startTime = currentTime; if (mCancelValidation) return true; } if (crc.getValue() != entry.mCRC32) { Log.e("EXPANSION ERROR", "CRC does not match for entry: " + entry.mFileName); Log.e("EXPANSION ERROR", "In file: " + entry.getZipFileName()); return false; } } finally { if (null != dis) { dis.close(); } } } } } catch (IOException e) { e.printStackTrace(); return false; } } return true; } @Override protected void onProgressUpdate(DownloadProgressInfo... values) { onDownloadProgress(values[0]); super.onProgressUpdate(values); } @Override protected void onPostExecute(Boolean result) { if (result) { mDownloadViewGroup.setVisibility(View.GONE); } else { mDownloadViewGroup.setVisibility(View.VISIBLE); } super.onPostExecute(result); } }; validationTask.execute(new Object()); }
From source file:com.redskyit.scriptDriver.RunTests.java
private void info(WebElement element, String selector, boolean verify) throws Exception { do {//from ww w . jav a 2 s . co m try { Point loc = element.getLocation(); Dimension size = element.getSize(); String tag = element.getTagName(); System.out .print(null == selector ? "test-id \"" + element.getAttribute("test-id") + "\"" : selector); System.out.print(" info"); System.out.print(" tag " + tag); System.out.print((element.isDisplayed() ? "" : " not") + " displayed"); System.out.print(" at " + loc.x + "," + loc.y); System.out.print(" size " + size.width + "," + size.height); System.out.print((element.isEnabled() ? "" : " not") + " enabled"); System.out.print((element.isSelected() ? "" : " not") + " selected"); if (tag.equals("input") || tag.equals("select")) { System.out.print(" check \"" + element.getAttribute("value") + "\""); } else { String text = tag.equals("textarea") ? element.getAttribute("value") : element.getText(); if (text.indexOf('\n') != -1) { CRC32 crc = new CRC32(); crc.update(text.getBytes()); System.out.print(" checksum \"crc32:" + crc.getValue() + "\""); } else { System.out.print(" check \"" + element.getText() + "\""); } } System.out.println(); return; } catch (StaleElementReferenceException e) { // If element has gone stale during a dump, ignore it if (!verify) return; // element has gone stale, re-select it System.out.println("// EXCEPTION : StaleElementReference"); } catch (Exception e) { if (verify) throw e; return; } sleepAndReselect(100); } while (_waitFor > 0 && (new Date()).getTime() < _waitFor); }
From source file:com.redskyit.scriptDriver.RunTests.java
private boolean compareStrings(String s1, String s2, boolean checksum) { if (checksum) { CRC32 crc = new CRC32(); crc.update(s1.getBytes());/*from w w w . ja v a 2 s. co m*/ return ("crc32:" + crc.getValue()).equals(s2); } return s1.equals(s2); }
From source file:org.commoncrawl.service.crawler.CrawlLog.java
public static void walkCrawlLogFile(File crawlLogPath, long startOffset) throws IOException { // and open the crawl log file ... RandomAccessFile inputStream = null; IOException exception = null; CRC32 crc = new CRC32(); CustomByteArrayOutputStream buffer = new CustomByteArrayOutputStream(1 << 17); byte[] syncBytesBuffer = new byte[SYNC_BYTES_SIZE]; // save position for potential debug output. long lastReadPosition = 0; try {//w w w. ja v a 2 s. c om inputStream = new RandomAccessFile(crawlLogPath, "rw"); // and a data input stream ... RandomAccessFile reader = inputStream; // seek to zero reader.seek(0L); // read the header ... LogFileHeader header = readLogFileHeader(reader); System.out.println("Header ItemCount:" + header._itemCount + " FileSize:" + header._fileSize); if (startOffset != 0L) { System.out.println("Preseeking to:" + startOffset); reader.seek(startOffset); } Configuration conf = new Configuration(); // read a crawl url from the stream... long recordCount = 0; while (inputStream.getFilePointer() < header._fileSize) { // System.out.println("PRE-SYNC SeekPos:"+ // inputStream.getFilePointer()); if (seekToNextSyncBytesPos(syncBytesBuffer, reader, header._fileSize)) { // System.out.println("POST-SYNC SeekPos:"+ // inputStream.getFilePointer()); lastReadPosition = inputStream.getFilePointer(); // skip sync inputStream.skipBytes(SYNC_BYTES_SIZE); // read length ... int urlDataLen = reader.readInt(); long urlDataCRC = reader.readLong(); if (urlDataLen > buffer.getBuffer().length) { buffer = new CustomByteArrayOutputStream(((urlDataLen / 65536) + 1) * 65536); } reader.read(buffer.getBuffer(), 0, urlDataLen); crc.reset(); crc.update(buffer.getBuffer(), 0, urlDataLen); long computedValue = crc.getValue(); // validate crc values ... if (computedValue != urlDataCRC) { LOG.error("CRC Mismatch Detected during HDFS transfer in CrawlLog:" + crawlLogPath.getAbsolutePath() + " FilePosition:" + lastReadPosition); inputStream.seek(lastReadPosition + 1); } else { if (recordCount++ % 10000 == 0) { // allocate a crawl url data structure CrawlURL url = new CrawlURL(); DataInputStream bufferReader = new DataInputStream( new ByteArrayInputStream(buffer.getBuffer(), 0, urlDataLen)); // populate it from the (in memory) data stream url.readFields(bufferReader); System.out.println("Record:" + recordCount + " At:" + lastReadPosition + " URL:" + url.getUrl() + " BuffSize:" + urlDataLen + " ContentLen:" + url.getContentRaw().getCount() + " LastModified:" + new Date(url.getLastAttemptTime()).toString()); } } } else { break; } } } catch (EOFException e) { LOG.error("Caught EOF Exception during read of local CrawlLog:" + crawlLogPath.getAbsolutePath() + " FilePosition:" + lastReadPosition); } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); exception = e; throw e; } finally { if (inputStream != null) inputStream.close(); } }
From source file:org.alfresco.repo.search.impl.lucene.index.IndexInfo.java
private void setStatusFromFile(FileChannel channel) throws IOException { if (channel.size() > 0) { channel.position(0);//from w w w . j av a2s .c o m ByteBuffer buffer; if (useNIOMemoryMapping) { MappedByteBuffer mbb = channel.map(MapMode.READ_ONLY, 0, channel.size()); mbb.load(); buffer = mbb; } else { buffer = ByteBuffer.wrap(new byte[(int) channel.size()]); channel.read(buffer); buffer.position(0); } buffer.position(0); long onDiskVersion = buffer.getLong(); if (version != onDiskVersion) { CRC32 crc32 = new CRC32(); crc32.update((int) (onDiskVersion >>> 32) & 0xFFFFFFFF); crc32.update((int) (onDiskVersion >>> 0) & 0xFFFFFFFF); int size = buffer.getInt(); crc32.update(size); LinkedHashMap<String, IndexEntry> newIndexEntries = new LinkedHashMap<String, IndexEntry>(); // Not all state is saved some is specific to this index so we // need to add the transient stuff. // Until things are committed they are not shared unless it is // prepared for (int i = 0; i < size; i++) { String indexTypeString = readString(buffer, crc32); IndexType indexType; try { indexType = IndexType.valueOf(indexTypeString); } catch (IllegalArgumentException e) { throw new IOException("Invalid type " + indexTypeString); } String name = readString(buffer, crc32); String parentName = readString(buffer, crc32); String txStatus = readString(buffer, crc32); TransactionStatus status; try { status = TransactionStatus.valueOf(txStatus); } catch (IllegalArgumentException e) { throw new IOException("Invalid status " + txStatus); } String mergeId = readString(buffer, crc32); long documentCount = buffer.getLong(); crc32.update((int) (documentCount >>> 32) & 0xFFFFFFFF); crc32.update((int) (documentCount >>> 0) & 0xFFFFFFFF); long deletions = buffer.getLong(); crc32.update((int) (deletions >>> 32) & 0xFFFFFFFF); crc32.update((int) (deletions >>> 0) & 0xFFFFFFFF); byte deleteOnlyNodesFlag = buffer.get(); crc32.update(deleteOnlyNodesFlag); boolean isDeletOnlyNodes = deleteOnlyNodesFlag == 1; if (!status.isTransient()) { newIndexEntries.put(name, new IndexEntry(indexType, name, parentName, status, mergeId, documentCount, deletions, isDeletOnlyNodes)); } } long onDiskCRC32 = buffer.getLong(); if (crc32.getValue() == onDiskCRC32) { for (IndexEntry entry : indexEntries.values()) { if (entry.getStatus().isTransient()) { newIndexEntries.put(entry.getName(), entry); } } version = onDiskVersion; indexEntries = newIndexEntries; } else { throw new IOException("Invalid file check sum"); } } } }
From source file:org.alfresco.repo.search.impl.lucene.index.IndexInfo.java
private void writeStatusToFile(FileChannel channel) throws IOException { long size = getBufferSize(); ByteBuffer buffer;//from ww w . ja v a 2 s . c o m if (useNIOMemoryMapping) { MappedByteBuffer mbb = channel.map(MapMode.READ_WRITE, 0, size); mbb.load(); buffer = mbb; } else { channel.truncate(size); buffer = ByteBuffer.wrap(new byte[(int) size]); } buffer.position(0); buffer.putLong(version); CRC32 crc32 = new CRC32(); crc32.update((int) (version >>> 32) & 0xFFFFFFFF); crc32.update((int) (version >>> 0) & 0xFFFFFFFF); buffer.putInt(indexEntries.size()); crc32.update(indexEntries.size()); for (IndexEntry entry : indexEntries.values()) { String entryType = entry.getType().toString(); writeString(buffer, crc32, entryType); writeString(buffer, crc32, entry.getName()); writeString(buffer, crc32, entry.getParentName()); String entryStatus = entry.getStatus().toString(); writeString(buffer, crc32, entryStatus); writeString(buffer, crc32, entry.getMergeId()); buffer.putLong(entry.getDocumentCount()); crc32.update((int) (entry.getDocumentCount() >>> 32) & 0xFFFFFFFF); crc32.update((int) (entry.getDocumentCount() >>> 0) & 0xFFFFFFFF); buffer.putLong(entry.getDeletions()); crc32.update((int) (entry.getDeletions() >>> 32) & 0xFFFFFFFF); crc32.update((int) (entry.getDeletions() >>> 0) & 0xFFFFFFFF); buffer.put(entry.isDeletOnlyNodes() ? (byte) 1 : (byte) 0); crc32.update(entry.isDeletOnlyNodes() ? new byte[] { (byte) 1 } : new byte[] { (byte) 0 }); } buffer.putLong(crc32.getValue()); if (useNIOMemoryMapping) { ((MappedByteBuffer) buffer).force(); } else { buffer.rewind(); channel.position(0); channel.write(buffer); } }