Example usage for java.util.zip CRC32 CRC32

Introduction

In this page you can find the example usage for java.util.zip CRC32 CRC32.

Prototype

public CRC32()

Source Link

Document

Creates a new CRC32 object.

Usage

From source file:org.commoncrawl.service.crawler.CrawlList.java

private static void appendTargetsToLogFile(File logFileName, IntrusiveList<CrawlTarget> list)
        throws IOException {

    LogFileHeader header = new LogFileHeader();

    boolean preExistingHeader = logFileName.exists();

    RandomAccessFile file = new RandomAccessFile(logFileName, "rw");

    try {//from w w w  .  j ava  2 s  .  c om
        long headerOffset = 0;

        if (preExistingHeader) {

            headerOffset = readLogFileHeader(file, header);

            if (header._writePos == 0) {
                file.seek(headerOffset);
            } else {
                // seelk to appropriate write position 
                file.seek(header._writePos);
            }
        } else {
            headerOffset = writeLogFileHeader(file, header);
        }

        CustomByteArrayOutputStream bufferOutputStream = new CustomByteArrayOutputStream(1 << 17);
        DataOutputStream dataOutputStream = new DataOutputStream(bufferOutputStream);
        CRC32 crc = new CRC32();

        for (CrawlTarget target : list) {

            PersistentCrawlTarget persistentTarget = target.createPersistentTarget();

            bufferOutputStream.reset();
            // write to intermediate stream ... 
            persistentTarget.write(dataOutputStream);
            // and crc the data ... 
            crc.reset();
            crc.update(bufferOutputStream.getBuffer(), 0, bufferOutputStream.size());
            // write out length first 
            file.writeInt(bufferOutputStream.size());
            //crc next
            long computedValue = crc.getValue();
            //TODO: waste of space - write 32 bit values as long because having problems with java sign promotion rules during read...
            file.writeLong(computedValue);
            // and then the data 
            file.write(bufferOutputStream.getBuffer(), 0, bufferOutputStream.size());
        }

        // now update header ... 
        header._itemCount += list.size();
        header._writePos = file.getFilePointer();

        // now write out header anew ... 
        writeLogFileHeader(file, header);

    } finally {
        if (file != null) {
            file.close();
        }
    }
}

From source file:org.commoncrawl.service.crawler.CrawlList.java

private static int readTargetsFromLogFile(CrawlList domain, File logFileName, int desiredReadAmount,
        IntrusiveList<CrawlTarget> targetsOut) throws IOException {

    int itemsRead = 0;

    if (logFileName.exists()) {

        RandomAccessFile file = new RandomAccessFile(logFileName, "rw");

        LogFileHeader header = new LogFileHeader();

        try {//  www  .  j  a v  a2s.  c om

            long headerOffset = readLogFileHeader(file, header);

            // seelk to appropriate write position 
            if (header._readPos != 0)
                file.seek(header._readPos);

            int itemsToRead = Math.min(desiredReadAmount, header._itemCount);

            PersistentCrawlTarget persistentTarget = new PersistentCrawlTarget();
            CRC32 crc = new CRC32();
            CustomByteArrayOutputStream buffer = new CustomByteArrayOutputStream(1 << 16);

            for (int i = 0; i < itemsToRead; ++i) {
                // read length ... 
                int urlDataLen = file.readInt();
                long urlDataCRC = file.readLong();

                buffer.reset();

                if (urlDataLen > buffer.getBuffer().length) {
                    buffer = new CustomByteArrayOutputStream(((urlDataLen / 65536) + 1) * 65536);
                }
                file.read(buffer.getBuffer(), 0, urlDataLen);
                crc.reset();
                crc.update(buffer.getBuffer(), 0, urlDataLen);

                long computedValue = crc.getValue();

                // validate crc values ... 
                if (computedValue != urlDataCRC) {
                    throw new IOException("Crawl Target Log File Corrupt");
                } else {
                    //populate a persistentTarget from the (in memory) data stream
                    DataInputStream bufferReader = new DataInputStream(
                            new ByteArrayInputStream(buffer.getBuffer(), 0, urlDataLen));

                    persistentTarget.clear();
                    persistentTarget.readFields(bufferReader);

                    //populate a new crawl target structure ... 
                    CrawlTarget newTarget = new CrawlTarget(domain, persistentTarget);

                    targetsOut.addTail(newTarget);
                }
            }

            itemsRead = itemsToRead;

            // now update header ... 
            header._itemCount -= itemsRead;
            // now if item count is non zero ... 
            if (header._itemCount != 0) {
                // set read cursor to next record location 
                header._readPos = file.getFilePointer();
            }
            // otherwise ... 
            else {
                // reset both cursors ... 
                header._readPos = 0;
                header._writePos = 0;
            }

            // now write out header anew ... 
            writeLogFileHeader(file, header);
        } finally {
            if (file != null) {
                file.close();
            }
        }
    }
    return itemsRead;
}

From source file:org.getlantern.firetweet.util.Utils.java

public static boolean hasAccountSignedWithOfficialKeys(final Context context) {
    if (context == null)
        return false;
    final Cursor cur = ContentResolverUtils.query(context.getContentResolver(), Accounts.CONTENT_URI,
            Accounts.COLUMNS, null, null, null);
    if (cur == null)
        return false;
    final String[] keySecrets = context.getResources()
            .getStringArray(R.array.values_official_consumer_secret_crc32);
    final ParcelableAccount.Indices indices = new ParcelableAccount.Indices(cur);
    cur.moveToFirst();/*w w w  .  j  a  va  2  s .  c  o m*/
    final CRC32 crc32 = new CRC32();
    try {
        while (!cur.isAfterLast()) {
            final String consumerSecret = cur.getString(indices.consumer_secret);
            if (consumerSecret != null) {
                final byte[] consumerSecretBytes = consumerSecret.getBytes(Charset.forName("UTF-8"));
                crc32.update(consumerSecretBytes, 0, consumerSecretBytes.length);
                final long value = crc32.getValue();
                crc32.reset();
                for (final String keySecret : keySecrets) {
                    if (Long.parseLong(keySecret, 16) == value)
                        return true;
                }
            }
            cur.moveToNext();
        }
    } finally {
        cur.close();
    }
    return false;
}