Example usage for java.util.zip Checksum update

Introduction

In this page you can find the example usage for java.util.zip Checksum update.

Prototype

public void update(byte[] b, int off, int len);

Source Link

Document

Updates the current checksum with the specified array of bytes.

Usage

From source file:org.apache.mnemonic.mapred.MneMapredBufferDataTest.java

@Test(enabled = true, dependsOnMethods = { "testWriteBufferData" })
public void testReadBufferData() throws Exception {
    long reccnt = 0L;
    long tsize = 0L;
    byte[] buf;//from   ww  w .j a v a2s  .co  m
    Checksum cs = new CRC32();
    cs.reset();
    File folder = new File(m_workdir.toString());
    File[] listfiles = folder.listFiles();
    for (int idx = 0; idx < listfiles.length; ++idx) {
        if (listfiles[idx].isFile()
                && listfiles[idx].getName().startsWith(MneConfigHelper.getBaseOutputName(m_conf, null))
                && listfiles[idx].getName().endsWith(MneConfigHelper.DEFAULT_FILE_EXTENSION)) {
            m_partfns.add(listfiles[idx].getName());
        }
    }
    Collections.sort(m_partfns); // keep the order for checksum
    for (int idx = 0; idx < m_partfns.size(); ++idx) {
        System.out.println(String.format("Verifying : %s", m_partfns.get(idx)));
        FileSplit split = new FileSplit(new Path(m_workdir, m_partfns.get(idx)), 0, 0L, new String[0]);
        InputFormat<NullWritable, MneDurableInputValue<DurableBuffer<?>>> inputFormat = new MneInputFormat<MneDurableInputValue<DurableBuffer<?>>, DurableBuffer<?>>();
        RecordReader<NullWritable, MneDurableInputValue<DurableBuffer<?>>> reader = inputFormat
                .getRecordReader((InputSplit) split, m_conf, null);
        NullWritable dbufkey = reader.createKey();
        MneDurableInputValue<DurableBuffer<?>> dbufval = null;
        while (true) {
            dbufval = reader.createValue();
            if (reader.next(dbufkey, dbufval)) {
                assert dbufval.getValue().getSize() == dbufval.getValue().get().capacity();
                dbufval.getValue().get().clear();
                buf = new byte[dbufval.getValue().get().capacity()];
                dbufval.getValue().get().get(buf);
                cs.update(buf, 0, buf.length);
                tsize += dbufval.getValue().getSize();
                ++reccnt;
            } else {
                break;
            }
        }
        reader.close();
    }
    AssertJUnit.assertEquals(m_reccnt, reccnt);
    AssertJUnit.assertEquals(m_totalsize, tsize);
    AssertJUnit.assertEquals(m_checksum, cs.getValue());
    System.out.println(String.format("The checksum of buffer is %d", m_checksum));
}

---

From source file:org.apache.nifi.processors.standard.TailFile.java

/**
 * Read new lines from the given FileChannel, copying it to the given Output
 * Stream. The Checksum is used in order to later determine whether or not
 * data has been consumed./*www .j a  v  a2 s  .  c om*/
 *
 * @param reader The FileChannel to read data from
 * @param buffer the buffer to use for copying data
 * @param out the OutputStream to copy the data to
 * @param checksum the Checksum object to use in order to calculate checksum
 * for recovery purposes
 *
 * @return The new position after the lines have been read
 * @throws java.io.IOException if an I/O error occurs.
 */
private long readLines(final FileChannel reader, final ByteBuffer buffer, final OutputStream out,
        final Checksum checksum) throws IOException {
    getLogger().debug("Reading lines starting at position {}", new Object[] { reader.position() });

    try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
        long pos = reader.position();
        long rePos = pos; // position to re-read

        int num;
        int linesRead = 0;
        boolean seenCR = false;
        buffer.clear();

        while (((num = reader.read(buffer)) != -1)) {
            buffer.flip();

            for (int i = 0; i < num; i++) {
                byte ch = buffer.get(i);

                switch (ch) {
                case '\n': {
                    baos.write(ch);
                    seenCR = false;
                    baos.writeTo(out);
                    final byte[] baosBuffer = baos.toByteArray();
                    checksum.update(baosBuffer, 0, baos.size());
                    if (getLogger().isTraceEnabled()) {
                        getLogger().trace("Checksum updated to {}", new Object[] { checksum.getValue() });
                    }

                    baos.reset();
                    rePos = pos + i + 1;
                    linesRead++;
                    break;
                }
                case '\r': {
                    baos.write(ch);
                    seenCR = true;
                    break;
                }
                default: {
                    if (seenCR) {
                        seenCR = false;
                        baos.writeTo(out);
                        final byte[] baosBuffer = baos.toByteArray();
                        checksum.update(baosBuffer, 0, baos.size());
                        if (getLogger().isTraceEnabled()) {
                            getLogger().trace("Checksum updated to {}", new Object[] { checksum.getValue() });
                        }

                        linesRead++;
                        baos.reset();
                        baos.write(ch);
                        rePos = pos + i;
                    } else {
                        baos.write(ch);
                    }
                }
                }
            }

            pos = reader.position();
        }

        if (rePos < reader.position()) {
            getLogger().debug("Read {} lines; repositioning reader from {} to {}",
                    new Object[] { linesRead, pos, rePos });
            reader.position(rePos); // Ensure we can re-read if necessary
        }

        return rePos;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »