List of usage examples for java.util.zip Checksum getValue
public long getValue();
From source file:org.apache.mnemonic.DurablePersonNGTest.java
@Test(dependsOnMethods = { "testGenPeople" })
public void testCheckPeople() throws RetrieveDurableEntityError {
NonVolatileMemAllocator act = new NonVolatileMemAllocator(
Utils.getNonVolatileMemoryAllocatorService("pmalloc"), 1024 * 1024 * 8, "./pobj_person.dat", false);
act.setBufferReclaimer(new Reclaim<ByteBuffer>() {
@Override/*from w w w .ja v a 2 s . c o m*/
public boolean reclaim(ByteBuffer mres, Long sz) {
System.out.println(String.format("Reclaim Memory Buffer: %X Size: %s",
System.identityHashCode(mres), null == sz ? "NULL" : sz.toString()));
return false;
}
});
act.setChunkReclaimer(new Reclaim<Long>() {
@Override
public boolean reclaim(Long mres, Long sz) {
System.out.println(String.format("Reclaim Memory Chunk: %X Size: %s",
System.identityHashCode(mres), null == sz ? "NULL" : sz.toString()));
return false;
}
});
Checksum pic_cs = new CRC32();
pic_cs.reset();
Checksum fp_cs = new CRC32();
fp_cs.reset();
long size;
byte[] buf;
long val;
for (long i = 0; i < cKEYCAPACITY; ++i) {
System.out.printf("----------Key %d--------------\n", i);
val = act.getHandler(i);
if (0L == val) {
break;
}
Person<Integer> person = PersonFactory.restore(act, val, true);
while (null != person) {
person.testOutput();
person.getPicture().get().clear();
buf = new byte[person.getPicture().get().capacity()];
person.getPicture().get().get(buf);
pic_cs.update(buf, 0, buf.length);
byte b;
for (int j = 0; j < person.getPreference().getSize(); ++j) {
b = unsafe.getByte(person.getPreference().get() + j);
fp_cs.update(b);
}
person = person.getMother();
}
}
act.close();
Assert.assertEquals(pic_cs.getValue(), pic_checksum);
Assert.assertEquals(fp_cs.getValue(), fp_checksum);
}
From source file:org.apache.mnemonic.ChunkBufferNGTest.java
@Test public void testGenChunkBuffers() { Checksum cs = new CRC32(); cs.reset();/*from w w w . j a v a2 s . c om*/ NonVolatileMemAllocator act = new NonVolatileMemAllocator( Utils.getNonVolatileMemoryAllocatorService("pmalloc"), 1024 * 1024 * 1024L, "./pmchunkbuffertest.dat", true); act.setChunkReclaimer(new Reclaim<Long>() { @Override public boolean reclaim(Long mres, Long sz) { System.out.println(String.format("Reclaim Memory Chunk: %X Size: %s", System.identityHashCode(mres), null == sz ? "NULL" : sz.toString())); return false; } }); DurableChunk<NonVolatileMemAllocator> mch; mch = act.createChunk(1000 * 1024 * 1024L); Assert.assertNotNull(mch); act.setHandler(m_keyid, mch.getHandler()); long bufcnt = mch.getSize() / m_bufsize; ChunkBuffer ckbuf; byte[] rdbytes; for (long idx = 0; idx < bufcnt; ++idx) { // System.err.println(String.format("---- bufcnt: %d, bufsize: %d, idx: %d", bufcnt, m_bufsize, idx)); ckbuf = mch.getChunkBuffer(idx * m_bufsize, m_bufsize); Assert.assertNotNull(ckbuf); rdbytes = RandomUtils.nextBytes(m_bufsize); Assert.assertNotNull(rdbytes); ckbuf.get().clear(); ckbuf.get().put(rdbytes); cs.update(rdbytes, 0, rdbytes.length); } m_checksum = cs.getValue(); m_count = bufcnt; act.close(); }
From source file:org.apache.mnemonic.DurablePersonNGTest.java
@Test(expectedExceptions = { OutOfHybridMemory.class })
public void testGenPeople() throws OutOfHybridMemory, RetrieveDurableEntityError {
Random rand = Utils.createRandom();
NonVolatileMemAllocator act = new NonVolatileMemAllocator(
Utils.getNonVolatileMemoryAllocatorService("pmalloc"), 1024L * 1024 * 1024, "./pobj_person.dat",
true);//from w w w . ja v a 2 s. co m
cKEYCAPACITY = act.handlerCapacity();
act.setBufferReclaimer(new Reclaim<ByteBuffer>() {
@Override
public boolean reclaim(ByteBuffer mres, Long sz) {
System.out.println(String.format("Reclaim Memory Buffer: %X Size: %s",
System.identityHashCode(mres), null == sz ? "NULL" : sz.toString()));
return false;
}
});
act.setChunkReclaimer(new Reclaim<Long>() {
@Override
public boolean reclaim(Long mres, Long sz) {
System.out.println(String.format("Reclaim Memory Chunk: %X Size: %s",
System.identityHashCode(mres), null == sz ? "NULL" : sz.toString()));
return false;
}
});
for (long i = 0; i < cKEYCAPACITY; ++i) {
act.setHandler(i, 0L);
}
Person<Integer> mother;
Person<Integer> person;
Checksum pic_cs = new CRC32();
pic_cs.reset();
Checksum fp_cs = new CRC32();
fp_cs.reset();
long keyidx = 0;
long val;
try {
while (true) {
// if (keyidx >= KEYCAPACITY) break;
keyidx %= cKEYCAPACITY;
System.out.printf("************ Generating People on Key %d ***********\n", keyidx);
val = act.getHandler(keyidx);
if (0L != val) {
PersonFactory.restore(act, val, true);
}
person = PersonFactory.create(act);
person.setAge((short) rand.nextInt(50));
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setPicture(genuptBuffer(act, pic_cs, genRandSize()), true);
person.setPreference(genuptChunk(act, fp_cs, genRandSize()), true);
act.setHandler(keyidx, person.getHandler());
pic_checksum = pic_cs.getValue();
fp_checksum = fp_cs.getValue();
for (int deep = 0; deep < rand.nextInt(100); ++deep) {
mother = PersonFactory.create(act);
mother.setAge((short) (50 + rand.nextInt(50)));
mother.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
mother.setPicture(genuptBuffer(act, pic_cs, genRandSize()), true);
mother.setPreference(genuptChunk(act, fp_cs, genRandSize()), true);
person.setMother(mother, true);
pic_checksum = pic_cs.getValue();
fp_checksum = fp_cs.getValue();
person = mother;
}
++keyidx;
}
} finally {
act.close();
}
}
From source file:com.cisco.dvbu.ps.common.util.CommonUtils.java
/** * Returns a sum of CRC32 checksums of all lines/rows in a file. * This method is used to compare files with the same lines/rows, which may be in different order, in which case we * still want to consider them equal (from the point of view of containing the same data) * In such case this method will return the same result. * * This is useful when the file contains results of a database query and we need to compare * results of two queries that may return the same data but in different order. * /*from w w w. j a va 2 s .co m*/ * @author SST * @param filePath file name with full path * @return sum of checksums of each line(row) from the input file * The type of this value could be long for files up to probably several GB in size. * BigInteger was chosen in case even bigger files are used. * @throws IOException */ public static BigInteger fileChecksumByRow(String filePath) throws IOException { BigInteger sumOfcheckSumValues = new BigInteger("0"); long currentLineCheckSumValue = 0L; Checksum checksum = new CRC32(); BufferedReader br = new BufferedReader(new FileReader(filePath)); String line; // System.out.println("currentLineCheckSumValue: "); while ((line = br.readLine()) != null) { // Read one line at a time byte bytes[] = line.getBytes(); checksum.reset(); checksum.update(bytes, 0, bytes.length); currentLineCheckSumValue = checksum.getValue(); // System.out.println(currentLineCheckSumValue); sumOfcheckSumValues = sumOfcheckSumValues.add(BigInteger.valueOf(currentLineCheckSumValue)); } br.close(); // System.out.println("fileChecksumByRow(): sumOfcheckSumValues = " + sumOfcheckSumValues); return sumOfcheckSumValues; }
From source file:org.apache.mnemonic.mapreduce.MneMapreduceBufferDataTest.java
@Test(enabled = true, dependsOnMethods = { "testWriteBufferData" })
public void testReadBufferData() throws Exception {
long reccnt = 0L;
long tsize = 0L;
byte[] buf;//from www.j a va2 s. c o m
Checksum cs = new CRC32();
cs.reset();
File folder = new File(m_workdir.toString());
File[] listfiles = folder.listFiles();
for (int idx = 0; idx < listfiles.length; ++idx) {
if (listfiles[idx].isFile()
&& listfiles[idx].getName().startsWith(MneConfigHelper.getBaseOutputName(m_conf, null))
&& listfiles[idx].getName().endsWith(MneConfigHelper.DEFAULT_FILE_EXTENSION)) {
m_partfns.add(listfiles[idx].getName());
}
}
Collections.sort(m_partfns); // keep the order for checksum
for (int idx = 0; idx < m_partfns.size(); ++idx) {
System.out.println(String.format("Verifying : %s", m_partfns.get(idx)));
FileSplit split = new FileSplit(new Path(m_workdir, m_partfns.get(idx)), 0, 0L, new String[0]);
InputFormat<NullWritable, MneDurableInputValue<DurableBuffer<?>>> inputFormat = new MneInputFormat<MneDurableInputValue<DurableBuffer<?>>, DurableBuffer<?>>();
RecordReader<NullWritable, MneDurableInputValue<DurableBuffer<?>>> reader = inputFormat
.createRecordReader(split, m_tacontext);
MneDurableInputValue<DurableBuffer<?>> dbufval = null;
while (reader.nextKeyValue()) {
dbufval = reader.getCurrentValue();
assert dbufval.getValue().getSize() == dbufval.getValue().get().capacity();
dbufval.getValue().get().clear();
buf = new byte[dbufval.getValue().get().capacity()];
dbufval.getValue().get().get(buf);
cs.update(buf, 0, buf.length);
tsize += dbufval.getValue().getSize();
++reccnt;
}
reader.close();
}
AssertJUnit.assertEquals(m_reccnt, reccnt);
AssertJUnit.assertEquals(m_totalsize, tsize);
AssertJUnit.assertEquals(m_checksum, cs.getValue());
System.out.println(String.format("The checksum of buffer is %d", m_checksum));
}
From source file:PngEncoder.java
/** * Writes the IDAT (Image data) chunks to the output stream. * * @param out the OutputStream to write the chunk to * @param csum the Checksum that is updated as data is written * to the passed-in OutputStream * @throws IOException if a problem is encountered writing the output *///from w ww. ja v a2 s . c o m private void writeIdatChunks(OutputStream out, Checksum csum) throws IOException { int rowWidth = width * outputBpp; // size of image data in a row in bytes. int row = 0; Deflater deflater = new Deflater(compressionLevel); ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); DeflaterOutputStream defOut = new DeflaterOutputStream(byteOut, deflater); byte[] filteredPixelQueue = new byte[rowWidth]; // Output Pixel Queues byte[][] outputPixelQueue = new byte[2][rowWidth]; Arrays.fill(outputPixelQueue[1], (byte) 0); int outputPixelQueueRow = 0; int outputPixelQueuePrevRow = 1; while (row < height) { if (filter == null) { defOut.write(0); translator.translate(outputPixelQueue[outputPixelQueueRow], row); defOut.write(outputPixelQueue[outputPixelQueueRow], 0, rowWidth); } else { defOut.write(filter.getType()); translator.translate(outputPixelQueue[outputPixelQueueRow], row); filter.filter(filteredPixelQueue, outputPixelQueue[outputPixelQueueRow], outputPixelQueue[outputPixelQueuePrevRow], outputBpp); defOut.write(filteredPixelQueue, 0, rowWidth); } ++row; outputPixelQueueRow = row & 1; outputPixelQueuePrevRow = outputPixelQueueRow ^ 1; } defOut.finish(); byteOut.close(); writeInt(out, byteOut.size()); csum.reset(); out.write(IDAT); byteOut.writeTo(out); writeInt(out, (int) csum.getValue()); }
From source file:org.apache.mnemonic.mapred.MneMapredBufferDataTest.java
@Test(enabled = true, dependsOnMethods = { "testWriteBufferData" })
public void testReadBufferData() throws Exception {
long reccnt = 0L;
long tsize = 0L;
byte[] buf;//from w ww . j a v a2 s . co m
Checksum cs = new CRC32();
cs.reset();
File folder = new File(m_workdir.toString());
File[] listfiles = folder.listFiles();
for (int idx = 0; idx < listfiles.length; ++idx) {
if (listfiles[idx].isFile()
&& listfiles[idx].getName().startsWith(MneConfigHelper.getBaseOutputName(m_conf, null))
&& listfiles[idx].getName().endsWith(MneConfigHelper.DEFAULT_FILE_EXTENSION)) {
m_partfns.add(listfiles[idx].getName());
}
}
Collections.sort(m_partfns); // keep the order for checksum
for (int idx = 0; idx < m_partfns.size(); ++idx) {
System.out.println(String.format("Verifying : %s", m_partfns.get(idx)));
FileSplit split = new FileSplit(new Path(m_workdir, m_partfns.get(idx)), 0, 0L, new String[0]);
InputFormat<NullWritable, MneDurableInputValue<DurableBuffer<?>>> inputFormat = new MneInputFormat<MneDurableInputValue<DurableBuffer<?>>, DurableBuffer<?>>();
RecordReader<NullWritable, MneDurableInputValue<DurableBuffer<?>>> reader = inputFormat
.getRecordReader((InputSplit) split, m_conf, null);
NullWritable dbufkey = reader.createKey();
MneDurableInputValue<DurableBuffer<?>> dbufval = null;
while (true) {
dbufval = reader.createValue();
if (reader.next(dbufkey, dbufval)) {
assert dbufval.getValue().getSize() == dbufval.getValue().get().capacity();
dbufval.getValue().get().clear();
buf = new byte[dbufval.getValue().get().capacity()];
dbufval.getValue().get().get(buf);
cs.update(buf, 0, buf.length);
tsize += dbufval.getValue().getSize();
++reccnt;
} else {
break;
}
}
reader.close();
}
AssertJUnit.assertEquals(m_reccnt, reccnt);
AssertJUnit.assertEquals(m_totalsize, tsize);
AssertJUnit.assertEquals(m_checksum, cs.getValue());
System.out.println(String.format("The checksum of buffer is %d", m_checksum));
}
From source file:org.apache.nifi.processors.standard.TailFile.java
/** * Read new lines from the given FileChannel, copying it to the given Output * Stream. The Checksum is used in order to later determine whether or not * data has been consumed.//from w ww . ja va 2 s . c o m * * @param reader The FileChannel to read data from * @param buffer the buffer to use for copying data * @param out the OutputStream to copy the data to * @param checksum the Checksum object to use in order to calculate checksum * for recovery purposes * * @return The new position after the lines have been read * @throws java.io.IOException if an I/O error occurs. */ private long readLines(final FileChannel reader, final ByteBuffer buffer, final OutputStream out, final Checksum checksum) throws IOException { getLogger().debug("Reading lines starting at position {}", new Object[] { reader.position() }); try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { long pos = reader.position(); long rePos = pos; // position to re-read int num; int linesRead = 0; boolean seenCR = false; buffer.clear(); while (((num = reader.read(buffer)) != -1)) { buffer.flip(); for (int i = 0; i < num; i++) { byte ch = buffer.get(i); switch (ch) { case '\n': { baos.write(ch); seenCR = false; baos.writeTo(out); final byte[] baosBuffer = baos.toByteArray(); checksum.update(baosBuffer, 0, baos.size()); if (getLogger().isTraceEnabled()) { getLogger().trace("Checksum updated to {}", new Object[] { checksum.getValue() }); } baos.reset(); rePos = pos + i + 1; linesRead++; break; } case '\r': { baos.write(ch); seenCR = true; break; } default: { if (seenCR) { seenCR = false; baos.writeTo(out); final byte[] baosBuffer = baos.toByteArray(); checksum.update(baosBuffer, 0, baos.size()); if (getLogger().isTraceEnabled()) { getLogger().trace("Checksum updated to {}", new Object[] { checksum.getValue() }); } linesRead++; baos.reset(); baos.write(ch); rePos = pos + i; } else { baos.write(ch); } } } } pos = reader.position(); } if (rePos < reader.position()) { getLogger().debug("Read {} lines; repositioning reader from {} to {}", new Object[] { linesRead, pos, rePos }); reader.position(rePos); // Ensure we can re-read if necessary } return rePos; } }