Example usage for java.util.zip CRC32 CRC32

Introduction

In this page you can find the example usage for java.util.zip CRC32 CRC32.

Prototype

public CRC32() 

Source Link

Document

Creates a new CRC32 object.

Usage

From source file:com.dcits.govsbu.southernbase.baseproject2.modules.utils.Digests.java

/**
 * crc32.//from w  w  w . j  a  v a2 s . c o  m
 */
public static int crc32(String input) {
    CRC32 crc32 = new CRC32();
    crc32.update(input.getBytes(Charsets.UTF8));
    return (int) crc32.getValue();
}

---

From source file:org.apache.mnemonic.mapred.MneMapredBufferDataTest.java

@Test(enabled = true)
public void testWriteBufferData() throws Exception {
    NullWritable nada = NullWritable.get();
    MneDurableOutputSession<DurableBuffer<?>> sess = new MneDurableOutputSession<DurableBuffer<?>>(null, m_conf,
            MneConfigHelper.DEFAULT_OUTPUT_CONFIG_PREFIX);
    MneDurableOutputValue<DurableBuffer<?>> mdvalue = new MneDurableOutputValue<DurableBuffer<?>>(sess);
    OutputFormat<NullWritable, MneDurableOutputValue<DurableBuffer<?>>> outputFormat = new MneOutputFormat<MneDurableOutputValue<DurableBuffer<?>>>();
    RecordWriter<NullWritable, MneDurableOutputValue<DurableBuffer<?>>> writer = outputFormat
            .getRecordWriter(m_fs, m_conf, null, null);
    DurableBuffer<?> dbuf = null;//w w  w  .j  a  v a2s. c om
    Checksum cs = new CRC32();
    cs.reset();
    for (int i = 0; i < m_reccnt; ++i) {
        dbuf = genupdDurableBuffer(sess, cs);
        Assert.assertNotNull(dbuf);
        writer.write(nada, mdvalue.of(dbuf));
    }
    m_checksum = cs.getValue();
    writer.close(null);
    sess.close();
}

---

From source file:me.j360.dubbo.modules.util.text.HashUtil.java

/**
 * crc32php64bitlong// w  w  w.  j a v  a 2  s . c om
 * 
 * Guavacrc32, longJDK
 */
public static long crc32AsLong(@NotNull byte[] input) {
    CRC32 crc32 = new CRC32();
    crc32.update(input);
    return crc32.getValue();
}

---

From source file:com.dcits.govsbu.southernbase.baseproject2.modules.utils.Digests.java

/**
 * crc32.//from w w w  .  ja  v  a  2  s.co  m
 */
public static int crc32(String input, Charset charset) {
    CRC32 crc32 = new CRC32();
    crc32.update(input.getBytes(charset));
    return (int) crc32.getValue();
}

---

From source file:net.sourceforge.squirrel_sql.fw.util.IOUtilitiesImpl.java

/**
 * @see net.sourceforge.squirrel_sql.fw.util.IOUtilities#getCheckSum(java.io.File)
 *//* ww w.  ja v a 2 s .  co m*/
public long getCheckSum(File f) throws IOException {
    CRC32 result = new CRC32();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(f);
        int len = 0;
        byte[] buffer = new byte[DISK_DATA_BUFFER_SIZE];
        while ((len = fis.read(buffer)) != -1) {
            result.update(buffer, 0, len);
        }
    } finally {
        closeInputStream(fis);
    }
    return result.getValue();
}

---

From source file:com.alexholmes.hdfsslurper.WorkerThread.java

private void process(FileStatus srcFileStatus) throws IOException, InterruptedException {

    Path stagingFile = null;//from ww w .ja v  a 2s . c om
    FileSystem destFs = null;
    String filenameBatchidDelimiter = config.getFileNameBatchIdDelimiter();

    try {
        FileSystem srcFs = srcFileStatus.getPath().getFileSystem(config.getConfig());

        // run a script which can change the name of the file as well as
        // write out a new version of the file
        //
        if (config.getWorkScript() != null) {
            Path newSrcFile = stageSource(srcFileStatus);
            srcFileStatus = srcFileStatus.getPath().getFileSystem(config.getConfig()).getFileStatus(newSrcFile);
        }

        Path srcFile = srcFileStatus.getPath();

        // get the target HDFS file
        //
        Path destFile = getHdfsTargetPath(srcFileStatus);

        if (config.getCodec() != null) {
            String ext = config.getCodec().getDefaultExtension();
            if (!destFile.getName().endsWith(ext)) {
                destFile = new Path(destFile.toString() + ext);
            }
        }

        destFs = destFile.getFileSystem(config.getConfig());

        // get the staging HDFS file
        //
        stagingFile = fileSystemManager.getStagingFile(srcFileStatus, destFile);
        String batchId = srcFile.toString().substring(
                srcFile.toString().lastIndexOf(filenameBatchidDelimiter) + 1, srcFile.toString().length());

        log.info("event#Copying source file '" + srcFile + "' to staging destination '" + stagingFile + "'"
                + "$batchId#" + batchId);

        // if the directory of the target file doesn't exist, attempt to
        // create it
        //
        Path destParentDir = destFile.getParent();
        if (!destFs.exists(destParentDir)) {
            log.info("event#Attempting creation of target directory: " + destParentDir.toUri());
            if (!destFs.mkdirs(destParentDir)) {
                throw new IOException("event#Failed to create target directory: " + destParentDir.toUri());
            }
        }

        // if the staging directory doesn't exist, attempt to create it
        //
        Path destStagingParentDir = stagingFile.getParent();
        if (!destFs.exists(destStagingParentDir)) {
            log.info("event#Attempting creation of staging directory: " + destStagingParentDir.toUri());
            if (!destFs.mkdirs(destStagingParentDir)) {
                throw new IOException("event#Failed to create staging directory: " + destParentDir.toUri());
            }
        }

        // copy the file
        //
        InputStream is = null;
        OutputStream os = null;
        CRC32 crc = new CRC32();
        try {
            is = new BufferedInputStream(srcFs.open(srcFile));
            if (config.isVerify()) {
                is = new CheckedInputStream(is, crc);
            }
            os = destFs.create(stagingFile);

            if (config.getCodec() != null) {
                os = config.getCodec().createOutputStream(os);
            }

            IOUtils.copyBytes(is, os, 4096, false);
        } finally {
            IOUtils.closeStream(is);
            IOUtils.closeStream(os);
        }

        long srcFileSize = srcFs.getFileStatus(srcFile).getLen();
        long destFileSize = destFs.getFileStatus(stagingFile).getLen();
        if (config.getCodec() == null && srcFileSize != destFileSize) {
            throw new IOException(
                    "event#File sizes don't match, source = " + srcFileSize + ", dest = " + destFileSize);
        }

        log.info("event#Local file size = " + srcFileSize + ", HDFS file size = " + destFileSize + "$batchId#"
                + batchId);

        if (config.isVerify()) {
            verify(stagingFile, crc.getValue());
        }

        if (destFs.exists(destFile)) {
            destFs.delete(destFile, false);
        }

        log.info("event#Moving staging file '" + stagingFile + "' to destination '" + destFile + "'"
                + "$batchId#" + batchId);
        if (!destFs.rename(stagingFile, destFile)) {
            throw new IOException("event#Failed to rename file");
        }

        if (config.isCreateLzopIndex() && destFile.getName().endsWith(lzopExt)) {
            Path lzoIndexPath = new Path(destFile.toString() + LzoIndex.LZO_INDEX_SUFFIX);
            if (destFs.exists(lzoIndexPath)) {
                log.info("event#Deleting index file as it already exists");
                destFs.delete(lzoIndexPath, false);
            }
            indexer.index(destFile);
        }

        fileSystemManager.fileCopyComplete(srcFileStatus);

    } catch (Throwable t) {
        log.error("event#Caught exception working on file " + srcFileStatus.getPath(), t);

        // delete the staging file if it still exists
        //
        try {
            if (destFs != null && destFs.exists(stagingFile)) {
                destFs.delete(stagingFile, false);
            }
        } catch (Throwable t2) {
            log.error("event#Failed to delete staging file " + stagingFile, t2);
        }

        fileSystemManager.fileCopyError(srcFileStatus);
    }

}

---

From source file:org.apache.mnemonic.mapreduce.MneMapreduceBufferDataTest.java

@Test(enabled = true)
public void testWriteBufferData() throws Exception {
    NullWritable nada = NullWritable.get();
    MneDurableOutputSession<DurableBuffer<?>> sess = new MneDurableOutputSession<DurableBuffer<?>>(m_tacontext,
            null, MneConfigHelper.DEFAULT_OUTPUT_CONFIG_PREFIX);
    MneDurableOutputValue<DurableBuffer<?>> mdvalue = new MneDurableOutputValue<DurableBuffer<?>>(sess);
    OutputFormat<NullWritable, MneDurableOutputValue<DurableBuffer<?>>> outputFormat = new MneOutputFormat<MneDurableOutputValue<DurableBuffer<?>>>();
    RecordWriter<NullWritable, MneDurableOutputValue<DurableBuffer<?>>> writer = outputFormat
            .getRecordWriter(m_tacontext);
    DurableBuffer<?> dbuf = null;/*w ww .  j a va  2s  .co  m*/
    Checksum cs = new CRC32();
    cs.reset();
    for (int i = 0; i < m_reccnt; ++i) {
        dbuf = genupdDurableBuffer(sess, cs);
        Assert.assertNotNull(dbuf);
        writer.write(nada, mdvalue.of(dbuf));
    }
    m_checksum = cs.getValue();
    writer.close(m_tacontext);
    sess.close();
}

---

From source file:org.klco.email2html.OutputWriter.java

/**
 * Adds the attachment to the EmailMessage. Call this method when the email
 * content has most likely already been loaded.
 * /*w  w  w. j  a v  a 2  s. c  o  m*/
 * @param containingMessage
 *            the Email Message to add the attachment to
 * @param part
 *            the content of the attachment
 * @throws IOException
 * @throws MessagingException
 */
public void addAttachment(EmailMessage containingMessage, Part part) throws IOException, MessagingException {
    log.trace("addAttachment");

    File attachmentFolder = new File(outputDir.getAbsolutePath() + File.separator + config.getImagesSubDir()
            + File.separator + FILE_DATE_FORMAT.format(containingMessage.getSentDate()));
    File attachmentFile = new File(attachmentFolder, part.getFileName());

    boolean addAttachment = true;
    boolean writeAttachment = false;
    if (!attachmentFolder.exists() || !attachmentFile.exists()) {
        log.warn("Attachment or folder missing, writing attachment {}", attachmentFile.getName());
        writeAttachment = true;
    }

    if (!writeAttachment && part.getContentType().toLowerCase().startsWith("image")) {
        for (Rendition rendition : renditions) {
            File renditionFile = new File(attachmentFolder, rendition.getName() + "-" + part.getFileName());
            if (!renditionFile.exists()) {
                log.warn("Rendition {} missing, writing attachment {}", renditionFile.getName(),
                        attachmentFile.getName());
                writeAttachment = true;
                break;
            }
        }
    }
    if (writeAttachment) {
        addAttachment = writeAttachment(containingMessage, part);
    } else {
        if (this.excludeDuplicates) {
            log.debug("Computing checksum");
            InputStream is = null;
            try {
                CRC32 checksum = new CRC32();
                is = new BufferedInputStream(new FileInputStream(attachmentFile));
                for (int read = is.read(); read != -1; read = is.read()) {
                    checksum.update(read);
                }
                long value = checksum.getValue();
                if (attachmentChecksums.contains(value)) {
                    addAttachment = false;
                } else {
                    attachmentChecksums.add(checksum.getValue());
                }
            } finally {
                IOUtils.closeQuietly(is);
            }
        }
    }
    if (addAttachment) {
        containingMessage.getAttachments().add(attachmentFile);
    } else {
        log.debug("Attachment is a duplicate, not adding as message attachment");
    }
}

---

From source file:com.guye.baffle.decoder.ArscData.java

public CRC32 createObfuscateFile(StringBlock tableBlock, StringBlock keyBlock, File file) throws IOException {
    FileOutputStream fileOutputStream = new FileOutputStream(file);
    CRC32 cksum = new CRC32();
    CheckedOutputStream checkedOutputStream = new CheckedOutputStream(fileOutputStream, cksum);
    LEDataOutputStream out = new LEDataOutputStream(checkedOutputStream);

    int tableStrChange = getmTableStrings().getSize() - tableBlock.getSize();
    int keyStrChange = getmSpecNames().getSize() - keyBlock.getSize();
    getmHeader().chunkSize -= (tableStrChange + keyStrChange);
    getmHeader().write(out);/*  ww w .  j av a  2  s  . c o m*/
    out.writeInt(1);
    tableBlock.write(out);
    getmPkgHeader().header.chunkSize -= keyStrChange;
    getmPkgHeader().write(out);
    getTypeNames().write(out);
    keyBlock.write(out);

    byte[] buff = new byte[1024];
    FileInputStream in = new FileInputStream(getFile());
    in.skip(getmResIndex());
    int len;
    while (((len = in.read(buff)) != -1)) {
        out.write(buff, 0, len);
    }

    in.close();
    out.close();
    checkedOutputStream.close();
    fileOutputStream.close();
    return cksum;
}

---

From source file:com.netflix.spinnaker.halyard.config.model.v1.node.Node.java

public void stageLocalFiles(Path outputPath) {
    if (!GlobalApplicationOptions.getInstance().isUseRemoteDaemon()) {
        return;/*from  ww  w . ja va 2  s.c om*/
    }
    localFiles().forEach(f -> {
        try {
            f.setAccessible(true);
            String fContent = (String) f.get(this);
            if (fContent != null) {
                CRC32 crc = new CRC32();
                crc.update(fContent.getBytes());
                String fPath = Paths
                        .get(outputPath.toAbsolutePath().toString(), Long.toHexString(crc.getValue()))
                        .toString();
                FileUtils.writeStringToFile(new File(fPath), fContent);
                f.set(this, fPath);
            }
        } catch (IllegalAccessException | IOException e) {
            throw new RuntimeException("Failed to get local files for node " + this.getNodeName(), e);
        } finally {
            f.setAccessible(false);
        }
    });
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• »