List of usage examples for java.io DataInputStream read
public final int read(byte b[], int off, int len) throws IOException
len bytes of data from the contained input stream into an array of bytes. From source file:org.nuxeo.ecm.core.blob.binary.AESBinaryManager.java
/** * Decrypts the given input stream into the given output stream. *///from w w w .ja v a2 s . c om protected void decrypt(InputStream in, OutputStream out) throws IOException { byte[] magic = new byte[FILE_MAGIC.length]; IOUtils.read(in, magic); if (!Arrays.equals(magic, FILE_MAGIC)) { throw new IOException("Invalid file (bad magic)"); } DataInputStream data = new DataInputStream(in); byte magicvers = data.readByte(); if (magicvers != FILE_VERSION_1) { throw new IOException("Invalid file (bad version)"); } byte usepb = data.readByte(); if (usepb == USE_PBKDF2) { if (!usePBKDF2) { throw new NuxeoException("File requires PBKDF2 password"); } } else if (usepb == USE_KEYSTORE) { if (usePBKDF2) { throw new NuxeoException("File requires keystore"); } } else { throw new IOException("Invalid file (bad use)"); } try { // secret key Key secret; if (usePBKDF2) { // read salt first int saltLen = data.readInt(); if (saltLen <= 0 || saltLen > MAX_SALT_LEN) { throw new NuxeoException("Invalid salt length: " + saltLen); } byte[] salt = new byte[saltLen]; data.read(salt, 0, saltLen); secret = generateSecretKey(salt); } else { secret = getSecretKey(); } // read IV int ivLen = data.readInt(); if (ivLen <= 0 || ivLen > MAX_IV_LEN) { throw new NuxeoException("Invalid IV length: " + ivLen); } byte[] iv = new byte[ivLen]; data.read(iv, 0, ivLen); // cipher Cipher cipher; cipher = Cipher.getInstance(AES_CBC_PKCS5_PADDING); cipher.init(Cipher.DECRYPT_MODE, secret, new IvParameterSpec(iv)); // read the encrypted data try (InputStream cipherIn = new CipherInputStream(in, cipher)) { IOUtils.copy(cipherIn, out); } catch (IOException e) { Throwable cause = e.getCause(); if (cause != null && cause instanceof BadPaddingException) { throw new NuxeoException(cause.getMessage(), e); } } } catch (GeneralSecurityException e) { throw new NuxeoException(e); } }
From source file:ffx.xray.parsers.MTZFilter.java
/** * {@inheritDoc}//from w ww. j av a 2 s . c o m */ @Override public boolean readFile(File mtzFile, ReflectionList reflectionList, DiffractionRefinementData refinementData, CompositeConfiguration properties) { int nRead, nIgnore, nRes, nFriedel, nCut; ByteOrder byteOrder = ByteOrder.nativeOrder(); FileInputStream fileInputStream; DataInputStream dataInputStream; boolean transpose = false; StringBuilder sb = new StringBuilder(); try { fileInputStream = new FileInputStream(mtzFile); dataInputStream = new DataInputStream(fileInputStream); byte headerOffset[] = new byte[4]; byte bytes[] = new byte[80]; int offset = 0; // Eat "MTZ" title. dataInputStream.read(bytes, offset, 4); String mtzstr = null; // Header offset. dataInputStream.read(headerOffset, offset, 4); // Machine stamp. dataInputStream.read(bytes, offset, 4); ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); int stamp = byteBuffer.order(ByteOrder.BIG_ENDIAN).getInt(); String stampString = Integer.toHexString(stamp); switch (stampString.charAt(0)) { case '1': case '3': if (byteOrder.equals(ByteOrder.LITTLE_ENDIAN)) { byteOrder = ByteOrder.BIG_ENDIAN; } break; case '4': if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) { byteOrder = ByteOrder.LITTLE_ENDIAN; } break; } byteBuffer = ByteBuffer.wrap(headerOffset); int headerOffsetI = byteBuffer.order(byteOrder).getInt(); // skip to header and parse dataInputStream.skipBytes((headerOffsetI - 4) * 4); for (Boolean parsing = true; parsing; dataInputStream.read(bytes, offset, 80)) { mtzstr = new String(bytes); parsing = parseHeader(mtzstr); } // column identifiers foString = sigFoString = rFreeString = null; if (properties != null) { foString = properties.getString("fostring", null); sigFoString = properties.getString("sigfostring", null); rFreeString = properties.getString("rfreestring", null); } h = k = l = fo = sigFo = rFree = -1; fPlus = sigFPlus = fMinus = sigFMinus = rFreePlus = rFreeMinus = -1; boolean print = true; parseColumns(print); if (h < 0 || k < 0 || l < 0) { String message = "Fatal error in MTZ file - no H K L indexes?\n"; logger.log(Level.SEVERE, message); return false; } // Reopen to start at beginning. fileInputStream = new FileInputStream(mtzFile); dataInputStream = new DataInputStream(fileInputStream); // Skip initial header. dataInputStream.skipBytes(80); // Check if HKLs need to be transposed or not. float data[] = new float[nColumns]; HKL mate = new HKL(); int nPosIgnore = 0; int nTransIgnore = 0; int nZero = 0; int none = 0; for (int i = 0; i < nReflections; i++) { for (int j = 0; j < nColumns; j++) { dataInputStream.read(bytes, offset, 4); byteBuffer = ByteBuffer.wrap(bytes); data[j] = byteBuffer.order(byteOrder).getFloat(); } int ih = (int) data[h]; int ik = (int) data[k]; int il = (int) data[l]; boolean friedel = reflectionList.findSymHKL(ih, ik, il, mate, false); HKL hklpos = reflectionList.getHKL(mate); if (hklpos == null) { nPosIgnore++; } friedel = reflectionList.findSymHKL(ih, ik, il, mate, true); HKL hkltrans = reflectionList.getHKL(mate); if (hkltrans == null) { nTransIgnore++; } if (rFree > 0) { if (((int) data[rFree]) == 0) { nZero++; } else if (((int) data[rFree]) == 1) { none++; } } if (rFreePlus > 0) { if (((int) data[rFreePlus]) == 0) { nZero++; } else if (((int) data[rFreePlus]) == 1) { none++; } } if (rFreeMinus > 0) { if (((int) data[rFreeMinus]) == 0) { nZero++; } else if (((int) data[rFreeMinus]) == 1) { none++; } } } if (nPosIgnore > nTransIgnore) { transpose = true; } if (none > (nZero * 2) && refinementData.rfreeflag < 0) { refinementData.setFreeRFlag(0); sb.append(format(" Setting R free flag to %d based on MTZ file data.\n", refinementData.rfreeflag)); } else if (nZero > (none * 2) && refinementData.rfreeflag < 0) { refinementData.setFreeRFlag(1); sb.append(format(" Setting R free flag to %d based on MTZ file data.\n", refinementData.rfreeflag)); } else if (refinementData.rfreeflag < 0) { refinementData.setFreeRFlag(0); sb.append(format(" Setting R free flag to MTZ default: %d\n", refinementData.rfreeflag)); } // reopen to start at beginning fileInputStream = new FileInputStream(mtzFile); dataInputStream = new DataInputStream(fileInputStream); // skip initial header dataInputStream.skipBytes(80); // read in data double anofSigF[][] = new double[refinementData.n][4]; for (int i = 0; i < refinementData.n; i++) { anofSigF[i][0] = anofSigF[i][1] = anofSigF[i][2] = anofSigF[i][3] = Double.NaN; } nRead = nIgnore = nRes = nFriedel = nCut = 0; for (int i = 0; i < nReflections; i++) { for (int j = 0; j < nColumns; j++) { dataInputStream.read(bytes, offset, 4); byteBuffer = ByteBuffer.wrap(bytes); data[j] = byteBuffer.order(byteOrder).getFloat(); } int ih = (int) data[h]; int ik = (int) data[k]; int il = (int) data[l]; boolean friedel = reflectionList.findSymHKL(ih, ik, il, mate, transpose); HKL hkl = reflectionList.getHKL(mate); if (hkl != null) { if (fo > 0 && sigFo > 0) { if (refinementData.fsigfcutoff > 0.0) { if ((data[fo] / data[sigFo]) < refinementData.fsigfcutoff) { nCut++; continue; } } if (friedel) { anofSigF[hkl.index()][2] = data[fo]; anofSigF[hkl.index()][3] = data[sigFo]; nFriedel++; } else { anofSigF[hkl.index()][0] = data[fo]; anofSigF[hkl.index()][1] = data[sigFo]; } } else { if (fPlus > 0 && sigFPlus > 0) { if (refinementData.fsigfcutoff > 0.0) { if ((data[fPlus] / data[sigFPlus]) < refinementData.fsigfcutoff) { nCut++; continue; } } anofSigF[hkl.index()][0] = data[fPlus]; anofSigF[hkl.index()][1] = data[sigFPlus]; } if (fMinus > 0 && sigFMinus > 0) { if (refinementData.fsigfcutoff > 0.0) { if ((data[fMinus] / data[sigFMinus]) < refinementData.fsigfcutoff) { nCut++; continue; } } anofSigF[hkl.index()][2] = data[fMinus]; anofSigF[hkl.index()][3] = data[sigFMinus]; } } if (rFree > 0) { refinementData.setFreeR(hkl.index(), (int) data[rFree]); } else { if (rFreePlus > 0 && rFreeMinus > 0) { // not sure what the correct thing to do here is? refinementData.setFreeR(hkl.index(), (int) data[rFreePlus]); } else if (rFreePlus > 0) { refinementData.setFreeR(hkl.index(), (int) data[rFreePlus]); } else if (rFreeMinus > 0) { refinementData.setFreeR(hkl.index(), (int) data[rFreeMinus]); } } nRead++; } else { HKL tmp = new HKL(ih, ik, il); if (!reflectionList.resolution .inInverseResSqRange(Crystal.invressq(reflectionList.crystal, tmp))) { nRes++; } else { nIgnore++; } } } // Set up fsigf from F+ and F-. refinementData.generate_fsigf_from_anofsigf(anofSigF); // Log results. if (logger.isLoggable(Level.INFO)) { sb.append(format(" MTZ file type (machine stamp): %s\n", stampString)); sb.append(format(" HKL data is %s\n", transpose ? "transposed" : "not transposed")); sb.append(format(" HKL read in: %d\n", nRead)); sb.append(format(" HKL read as friedel mates: %d\n", nFriedel)); sb.append(format(" HKL NOT read in (too high resolution): %d\n", nRes)); sb.append(format(" HKL NOT read in (not in internal list?): %d\n", nIgnore)); sb.append(format(" HKL NOT read in (F/sigF cutoff): %d\n", nCut)); sb.append(format(" HKL in internal list: %d\n", reflectionList.hkllist.size())); logger.info(sb.toString()); } if (rFree < 0 && rFreePlus < 0 && rFreeMinus < 0) { refinementData.generateRFree(); } } catch (EOFException e) { String message = " MTZ end of file reached."; logger.log(Level.WARNING, message, e); return false; } catch (IOException e) { String message = " MTZ IO Exception."; logger.log(Level.WARNING, message, e); return false; } return true; }
From source file:org.apache.hadoop.hdfs.server.datanode.DWRRDataXceiver.java
private MD5Hash calcPartialBlockChecksum(ExtendedBlock block, long requestLength, DataChecksum checksum, DataInputStream checksumIn) throws IOException { final int bytesPerCRC = checksum.getBytesPerChecksum(); final int csize = checksum.getChecksumSize(); final byte[] buffer = new byte[4 * 1024]; MessageDigest digester = MD5Hash.getDigester(); long remaining = requestLength / bytesPerCRC * csize; for (int toDigest = 0; remaining > 0; remaining -= toDigest) { toDigest = checksumIn.read(buffer, 0, (int) Math.min(remaining, buffer.length)); if (toDigest < 0) { break; }//from w w w . j a v a 2 s . co m digester.update(buffer, 0, toDigest); } int partialLength = (int) (requestLength % bytesPerCRC); if (partialLength > 0) { byte[] buf = new byte[partialLength]; final InputStream blockIn = datanode.data.getBlockInputStream(block, requestLength - partialLength); try { // Get the CRC of the partialLength. IOUtils.readFully(blockIn, buf, 0, partialLength); } finally { IOUtils.closeStream(blockIn); } checksum.update(buf, 0, partialLength); byte[] partialCrc = new byte[csize]; checksum.writeValue(partialCrc, 0, true); digester.update(partialCrc); } return new MD5Hash(digester.digest()); }
From source file:it.classhidra.core.controller.bean.java
public boolean initJsonPart(HttpServletRequest request) throws bsControllerException { boolean isJson = false; HashMap parameters = new HashMap(); DataInputStream in = null; try {//from www .j a va 2 s .com in = new DataInputStream(request.getInputStream()); int formDataLength = request.getContentLength(); byte dataBytes[] = new byte[formDataLength]; int bytesRead = 0; int totalBytesRead = 0; while (totalBytesRead < formDataLength && totalBytesRead > -1) { bytesRead = in.read(dataBytes, totalBytesRead, formDataLength); totalBytesRead += bytesRead; } String json = new String(dataBytes, 0, dataBytes.length).trim(); if (json.charAt(0) == '{' && json.charAt(json.length() - 1) == '}') isJson = true; if (isJson) { if (json.charAt(0) == '{' && json.length() > 0) json = json.substring(1, json.length()); if (json.charAt(json.length() - 1) == '}' && json.length() > 0) json = json.substring(0, json.length() - 1); StringTokenizer st = new StringTokenizer(json, ","); while (st.hasMoreTokens()) { String pair = st.nextToken(); StringTokenizer st1 = new StringTokenizer(pair, ":"); String key = null; String value = null; if (st1.hasMoreTokens()) key = st1.nextToken(); if (st1.hasMoreTokens()) value = st1.nextToken(); if (key != null && value != null) { key = key.trim(); if (key.charAt(0) == '"' && key.length() > 0) key = key.substring(1, key.length()); if (key.charAt(key.length() - 1) == '"' && key.length() > 0) key = key.substring(0, key.length() - 1); value = value.trim(); if (value.charAt(0) == '"' && value.length() > 0) value = value.substring(1, value.length()); if (value.charAt(value.length() - 1) == '"' && value.length() > 0) value = value.substring(0, value.length() - 1); parameters.put(key, value); } } } } catch (Exception e) { } finally { try { in.close(); } catch (Exception ex) { } } if (isJson) initPartFromMap(parameters); return isJson; }
From source file:it.classhidra.core.controller.bsController.java
public static String getPropertyMultipart(String key, HttpServletRequest req) { try {/*from w ww . j a v a2 s. c o m*/ String file = (String) req.getAttribute("multipart/form-data"); DataInputStream in = null; String contentType = req.getContentType(); if (contentType != null && contentType.indexOf("multipart/form-data") != -1) { if (file == null) { in = new DataInputStream(req.getInputStream()); int formDataLength = req.getContentLength(); byte dataBytes[] = new byte[formDataLength]; int bytesRead = 0; int totalBytesRead = 0; while (totalBytesRead < formDataLength) { bytesRead = in.read(dataBytes, totalBytesRead, formDataLength); totalBytesRead += bytesRead; } file = new String(dataBytes, 0, dataBytes.length, "ASCII"); in.close(); req.setAttribute("multipart/form-data", file); } String check = "Content-Disposition: form-data; name=\"" + key + "\""; int pos = file.indexOf(check); if (pos > -1) { int pos1 = file.indexOf("-----------------------------", pos); if (pos1 > -1) { String result = file.substring(pos + check.length(), pos1); result = result.replace('\n', ' ').replace('\r', ' '); return result.trim(); } } } } catch (Exception e) { new bsControllerException(e, iStub.log_DEBUG); return null; } return null; }
From source file:org.apache.hadoop.hive.service.HSSessionItem.java
public byte[] downloadModule(String user, String moduleName) throws HiveServerException { String module = ""; String fname = getHome() + "/pl/lib/" + user + "/" + moduleName; File f;//w w w .java 2 s . c o m String dname = getHome() + "/pl/lib/" + user; File d = new File(dname); if (moduleName.startsWith("gxx_")) { fname = getHome() + "/pl/lib/global/" + moduleName; } else if (moduleName.contains("/")) { fname = getHome() + "/pl/lib/" + moduleName; } if (!d.exists()) { if (!d.mkdir()) { l4j.error(getSessionName() + " try to mkdir " + dname + " failed."); throw new HiveServerException("Create user library failed."); } } f = new File(fname); if (f.canRead() == false) { throw new HiveServerException("Try to read file " + fname + " failed."); } int size = 0; long size_org = f.length(); if (size_org > Integer.MAX_VALUE) throw new HiveServerException("File is too large, no more than 2GB"); else size = (int) size_org; byte[] bytes = new byte[size]; try { FileInputStream fis = new FileInputStream(f); DataInputStream dis = new DataInputStream(fis); int read = 0; int numRead = 0; while (read < bytes.length && (numRead = dis.read(bytes, read, bytes.length - read)) >= 0) { read = read + numRead; } fis.close(); } catch (java.io.IOException ex) { l4j.error("IO Exception ", ex); throw new HiveServerException("Read file " + fname + " failed w/ " + ex); } return bytes; }
From source file:net.sergetk.mobile.lcdui.BitmapFont.java
/** * Creates a new font from the resource. The capacity of the color cache defines maximum size of * the color cache.// w ww . ja v a2 s.c o m * * @param fontPath * the resource name * @param colorCacheCapacity * the maximum color cache size */ public BitmapFont(String fontPath, int colorCacheCapacity) { this.style = Font.STYLE_PLAIN; this.currentColor = 0; this.colorCache = new CacheEntry[colorCacheCapacity]; this.colorUsageCounts = new IntHashMap(colorCacheCapacity * 2); try { InputStream input = new Object().getClass().getResourceAsStream(fontPath); if (input == null) { throw new IOException(); } DataInputStream data = new DataInputStream(input); int streamLen = data.available(); this.fontFilePath = fontPath; this.version = data.readByte(); this.height = data.readByte(); this.baseline = data.readByte(); this.xIndent = data.readByte(); this.yIndent = data.readByte(); this.spaceWidth = data.readByte(); characterMap = data.readUTF(); int count = characterMap.length(); // read characters widthes this.widths = new int[count]; this.x = new int[count]; this.y = new int[count]; for (int i = 0; i < count; i++) { widths[i] = data.readByte(); } baseImage = null; // the original implementation supported multiple-images // in the font file, but this is not necessary. Because I do // not want to change the encoding, I am leaving this byte that // used to represent the number of PNGs in the file data.skipBytes(1); short pngLen = data.readShort(); byte[] buffer = new byte[pngLen]; data.read(buffer, 0, pngLen); this.pngOffset = (short) (streamLen - pngLen); baseImage = Image.createImage(buffer, 0, pngLen); currentImage = baseImage; // calculate characters coordinates int curX = 0, curY = 0; for (int i = 0; i < count; i++) { if (widths[i] < 0) { // negative width points to another character int sourceIndex = -widths[i]; widths[i] = widths[sourceIndex]; x[i] = x[sourceIndex]; y[i] = y[sourceIndex]; } else { x[i] = curX; y[i] = curY; curX += widths[i]; } } if (defaultFont == null) defaultFont = this; } catch (IOException e) { // Log.warn("IOException reading font: ", e); System.err.println("IOException reading font: " + e.getMessage()); e.printStackTrace(); } }
From source file:org.apache.giraph.graph.BspServiceMaster.java
/** * Read the finalized checkpoint file and associated metadata files for the * checkpoint. Modifies the {@link PartitionOwner} objects to get the * checkpoint prefixes. It is an optimization to prevent all workers from * searching all the files. Also read in the aggregator data from the * finalized checkpoint file and setting it. * * @param superstep Checkpoint set to examine. * @param partitionOwners Partition owners to modify with checkpoint * prefixes/*from w w w .j a v a2s . c o m*/ * @throws IOException * @throws InterruptedException * @throws KeeperException */ private void prepareCheckpointRestart(long superstep, Collection<PartitionOwner> partitionOwners) throws IOException, KeeperException, InterruptedException { FileSystem fs = getFs(); List<Path> validMetadataPathList = new ArrayList<Path>(); String finalizedCheckpointPath = getCheckpointBasePath(superstep) + CHECKPOINT_FINALIZED_POSTFIX; DataInputStream finalizedStream = fs.open(new Path(finalizedCheckpointPath)); int prefixFileCount = finalizedStream.readInt(); for (int i = 0; i < prefixFileCount; ++i) { String metadataFilePath = finalizedStream.readUTF() + CHECKPOINT_METADATA_POSTFIX; validMetadataPathList.add(new Path(metadataFilePath)); } // Set the merged aggregator data if it exists. int aggregatorDataSize = finalizedStream.readInt(); if (aggregatorDataSize > 0) { byte[] aggregatorZkData = new byte[aggregatorDataSize]; int actualDataRead = finalizedStream.read(aggregatorZkData, 0, aggregatorDataSize); if (actualDataRead != aggregatorDataSize) { throw new RuntimeException("prepareCheckpointRestart: Only read " + actualDataRead + " of " + aggregatorDataSize + " aggregator bytes from " + finalizedCheckpointPath); } String mergedAggregatorPath = getMergedAggregatorPath(getApplicationAttempt(), superstep - 1); if (LOG.isInfoEnabled()) { LOG.info("prepareCheckpointRestart: Reloading merged " + "aggregator " + "data '" + Arrays.toString(aggregatorZkData) + "' to previous checkpoint in path " + mergedAggregatorPath); } if (getZkExt().exists(mergedAggregatorPath, false) == null) { getZkExt().createExt(mergedAggregatorPath, aggregatorZkData, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); } else { getZkExt().setData(mergedAggregatorPath, aggregatorZkData, -1); } } masterCompute.readFields(finalizedStream); finalizedStream.close(); Map<Integer, PartitionOwner> idOwnerMap = new HashMap<Integer, PartitionOwner>(); for (PartitionOwner partitionOwner : partitionOwners) { if (idOwnerMap.put(partitionOwner.getPartitionId(), partitionOwner) != null) { throw new IllegalStateException("prepareCheckpointRestart: Duplicate partition " + partitionOwner); } } // Reading the metadata files. Simply assign each partition owner // the correct file prefix based on the partition id. for (Path metadataPath : validMetadataPathList) { String checkpointFilePrefix = metadataPath.toString(); checkpointFilePrefix = checkpointFilePrefix.substring(0, checkpointFilePrefix.length() - CHECKPOINT_METADATA_POSTFIX.length()); DataInputStream metadataStream = fs.open(metadataPath); long partitions = metadataStream.readInt(); for (long i = 0; i < partitions; ++i) { long dataPos = metadataStream.readLong(); int partitionId = metadataStream.readInt(); PartitionOwner partitionOwner = idOwnerMap.get(partitionId); if (LOG.isInfoEnabled()) { LOG.info("prepareSuperstepRestart: File " + metadataPath + " with position " + dataPos + ", partition id = " + partitionId + " assigned to " + partitionOwner); } partitionOwner.setCheckpointFilesPrefix(checkpointFilePrefix); } metadataStream.close(); } }
From source file:clueweb09.WarcRecord.java
private static byte[] readNextRecord(DataInputStream in, StringBuffer headerBuffer) throws IOException { if (in == null) { return null; }/* w w w.j a va2 s . co m*/ if (headerBuffer == null) { return null; } String line = null; boolean foundMark = false; byte[] retContent = null; // cannot be using a buffered reader here!!!! // just read the header // first - find our WARC header while ((!foundMark) && ((line = readLineFromInputStream(in)) != null)) { if (line.startsWith(WARC_VERSION)) { WARC_VERSION_LINE = line; foundMark = true; } } // no WARC mark? if (!foundMark) { return null; } // LOG.info("Found WARC_VERSION"); int contentLength = -1; // read until we see contentLength then an empty line // (to handle malformed ClueWeb09 headers that have blank lines) // get the content length and set our retContent for (line = readLineFromInputStream(in).trim(); line.length() > 0 || contentLength < 0; line = readLineFromInputStream(in).trim()) { if (line.length() > 0) { headerBuffer.append(line); headerBuffer.append(LINE_ENDING); // find the content length designated by Content-Length: <length> String[] parts = line.split(":", 2); if (parts.length == 2 && parts[0].equals("Content-Length")) { try { contentLength = Integer.parseInt(parts[1].trim()); // LOG.info("WARC record content length: " + contentLength); } catch (NumberFormatException nfEx) { contentLength = -1; } } } } // now read the bytes of the content retContent = new byte[contentLength]; int totalWant = contentLength; int totalRead = 0; // // LOOP TO REMOVE LEADING CR * LF // To prevent last few characters from being cut off of the content // when reading // while ((totalRead == 0) && (totalRead < contentLength)) { byte CR = in.readByte(); byte LF = in.readByte(); if ((CR != 13) && (LF != 10)) { retContent[0] = CR; retContent[1] = LF; totalRead = 2; totalWant = contentLength - totalRead; } } // // // while (totalRead < contentLength) { try { int numRead = in.read(retContent, totalRead, totalWant); if (numRead < 0) { return null; } else { totalRead += numRead; totalWant = contentLength - totalRead; } // end if (numRead < 0) / else } catch (EOFException eofEx) { // resize to what we have if (totalRead > 0) { byte[] newReturn = new byte[totalRead]; System.arraycopy(retContent, 0, newReturn, 0, totalRead); return newReturn; } else { return null; } } // end try/catch (EOFException) } // end while (totalRead < contentLength) return retContent; }
From source file:cwru.mjq.WarcRecord.java
private static byte[] readNextRecord(DataInputStream in, StringBuilder headerBuffer) throws IOException { if (in == null) { return null; }/*from ww w.jav a 2s .c o m*/ if (headerBuffer == null) { return null; } String line; boolean foundMark = false; byte[] retContent; // cannot be using a buffered reader here!!!! // just read the header // first - find our WARC header while ((!foundMark) && ((line = readLineFromInputStream(in)) != null)) { if (line.startsWith(WARC_VERSION)) { WARC_VERSION_LINE = line; foundMark = true; } } // no WARC mark? if (!foundMark) { return null; } // LOG.info("Found WARC_VERSION"); int contentLength = -1; // read until we see contentLength then an empty line // (to handle malformed ClueWeb09 headers that have blank lines) // get the content length and set our retContent for (line = readLineFromInputStream(in).trim(); line.length() > 0 || contentLength < 0; line = readLineFromInputStream(in).trim()) { if (line.length() > 0) { headerBuffer.append(line); headerBuffer.append(LINE_ENDING); // find the content length designated by Content-Length: // <length> String[] parts = line.split(":", 2); if (parts.length == 2 && parts[0].equals(CONTENT_LENGTH)) { try { contentLength = Integer.parseInt(parts[1].trim()); // LOG.info("WARC record content length: " + // contentLength); } catch (NumberFormatException nfEx) { contentLength = -1; } } } } // now read the bytes of the content retContent = new byte[contentLength]; int totalWant = contentLength; int totalRead = 0; // // LOOP TO REMOVE LEADING CR * LF // To prevent last few characters from being cut off of the content // when reading // while ((totalRead == 0) && (totalRead < contentLength)) { byte CR = in.readByte(); byte LF = in.readByte(); if ((CR != 13) && (LF != 10)) { retContent[0] = CR; retContent[1] = LF; totalRead = 2; totalWant = contentLength - totalRead; } } // // // while (totalRead < contentLength) { try { int numRead = in.read(retContent, totalRead, totalWant); if (numRead < 0) { return null; } else { totalRead += numRead; totalWant = contentLength - totalRead; } // end if (numRead < 0) / else } catch (EOFException eofEx) { // resize to what we have if (totalRead > 0) { byte[] newReturn = new byte[totalRead]; System.arraycopy(retContent, 0, newReturn, 0, totalRead); return newReturn; } else { return null; } } // end try/catch (EOFException) } // end while (totalRead < contentLength) return retContent; }