List of usage examples for java.io BufferedInputStream mark
public synchronized void mark(int readlimit)
mark method of InputStream. From source file:org.apereo.portal.io.xml.JaxbPortalDataHandlerService.java
protected void importDataArchive(Resource archive, InputStream resourceStream, BatchImportOptions options) { BufferedInputStream bufferedResourceStream = null; try {/*from w w w .j av a 2 s . co m*/ //Make sure the stream is buffered if (resourceStream instanceof BufferedInputStream) { bufferedResourceStream = (BufferedInputStream) resourceStream; } else { bufferedResourceStream = new BufferedInputStream(resourceStream); } //Buffer up to 100MB, bad things will happen if we bust this buffer. //TODO see if there is a buffered stream that will write to a file once the buffer fills up bufferedResourceStream.mark(100 * 1024 * 1024); final MediaType type = getMediaType(bufferedResourceStream, archive.getFilename()); if (MT_JAVA_ARCHIVE.equals(type)) { final ArchiveInputStream archiveStream = new JarArchiveInputStream(bufferedResourceStream); importDataArchive(archive, archiveStream, options); } else if (MediaType.APPLICATION_ZIP.equals(type)) { final ArchiveInputStream archiveStream = new ZipArchiveInputStream(bufferedResourceStream); importDataArchive(archive, archiveStream, options); } else if (MT_CPIO.equals(type)) { final ArchiveInputStream archiveStream = new CpioArchiveInputStream(bufferedResourceStream); importDataArchive(archive, archiveStream, options); } else if (MT_AR.equals(type)) { final ArchiveInputStream archiveStream = new ArArchiveInputStream(bufferedResourceStream); importDataArchive(archive, archiveStream, options); } else if (MT_TAR.equals(type)) { final ArchiveInputStream archiveStream = new TarArchiveInputStream(bufferedResourceStream); importDataArchive(archive, archiveStream, options); } else if (MT_BZIP2.equals(type)) { final CompressorInputStream compressedStream = new BZip2CompressorInputStream( bufferedResourceStream); importDataArchive(archive, compressedStream, options); } else if (MT_GZIP.equals(type)) { final CompressorInputStream compressedStream = new GzipCompressorInputStream( bufferedResourceStream); importDataArchive(archive, compressedStream, options); } else if (MT_PACK200.equals(type)) { final CompressorInputStream compressedStream = new Pack200CompressorInputStream( bufferedResourceStream); importDataArchive(archive, compressedStream, options); } else if (MT_XZ.equals(type)) { final CompressorInputStream compressedStream = new XZCompressorInputStream(bufferedResourceStream); importDataArchive(archive, compressedStream, options); } else { throw new RuntimeException("Unrecognized archive media type: " + type); } } catch (IOException e) { throw new RuntimeException("Could not load InputStream for resource: " + archive, e); } finally { IOUtils.closeQuietly(bufferedResourceStream); } }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.por.PORFileReader.java
private File decodeHeader(BufferedInputStream stream) throws IOException { File tempPORfile = null;/*from w w w .j a v a 2 s . c om*/ if (stream == null) { throw new IllegalArgumentException("file == null!"); } byte[] headerByes = new byte[POR_HEADER_SIZE]; if (stream.markSupported()) { stream.mark(1000); } int nbytes = stream.read(headerByes, 0, POR_HEADER_SIZE); //printHexDump(headerByes, "hex dump of the byte-array"); if (nbytes == 0) { throw new IOException("decodeHeader: reading failure"); } else if (nbytes < 491) { // Size test: by defnition, it must have at least // 491-byte header, i.e., the file size less than this threshold // is not a POR file dbgLog.fine("this file is NOT spss-por type"); throw new IllegalArgumentException("file is not spss-por type"); } // rewind the current reading position back to the beginning if (stream.markSupported()) { stream.reset(); } // line-terminating characters are usually one or two by defnition // however, a POR file saved by a genuine SPSS for Windows // had a three-character line terminator, i.e., failed to remove the // original file's one-character terminator when it was opened, and // saved it with the default two-character terminator without // removing original terminators. So we have to expect such a rare // case // // terminator // windows [0D0A]=> [1310] = [CR/LF] // unix [0A] => [10] // mac [0D] => [13] // 3char [0D0D0A]=> [131310] spss for windows rel 15 // // terminating characters should be found at the following // column positions[counting from 0]: // unix case: [0A] : [80], [161], [242], [323], [404], [485] // windows case: [0D0A] : [81], [163], [245], [327], [409], [491] // : [0D0D0A] : [82], [165], [248], [331], [414], [495] // convert b into a ByteBuffer ByteBuffer buff = ByteBuffer.wrap(headerByes); byte[] nlch = new byte[36]; int pos1; int pos2; int pos3; int ucase = 0; int wcase = 0; int mcase = 0; int three = 0; int nolines = 6; int nocols = 80; for (int i = 0; i < nolines; ++i) { int baseBias = nocols * (i + 1); // 1-char case pos1 = baseBias + i; buff.position(pos1); dbgLog.finer("\tposition(1)=" + buff.position()); int j = 6 * i; nlch[j] = buff.get(); if (nlch[j] == 10) { ucase++; } else if (nlch[j] == 13) { mcase++; } // 2-char case pos2 = baseBias + 2 * i; buff.position(pos2); dbgLog.finer("\tposition(2)=" + buff.position()); nlch[j + 1] = buff.get(); nlch[j + 2] = buff.get(); // 3-char case pos3 = baseBias + 3 * i; buff.position(pos3); dbgLog.finer("\tposition(3)=" + buff.position()); nlch[j + 3] = buff.get(); nlch[j + 4] = buff.get(); nlch[j + 5] = buff.get(); dbgLog.finer(i + "-th iteration position =" + nlch[j] + "\t" + nlch[j + 1] + "\t" + nlch[j + 2]); dbgLog.finer(i + "-th iteration position =" + nlch[j + 3] + "\t" + nlch[j + 4] + "\t" + nlch[j + 5]); if ((nlch[j + 3] == 13) && (nlch[j + 4] == 13) && (nlch[j + 5] == 10)) { three++; } else if ((nlch[j + 1] == 13) && (nlch[j + 2] == 10)) { wcase++; } buff.rewind(); } boolean windowsNewLine = true; if (three == nolines) { windowsNewLine = false; // lineTerminator = "0D0D0A" } else if ((ucase == nolines) && (wcase < nolines)) { windowsNewLine = false; // lineTerminator = "0A" } else if ((ucase < nolines) && (wcase == nolines)) { windowsNewLine = true; //lineTerminator = "0D0A" } else if ((mcase == nolines) && (wcase < nolines)) { windowsNewLine = false; //lineTerminator = "0D" } buff.rewind(); int PORmarkPosition = POR_MARK_POSITION_DEFAULT; if (windowsNewLine) { PORmarkPosition = PORmarkPosition + 5; } else if (three == nolines) { PORmarkPosition = PORmarkPosition + 10; } byte[] pormark = new byte[8]; buff.position(PORmarkPosition); buff.get(pormark, 0, 8); String pormarks = new String(pormark); //dbgLog.fine("pormark =>" + pormarks + "<-"); dbgLog.fine( "pormark[hex: 53 50 53 53 50 4F 52 54 == SPSSPORT] =>" + new String(Hex.encodeHex(pormark)) + "<-"); if (pormarks.equals(POR_MARK)) { dbgLog.fine("POR ID toke test: Passed"); init(); smd.getFileInformation().put("mimeType", MIME_TYPE); smd.getFileInformation().put("fileFormat", MIME_TYPE); } else { dbgLog.fine("this file is NOT spss-por type"); throw new IllegalArgumentException("decodeHeader: POR ID token was not found"); } // save the POR file without new line characters FileOutputStream fileOutPOR = null; Writer fileWriter = null; // Scanner class can handle three-character line-terminator Scanner porScanner = null; try { tempPORfile = File.createTempFile("tempPORfile.", ".por"); fileOutPOR = new FileOutputStream(tempPORfile); fileWriter = new BufferedWriter(new OutputStreamWriter(fileOutPOR, "utf8")); porScanner = new Scanner(stream); // Because 64-bit and 32-bit machines decode POR's first 40-byte // sequence differently, the first 5 leader lines are skipped from // the new-line-stripped file int lineCounter = 0; while (porScanner.hasNextLine()) { lineCounter++; if (lineCounter <= 5) { String line = porScanner.nextLine().toString(); dbgLog.fine("line=" + lineCounter + ":" + line.length() + ":" + line); } else { fileWriter.write(porScanner.nextLine().toString()); } } } finally { try { if (fileWriter != null) { fileWriter.close(); } } catch (IOException ex) { ex.printStackTrace(); } if (porScanner != null) { porScanner.close(); } } return tempPORfile; }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader.java
private File decodeHeader(BufferedInputStream stream) throws IOException { dbgLog.fine("decodeHeader(): start"); File tempPORfile = null;/*from w ww. ja v a 2s .co m*/ if (stream == null) { throw new IllegalArgumentException("file == null!"); } byte[] headerByes = new byte[POR_HEADER_SIZE]; if (stream.markSupported()) { stream.mark(1000); } int nbytes = stream.read(headerByes, 0, POR_HEADER_SIZE); //printHexDump(headerByes, "hex dump of the byte-array"); if (nbytes == 0) { throw new IOException("decodeHeader: reading failure"); } else if (nbytes < 491) { // Size test: by defnition, it must have at least // 491-byte header, i.e., the file size less than this threshold // is not a POR file dbgLog.fine("this file is NOT spss-por type"); throw new IllegalArgumentException("file is not spss-por type"); } // rewind the current reading position back to the beginning if (stream.markSupported()) { stream.reset(); } // line-terminating characters are usually one or two by defnition // however, a POR file saved by a genuine SPSS for Windows // had a three-character line terminator, i.e., failed to remove the // original file's one-character terminator when it was opened, and // saved it with the default two-character terminator without // removing original terminators. So we have to expect such a rare // case // // terminator // windows [0D0A]=> [1310] = [CR/LF] // unix [0A] => [10] // mac [0D] => [13] // 3char [0D0D0A]=> [131310] spss for windows rel 15 // // terminating characters should be found at the following // column positions[counting from 0]: // unix case: [0A] : [80], [161], [242], [323], [404], [485] // windows case: [0D0A] : [81], [163], [245], [327], [409], [491] // : [0D0D0A] : [82], [165], [248], [331], [414], [495] // convert b into a ByteBuffer ByteBuffer buff = ByteBuffer.wrap(headerByes); byte[] nlch = new byte[36]; int pos1; int pos2; int pos3; int ucase = 0; int wcase = 0; int mcase = 0; int three = 0; int nolines = 6; int nocols = 80; for (int i = 0; i < nolines; ++i) { int baseBias = nocols * (i + 1); // 1-char case pos1 = baseBias + i; buff.position(pos1); dbgLog.finer("\tposition(1)=" + buff.position()); int j = 6 * i; nlch[j] = buff.get(); if (nlch[j] == 10) { ucase++; } else if (nlch[j] == 13) { mcase++; } // 2-char case pos2 = baseBias + 2 * i; buff.position(pos2); dbgLog.finer("\tposition(2)=" + buff.position()); nlch[j + 1] = buff.get(); nlch[j + 2] = buff.get(); // 3-char case pos3 = baseBias + 3 * i; buff.position(pos3); dbgLog.finer("\tposition(3)=" + buff.position()); nlch[j + 3] = buff.get(); nlch[j + 4] = buff.get(); nlch[j + 5] = buff.get(); dbgLog.finer(i + "-th iteration position =" + nlch[j] + "\t" + nlch[j + 1] + "\t" + nlch[j + 2]); dbgLog.finer(i + "-th iteration position =" + nlch[j + 3] + "\t" + nlch[j + 4] + "\t" + nlch[j + 5]); if ((nlch[j + 3] == 13) && (nlch[j + 4] == 13) && (nlch[j + 5] == 10)) { three++; } else if ((nlch[j + 1] == 13) && (nlch[j + 2] == 10)) { wcase++; } buff.rewind(); } boolean windowsNewLine = true; if (three == nolines) { windowsNewLine = false; // lineTerminator = "0D0D0A" } else if ((ucase == nolines) && (wcase < nolines)) { windowsNewLine = false; // lineTerminator = "0A" } else if ((ucase < nolines) && (wcase == nolines)) { windowsNewLine = true; //lineTerminator = "0D0A" } else if ((mcase == nolines) && (wcase < nolines)) { windowsNewLine = false; //lineTerminator = "0D" } buff.rewind(); int PORmarkPosition = POR_MARK_POSITION_DEFAULT; if (windowsNewLine) { PORmarkPosition = PORmarkPosition + 5; } else if (three == nolines) { PORmarkPosition = PORmarkPosition + 10; } byte[] pormark = new byte[8]; buff.position(PORmarkPosition); buff.get(pormark, 0, 8); String pormarks = new String(pormark); //dbgLog.fine("pormark =>" + pormarks + "<-"); dbgLog.fine( "pormark[hex: 53 50 53 53 50 4F 52 54 == SPSSPORT] =>" + new String(Hex.encodeHex(pormark)) + "<-"); if (pormarks.equals(POR_MARK)) { dbgLog.fine("POR ID toke test: Passed"); init(); dataTable.setOriginalFileFormat(MIME_TYPE); dataTable.setUnf("UNF:6:NOTCALCULATED"); } else { dbgLog.fine("this file is NOT spss-por type"); throw new IllegalArgumentException("decodeHeader: POR ID token was not found"); } // save the POR file without new line characters FileOutputStream fileOutPOR = null; Writer fileWriter = null; // Scanner class can handle three-character line-terminator Scanner porScanner = null; try { tempPORfile = File.createTempFile("tempPORfile.", ".por"); fileOutPOR = new FileOutputStream(tempPORfile); fileWriter = new BufferedWriter(new OutputStreamWriter(fileOutPOR, "utf8")); porScanner = new Scanner(stream); // Because 64-bit and 32-bit machines decode POR's first 40-byte // sequence differently, the first 5 leader lines are skipped from // the new-line-stripped file int lineCounter = 0; while (porScanner.hasNextLine()) { lineCounter++; if (lineCounter <= 5) { String line = porScanner.nextLine(); dbgLog.fine("line=" + lineCounter + ":" + line.length() + ":" + line); } else { fileWriter.write(porScanner.nextLine()); } } } finally { try { if (fileWriter != null) { fileWriter.close(); } } catch (IOException ex) { ex.printStackTrace(); } if (porScanner != null) { porScanner.close(); } } return tempPORfile; }
From source file:org.lockss.remote.RemoteApi.java
public BatchAuStatus processSavedConfigProps(BufferedInputStream auTxtStream) throws IOException, InvalidAuConfigBackupFile { int commentLen = AU_BACKUP_FILE_COMMENT.length(); // There is apparently hidden buffering in the InputStreamReader's // StreamDecoder which throws off our calculation as to how much // auTxtStream needs to buffer, so use a large number auTxtStream.mark(paramBackupStreamMarkSize); BufferedReader rdr = new BufferedReader(new InputStreamReader(auTxtStream, Constants.DEFAULT_ENCODING), commentLen * 2);// w w w . j a va 2s . c om // We really want rdr.readLine(), but we need to limit amount it reads // (in case it doesn't find newline) char[] buf = new char[commentLen]; int chars = StreamUtil.readChars(rdr, buf, commentLen); log.debug3("chars: " + chars); if (chars == 0) { throw new InvalidAuConfigBackupFile("Uploaded file is empty"); } String line1 = new String(buf); if (chars < commentLen || !line1.startsWith(AU_BACKUP_FILE_COMMENT)) { log.debug("line1: " + line1); throw new InvalidAuConfigBackupFile("Uploaded file does not appear to be a saved AU configuration"); } try { auTxtStream.reset(); } catch (IOException e) { throw new IOException("Internal error: please report \"Insufficient buffering for restore\"."); } Properties allAuProps = new Properties(); try { allAuProps.load(auTxtStream); } catch (Exception e) { log.warning("Loading AU config backup file", e); throw new InvalidAuConfigBackupFile("Uploaded file has illegal format: " + e.getMessage()); } Configuration allAuConfig = ConfigManager.fromPropertiesUnsealed(allAuProps); int ver = checkLegalAuConfigTree(allAuConfig); return batchProcessAus(false, BATCH_ADD_RESTORE, allAuConfig, null); }
From source file:tufts.vue.URLResource.java
private Properties scrapeHTMLmetaData(URLConnection connection, int maxSearchBytes) throws java.io.IOException { Properties metaData = new Properties(); InputStream byteStream = connection.getInputStream(); if (DEBUG.DND && DEBUG.META) { System.err.println("Getting headers from " + connection); System.err.println("Headers: " + connection.getHeaderFields()); }//from w ww. j a v a 2 s .c o m // note: be sure to call getContentType and don't rely on getting it from the HeaderFields map, // as sometimes it's set by the OS for a file:/// URL when there are no header fields (no http server) // (actually, this is set by java via a mime type table based on file extension, or a guess based on the stream) if (DEBUG.DND) System.err.println("*** getting contentType & encoding..."); final String contentType = connection.getContentType(); final String contentEncoding = connection.getContentEncoding(); final int contentLength = connection.getContentLength(); if (DEBUG.DND) System.err.println("*** contentType [" + contentType + "]"); if (DEBUG.DND) System.err.println("*** contentEncoding [" + contentEncoding + "]"); if (DEBUG.DND) System.err.println("*** contentLength [" + contentLength + "]"); setProperty("url.contentType", contentType); setProperty("url.contentEncoding", contentEncoding); if (contentLength >= 0) setProperty("url.contentLength", contentLength); //if (contentType.toLowerCase().startsWith("text/html") == false) { if (!isHTML()) { // we only currently handle HTML if (DEBUG.Enabled) System.err.println("*** contentType [" + contentType + "] not HTML; skipping title extraction"); return metaData; } if (DEBUG.DND) System.err.println("*** scanning for HTML meta-data..."); try { final BufferedInputStream bufStream = new BufferedInputStream(byteStream, maxSearchBytes); bufStream.mark(maxSearchBytes); final byte[] byteBuffer = new byte[maxSearchBytes]; int bytesRead = 0; int len = 0; // BufferedInputStream still won't read thru a block, so we need to allow // a few reads here to get thru a couple of blocks, so we can get up to // our maxbytes (e.g., a common return chunk count is 1448 bytes, presumably related to the MTU) do { int max = maxSearchBytes - bytesRead; len = bufStream.read(byteBuffer, bytesRead, max); System.out.println("*** read " + len); if (len > 0) bytesRead += len; else if (len < 0) break; } while (len > 0 && bytesRead < maxSearchBytes); if (DEBUG.DND) System.out.println("*** Got total chars: " + bytesRead); String html = new String(byteBuffer, 0, bytesRead); if (DEBUG.DND && DEBUG.META) System.out.println("*** HTML-STRING[" + html + "]"); // first, look for a content encoding, so we can search for and get the title // on a properly encoded character stream String charset = null; Matcher cm = Content_Charset_Regex.matcher(html); if (cm.lookingAt()) { charset = cm.group(1); if (DEBUG.DND) System.err.println("*** found HTML specified charset [" + charset + "]"); setProperty("charset", charset); } if (charset == null && contentEncoding != null) { if (DEBUG.DND || true) System.err.println("*** no charset found: using contentEncoding charset " + contentEncoding); charset = contentEncoding; } final String decodedHTML; if (charset != null) { bufStream.reset(); InputStreamReader decodedStream = new InputStreamReader(bufStream, charset); //InputStreamReader decodedStream = new InputStreamReader(new ByteArrayInputStream(byteBuffer), charset); if (true || DEBUG.DND) System.out.println("*** decoding bytes into characters with official encoding " + decodedStream.getEncoding()); setProperty("contentEncoding", decodedStream.getEncoding()); char[] decoded = new char[bytesRead]; int decodedChars = decodedStream.read(decoded); decodedStream.close(); if (true || DEBUG.DND) System.err.println("*** " + decodedChars + " characters decoded using " + charset); decodedHTML = new String(decoded, 0, decodedChars); } else decodedHTML = html; // we'll just have to go with the default platform charset... // these needed to be left open till the decodedStream was done, which // although it should never need to read beyond what's already buffered, // some internal java code has checks that make sure the underlying stream // isn't closed, even it it isn't used. byteStream.close(); bufStream.close(); Matcher m = HTML_Title_Regex.matcher(decodedHTML); if (m.lookingAt()) { String title = m.group(1); if (true || DEBUG.DND) System.err.println("*** found title [" + title + "]"); metaData.put("title", title.trim()); } } catch (Throwable e) { System.err.println("scrapeHTMLmetaData: " + e); if (DEBUG.DND) e.printStackTrace(); } if (DEBUG.DND || DEBUG.Enabled) System.err.println("*** scrapeHTMLmetaData returning [" + metaData + "]"); return metaData; }
From source file:org.jzkit.a2j.codec.runtime.ChunkingBERInputStream.java
public int completeConstructedType(BufferedInputStream in, ByteArrayOutputStream baos, int level) throws java.io.IOException { int bytes_written = 0; int first_byte = in.read(); if (first_byte == -1) throw new java.io.IOException("EOF"); byte c = (byte) first_byte; // byte c = (byte)in.read(); baos.write(c);/*from w w w . ja v a 2 s .c o m*/ bytes_written++; c &= 0xFF; int next_tag_class = c & 0xC0; boolean next_is_constructed = (c & 0x20) != 0; boolean next_is_indefinite = false; int next_tag_number = c & 0x1F; // If there are multiple octets to encode the tag if (next_tag_number == 0x1F) { next_tag_number = 0; do { c = (byte) in.read(); baos.write(c); bytes_written++; // Shift value 7 bits left next_tag_number = next_tag_number << 7; // Merge with the octets we just got next_tag_number = (next_tag_number | (c & 0x7F)); } while ((c & 0x80) != 0); } // dbg("tag: "+next_tag_number+" class:"+next_tag_class, level); int datalen; byte lenpart = (byte) in.read(); baos.write(lenpart); bytes_written++; // System.err.println("First len octet is "+lenpart); if ((lenpart & 0x80) == 0) // If bit 8 is 0 { // Single octet length encoding // System.err.println("Single octet length encoding"); datalen = lenpart; next_is_indefinite = false; } else if ((lenpart & 0x7F) == 0) // Otherwise we are multiple octets (Maybe 0, which = indefinite) { // System.err.println("Indefinite length encoding"); next_is_indefinite = true; datalen = 0; } else { next_is_indefinite = false; // System.err.println("Multiple octet length encoding ("+(lenpart & 0x7F )+"octets)"); lenpart &= 0x7F; datalen = 0; while (lenpart-- > 0) { byte lenbyte = (byte) in.read(); datalen = (datalen << 8) | (lenbyte & 0xFF); baos.write(lenbyte); bytes_written++; } } // System.err.print(" indefinite: "+next_is_indefinite+" cons:"+next_is_constructed+" len:"+datalen); // OK, len now contains the size of the octets. // If it's definite length encoding, just copy that many bytes if (next_is_indefinite) { if (next_is_constructed) { // System.err.print(" {\n"); // Peek ahead looking for terminating octets. boolean more_data = true; in.mark(5); byte i1 = (byte) in.read(); byte i2 = (byte) in.read(); in.reset(); if ((i1 == 0) && (i2 == 0)) { more_data = false; } while (more_data) { completeConstructedType(in, baos, level + 1); in.mark(5); i1 = (byte) in.read(); i2 = (byte) in.read(); in.reset(); if ((i1 == 0) && (i2 == 0)) { more_data = false; } } // Better consume the terminating octets. in.read(); in.read(); baos.write(0); bytes_written++; baos.write(0); bytes_written++; // dbg("} ("+bytes_written+")\n",level); } else { // Indefinite length primitive type // System.err.print(" Indefinite length primitive type"); byte b1 = (byte) in.read(); baos.write(b1); bytes_written++; byte b2 = (byte) in.read(); baos.write(b2); bytes_written++; while (!((b1 == 0) && (b2 == 0))) { b1 = b2; b2 = (byte) in.read(); baos.write(b2); bytes_written++; } // System.err.println("("+bytes_written+")"); } } else { // System.err.println("copy definite length encoding, remain="+datalen); if (next_is_constructed) { // System.err.println(" {"); while (datalen > 0) { int child_len = completeConstructedType(in, baos, level + 1); datalen -= child_len; bytes_written += child_len; } // dbg("} ("+bytes_written+")\n",level); } else { // System.err.print(" Definite length primitive type"); byte[] buff = new byte[4096]; while (datalen > 0) { int bytes_read = in.read(buff, 0, datalen > 4096 ? 4096 : datalen); // System.err.println("processed "+bytes_read); baos.write(buff, 0, bytes_read); datalen -= bytes_read; bytes_written += bytes_read; // System.err.println("("+bytes_written+")"); } } } return bytes_written; }
From source file:com.microsoft.tfs.core.clients.versioncontrol.engines.internal.workers.CheckinWorker.java
/** * Upload the given source file to the Team Foundation Server. If an * exception is thrown, no clean-up of the source file is performed (the * caller must handle this condition)./*from w w w .j av a2 s .c o m*/ * * @param uploadFile * the compressed or uncompressed local file that will be sent to the * server, which must exist (must not be <code>null</code> or empty) * @param parts * partially populated content parts. The Range part and file part * are filled in for each chunk here. * @param contentType * the type of the file content (compressed or uncompressed). * @throws CoreCancelException * if the upload was cancelled by the user via the * {@link TaskMonitor}. */ private void retryableUpload(final File uploadFile, final Part[] parts, final String contentType) throws SocketException, CoreCancelException { Check.notNull(uploadFile, "uploadFile"); //$NON-NLS-1$ Check.notNullOrEmpty(parts, "parts"); //$NON-NLS-1$ Check.notNullOrEmpty(contentType, "contentType"); //$NON-NLS-1$ PostMethod method = null; InputStream fileStream = null; BufferedInputStream bufferedStream = null; try { final long uploadFileLength = uploadFile.length(); long uploadFilePos = 0; boolean aChunkHasBeenRetried = false; fileStream = new FileInputStream(uploadFile); bufferedStream = new BufferedInputStream(fileStream); do { final long bytesLeft = uploadFileLength - uploadFilePos; final long chunkSize = 0 < MAX_CHUNK_SIZE && MAX_CHUNK_SIZE < bytesLeft ? MAX_CHUNK_SIZE : bytesLeft; final CancellableFilePart filePart; if (chunkSize == uploadFileLength) { /* * The calculated chunk size can be equal to the size of the * file only if: * * (1) this is the first iteration of the loop, and * * (2) the chunk size is zero, i.e. chunked upload is not * allowed, or the entire file is small and fits in a single * chunk. * * In this case we use the full file upload not bothering * with chunks at all. */ filePart = new CancellableFilePart("content", "item", uploadFile, contentType, null); //$NON-NLS-1$ //$NON-NLS-2$ /* * NOTE We construct the file part in a special way so the * character set is never sent to the server (TFS can't * handle that header, and it causes an internal server * error). If we construct the CancellableFilePart object * with a null charset, the header is still included, and * its value is the default charset. If we invoke * setCharSet() with null, the header is never supplied * (which is what we desire). * * Also, we use the file name "item" to match Visual * Studio's implementation. Sending the actual file name * doesn't seem to hurt, but appears to be ignored by the * server. */ filePart.setCharSet(null); } else { /* * Chunked upload. We mark the current position in the * buffered stream to allow re-sending of the chunk in case * redirection or authentication is required on the lower * HTTP Client level. */ bufferedStream.mark(MAX_CHUNK_SIZE); filePart = new CancellableChunkPart(uploadFile, bufferedStream, contentType, chunkSize); } /* * The Range part. NOTE There must be the extra newline after * the range line, or it will cause an internal server error. */ parts[5] = new StringPart(VersionControlConstants.RANGE_FIELD, "bytes=" //$NON-NLS-1$ + uploadFilePos + "-" //$NON-NLS-1$ + (uploadFilePos + chunkSize - 1) + "/" //$NON-NLS-1$ + uploadFile.length() + "\r\n", //$NON-NLS-1$ partCharSet); /* * The File part. It could be the entire file or its chunk. */ parts[6] = filePart; int attempt = 0; do { attempt++; if (TaskMonitorService.getTaskMonitor().isCanceled()) { throw new CoreCancelException(); } try { final String messageFormat = MessageFormat.format( Messages.getString("CheckinWorker.UploadFileProgressFormat_SKIPVALIDATE"), //$NON-NLS-1$ change.getServerItem()); final FileProcessingProgressMonitorAdapter monitor = new FileProcessingProgressMonitorAdapter( userCancellationMonitor, uploadFilePos, uploadFileLength, messageFormat); TaskMonitorService.pushTaskMonitor(monitor); /* * Connect to the server. */ method = client.beginUploadRequest(); /* * Create the multi-part request entity that wraps our * parts. */ method.setRequestEntity(new MultipartRequestEntity(parts, method.getParams())); client.executeUploadRequest(method); /* * Uploaded successfully */ uploadFilePos += chunkSize; break; } catch (final SocketException e) { if (attempt == MAX_CHUNK_RETRY_ATTEMPTS || chunkSize == uploadFileLength || client .getServiceLevel().getValue() < WebServiceLevel.TFS_2012_QU1_1.getValue()) { /* * We're here because: * * i. We already have retried the chunk allowed * number of times or * * ii. It does not make sense to retry because the * server does not support chunk retrying or * * iii. We're uploading the entire file as one chunk * * The caller might wish to retry the entire file */ throw e; } else { /* * Let's retry this chunk. On the pre-Dev12.M46 * servers that could cause VersionControlException * on the same or the following chunks and we'll * need to retry the entire file. */ aChunkHasBeenRetried = true; } } catch (final VersionControlException e) { if (aChunkHasBeenRetried) { /* * Most likely this is a pre-Dev12.M46 server that * does not support chunk retrying. * * TODO. We might need to perform some deeper * analysis of the exception as VS does. */ throw new SocketException( "This version of the TFS Server does not support chunk retrying."); //$NON-NLS-1$ } else { throw e; } } finally { if (method != null) { client.finishUploadRequest(method); } TaskMonitorService.popTaskMonitor(); } } while (true); } while (uploadFilePos < uploadFileLength); } catch (final CancellableFilePart.SendDataCancellationException e) { /* * This is thrown by the CancellableFilePart because it must throw * an IOException to work inside Commons HTTP. Treat like a normal * cancel exception. */ throw new CoreCancelException(); } catch (final SocketException e) { /* * Throw this so it can be caught and the upload retried. */ throw e; } catch (final IOException e) { throw new VersionControlException(e); } finally { if (bufferedStream != null) { try { bufferedStream.close(); } catch (final IOException e) { // Do nothing. } } /* * Surprisingly the BufferedStream class does not close the * underlying file stream. So, we have to close it explicitly. */ if (fileStream != null) { try { fileStream.close(); } catch (final IOException e) { // Do nothing. } } } }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordType999(BufferedInputStream stream) throws IOException { dbgLog.fine("decodeRecordType999(): start"); try {//from w w w . j a v a 2 s . com if (stream == null) { throw new IllegalArgumentException("RT999: stream == null!"); } // first check the 4-byte header value //if (stream.markSupported()){ stream.mark(1000); //} // 999.0 check the first 4 bytes byte[] headerCodeRt999 = new byte[LENGTH_RECORD_TYPE999_CODE]; //dbgLog.fine("RT999: stream position="+stream.pos); int nbytes_rt999 = stream.read(headerCodeRt999, 0, LENGTH_RECORD_TYPE999_CODE); // to-do check against nbytes //printHexDump(headerCodeRt999, "RT999 header test"); ByteBuffer bb_header_code_rt999 = ByteBuffer.wrap(headerCodeRt999, 0, LENGTH_RECORD_TYPE999_CODE); if (isLittleEndian) { bb_header_code_rt999.order(ByteOrder.LITTLE_ENDIAN); } int intRT999test = bb_header_code_rt999.getInt(); dbgLog.fine("header test value: RT999=" + intRT999test); if (intRT999test != 999) { //if (stream.markSupported()){ dbgLog.fine("intRT999test failed=" + intRT999test); stream.reset(); throw new IOException("RT999:Header value(999) was not correctly detected:" + intRT999test); //} } // 999.1 check 4-byte integer Filler block byte[] length_filler = new byte[LENGTH_RT999_FILLER]; int nbytes_rt999_1 = stream.read(length_filler, 0, LENGTH_RT999_FILLER); // to-do check against nbytes //printHexDump(length_how_many_line_bytes, "RT999 how_many_line_bytes"); ByteBuffer bb_filler = ByteBuffer.wrap(length_filler, 0, LENGTH_RT999_FILLER); if (isLittleEndian) { bb_filler.order(ByteOrder.LITTLE_ENDIAN); } int rt999filler = bb_filler.getInt(); dbgLog.fine("rt999filler=" + rt999filler); if (rt999filler == 0) { dbgLog.fine("the end of the dictionary section"); } else { throw new IOException("RT999: failed to detect the end mark(0): value=" + rt999filler); } // missing value processing concerning HIGHEST/LOWEST values Set<Map.Entry<String, InvalidData>> msvlc = invalidDataTable.entrySet(); for (Iterator<Map.Entry<String, InvalidData>> itc = msvlc.iterator(); itc.hasNext();) { Map.Entry<String, InvalidData> et = itc.next(); String variable = et.getKey(); dbgLog.fine("variable=" + variable); InvalidData invalidDataInfo = et.getValue(); if (invalidDataInfo.getInvalidRange() != null && !invalidDataInfo.getInvalidRange().isEmpty()) { if (invalidDataInfo.getInvalidRange().get(0).equals(OBSTypeHexValue.get("LOWEST"))) { dbgLog.fine("1st value is LOWEST"); invalidDataInfo.getInvalidRange().set(0, "LOWEST"); } else if (invalidDataInfo.getInvalidRange().get(1).equals(OBSTypeHexValue.get("HIGHEST"))) { dbgLog.fine("2nd value is HIGHEST"); invalidDataInfo.getInvalidRange().set(1, "HIGHEST"); } } } dbgLog.fine("invalidDataTable:\n" + invalidDataTable); // TODO: take care of the invalid data! - add the appropriate // value labels (?) // should it be done here, or at the end of ingest? // -- L.A. 4.0 alpha ///smd.setInvalidDataTable(invalidDataTable); } catch (IOException ex) { //ex.printStackTrace(); //exit(1); throw ex; } dbgLog.fine("decodeRecordType999(): end"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeHeader(BufferedInputStream stream) throws IOException { dbgLog.fine("decodeHeader(): start"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }/*from w ww . j a v a 2 s. com*/ // the length of the magic number is 4 (1-byte character * 4) // its value is expected to be $FL2 byte[] b = new byte[SAV_MAGIC_NUMBER_LENGTH]; try { if (stream.markSupported()) { stream.mark(100); } int nbytes = stream.read(b, 0, SAV_MAGIC_NUMBER_LENGTH); if (nbytes == 0) { throw new IOException(); } } catch (IOException ex) { //ex.printStackTrace(); throw ex; } //printHexDump(b, "hex dump of the byte-array"); String hdr4sav = new String(b); dbgLog.fine("from string=" + hdr4sav); if (hdr4sav.equals(SAV_FILE_SIGNATURE)) { dbgLog.fine("this file is spss-sav type"); // initialize version-specific parameter init(); dataTable.setOriginalFileFormat(MIME_TYPE[0]); dataTable.setUnf("UNF:6:"); } else { dbgLog.fine("this file is NOT spss-sav type"); throw new IllegalArgumentException("given file is not spss-sav type"); } dbgLog.fine("***** decodeHeader(): end *****"); }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReader.java
void decodeRecordType6(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeRecordType6(): start *****"); try {//from w ww . j av a 2 s .c o m if (stream == null) { throw new IllegalArgumentException("stream == null!"); } // this section is optional; so let's first check the 4-byte header // value and see what type it is. //if (stream.markSupported()){ // -- ? L.A. 4.0 alpha stream.mark(1000); //} // 6.0 check the first 4 bytes byte[] headerCodeRt6 = new byte[LENGTH_RECORD_TYPE6_CODE]; int nbytes_rt6 = stream.read(headerCodeRt6, 0, LENGTH_RECORD_TYPE6_CODE); // to-do check against nbytes //printHexDump(headerCodeRt6, "RT6 header test"); ByteBuffer bb_header_code_rt6 = ByteBuffer.wrap(headerCodeRt6, 0, LENGTH_RECORD_TYPE6_CODE); if (isLittleEndian) { bb_header_code_rt6.order(ByteOrder.LITTLE_ENDIAN); } int intRT6test = bb_header_code_rt6.getInt(); dbgLog.fine("RT6: header test value=" + intRT6test); if (intRT6test != 6) { //if (stream.markSupported()){ //out.print("iteration="+safteyCounter); //dbgLog.fine("iteration="+safteyCounter); dbgLog.fine("intRT6test failed=" + intRT6test); stream.reset(); return; //} } // 6.1 check 4-byte integer that tells how many lines follow byte[] length_how_many_line_bytes = new byte[LENGTH_RT6_HOW_MANY_LINES]; int nbytes_rt6_1 = stream.read(length_how_many_line_bytes, 0, LENGTH_RT6_HOW_MANY_LINES); // to-do check against nbytes //printHexDump(length_how_many_line_bytes, "RT6 how_many_line_bytes"); ByteBuffer bb_how_many_lines = ByteBuffer.wrap(length_how_many_line_bytes, 0, LENGTH_RT6_HOW_MANY_LINES); if (isLittleEndian) { bb_how_many_lines.order(ByteOrder.LITTLE_ENDIAN); } int howManyLinesRt6 = bb_how_many_lines.getInt(); dbgLog.fine("how Many lines follow=" + howManyLinesRt6); // 6.2 read 80-char-long lines String[] documentRecord = new String[howManyLinesRt6]; for (int i = 0; i < howManyLinesRt6; i++) { byte[] line = new byte[80]; int nbytes_rt6_line = stream.read(line); documentRecord[i] = StringUtils.stripEnd( new String(Arrays.copyOfRange(line, 0, LENGTH_RT6_DOCUMENT_LINE), defaultCharSet), " "); dbgLog.fine(i + "-th line =" + documentRecord[i] + "<-"); } dbgLog.fine("documentRecord:\n" + StringUtils.join(documentRecord, "\n")); } catch (IOException ex) { //ex.printStackTrace(); throw ex; } dbgLog.fine("decodeRecordType6(): end"); }