List of usage examples for java.io FileInputStream skip
public long skip(long n) throws IOException
n bytes of data from the input stream. From source file:com.att.api.rest.RESTClient.java
/** * Gets MIME type for specified file./*from w w w .ja v a 2 s . c o m*/ * * <p> * MIME type calculated by doing a very simple check based on file header. * </p> * * Currently supports checking for the following formats: * <ul> * <li>AMR</li> * <li>AMR-WB</li> * <li>WAV</li> * <li>Speex</li> * </ul> * * @param file file to check for MIME type * @return String MIME type * @throws IOException if there is a problem reading the specified file */ private String getMIMEType(File file) throws IOException { // AMR/AMR-WB check will be done according to RFC3267 // (http://www.ietf.org/rfc/rfc3267.txt?number=3267) final byte[] AMRHeader = { '#', '!', 'A', 'M', 'R' }; final byte[] AMRWBExtension = { '-', 'W', 'B' }; final byte[] RIFFHeader = { 'R', 'I', 'F', 'F' }; final byte[] WAVEHeader = { 'W', 'A', 'V', 'E' }; // Check for Speex in Ogg files. Ogg will be checked according to // RFC3533 (http://www.ietf.org/rfc/rfc3533.txt). Speex will be checked // according to the format specified the speex manual // (www.speex.org/docs/manual/speex-manual/node8.html) final byte[] OggHeader = { 'O', 'g', 'g', 'S' }; final byte[] SpeexHeader = { 'S', 'p', 'e', 'e', 'x', ' ', ' ', ' ' }; final byte[] header = new byte[4]; FileInputStream fStream = null; String contentType = null; try { fStream = new FileInputStream(file); // Read the first 4 bytes int bytesRead = fStream.read(header, 0, 4); if (bytesRead >= 4 && Arrays.equals(header, RIFFHeader)) { // read more bytes to determine if it's a wav file if (fStream.skip(4) >= 4) { // size if wav structure bytesRead = fStream.read(header, 0, 4); // wav header if (bytesRead >= 4 && Arrays.equals(header, WAVEHeader)) { contentType = "audio/wav"; } } } else if (Arrays.equals(header, OggHeader) && fStream.skip(24) >= 24) { // first 28 bytes are ogg. Afterwards should be speex header. final byte[] headerExt = new byte[8]; bytesRead = fStream.read(headerExt, 0, 8); if (bytesRead >= 8 && Arrays.equals(headerExt, SpeexHeader)) { contentType = "audio/x-speex"; } } // try looking for AMR final byte[] testHeader = new byte[5]; for (int i = 0; i < header.length; ++i) { testHeader[i] = header[i]; } bytesRead = fStream.read(testHeader, 4, 1); if (bytesRead >= 1 && Arrays.equals(testHeader, AMRHeader)) { final byte[] headerExt = new byte[3]; bytesRead = fStream.read(headerExt, 0, 3); if (bytesRead >= 3 && Arrays.equals(headerExt, AMRWBExtension)) { contentType = "audio/amr-wb"; } else { contentType = "audio/amr"; } } } catch (IOException ioe) { throw ioe; // pass along exception } finally { if (fStream != null) { fStream.close(); } } return contentType; }
From source file:com.funambol.foundation.items.manager.FileDataObjectManagerTest.java
/** * Reads the source file from the given lower bound to the upper bound and * writes those bytes in the destination file. * * @param source the source file/*from w ww .ja v a 2 s . co m*/ * @param dest the destination file * @param from the lower bound * @param to the upper bound * @throws IOException if the source file cannot be read or the destination * file cannot be create or write */ private void splitFile(String source, String dest, long from, long to) throws IOException { FileInputStream inputFile = new FileInputStream(source); FileOutputStream outputFile = new FileOutputStream(dest, false); if (from > 0) { if (inputFile.skip(from) != from) { throw new IOException("Skipped few bytes than expected."); } } byte[] buffer = new byte[1024]; int bytesRead = 0; long totalAmount = from; while ((bytesRead = inputFile.read(buffer)) > 0) { int reallyWrittenBytes = buffer.length; if (to <= 0) { outputFile.write(buffer); } else { if (totalAmount + bytesRead > to) { reallyWrittenBytes = (int) ((to - totalAmount)); outputFile.write(buffer, 0, reallyWrittenBytes); } else { outputFile.write(buffer); } } totalAmount += reallyWrittenBytes; if (totalAmount >= to) { break; } } outputFile.flush(); outputFile.close(); }
From source file:org.apache.pdfbox.pdfparser.NonSequentialPDFParser.java
/** Looks for and parses startxref. We first look for last '%%EOF' marker * (within last {@link #DEFAULT_TRAIL_BYTECOUNT} bytes (or range set via * {@link #setEOFLookupRange(int)}) and go back to find <code>startxref</code>. */ private final long getStartxrefOffset() throws IOException { byte[] buf;//w w w . ja va 2 s . c o m long skipBytes; // ---- read trailing bytes into buffer final long fileLen = pdfFile.length(); FileInputStream fIn = null; try { fIn = new FileInputStream(pdfFile); final int trailByteCount = (fileLen < readTrailBytes) ? (int) fileLen : readTrailBytes; buf = new byte[trailByteCount]; fIn.skip(skipBytes = fileLen - trailByteCount); int off = 0; int readBytes; while (off < trailByteCount) { readBytes = fIn.read(buf, off, trailByteCount - off); // in order to not get stuck in a loop we check readBytes (this should never happen) if (readBytes < 1) { throw new IOException( "No more bytes to read for trailing buffer, but expected: " + (trailByteCount - off)); } off += readBytes; } } finally { if (fIn != null) { try { fIn.close(); } catch (IOException ioe) { } } } // ---- find last '%%EOF' int bufOff = lastIndexOf(EOF_MARKER, buf, buf.length); if (bufOff < 0) { throw new IOException("Missing end of file marker '" + (new String(EOF_MARKER)) + "'"); } // ---- find last startxref preceding EOF marker bufOff = lastIndexOf(STARTXREF_MARKER, buf, bufOff); if (bufOff < 0) { throw new IOException("Missing 'startxref' marker."); } return skipBytes + bufOff; }
From source file:shelper.ifmock.HttpMock.java
/** * Serves file from homeDir and its' subdirectories (only). * Uses only URI, ignores all headers and HTTP parameters. *//* ww w . j a v a 2 s. c o m*/ public Response serveFile(String uri, Properties header, File homeDir, boolean allowDirectoryListing) { Response res = null; // Make sure we won't die of an exception later if (!homeDir.isDirectory()) res = new Response(HTTP_INTERNALERROR, MIME_PLAINTEXT, "INTERNAL ERRROR: serveFile(): given homeDir is not a directory."); if (res == null) { // Remove URL arguments uri = uri.trim().replace(File.separatorChar, '/'); if (uri.indexOf('?') >= 0) uri = uri.substring(0, uri.indexOf('?')); // Prohibit getting out of current directory if (uri.startsWith("..") || uri.endsWith("..") || uri.indexOf("../") >= 0) res = new Response(HTTP_FORBIDDEN, MIME_PLAINTEXT, "FORBIDDEN: Won't serve ../ for security reasons."); } File f = new File(homeDir, uri); if (res == null && !f.exists()) res = new Response(HTTP_NOTFOUND, MIME_PLAINTEXT, "Error 404, file not found."); // List the directory, if necessary if (res == null && f.isDirectory()) { // Browsers get confused without '/' after the // directory, send a redirect. if (!uri.endsWith("/")) { uri += "/"; res = new Response(HTTP_REDIRECT, MIME_HTML, "<html><body>Redirected: <a href=\"" + uri + "\">" + uri + "</a></body></html>"); res.addHeader("Location", uri); } if (res == null) { // First try index.html and index.htm if (new File(f, "index.html").exists()) f = new File(homeDir, uri + "/index.html"); else if (new File(f, "index.htm").exists()) f = new File(homeDir, uri + "/index.htm"); // No index file, list the directory if it is readable else if (allowDirectoryListing && f.canRead()) { String[] files = f.list(); String msg = "<html><body><h1>Directory " + uri + "</h1><br/>"; if (uri.length() > 1) { String u = uri.substring(0, uri.length() - 1); int slash = u.lastIndexOf('/'); if (slash >= 0 && slash < u.length()) msg += "<b><a href=\"" + uri.substring(0, slash + 1) + "\">..</a></b><br/>"; } if (files != null) { for (int i = 0; i < files.length; ++i) { File curFile = new File(f, files[i]); boolean dir = curFile.isDirectory(); if (dir) { msg += "<b>"; files[i] += "/"; } msg += "<a href=\"" + encodeUri(uri + files[i]) + "\">" + files[i] + "</a>"; // Show file size if (curFile.isFile()) { long len = curFile.length(); msg += " <font size=2>("; if (len < 1024) msg += len + " bytes"; else if (len < 1024 * 1024) msg += len / 1024 + "." + (len % 1024 / 10 % 100) + " KB"; else msg += len / (1024 * 1024) + "." + len % (1024 * 1024) / 10 % 100 + " MB"; msg += ")</font>"; } msg += "<br/>"; if (dir) msg += "</b>"; } } msg += "</body></html>"; res = new Response(HTTP_OK, MIME_HTML, msg); } else { res = new Response(HTTP_FORBIDDEN, MIME_PLAINTEXT, "FORBIDDEN: No directory listing."); } } } try { if (res == null) { // Get MIME type from file name extension, if possible String mime = null; int dot = f.getCanonicalPath().lastIndexOf('.'); if (dot >= 0) mime = (String) theMimeTypes.get(f.getCanonicalPath().substring(dot + 1).toLowerCase()); if (mime == null) mime = MIME_DEFAULT_BINARY; // Calculate etag String etag = Integer .toHexString((f.getAbsolutePath() + f.lastModified() + "" + f.length()).hashCode()); // Support (simple) skipping: long startFrom = 0; long endAt = -1; String range = header.getProperty("range"); if (range != null) { if (range.startsWith("bytes=")) { range = range.substring("bytes=".length()); int minus = range.indexOf('-'); try { if (minus > 0) { startFrom = Long.parseLong(range.substring(0, minus)); endAt = Long.parseLong(range.substring(minus + 1)); } } catch (NumberFormatException nfe) { } } } // Change return code and add Content-Range header when skipping is requested long fileLen = f.length(); if (range != null && startFrom >= 0) { if (startFrom >= fileLen) { res = new Response(HTTP_RANGE_NOT_SATISFIABLE, MIME_PLAINTEXT, ""); res.addHeader("Content-Range", "bytes 0-0/" + fileLen); res.addHeader("ETag", etag); } else { if (endAt < 0) endAt = fileLen - 1; long newLen = endAt - startFrom + 1; if (newLen < 0) newLen = 0; final long dataLen = newLen; FileInputStream fis = new FileInputStream(f) { @Override public int available() throws IOException { return (int) dataLen; } }; fis.skip(startFrom); res = new Response(HTTP_PARTIALCONTENT, mime, fis); res.addHeader("Content-Length", "" + dataLen); res.addHeader("Content-Range", "bytes " + startFrom + "-" + endAt + "/" + fileLen); res.addHeader("ETag", etag); } } else { res = new Response(HTTP_OK, mime, new FileInputStream(f)); res.addHeader("Content-Length", "" + fileLen); res.addHeader("ETag", etag); } } } catch (IOException ioe) { res = new Response(HTTP_FORBIDDEN, MIME_PLAINTEXT, "FORBIDDEN: Reading file failed."); } res.addHeader("Accept-Ranges", "bytes"); // Announce that the file server accepts partial content requestes return res; }
From source file:fr.cmoatoto.multishare.receiver.NanoHTTPDReceiver.java
/** * Serves file from homeDir and its' subdirectories (only). Uses only URI, ignores all headers and HTTP parameters. *//* ww w. ja va 2 s . c o m*/ public Response serveFile(String uri, Properties header, File homeDir, boolean allowDirectoryListing) { Response res = null; // Make sure we won't die of an exception later if (!homeDir.isDirectory()) res = new Response(HTTP_INTERNALERROR, MIME_PLAINTEXT, "INTERNAL ERRROR: serveFile(): given homeDir is not a directory."); if (res == null) { // Remove URL arguments uri = uri.trim().replace(File.separatorChar, '/'); if (uri.indexOf('?') >= 0) uri = uri.substring(0, uri.indexOf('?')); // Prohibit getting out of current directory if (uri.startsWith("..") || uri.endsWith("..") || uri.indexOf("../") >= 0) res = new Response(HTTP_FORBIDDEN, MIME_PLAINTEXT, "FORBIDDEN: Won't serve ../ for security reasons."); } File f = new File(homeDir, uri); if (res == null && !f.exists()) res = new Response(HTTP_NOTFOUND, MIME_PLAINTEXT, "Error 404, file not found."); // List the directory, if necessary if (res == null && f.isDirectory()) { // Browsers get confused without '/' after the // directory, send a redirect. if (!uri.endsWith("/")) { uri += "/"; res = new Response(HTTP_REDIRECT, MIME_HTML, "<html><body>Redirected: <a href=\"" + uri + "\">" + uri + "</a></body></html>"); res.addHeader("Location", uri); } if (res == null) { // First try index.html and index.htm if (new File(f, "index.html").exists()) f = new File(homeDir, uri + "/index.html"); else if (new File(f, "index.htm").exists()) f = new File(homeDir, uri + "/index.htm"); // No index file, list the directory if it is readable else if (allowDirectoryListing && f.canRead()) { String[] files = f.list(); String msg = "<html><body><h1>Directory " + uri + "</h1><br/>"; if (uri.length() > 1) { String u = uri.substring(0, uri.length() - 1); int slash = u.lastIndexOf('/'); if (slash >= 0 && slash < u.length()) msg += "<b><a href=\"" + uri.substring(0, slash + 1) + "\">..</a></b><br/>"; } if (files != null) { for (int i = 0; i < files.length; ++i) { File curFile = new File(f, files[i]); boolean dir = curFile.isDirectory(); if (dir) { msg += "<b>"; files[i] += "/"; } msg += "<a href=\"" + encodeUri(uri + files[i]) + "\">" + files[i] + "</a>"; // Show file size if (curFile.isFile()) { long len = curFile.length(); msg += " <font size=2>("; if (len < 1024) msg += len + " bytes"; else if (len < 1024 * 1024) msg += len / 1024 + "." + (len % 1024 / 10 % 100) + " KB"; else msg += len / (1024 * 1024) + "." + len % (1024 * 1024) / 10 % 100 + " MB"; msg += ")</font>"; } msg += "<br/>"; if (dir) msg += "</b>"; } } msg += "</body></html>"; res = new Response(HTTP_OK, MIME_HTML, msg); } else { res = new Response(HTTP_FORBIDDEN, MIME_PLAINTEXT, "FORBIDDEN: No directory listing."); } } } try { if (res == null) { // Get MIME type from file name extension, if possible String mime = null; int dot = f.getCanonicalPath().lastIndexOf('.'); if (dot >= 0) mime = (String) theMimeTypes.get(f.getCanonicalPath().substring(dot + 1).toLowerCase()); if (mime == null) mime = MIME_DEFAULT_BINARY; // Calculate etag String etag = Integer .toHexString((f.getAbsolutePath() + f.lastModified() + "" + f.length()).hashCode()); // Support (simple) skipping: long startFrom = 0; long endAt = -1; String range = header.getProperty("range"); if (range != null) { if (range.startsWith("bytes=")) { range = range.substring("bytes=".length()); int minus = range.indexOf('-'); try { if (minus > 0) { startFrom = Long.parseLong(range.substring(0, minus)); endAt = Long.parseLong(range.substring(minus + 1)); } } catch (NumberFormatException nfe) { } } } // Change return code and add Content-Range header when skipping // is requested long fileLen = f.length(); if (range != null && startFrom >= 0) { if (startFrom >= fileLen) { res = new Response(HTTP_RANGE_NOT_SATISFIABLE, MIME_PLAINTEXT, ""); res.addHeader("Content-Range", "bytes 0-0/" + fileLen); if (mime.startsWith("application/")) res.addHeader("Content-Disposition", "attachment; filename=\"" + f.getName() + "\""); res.addHeader("ETag", etag); } else { if (endAt < 0) endAt = fileLen - 1; long newLen = endAt - startFrom + 1; if (newLen < 0) newLen = 0; final long dataLen = newLen; FileInputStream fis = new FileInputStream(f) { public int available() throws IOException { return (int) dataLen; } }; fis.skip(startFrom); res = new Response(HTTP_PARTIALCONTENT, mime, fis); res.addHeader("Content-Length", "" + dataLen); res.addHeader("Content-Range", "bytes " + startFrom + "-" + endAt + "/" + fileLen); if (mime.startsWith("application/")) res.addHeader("Content-Disposition", "attachment; filename=\"" + f.getName() + "\""); res.addHeader("ETag", etag); } } else { if (etag.equals(header.getProperty("if-none-match"))) res = new Response(HTTP_NOTMODIFIED, mime, ""); else { res = new Response(HTTP_OK, mime, new FileInputStream(f)); res.addHeader("Content-Length", "" + fileLen); if (mime.startsWith("application/")) res.addHeader("Content-Disposition", "attachment; filename=\"" + f.getName() + "\""); res.addHeader("ETag", etag); } } } } catch (IOException ioe) { res = new Response(HTTP_FORBIDDEN, MIME_PLAINTEXT, "FORBIDDEN: Reading file failed."); } res.addHeader("Accept-Ranges", "bytes"); // Announce that the file // server accepts partial // content requestes return res; }
From source file:org.apache.hadoop.mapred.TaskLogsTruncater.java
/** * Truncate the log file of this task-attempt so that only the last retainSize * many bytes of each log file is retained and the log file is reduced in size * saving disk space./* w w w . j a v a 2 s . c o m*/ * * @param taskID Task whose logs need to be truncated * @param oldLogFileDetail contains the original log details for the attempt * @param taskRetainSize retain-size * @param tmpFileOutputStream New log file to write to. Already opened in append * mode. * @param logFileInputStream Original log file to read from. * @return * @throws IOException */ private LogFileDetail truncateALogFileOfAnAttempt(final TaskAttemptID taskID, final LogFileDetail oldLogFileDetail, final long taskRetainSize, final FileOutputStream tmpFileOutputStream, final FileInputStream logFileInputStream, final LogName logName) throws IOException { LogFileDetail newLogFileDetail = new LogFileDetail(); long logSize = 0; // ///////////// Truncate log file /////////////////////// // New location of log file is same as the old newLogFileDetail.location = oldLogFileDetail.location; if (taskRetainSize > MINIMUM_RETAIN_SIZE_FOR_TRUNCATION && oldLogFileDetail.length > taskRetainSize) { LOG.info("Truncating " + logName + " logs for " + taskID + " from " + oldLogFileDetail.length + "bytes to " + taskRetainSize + "bytes."); logSize = taskRetainSize; byte[] truncatedMsgBytes = TRUNCATED_MSG.getBytes(); tmpFileOutputStream.write(truncatedMsgBytes); newLogFileDetail.length += truncatedMsgBytes.length; } else { LOG.debug("No truncation needed for " + logName + " logs for " + taskID + " length is " + oldLogFileDetail.length + " retain size " + taskRetainSize + "bytes."); logSize = oldLogFileDetail.length; } long bytesSkipped = logFileInputStream.skip(oldLogFileDetail.length - logSize); if (bytesSkipped != oldLogFileDetail.length - logSize) { throw new IOException("Erroneously skipped " + bytesSkipped + " instead of the expected " + (oldLogFileDetail.length - logSize) + " while truncating " + logName + " logs for " + taskID); } long alreadyRead = 0; while (alreadyRead < logSize) { byte tmpBuf[]; // Temporary buffer to read logs if (logSize - alreadyRead >= DEFAULT_BUFFER_SIZE) { tmpBuf = new byte[DEFAULT_BUFFER_SIZE]; } else { tmpBuf = new byte[(int) (logSize - alreadyRead)]; } int bytesRead = logFileInputStream.read(tmpBuf); if (bytesRead < 0) { break; } else { alreadyRead += bytesRead; } tmpFileOutputStream.write(tmpBuf); } newLogFileDetail.length += logSize; // ////// End of truncating log file /////////////////////// return newLogFileDetail; }
From source file:org.apache.hadoop.mapred.TestConcatenatedCompressedInput.java
/** * Test using the raw Inflater codec for reading gzip files. *//*from w w w . j a v a2 s . c o m*/ @Test public void testPrototypeInflaterGzip() throws IOException { CompressionCodec gzip = new GzipCodec(); // used only for file extension localFs.delete(workDir, true); // localFs = FileSystem instance System.out.println(COLOR_BR_BLUE + "testPrototypeInflaterGzip() using " + "non-native/Java Inflater and manual gzip header/trailer parsing" + COLOR_NORMAL); // copy prebuilt (correct!) version of concat.gz to HDFS final String fn = "concat" + gzip.getDefaultExtension(); Path fnLocal = new Path(System.getProperty("test.concat.data", "/tmp"), fn); Path fnHDFS = new Path(workDir, fn); localFs.copyFromLocalFile(fnLocal, fnHDFS); final FileInputStream in = new FileInputStream(fnLocal.toString()); assertEquals("concat bytes available", 148, in.available()); // should wrap all of this header-reading stuff in a running-CRC wrapper // (did so in BuiltInGzipDecompressor; see below) byte[] compressedBuf = new byte[256]; int numBytesRead = in.read(compressedBuf, 0, 10); assertEquals("header bytes read", 10, numBytesRead); assertEquals("1st byte", 0x1f, compressedBuf[0] & 0xff); assertEquals("2nd byte", 0x8b, compressedBuf[1] & 0xff); assertEquals("3rd byte (compression method)", 8, compressedBuf[2] & 0xff); byte flags = (byte) (compressedBuf[3] & 0xff); if ((flags & 0x04) != 0) { // FEXTRA numBytesRead = in.read(compressedBuf, 0, 2); assertEquals("XLEN bytes read", 2, numBytesRead); int xlen = ((compressedBuf[1] << 8) | compressedBuf[0]) & 0xffff; in.skip(xlen); } if ((flags & 0x08) != 0) { // FNAME while ((numBytesRead = in.read()) != 0) { assertFalse("unexpected end-of-file while reading filename", numBytesRead == -1); } } if ((flags & 0x10) != 0) { // FCOMMENT while ((numBytesRead = in.read()) != 0) { assertFalse("unexpected end-of-file while reading comment", numBytesRead == -1); } } if ((flags & 0xe0) != 0) { // reserved assertTrue("reserved bits are set??", (flags & 0xe0) == 0); } if ((flags & 0x02) != 0) { // FHCRC numBytesRead = in.read(compressedBuf, 0, 2); assertEquals("CRC16 bytes read", 2, numBytesRead); int crc16 = ((compressedBuf[1] << 8) | compressedBuf[0]) & 0xffff; } // ready to go! next bytes should be start of deflated stream, suitable // for Inflater numBytesRead = in.read(compressedBuf); // Inflater docs refer to a "dummy byte": no clue what that's about; // appears to work fine without one byte[] uncompressedBuf = new byte[256]; Inflater inflater = new Inflater(true); inflater.setInput(compressedBuf, 0, numBytesRead); try { int numBytesUncompressed = inflater.inflate(uncompressedBuf); String outString = new String(uncompressedBuf, 0, numBytesUncompressed, "UTF-8"); System.out.println("uncompressed data of first gzip member = [" + outString + "]"); } catch (java.util.zip.DataFormatException ex) { throw new IOException(ex.getMessage()); } in.close(); }
From source file:org.commoncrawl.service.listcrawler.CacheManager.java
/** * queue a cache load request via a background thread * //from w w w . j av a2 s. c o m * @param loadRequest */ private CacheItem queueLocalCacheLoadRequestInWorkerThread(final CacheLoadRequest loadRequest) { // LOG.info("#### ENTERING queueLocalCacheLoadRequestInWorkerThread"); try { CacheItem loadResult = null; // queue up requests into the thread pool executor (for now) for (final Long location : loadRequest._loacations) { LOG.info("### Local Cache Loader Called. Acquiring Semaphore"); getLocalLogAccessSemaphore().acquireUninterruptibly(); LOG.info("### Local Cache Loader Called. Acquired Semaphore"); // now set up and exception handler block to ensure that we release semaphore try { // LOG.info("### Item Loading Item for URL:" + loadRequest._targetURL + " at Pos:" + location.longValue()); // now that we have acquire the semaphore ... validate position against current log file offset ... if (location < _localLogStartOffset) { LOG.error("### Load Request for Potentially Flushed Item. Location Request:" + location + " Current LogStartOffset:" + _localLogStartOffset); } else { long timeStart = System.currentTimeMillis(); // we got a location ... initiate a disk read to fetch the serialized CacheItem FileInputStream file = new FileInputStream(getActiveLogFilePath()); try { // seek to item location ... file.skip(location.longValue() - _localLogStartOffset); loadResult = loadCacheItemFromDisk(file, loadRequest._targetURL, location.longValue()); if (loadResult != null) { break; } } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); } finally { // file.getFD().sync(); file.close(); } } } finally { LOG.info("### Local Cache Loader Releasing Semaphore"); getLocalLogAccessSemaphore().release(); } } if (loadResult != null) { LOG.info("### Item Load Request for URL:" + loadRequest._targetURL + " Succeeded. Initiating Callback"); // reset pending to zero so no other load requests satisfy callback loadRequest._pendingItemCount = 0; // loadRequest._callback.cacheItemAvailable(loadRequest._targetURL,loadResult); return loadResult; } else { // on failure reduce pending count ... loadRequest._pendingItemCount--; if (loadRequest._pendingItemCount == 0) { LOG.info("### Item Load Request for URL:" + loadRequest._targetURL + " Failed with No-Item-Found"); // if pending zero ... initiate failure callback // loadRequest._callback.cacheItemNotFound(loadRequest._targetURL); return null; } } } catch (Exception e) { LOG.info("### Item Load Request for URL:" + loadRequest._targetURL + " Failed with FAILURE Reason:" + CCStringUtils.stringifyException(e)); // if pending zero ... initiate failure callback //loadRequest._callback.cacheItemNotFound(loadRequest._targetURL); } finally { //LOG.info("#### EXITING queueLocalCacheLoadRequestInWorkerThread"); } return null; }
From source file:org.commoncrawl.service.listcrawler.CacheManager.java
/** * queue a cache load request via a background thread * /*from ww w. j av a 2 s . c om*/ * @param loadRequest */ private void queueLocalCacheLoadRequest(final CacheLoadRequest loadRequest) { // queue up requests into the thread pool executor (for now) for (final Long location : loadRequest._loacations) { _cacheLoadThreadPool.submit(new ConcurrentTask<CacheItem>(_eventLoop, new Callable<CacheItem>() { @Override public CacheItem call() throws Exception { //LOG.info("### Local Cache Loader Called. Acquiring Semaphore"); getLocalLogAccessSemaphore().acquireUninterruptibly(); //LOG.info("### Local Cache Loader Called. Acquired Semaphore"); // now set up and exception handler block to ensure that we release semaphore try { //LOG.info("### Item Loading Item for URL:" + loadRequest._targetURL + " at Pos:" + location.longValue()); // now that we have acquire the semaphore ... validate position against current log file offset ... if (location < _localLogStartOffset) { LOG.error("### Load Request for Potentially Flushed Item. Location Request:" + location + " Current LogStartOffset:" + _localLogStartOffset); } else { long timeStart = System.currentTimeMillis(); // we got a location ... initiate a disk read to fetch the serialized CacheItem FileInputStream file = new FileInputStream(getActiveLogFilePath()); try { // seek to item location ... file.skip(location.longValue() - _localLogStartOffset); return loadCacheItemFromDisk(file, loadRequest._targetURL, location.longValue()); } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); } finally { // file.getFD().sync(); file.close(); } } return null; } finally { //LOG.info("### Local Cache Loader Releasing Semaphore"); getLocalLogAccessSemaphore().release(); } } }, new CompletionCallback<CacheItem>() { @Override public void taskComplete(CacheItem loadResult) { if (loadResult != null) { // LOG.info("### Item Load Request for URL:" + loadRequest._targetURL + " Succeeded. Initiating Callback"); // reset pending to zero so no other load requests satisfy callback loadRequest._pendingItemCount = 0; loadRequest._callback.cacheItemAvailable(loadRequest._targetURL, loadResult); } else { // on failure reduce pending count ... loadRequest._pendingItemCount--; if (loadRequest._pendingItemCount == 0) { // LOG.info("### Item Load Request for URL:" + loadRequest._targetURL + " Failed with No-Item-Found"); // if pending zero ... initiate failure callback loadRequest._callback.cacheItemNotFound(loadRequest._targetURL); } } } @Override public void taskFailed(Exception e) { // on failure reduce pending count ... loadRequest._pendingItemCount--; if (loadRequest._pendingItemCount == 0) { // LOG.info("### Item Load Request for URL:" + loadRequest._targetURL + " Failed with FAILURE Reason:" + CCStringUtils.stringifyException(e)); // if pending zero ... initiate failure callback loadRequest._callback.cacheItemNotFound(loadRequest._targetURL); } } })); } }
From source file:com.google.ytd.SubmitActivity.java
private String gdataUpload(File file, String uploadUrl, int start, int end) throws IOException { int chunk = end - start + 1; int bufferSize = 1024; byte[] buffer = new byte[bufferSize]; FileInputStream fileStream = new FileInputStream(file); HttpURLConnection urlConnection = getGDataUrlConnection(uploadUrl); // some mobile proxies do not support PUT, using X-HTTP-Method-Override to get around this problem if (isFirstRequest()) { Log.d(LOG_TAG, String.format("Uploaded %d bytes so far, using POST method.", (int) totalBytesUploaded)); urlConnection.setRequestMethod("POST"); } else {// w w w .ja v a 2 s . com urlConnection.setRequestMethod("POST"); urlConnection.setRequestProperty("X-HTTP-Method-Override", "PUT"); Log.d(LOG_TAG, String.format("Uploaded %d bytes so far, using POST with X-HTTP-Method-Override PUT method.", (int) totalBytesUploaded)); } urlConnection.setDoOutput(true); urlConnection.setFixedLengthStreamingMode(chunk); urlConnection.setRequestProperty("Content-Type", "video/3gpp"); urlConnection.setRequestProperty("Content-Range", String.format("bytes %d-%d/%d", start, end, file.length())); Log.d(LOG_TAG, urlConnection.getRequestProperty("Content-Range")); OutputStream outStreamWriter = urlConnection.getOutputStream(); fileStream.skip(start); int bytesRead; int totalRead = 0; while ((bytesRead = fileStream.read(buffer, 0, bufferSize)) != -1) { outStreamWriter.write(buffer, 0, bytesRead); totalRead += bytesRead; this.totalBytesUploaded += bytesRead; double percent = (totalBytesUploaded / currentFileSize) * 99; /* Log.d(LOG_TAG, String.format( "fileSize=%f totalBytesUploaded=%f percent=%f", currentFileSize, totalBytesUploaded, percent)); */ dialog.setProgress((int) percent); if (totalRead == (end - start + 1)) { break; } } outStreamWriter.close(); int responseCode = urlConnection.getResponseCode(); Log.d(LOG_TAG, "responseCode=" + responseCode); Log.d(LOG_TAG, "responseMessage=" + urlConnection.getResponseMessage()); try { if (responseCode == 201) { String videoId = parseVideoId(urlConnection.getInputStream()); String latLng = null; if (this.videoLocation != null) { latLng = String.format("lat=%f lng=%f", this.videoLocation.getLatitude(), this.videoLocation.getLongitude()); } submitToYtdDomain(this.ytdDomain, this.assignmentId, videoId, this.youTubeName, SubmitActivity.this.clientLoginToken, getTitleText(), getDescriptionText(), this.dateTaken, latLng, this.tags); dialog.setProgress(100); return videoId; } else if (responseCode == 200) { Set<String> keySet = urlConnection.getHeaderFields().keySet(); String keys = urlConnection.getHeaderFields().keySet().toString(); Log.d(LOG_TAG, String.format("Headers keys %s.", keys)); for (String key : keySet) { Log.d(LOG_TAG, String.format("Header key %s value %s.", key, urlConnection.getHeaderField(key))); } Log.w(LOG_TAG, "Received 200 response during resumable uploading"); throw new IOException(String.format("Unexpected response code : responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage())); } else { if ((responseCode + "").startsWith("5")) { String error = String.format("responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage()); Log.w(LOG_TAG, error); // TODO - this exception will trigger retry mechanism to kick in // TODO - even though it should not, consider introducing a new type so // TODO - resume does not kick in upon 5xx throw new IOException(error); } else if (responseCode == 308) { // OK, the chunk completed succesfully Log.d(LOG_TAG, String.format("responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage())); } else { // TODO - this case is not handled properly yet Log.w(LOG_TAG, String.format("Unexpected return code : %d %s while uploading :%s", responseCode, urlConnection.getResponseMessage(), uploadUrl)); } } } catch (ParserConfigurationException e) { e.printStackTrace(); } catch (SAXException e) { e.printStackTrace(); } return null; }