List of usage examples for org.apache.commons.io IOUtils copyLarge
public static long copyLarge(Reader input, Writer output) throws IOException
Reader to a Writer. From source file:fr.gael.dhus.datastore.processing.ProcessingManager.java
private void upload(String url, final String username, final String password, final File dest) { String remote_base_dir;/*from w w w . ja v a 2 s .com*/ try { remote_base_dir = (new URL(url)).getPath(); } catch (MalformedURLException e1) { LOGGER.error("Problem during upload", e1); return; } final String remoteBaseDir = remote_base_dir; Scanner scanner = scannerFactory.getScanner(url, username, password, null); // Get all files supported scanner.setUserPattern(".*"); scanner.setForceNavigate(true); scanner.getScanList().addListener(new Listener<URLExt>() { @Override public void addedElement(Event<URLExt> e) { URLExt element = e.getElement(); String remote_path = element.getUrl().getPath(); String remoteBase = remoteBaseDir; if (remoteBase.endsWith("/")) { remoteBase = remoteBase.substring(0, remoteBase.length() - 1); } String local_path_dir = remote_path .replaceFirst(remoteBase.substring(0, remoteBase.lastIndexOf("/") + 1), ""); File local_path = new File(dest, local_path_dir); if (!local_path.getParentFile().exists()) { LOGGER.info("Creating directory \"" + local_path.getParentFile().getPath() + "\"."); local_path.getParentFile().mkdirs(); local_path.getParentFile().setWritable(true); } BufferedInputStream bis = null; InputStream is = null; FileOutputStream fos = null; BufferedOutputStream bos = null; int retry = 3; boolean source_remove = cfgManager.getFileScannersCronConfiguration().isSourceRemove(); if (!element.isDirectory()) { DrbNode node = DrbFactory.openURI(element.getUrl().toExternalForm()); long start = System.currentTimeMillis(); do { try { LOGGER.info( "Transfering remote file \"" + remote_path + "\" into \"" + local_path + "\"."); if ((node instanceof DrbNodeSpi) && (((DrbNodeSpi) node).hasImpl(File.class))) { File source = (File) ((DrbNodeSpi) node).getImpl(File.class); { if (source_remove) moveFile(source, local_path); else copyFile(source, local_path); } } else // Case of Use Transfer class to run if ((node instanceof DrbNodeSpi) && (((DrbNodeSpi) node).hasImpl(Transfer.class))) { fos = new FileOutputStream(local_path); bos = new BufferedOutputStream(fos); Transfer t = (Transfer) ((DrbNodeSpi) node).getImpl(Transfer.class); t.copy(bos); try { if (cfgManager.getFileScannersCronConfiguration().isSourceRemove()) t.remove(); } catch (IOException ioe) { LOGGER.error("Unable to remove " + local_path.getPath(), ioe); } } else { if ((node instanceof DrbNodeSpi) && (((DrbNodeSpi) node).hasImpl(InputStream.class))) { is = (InputStream) ((DrbNodeSpi) node).getImpl(InputStream.class); } else is = element.getUrl().openStream(); bis = new BufferedInputStream(is); fos = new FileOutputStream(local_path); bos = new BufferedOutputStream(fos); IOUtils.copyLarge(bis, bos); } // Prepare message long stop = System.currentTimeMillis(); long delay_ms = stop - start; long size = local_path.length(); String message = " in " + delay_ms + "ms"; if ((size > 0) && (delay_ms > 0)) message += " at " + ((size / (1024 * 1024)) / ((float) delay_ms / 1000.0)) + "MB/s"; LOGGER.info("Copy of " + node.getName() + " completed" + message); retry = 0; } catch (Exception excp) { if ((retry - 1) <= 0) { LOGGER.error("Cannot copy " + node.getName() + " aborted."); throw new RuntimeException("Transfer Aborted.", excp); } else { LOGGER.warn("Cannot copy " + node.getName() + " retrying... (" + excp.getMessage() + ")"); try { Thread.sleep(1000); } catch (InterruptedException e1) { // Do nothing. } } } finally { try { if (bos != null) bos.close(); if (fos != null) fos.close(); if (bis != null) bis.close(); if (is != null) is.close(); } catch (IOException exp) { LOGGER.error("Error while closing copy streams."); } } } while (--retry > 0); } else { if (!local_path.exists()) { LOGGER.info("Creating directory \"" + local_path.getPath() + "\"."); local_path.mkdirs(); local_path.setWritable(true); } return; } } @Override public void removedElement(Event<URLExt> e) { } }); try { scanner.scan(); // Remove root product if required. if (cfgManager.getFileScannersCronConfiguration().isSourceRemove()) { try { DrbNode node = DrbFactory.openURI(url); if (node instanceof DrbNodeSpi) { DrbNodeSpi spi = (DrbNodeSpi) node; if (spi.hasImpl(File.class)) { FileUtils.deleteQuietly((File) spi.getImpl(File.class)); } else if (spi.hasImpl(Transfer.class)) { ((Transfer) spi.getImpl(Transfer.class)).remove(); } else { LOGGER.error("Root product note removed (TBC)"); } } } catch (Exception e) { LOGGER.warn("Cannot remove input source (" + e.getMessage() + ")."); } } } catch (Exception e) { if (e instanceof InterruptedException) LOGGER.error("Process interrupted by user"); else LOGGER.error("Error while uploading product", e); // If something get wrong during upload: do not keep any residual // data locally. LOGGER.warn("Remove residual uploaded data :" + dest.getPath()); FileUtils.deleteQuietly(dest); throw new UnsupportedOperationException("Error during scan.", e); } }
From source file:fr.gael.dhus.datastore.processing.ProcessingManager.java
private void copyFile(File source, File dest) throws IOException, NoSuchAlgorithmException { String[] algorithms = cfgManager.getDownloadConfiguration().getChecksumAlgorithms().split(","); FileInputStream fis = null;/* w w w . j av a 2 s . com*/ FileOutputStream fos = null; MultipleDigestInputStream dis = null; try { fis = new FileInputStream(source); fos = new FileOutputStream(dest); Boolean compute_checksum = UnZip.supported(dest.getPath()); if (compute_checksum) { dis = new MultipleDigestInputStream(fis, algorithms); IOUtils.copyLarge(dis, fos); // Write the checksums if any for (String algorithm : algorithms) { String chk = dis.getMessageDigestAsHexadecimalString(algorithm); FileUtils.write(new File(dest.getPath() + "." + algorithm), chk); } } else IOUtils.copyLarge(fis, fos); } finally { IOUtils.closeQuietly(fos); IOUtils.closeQuietly(dis); IOUtils.closeQuietly(fis); } if (source.length() != dest.length()) { throw new IOException("Failed to copy full contents from '" + source + "' to '" + dest + "'"); } }
From source file:com.example.util.FileUtils.java
/** * Copy bytes from a <code>File</code> to an <code>OutputStream</code>. * <p>//from w w w.ja v a2 s. c o m * This method buffers the input internally, so there is no need to use a <code>BufferedInputStream</code>. * </p> * * @param input * the <code>File</code> to read from * @param output * the <code>OutputStream</code> to write to * @return the number of bytes copied * @throws NullPointerException * if the input or output is null * @throws IOException * if an I/O error occurs * @since 2.1 */ public static long copyFile(File input, OutputStream output) throws IOException { final FileInputStream fis = new FileInputStream(input); try { return IOUtils.copyLarge(fis, output); } finally { fis.close(); } }
From source file:com.android.email.mail.store.ImapFolder.java
/** * Appends the given messages to the selected folder. This implementation also determines * the new UID of the given message on the IMAP server and sets the Message's UID to the * new server UID.//from w w w . j av a 2 s. co m * @param message Message * @param noTimeout Set to true on manual syncs, disables the timeout after sending the message * content to the server */ @Override public void appendMessage(final Context context, final Message message, final boolean noTimeout) throws MessagingException { checkOpen(); try { // Create temp file /** * We need to know the encoded message size before we upload it, and encoding * attachments as Base64, possibly reading from a slow provider, is a non-trivial * operation. So we write the contents to a temp file while measuring the size, * and then use that temp file and size to do the actual upsync. * For context, most classic email clients would store the message in RFC822 format * internally, and so would not need to do this on-the-fly. */ final File tempDir = context.getExternalCacheDir(); final File tempFile = File.createTempFile("IMAPupsync", ".eml", tempDir); // Delete here so we don't leave the file lingering. We've got a handle to it so we // can still use it. final boolean deleteSuccessful = tempFile.delete(); if (!deleteSuccessful) { LogUtils.w(LogUtils.TAG, "Could not delete temp file %s", tempFile.getAbsolutePath()); } final OutputStream tempOut = new FileOutputStream(tempFile); // Create output count while writing temp file final CountingOutputStream out = new CountingOutputStream(tempOut); final EOLConvertingOutputStream eolOut = new EOLConvertingOutputStream(out); message.writeTo(eolOut); eolOut.flush(); // Create flag list (most often this will be "\SEEN") String flagList = ""; Flag[] flags = message.getFlags(); if (flags.length > 0) { StringBuilder sb = new StringBuilder(); for (final Flag flag : flags) { if (flag == Flag.SEEN) { sb.append(" " + ImapConstants.FLAG_SEEN); } else if (flag == Flag.FLAGGED) { sb.append(" " + ImapConstants.FLAG_FLAGGED); } } if (sb.length() > 0) { flagList = sb.substring(1); } } mConnection.sendCommand( String.format(Locale.US, ImapConstants.APPEND + " \"%s\" (%s) {%d}", ImapStore.encodeFolderName(mName, mStore.mPathPrefix), flagList, out.getCount()), false); ImapResponse response; do { final int socketTimeout = mConnection.mTransport.getSoTimeout(); try { // Need to set the timeout to unlimited since we might be upsyncing a pretty // big attachment so who knows how long it'll take. It would sure be nice // if this only timed out after the send buffer drained but welp. if (noTimeout) { // For now, only unset the timeout if we're doing a manual sync mConnection.mTransport.setSoTimeout(0); } response = mConnection.readResponse(); if (response.isContinuationRequest()) { final OutputStream transportOutputStream = mConnection.mTransport.getOutputStream(); IOUtils.copyLarge(new FileInputStream(tempFile), transportOutputStream); transportOutputStream.write('\r'); transportOutputStream.write('\n'); transportOutputStream.flush(); } else if (!response.isTagged()) { handleUntaggedResponse(response); } } finally { mConnection.mTransport.setSoTimeout(socketTimeout); } } while (!response.isTagged()); // TODO Why not check the response? /* * Try to recover the UID of the message from an APPENDUID response. * e.g. 11 OK [APPENDUID 2 238268] APPEND completed */ final ImapList appendList = response.getListOrEmpty(1); if ((appendList.size() >= 3) && appendList.is(0, ImapConstants.APPENDUID)) { String serverUid = appendList.getStringOrEmpty(2).getString(); if (!TextUtils.isEmpty(serverUid)) { message.setUid(serverUid); return; } } /* * Try to find the UID of the message we just appended using the * Message-ID header. If there are more than one response, take the * last one, as it's most likely the newest (the one we just uploaded). */ final String messageId = message.getMessageId(); if (messageId == null || messageId.length() == 0) { return; } // Most servers don't care about parenthesis in the search query [and, some // fail to work if they are used] String[] uids = searchForUids(String.format(Locale.US, "HEADER MESSAGE-ID %s", messageId)); if (uids.length > 0) { message.setUid(uids[0]); } // However, there's at least one server [AOL] that fails to work unless there // are parenthesis, so, try this as a last resort uids = searchForUids(String.format(Locale.US, "(HEADER MESSAGE-ID %s)", messageId)); if (uids.length > 0) { message.setUid(uids[0]); } } catch (IOException ioe) { throw ioExceptionHandler(mConnection, ioe); } finally { destroyResponses(); } }
From source file:de.juwimm.cms.remote.ContentServiceSpringImpl.java
private String storeEditionFile(InputStream in) throws IOException { String dir = getTizzitPropertiesBeanSpring().getDatadir() + File.separatorChar + "edition"; if (log.isInfoEnabled()) log.info("Storing Edition File in: " + dir); File fDir = new File(dir); fDir.mkdirs();//from w w w . ja v a2 s . co m if (log.isInfoEnabled()) log.info("deploy dir created"); DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); String date = dateFormat.format(new Date()); File storedEditionFile = new File( fDir.getAbsolutePath() + File.separatorChar + "edition_import_" + date + ".xml.gz"); if (log.isInfoEnabled()) log.info("edition file is named: " + storedEditionFile.getAbsolutePath()); //File.createTempFile("edition_import_" + date, ".xml.gz", fDir); FileOutputStream out = new FileOutputStream(storedEditionFile); IOUtils.copyLarge(in, out); IOUtils.closeQuietly(out); IOUtils.closeQuietly(in); if (log.isInfoEnabled()) log.info("Storing Edition File finished"); return storedEditionFile.getAbsolutePath(); }
From source file:de.juwimm.cms.remote.ContentServiceSpringImpl.java
private File storeTempFile(InputStream in, String name) throws IOException { String dir = getTizzitPropertiesBeanSpring().getDatadir() + File.separatorChar + "tmp"; File fDir = new File(dir); fDir.mkdirs();//from w w w .j av a 2s .c o m File storedEditionFile = File.createTempFile(name, "bak", fDir); FileOutputStream out = new FileOutputStream(storedEditionFile); IOUtils.copyLarge(in, out); IOUtils.closeQuietly(out); IOUtils.closeQuietly(in); if (log.isDebugEnabled()) { log.debug("Stored document file stream temporarily in: " + storedEditionFile.getAbsolutePath()); } return storedEditionFile; }
From source file:de.juwimm.cms.remote.ViewServiceSpringImpl.java
private String storeSiteFile(InputStream in) throws IOException { String dir = getTizzitPropertiesBeanSpring().getDatadir() + File.separatorChar + "editions"; File fDir = new File(dir); fDir.mkdirs();//from w w w . j a v a 2 s .co m File storedEditionFile = File.createTempFile("edition_import_", ".xml.gz", fDir); FileOutputStream out = new FileOutputStream(storedEditionFile); IOUtils.copyLarge(in, out); IOUtils.closeQuietly(out); IOUtils.closeQuietly(in); return storedEditionFile.getAbsolutePath(); }
From source file:de.juwimm.cms.util.Communication.java
public void createEditionForExport(File outputFile, int viewComponentIdWithUnit) throws Exception { log.info("createEditionForExport "); InputStream edition = null;/*from w w w . jav a 2s.co m*/ if (viewComponentIdWithUnit <= 0) { edition = getClientService().exportEditionFull(); } else { edition = getClientService().exportEditionUnit(Integer.valueOf(viewComponentIdWithUnit)); } log.info("got answer... "); if (log.isDebugEnabled()) log.debug("tmpFile " + outputFile.getName()); FileOutputStream fos = new FileOutputStream(outputFile); IOUtils.copyLarge(edition, fos); IOUtils.closeQuietly(edition); IOUtils.closeQuietly(fos); outputFile = null; System.gc(); }
From source file:de.juwimm.cms.util.Communication.java
public void createViewComponentForExport(File output, int viewComponentId) throws Exception { InputStream viewComponentStream = null; /**set view component stream*/ viewComponentStream = getClientService().exportViewComponent(viewComponentId); FileOutputStream fileOutput = new FileOutputStream(output); IOUtils.copyLarge(viewComponentStream, fileOutput); IOUtils.closeQuietly(viewComponentStream); IOUtils.closeQuietly(fileOutput);//from ww w . j a va 2s . c o m output = null; System.gc(); }
From source file:net.voidfunction.rm.common.FileServlet.java
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { request.getSession().setMaxInactiveInterval(120); response.setHeader("Date", HTTPUtils.getServerTime(0)); // Parse the filename and the ID out of the URL String[] urlParts = request.getRequestURI().substring(1).split("/"); if (urlParts.length < 2) { response.setStatus(HttpServletResponse.SC_BAD_REQUEST); return;/*w w w . ja v a2s . c o m*/ } String fileID = urlParts[1]; String fileName = ""; if (urlParts.length > 2) fileName = urlParts[2]; String logOut = "File " + fileID + " (" + fileName + ") requested by " + request.getRemoteHost() + " [Result: "; RMFile file = node.getFileRepository().getFileById(fileID); if (file == null) { // File with given ID not found - no redirect for you. logOut += "Not found]"; node.getLog().info(logOut); response.setStatus(HttpServletResponse.SC_NOT_FOUND); response.getWriter().write("<b>404 Not Found</b><br/>Could not find a file with ID " + fileID); return; } boolean workerDL = (fileName.equals("Worker-Download")); if (workerDL) logOut += " (Worker Download) "; // Let the download listener know, if any, but don't count worker downloads if (dlListener != null && !workerDL) dlListener.fileDownloaded(file); String redirURL = null; if (locator != null) redirURL = (String) request.getSession().getAttribute("fileURL-" + fileID); if (redirURL == null && locator != null) redirURL = locator.locateURL(fileID, fileName); if (redirURL != null) { node.getLog().debug("Found redirect URL: " + redirURL); request.getSession().setAttribute("fileURL-" + fileID, redirURL); // Redirect to the new URL logOut += "Redirect]"; node.getLog().info(logOut); response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY); response.setHeader("Location", redirURL); } else { // We have to try to find it ourselves logOut += "Found locally]"; node.getLog().info(logOut); // Caching magic - we can safely assume the file won't change String etag = Hex.encodeHexString(file.getHash()); response.setHeader("ETag", etag); String ifModifiedSince = request.getHeader("If-Modified-Since"); String ifNoneMatch = request.getHeader("If-None-Match"); boolean etagMatch = (ifNoneMatch != null) && (ifNoneMatch.equals(etag)); if (ifModifiedSince != null || etagMatch) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); response.setHeader("Last-Modified", ifModifiedSince); } else { // Send the HTTP response and file data response.setStatus(HttpServletResponse.SC_OK); response.setHeader("Expires", HTTPUtils.getServerTime(3600)); response.setHeader("Cache-Control", "max-age=3600"); response.setContentType(file.getMimetype()); response.setHeader("Content-Length", String.valueOf(file.getSize())); // Stream the file data to the output stream using Apache IOUtils InputStream fileIn = node.getFileRepository().getFileData(fileID); IOUtils.copyLarge(fileIn, response.getOutputStream()); } } }