List of usage examples for org.apache.commons.io IOUtils copyLarge
public static long copyLarge(Reader input, Writer output) throws IOException
Reader to a Writer. From source file:fedora.server.storage.lowlevel.akubra.AkubraLowlevelStorage.java
private static long copy(InputStream source, OutputStream sink) { try {// w w w .j a v a 2 s . c o m return IOUtils.copyLarge(source, sink); } catch (IOException e) { throw new FaultException("System error copying stream", e); } finally { IOUtils.closeQuietly(source); IOUtils.closeQuietly(sink); } }
From source file:com.datatorrent.contrib.hdht.HDHTWalManager.java
public void copyPreviousWalFiles(List<PreviousWALDetails> parentWals, Set<PreviousWALDetails> alreadyCopiedWals) { try {/*from w ww . jav a 2s. c o m*/ PreviousWALDetails parentWal = parentWals.iterator().next(); // Copy Files to new WAL location for (long i = parentWal.getStartPosition().fileId; i <= parentWal.getEndPosition().fileId; i++) { DataInputStream in = bfs.getInputStream(parentWal.getWalKey(), WAL_FILE_PREFIX + i); DataOutputStream out = bfs.getOutputStream(walKey, WAL_FILE_PREFIX + i); IOUtils.copyLarge(in, out); in.close(); out.close(); } } catch (IOException e) { throw Throwables.propagate(e); } }
From source file:com.linkedin.pinot.controller.api.resources.LLCSegmentCompletionHandlers.java
@Nullable private String uploadSegment(FormDataMultiPart multiPart, String instanceId, String segmentName, boolean isSplitCommit) { try {/*from w w w .ja va 2 s. c o m*/ Map<String, List<FormDataBodyPart>> map = multiPart.getFields(); if (!PinotSegmentUploadRestletResource.validateMultiPart(map, segmentName)) { return null; } String name = map.keySet().iterator().next(); FormDataBodyPart bodyPart = map.get(name).get(0); FileUploadPathProvider provider = new FileUploadPathProvider(_controllerConf); File tmpFile = new File(provider.getFileUploadTmpDir(), name + "." + UUID.randomUUID().toString()); tmpFile.deleteOnExit(); try (InputStream inputStream = bodyPart.getValueAs(InputStream.class); OutputStream outputStream = new FileOutputStream(tmpFile)) { IOUtils.copyLarge(inputStream, outputStream); } LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); final String rawTableName = llcSegmentName.getTableName(); final File tableDir = new File(provider.getBaseDataDir(), rawTableName); File segmentFile; if (isSplitCommit) { String uniqueSegmentFileName = SegmentCompletionUtils.generateSegmentFileName(segmentName); segmentFile = new File(tableDir, uniqueSegmentFileName); } else { segmentFile = new File(tableDir, segmentName); } if (isSplitCommit) { FileUtils.moveFile(tmpFile, segmentFile); } else { // Multiple threads can reach this point at the same time, if the following scenario happens // The server that was asked to commit did so very slowly (due to network speeds). Meanwhile the FSM in // SegmentCompletionManager timed out, and allowed another server to commit, which did so very quickly (somehow // the network speeds changed). The second server made it through the FSM and reached this point. // The synchronization below takes care that exactly one file gets moved in place. // There are still corner conditions that are not handled correctly. For example, // 1. What if the offset of the faster server was different? // 2. We know that only the faster server will get to complete the COMMIT call successfully. But it is possible // that the race to this statement is won by the slower server, and so the real segment that is in there is that // of the slower server. // In order to overcome controller restarts after the segment is renamed, but before it is committed, we DO need to // check for existing segment file and remove it. So, the block cannot be removed altogether. // For now, we live with these corner cases. Once we have split-commit enabled and working, this code will no longer // be used. synchronized (SegmentCompletionManager.getInstance()) { if (segmentFile.exists()) { LOGGER.warn("Segment file {} exists. Replacing with upload from {}", segmentFile.getAbsolutePath(), instanceId); FileUtils.deleteQuietly(segmentFile); } FileUtils.moveFile(tmpFile, segmentFile); } } LOGGER.info("Moved file {} to {}", tmpFile.getAbsolutePath(), segmentFile.getAbsolutePath()); return new URI(SCHEME + segmentFile.getAbsolutePath(), /* boolean escaped */ false).toString(); } catch (InvalidControllerConfigException e) { LOGGER.error("Invalid controller config exception from instance {} for segment {}", instanceId, segmentName, e); return null; } catch (IOException e) { LOGGER.error("File upload exception from instance {} for segment {}", instanceId, segmentName, e); return null; } finally { multiPart.cleanup(); } }
From source file:de.huxhorn.sulky.blobs.impl.BlobRepositoryImpl.java
private String copyAndHash(InputStream input, File into) throws IOException { MessageDigest digest = createMessageDigest(); DigestInputStream dis = new DigestInputStream(input, digest); IOException ex;//from www . j a va2 s. c om FileOutputStream fos = null; try { fos = new FileOutputStream(into); IOUtils.copyLarge(dis, fos); byte[] hash = digest.digest(); Formatter formatter = new Formatter(); for (byte b : hash) { formatter.format("%02x", b); } return formatter.toString(); } catch (IOException e) { ex = e; } finally { IOUtils.closeQuietly(dis); IOUtils.closeQuietly(fos); } if (logger.isWarnEnabled()) logger.warn("Couldn't retrieve data from input!", ex); deleteTempFile(into); throw ex; }
From source file:com.kixeye.chassis.transport.websocket.ActionInvokingWebSocket.java
/** * Gets the websocket session.//from w w w. j a v a 2 s. com * * @return * @throws IOException * @throws GeneralSecurityException */ protected Future<Void> sendContent(InputStream inputStream) throws IOException, GeneralSecurityException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); IOUtils.copyLarge(inputStream, baos); // generate blob byte[] contentBlob = serDe.serialize(baos.toByteArray()); // check if we need to do psk encryption contentBlob = pskFrameProcessor.processOutgoing(contentBlob, 0, contentBlob.length); return session.getRemote().sendBytesByFuture(ByteBuffer.wrap(contentBlob)); }
From source file:fr.gael.dhus.datastore.processing.impl.ProcessProductTransfer.java
private void copyFile(File source, File dest, boolean compute_checksum) throws IOException, NoSuchAlgorithmException { String[] algorithms = cfgManager.getDownloadConfiguration().getChecksumAlgorithms().split(","); FileInputStream fis = null;//from ww w . ja v a 2 s .c om FileOutputStream fos = null; MultipleDigestInputStream dis = null; try { fis = new FileInputStream(source); fos = new FileOutputStream(dest); if (compute_checksum) { dis = new MultipleDigestInputStream(fis, algorithms); IOUtils.copyLarge(dis, fos); // Write the checksums if any for (String algorithm : algorithms) { String chk = dis.getMessageDigestAsHexadecimalString(algorithm); FileUtils.write(new File(dest.getPath() + "." + algorithm), chk); } } else IOUtils.copyLarge(fis, fos); } finally { IOUtils.closeQuietly(fos); IOUtils.closeQuietly(dis); IOUtils.closeQuietly(fis); } if (source.length() != dest.length()) { throw new IOException("Failed to copy full contents from '" + source + "' to '" + dest + "'"); } }
From source file:com.datatorrent.contrib.hdht.HDHTWalManager.java
/** * Copy old WAL files to current location from startPosition to End Position in old WAL. * @param startPosition/*from www. j av a2s.c o m*/ * @param endPosition * @param oldWalKey */ public void copyWALFiles(WalPosition startPosition, WalPosition endPosition, long oldWalKey) { try { for (long i = startPosition.fileId; i < endPosition.fileId; i++) { if (bfs.exists(oldWalKey, WAL_FILE_PREFIX + i)) { DataInputStream in = bfs.getInputStream(oldWalKey, WAL_FILE_PREFIX + i); DataOutputStream out = bfs.getOutputStream(walKey, WAL_FILE_PREFIX + walFileId); IOUtils.copyLarge(in, out); in.close(); out.close(); walFileId++; } } // Copy last file upto end position offset copyWalPart(startPosition, endPosition, oldWalKey); if (maxWalFileSize > 0 && walSize > maxWalFileSize) { writer.close(); writer = null; walFileId++; walSize = 0; } } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:ch.entwine.weblounge.contentrepository.impl.fs.FileSystemContentRepository.java
/** * {@inheritDoc}/*from w w w . j a v a 2 s.co m*/ * * @see ch.entwine.weblounge.contentrepository.impl.AbstractWritableContentRepository#storeResourceContent(ch.entwine.weblounge.common.content.ResourceURI, * ch.entwine.weblounge.common.content.ResourceContent, * java.io.InputStream) */ @Override protected ResourceContent storeResourceContent(ResourceURI uri, ResourceContent content, InputStream is) throws ContentRepositoryException, IOException { if (is == null) return content; File contentFile = uriToContentFile(uri, content); OutputStream os = null; try { FileUtils.forceMkdir(contentFile.getParentFile()); if (!contentFile.exists()) contentFile.createNewFile(); os = new FileOutputStream(contentFile); IOUtils.copyLarge(is, os); } finally { IOUtils.closeQuietly(is); IOUtils.closeQuietly(os); } // Set the size content.setSize(contentFile.length()); return content; }
From source file:com.linkedin.pinot.controller.api.resources.PinotSegmentUploadRestletResource.java
private SuccessResponse uploadSegmentInternal(FormDataMultiPart multiPart, String segmentJsonStr, boolean enableParallelPushProtection, HttpHeaders headers, Request request) { File tempTarredSegmentFile = null; File tempSegmentDir = null;/*from ww w.j a v a2s .c om*/ if (headers != null) { // TODO: Add these headers into open source hadoop jobs LOGGER.info("HTTP Header {} is {}", CommonConstants.Controller.SEGMENT_NAME_HTTP_HEADER, headers.getRequestHeader(CommonConstants.Controller.SEGMENT_NAME_HTTP_HEADER)); LOGGER.info("HTTP Header {} is {}", CommonConstants.Controller.TABLE_NAME_HTTP_HEADER, headers.getRequestHeader(CommonConstants.Controller.TABLE_NAME_HTTP_HEADER)); } try { FileUploadPathProvider provider = new FileUploadPathProvider(_controllerConf); String tempSegmentName = "tmp-" + System.nanoTime(); tempTarredSegmentFile = new File(provider.getFileUploadTmpDir(), tempSegmentName); tempSegmentDir = new File(provider.getTmpUntarredPath(), tempSegmentName); // Get upload type String uploadTypeStr = headers.getHeaderString(FileUploadDownloadClient.CustomHeaders.UPLOAD_TYPE); FileUploadDownloadClient.FileUploadType uploadType; if (uploadTypeStr != null) { uploadType = FileUploadDownloadClient.FileUploadType.valueOf(uploadTypeStr); } else { uploadType = FileUploadDownloadClient.FileUploadType.getDefaultUploadType(); } String downloadURI = null; switch (uploadType) { case JSON: case URI: // Get download URI try { downloadURI = getDownloadUri(uploadType, headers, segmentJsonStr); } catch (Exception e) { throw new ControllerApplicationException(LOGGER, "Failed to get download URI", Response.Status.BAD_REQUEST, e); } // Get segment fetcher based on the download URI SegmentFetcher segmentFetcher; try { segmentFetcher = SegmentFetcherFactory.getInstance().getSegmentFetcherBasedOnURI(downloadURI); } catch (URISyntaxException e) { throw new ControllerApplicationException(LOGGER, "Caught exception while parsing download URI: " + downloadURI, Response.Status.BAD_REQUEST, e); } if (segmentFetcher == null) { throw new ControllerApplicationException(LOGGER, "Failed to get segment fetcher for download URI: " + downloadURI, Response.Status.BAD_REQUEST); } // Download segment tar file to local segmentFetcher.fetchSegmentToLocal(downloadURI, tempTarredSegmentFile); break; case TAR: try { Map<String, List<FormDataBodyPart>> map = multiPart.getFields(); if (!validateMultiPart(map, null)) { throw new ControllerApplicationException(LOGGER, "Invalid multi-part form", Response.Status.BAD_REQUEST); } FormDataBodyPart bodyPart = map.values().iterator().next().get(0); try (InputStream inputStream = bodyPart.getValueAs(InputStream.class); OutputStream outputStream = new FileOutputStream(tempTarredSegmentFile)) { IOUtils.copyLarge(inputStream, outputStream); } } finally { multiPart.cleanup(); } break; default: throw new UnsupportedOperationException("Unsupported upload type: " + uploadType); } // While there is TarGzCompressionUtils.unTarOneFile, we use unTar here to unpack all files // in the segment in order to ensure the segment is not corrupted TarGzCompressionUtils.unTar(tempTarredSegmentFile, tempSegmentDir); File[] files = tempSegmentDir.listFiles(); Preconditions.checkState(files != null && files.length == 1); File indexDir = files[0]; SegmentMetadata segmentMetadata = new SegmentMetadataImpl(indexDir); String segmentName = segmentMetadata.getName(); String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(segmentMetadata.getTableName()); String clientAddress = InetAddress.getByName(request.getRemoteAddr()).getHostName(); LOGGER.info("Processing upload request for segment: {} of table: {} from client: {}", segmentName, offlineTableName, clientAddress); uploadSegment(indexDir, segmentMetadata, tempTarredSegmentFile, downloadURI, provider, enableParallelPushProtection, headers); return new SuccessResponse( "Successfully uploaded segment: " + segmentName + " of table: " + offlineTableName); } catch (WebApplicationException e) { throw e; } catch (Exception e) { _controllerMetrics.addMeteredGlobalValue(ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR, 1L); throw new ControllerApplicationException(LOGGER, "Caught internal server exception while uploading segment", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { FileUtils.deleteQuietly(tempTarredSegmentFile); FileUtils.deleteQuietly(tempSegmentDir); } }
From source file:eu.planets_project.services.utils.ZipUtils.java
/** * Inserts a file into a given zip at a location specified by targetPath. * If toInsert points to a folder, all files in this folder will be added to the zip. * If the zip already contains a file specified by targetPath, the existing entry (and all files contained in it) * will be deleted and toInsert (and all contained files) is inserted. * /*from w w w . ja v a2s.c o m*/ * @param zipFile the zip file where the file toInsert will be added. * @param toInsert the file to add to the zip * @param targetPath the location the file should have in this zip * @return the modified zip, containing the file toInsert. */ public static File insertFileInto(File zipFile, File toInsert, String targetPath) { Zip64File zip64File = null; try { boolean compress = false; zip64File = new Zip64File(zipFile); FileEntry testEntry = getFileEntry(zip64File, targetPath); if (testEntry != null && testEntry.getMethod() == FileEntry.iMETHOD_DEFLATED) { compress = true; } processAndCreateFolderEntries(zip64File, parseTargetPath(targetPath, toInsert), compress); if (testEntry != null) { log.info("[insertFileInto] Entry exists: " + testEntry.getName()); log.info("[insertFileInto] Will delete this entry before inserting: " + toInsert.getName()); if (!testEntry.isDirectory()) { zip64File.delete(testEntry.getName()); } else { log.info("[insertFileInto] Entry is a directory. " + "Will delete all files contained in this entry and insert " + toInsert.getName() + "and all nested files."); if (!targetPath.contains("/")) { targetPath = targetPath + "/"; } deleteFileEntry(zip64File, testEntry); log.info("[insertFileInto] Entry successfully deleted."); } log.info("[insertFileInto] Writing new Entry: " + targetPath); EntryOutputStream out = null; if (!compress) { out = zip64File.openEntryOutputStream(targetPath, FileEntry.iMETHOD_STORED, new Date(toInsert.lastModified())); } else { out = zip64File.openEntryOutputStream(targetPath, FileEntry.iMETHOD_DEFLATED, new Date(toInsert.lastModified())); } if (toInsert.isDirectory()) { out.flush(); out.close(); log.info("[insertFileInto] Finished writing entry: " + targetPath); List<String> containedPaths = normalizePaths(toInsert); List<File> containedFiles = listAllFilesAndFolders(toInsert, new ArrayList<File>()); log.info("[insertFileInto] Added entry is a folder."); log.info("[insertFileInto] Adding all nested files: "); for (int i = 0; i < containedPaths.size(); i++) { File currentFile = containedFiles.get(i); String currentPath = targetPath.replace("/", "") + File.separator + containedPaths.get(i); EntryOutputStream loop_out = null; if (!compress) { loop_out = zip64File.openEntryOutputStream(currentPath, FileEntry.iMETHOD_STORED, new Date(currentFile.lastModified())); } else { loop_out = zip64File.openEntryOutputStream(currentPath, FileEntry.iMETHOD_DEFLATED, new Date(currentFile.lastModified())); } if (currentFile.isFile()) { InputStream loop_in = new FileInputStream(currentFile); IOUtils.copyLarge(loop_in, loop_out); loop_in.close(); } log.info("[insertFileInto] Added: " + currentPath); loop_out.flush(); loop_out.close(); } } else { InputStream in = new FileInputStream(toInsert); IOUtils.copyLarge(in, out); in.close(); out.flush(); out.close(); } } else { EntryOutputStream out = null; if (!compress) { out = zip64File.openEntryOutputStream(targetPath, FileEntry.iMETHOD_STORED, new Date(toInsert.lastModified())); } else { out = zip64File.openEntryOutputStream(targetPath, FileEntry.iMETHOD_DEFLATED, new Date(toInsert.lastModified())); } if (toInsert.isDirectory()) { out.flush(); out.close(); log.info("[insertFileInto] Finished writing entry: " + targetPath); List<String> containedPaths = normalizePaths(toInsert); List<File> containedFiles = listAllFilesAndFolders(toInsert, new ArrayList<File>()); log.info("[insertFileInto] Added entry is a folder."); log.info("[insertFileInto] Adding all nested files: "); for (int i = 0; i < containedPaths.size(); i++) { File currentFile = containedFiles.get(i); String currentPath = targetPath.replace("/", "") + File.separator + containedPaths.get(i); EntryOutputStream loop_out = null; if (!compress) { loop_out = zip64File.openEntryOutputStream(currentPath, FileEntry.iMETHOD_STORED, new Date(currentFile.lastModified())); } else { loop_out = zip64File.openEntryOutputStream(currentPath, FileEntry.iMETHOD_DEFLATED, new Date(currentFile.lastModified())); } if (currentFile.isFile()) { InputStream loop_in = new FileInputStream(currentFile); IOUtils.copyLarge(loop_in, loop_out); loop_in.close(); } log.info("[insertFileInto] Added: " + currentPath); loop_out.flush(); loop_out.close(); } } else { InputStream in = new FileInputStream(toInsert); IOUtils.copyLarge(in, out); in.close(); out.flush(); out.close(); } } log.info("[insertFileInto] Done! Added " + toInsert.getName() + " to zip."); zip64File.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return new File(zip64File.getDiskFile().getFileName()); }