List of usage examples for org.apache.commons.io IOUtils copyLarge
public static long copyLarge(Reader input, Writer output) throws IOException
Reader to a Writer. From source file:ddf.catalog.source.opensearch.OpenSearchSource.java
@Override public SourceResponse query(QueryRequest queryRequest) throws UnsupportedQueryException { String methodName = "query"; LOGGER.trace(methodName);//from w w w .j a va2s. co m Serializable metacardId = queryRequest.getPropertyValue(Metacard.ID); SourceResponseImpl response = null; Subject subject = null; WebClient restWebClient = null; if (queryRequest.hasProperties()) { Object subjectObj = queryRequest.getProperties().get(SecurityConstants.SECURITY_SUBJECT); subject = (Subject) subjectObj; } restWebClient = factory.getWebClientForSubject(subject); Query query = queryRequest.getQuery(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Received query: " + query); } boolean canDoOpenSearch = setOpenSearchParameters(query, subject, restWebClient); if (canDoOpenSearch) { InputStream responseStream = performRequest(restWebClient); response = new SourceResponseImpl(queryRequest, new ArrayList<Result>()); if (responseStream != null) { response = processResponse(responseStream, queryRequest); } } else { if (StringUtils.isEmpty((String) metacardId)) { OpenSearchFilterVisitor visitor = new OpenSearchFilterVisitor(); query.accept(visitor, null); metacardId = visitor.getMetacardId(); } restWebClient = newRestClient(query, (String) metacardId, false, subject); if (restWebClient != null) { InputStream responseStream = performRequest(restWebClient); Metacard metacard = null; List<Result> resultQueue = new ArrayList<Result>(); try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(1000000)) { if (responseStream != null) { IOUtils.copyLarge(responseStream, fileBackedOutputStream); InputTransformer inputTransformer = null; try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) { inputTransformer = getInputTransformer(inputStream); } catch (IOException e) { LOGGER.debug("Problem with transformation.", e); } if (inputTransformer != null) { try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) { metacard = inputTransformer.transform(inputStream); } catch (IOException e) { LOGGER.debug("Problem with transformation.", e); } } } } catch (IOException | CatalogTransformerException e) { LOGGER.debug("Problem with transformation.", e); } if (metacard != null) { metacard.setSourceId(getId()); ResultImpl result = new ResultImpl(metacard); resultQueue.add(result); response = new SourceResponseImpl(queryRequest, resultQueue); response.setHits(resultQueue.size()); } } } LOGGER.trace(methodName); return response; }
From source file:de.uni_luebeck.inb.knowarc.usecases.invocation.local.LocalUseCaseInvocation.java
@Override public String setOneInput(ReferenceService referenceService, T2Reference t2Reference, ScriptInput input) throws InvocationException { if (input.getCharsetName() == null) { input.setCharsetName(Charset.defaultCharset().name()); }/*from www . j a va 2 s . com*/ String target = null; String targetSuffix = null; if (input.isFile()) { targetSuffix = input.getTag(); } else if (input.isTempFile()) { targetSuffix = "tempfile." + (nTempFiles++) + ".tmp"; } if (input.isBinary()) { return setOneBinaryInput(referenceService, t2Reference, input, targetSuffix); } logger.info("Target is " + target); if (input.isFile() || input.isTempFile()) { target = tempDir.getAbsolutePath() + "/" + targetSuffix; // Try to get it as a file Reader r; Writer w; FileReference fileRef = getAsFileReference(referenceService, t2Reference); if (fileRef != null) { if (!input.isForceCopy()) { if (linkCommand != null) { String source = fileRef.getFile().getAbsolutePath(); String actualLinkCommand = getActualOsCommand(linkCommand, source, targetSuffix, target); logger.info("Link command is " + actualLinkCommand); String[] splitCmds = actualLinkCommand.split(" "); ProcessBuilder builder = new ProcessBuilder(splitCmds); builder.directory(tempDir); try { int code = builder.start().waitFor(); if (code == 0) { return target; } else { logger.error("Link command gave errorcode: " + code); } } catch (InterruptedException e) { // go through } catch (IOException e) { // go through } } } if (fileRef.getDataNature().equals(ReferencedDataNature.TEXT)) { r = new InputStreamReader(fileRef.openStream(this.getContext()), Charset.forName(fileRef.getCharset())); } else { try { r = new FileReader(fileRef.getFile()); } catch (FileNotFoundException e) { throw new InvocationException(e); } } } else { r = new InputStreamReader(getAsStream(referenceService, t2Reference)); } try { w = new OutputStreamWriter(new FileOutputStream(target), input.getCharsetName()); } catch (UnsupportedEncodingException e) { throw new InvocationException(e); } catch (FileNotFoundException e) { throw new InvocationException(e); } try { IOUtils.copyLarge(r, w); } catch (IOException e) { throw new InvocationException(e); } try { r.close(); w.close(); } catch (IOException e) { throw new InvocationException(e); } return target; } else { String value = (String) referenceService.renderIdentifier(t2Reference, String.class, this.getContext()); return value; } }
From source file:de.extra.client.plugins.responseprocessplugin.filesystem.FileSystemResponseProcessPlugin.java
private boolean saveBodyToFilesystem(final String responseId, final DataHandler dataHandler) throws IOException { final boolean erfolgreichGespeichert = false; final StringBuffer dateiName = new StringBuffer(); dateiName.append(baueDateiname());//from www . j a v a2 s .c o m dateiName.append("-"); dateiName.append(responseId); final File responseFile = new File(eingangOrdner, dateiName.toString()); final FileOutputStream fileOutputStream = new FileOutputStream(responseFile); IOUtils.copyLarge(dataHandler.getInputStream(), fileOutputStream); transportObserver.responseDataForwarded(responseFile.getAbsolutePath(), responseFile.length()); if (LOG.isTraceEnabled()) { LOG.trace("Dateiname: '" + dateiName + "'"); } return erfolgreichGespeichert; }
From source file:fr.gael.dhus.datastore.processing.impl.ProcessProductTransfer.java
private void upload(String url, final String username, final String password, final File dest, final boolean compute_checksum) { String remote_base_dir;/*from w w w . j a va 2s .co m*/ try { remote_base_dir = (new URL(url)).getPath(); } catch (MalformedURLException e1) { logger.error("Problem during upload", e1); return; } final String remote_base = remote_base_dir; Scanner scanner = scannerFactory.getScanner(url, username, password, null); // Get all files supported scanner.setUserPattern(".*"); scanner.setForceNavigate(true); scanner.getScanList().addListener(new Listener<URLExt>() { @Override public void addedElement(Event<URLExt> e) { URLExt element = e.getElement(); String remote_path = element.getUrl().getPath(); String local_filename = ScannerFactory.getFileFromPath(remote_path); String local_path_dir = ""; if (!remote_base.equals(remote_path)) local_path_dir = remote_path.replaceFirst(ScannerFactory.getParentPath(remote_base), ""); else local_path_dir = local_filename; File local_path = new File(dest, local_path_dir); if (!local_path.getParentFile().exists()) { logger.info("Creating directory \"" + local_path.getParentFile().getPath() + "\"."); local_path.getParentFile().mkdirs(); local_path.getParentFile().setWritable(true); } BufferedInputStream bis = null; InputStream is = null; FileOutputStream fos = null; BufferedOutputStream bos = null; int retry = 3; boolean source_remove = cfgManager.getFileScannersCronConfiguration().isSourceRemove(); if (!element.isDirectory()) { DrbNode node = DrbFactory.openURI(element.getUrl().toExternalForm()); long start = System.currentTimeMillis(); do { try { logger.info( "Transfering remote file \"" + remote_path + "\" into \"" + local_path + "\"."); if ((node instanceof DrbNodeSpi) && (((DrbNodeSpi) node).hasImpl(File.class))) { File source = (File) ((DrbNodeSpi) node).getImpl(File.class); { if (source_remove) moveFile(source, local_path, compute_checksum); else copyFile(source, local_path, compute_checksum); } } else // Case of Use Transfer class to run if ((node instanceof DrbNodeSpi) && (((DrbNodeSpi) node).hasImpl(Transfer.class))) { fos = new FileOutputStream(local_path); bos = new BufferedOutputStream(fos); Transfer t = (Transfer) ((DrbNodeSpi) node).getImpl(Transfer.class); t.copy(bos); try { if (cfgManager.getFileScannersCronConfiguration().isSourceRemove()) t.remove(); } catch (IOException ioe) { logger.error("Unable to remove " + local_path.getPath(), ioe); } } else { if ((node instanceof DrbNodeSpi) && (((DrbNodeSpi) node).hasImpl(InputStream.class))) { is = (InputStream) ((DrbNodeSpi) node).getImpl(InputStream.class); } else is = element.getUrl().openStream(); bis = new BufferedInputStream(is); fos = new FileOutputStream(local_path); bos = new BufferedOutputStream(fos); IOUtils.copyLarge(bis, bos); } // Prepare message long stop = System.currentTimeMillis(); long delay_ms = stop - start; long size = local_path.length(); String message = " in " + delay_ms + "ms"; if ((size > 0) && (delay_ms > 0)) message += " at " + ((size / (1024 * 1024)) / ((float) delay_ms / 1000.0)) + "MB/s"; logger.info("Copy of " + node.getName() + " completed" + message); retry = 0; } catch (Exception excp) { if ((retry - 1) <= 0) { logger.error("Cannot copy " + node.getName() + " aborted."); throw new RuntimeException("Transfer Aborted.", excp); } else { logger.warn("Cannot copy " + node.getName() + " retrying... (" + excp.getMessage() + ")"); try { Thread.sleep(1000); } catch (InterruptedException e1) { // Do nothing. } } } finally { try { if (bos != null) bos.close(); if (fos != null) fos.close(); if (bis != null) bis.close(); if (is != null) is.close(); } catch (IOException exp) { logger.error("Error while closing copy streams."); } } } while (--retry > 0); } else { if (!local_path.exists()) { logger.info("Creating directory \"" + local_path.getPath() + "\"."); local_path.mkdirs(); local_path.setWritable(true); } return; } } @Override public void removedElement(Event<URLExt> e) { } }); try { scanner.scan(); // Remove root product if required. if (cfgManager.getFileScannersCronConfiguration().isSourceRemove()) { try { DrbNode node = DrbFactory.openURI(url); if (node instanceof DrbNodeSpi) { DrbNodeSpi spi = (DrbNodeSpi) node; if (spi.hasImpl(File.class)) { FileUtils.deleteQuietly((File) spi.getImpl(File.class)); } else if (spi.hasImpl(Transfer.class)) { ((Transfer) spi.getImpl(Transfer.class)).remove(); } else { logger.error("Root product note removed (TBC)"); } } } catch (Exception e) { logger.warn("Cannot remove input source (" + e.getMessage() + ")."); } } } catch (Exception e) { if (e instanceof InterruptedException) logger.error("Process interrupted by user"); else logger.error("Error while uploading product", e); // If something get wrong during upload: do not keep any residual // data locally. logger.warn("Remove residual uploaded data :" + dest.getPath()); FileUtils.deleteQuietly(dest); throw new UnsupportedOperationException("Error during scan.", e); } }
From source file:it.greenvulcano.util.bin.BinaryUtils.java
/** * Read all data from a source <code>InputStream</code> and stores them into * a file on the local filesystem.<br> * It is assumed that the source <code>InputStream</code> will be closed by * the caller of this method./*from w ww . j a v a2 s. c o m*/ * * @param in * the <code>InputStream</code> to read from * @param filename * the name of the file to be written to * @throws IOException * if any I/O error occurs */ public static void inputStreamToFile(InputStream in, String filename) throws IOException { OutputStream fOut = null; try { filename = TextUtils.adjustPath(filename); fOut = new FileOutputStream(filename); IOUtils.copyLarge(in, fOut); } finally { if (fOut != null) { fOut.close(); } } }
From source file:ddf.catalog.source.opensearch.impl.OpenSearchSource.java
@Override public SourceResponse query(QueryRequest queryRequest) throws UnsupportedQueryException { String methodName = "query"; LOGGER.trace(methodName);/*from ww w . j a v a 2s . c om*/ Serializable metacardId = queryRequest.getPropertyValue(Metacard.ID); SourceResponseImpl response = null; Subject subject = null; WebClient restWebClient; if (queryRequest.hasProperties()) { Object subjectObj = queryRequest.getProperties().get(SecurityConstants.SECURITY_SUBJECT); subject = (Subject) subjectObj; } restWebClient = factory.getWebClientForSubject(subject); Query query = queryRequest.getQuery(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Received query: {}", query); } boolean canDoOpenSearch = setOpenSearchParameters(queryRequest, subject, restWebClient); if (canDoOpenSearch) { InputStream responseStream = performRequest(restWebClient); response = new SourceResponseImpl(queryRequest, new ArrayList<>()); if (responseStream != null) { response = processResponse(responseStream, queryRequest); } } else { if (StringUtils.isEmpty((String) metacardId)) { OpenSearchFilterVisitorObject openSearchFilterVisitorObject = (OpenSearchFilterVisitorObject) query .accept(openSearchFilterVisitor, new OpenSearchFilterVisitorObject()); metacardId = openSearchFilterVisitorObject.getId(); } restWebClient = newRestClient(query, (String) metacardId, false, subject); if (restWebClient != null) { InputStream responseStream = performRequest(restWebClient); Metacard metacard = null; List<Result> resultQueue = new ArrayList<>(); try (TemporaryFileBackedOutputStream fileBackedOutputStream = new TemporaryFileBackedOutputStream()) { if (responseStream != null) { IOUtils.copyLarge(responseStream, fileBackedOutputStream); InputTransformer inputTransformer = null; try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) { inputTransformer = getInputTransformer(inputStream); } catch (IOException e) { LOGGER.debug("Problem with transformation.", e); } if (inputTransformer != null) { try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) { metacard = inputTransformer.transform(inputStream); } catch (IOException e) { LOGGER.debug("Problem with transformation.", e); } } } } catch (IOException | CatalogTransformerException e) { LOGGER.debug("Problem with transformation.", e); } if (metacard != null) { metacard.setSourceId(getId()); ResultImpl result = new ResultImpl(metacard); resultQueue.add(result); response = new SourceResponseImpl(queryRequest, resultQueue); response.setHits(resultQueue.size()); } } } LOGGER.trace(methodName); return response; }
From source file:de.tu.darmstadt.lt.ner.preprocessing.GermaNERMain.java
private static void setModelDir() throws IOException, FileNotFoundException { modelDirectory = (Configuration.modelDir == null || Configuration.modelDir.isEmpty()) ? new File("output") : new File(Configuration.modelDir); modelDirectory.mkdirs();/*from ww w .j a v a 2 s. c o m*/ if (!new File(modelDirectory, "model.jar").exists()) { IOUtils.copyLarge(ClassLoader.getSystemResourceAsStream("model/model.jar"), new FileOutputStream(new File(modelDirectory, "model.jar"))); } if (!new File(modelDirectory, "MANIFEST.MF").exists()) { IOUtils.copyLarge(ClassLoader.getSystemResourceAsStream("model/MANIFEST.MF"), new FileOutputStream(new File(modelDirectory, "MANIFEST.MF"))); } if (!new File(modelDirectory, "feature.xml").exists()) { IOUtils.copyLarge(ClassLoader.getSystemResourceAsStream("feature/feature.xml"), new FileOutputStream(new File(modelDirectory, "feature.xml"))); } }
From source file:de.uni_luebeck.inb.knowarc.usecases.invocation.local.LocalUseCaseInvocation.java
@Override protected void submit_generate_job_inner() throws InvocationException { tags.put("uniqueID", "" + getSubmissionID()); String command = usecase.getCommand(); for (String cur : tags.keySet()) { command = command.replaceAll("\\Q%%" + cur + "%%\\E", Matcher.quoteReplacement(tags.get(cur))); }/*w w w.ja va 2 s . c o m*/ List<String> cmds = new ArrayList<String>(); if ((shellPrefix != null) && !shellPrefix.isEmpty()) { String[] prefixCmds = shellPrefix.split(" "); for (int i = 0; i < prefixCmds.length; i++) { cmds.add(prefixCmds[i]); } cmds.add(command); } else { String[] splitCmds = command.split(" "); for (int i = 0; i < splitCmds.length; i++) { cmds.add(splitCmds[i]); } } ProcessBuilder builder = new ProcessBuilder(cmds); builder.directory(tempDir); for (int i = 0; i < cmds.size(); i++) { logger.info("cmds[" + i + "] = " + cmds.get(i)); } logger.info("Command is " + command + " in directory " + tempDir); try { running = builder.start(); if (stdInReader != null) { BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(running.getOutputStream())); IOUtils.copyLarge(stdInReader, writer); writer.close(); } } catch (IOException e) { throw new InvocationException(e); } }
From source file:edu.jhu.pha.vospace.node.ContainerNode.java
protected void tarContainer(String parent, TarOutputStream out) throws IOException { NodesList childrenList = getDirectChildren(false, 0, -1); List<Node> children = childrenList.getNodesList(); parent = parent + getUri().getNodePath().getNodeName() + "/"; for (Node child : children) { if (child.getType() == NodeType.CONTAINER_NODE) { out.putNextEntry(/*w w w. j av a2s . c o m*/ new TarEntry(TarHeader.createHeader(parent + child.getUri().getNodePath().getNodeName(), 0, child.getNodeInfo().getMtime().getTime() / 1000, true))); ((ContainerNode) child).tarContainer(parent, out); } else { try { InputStream nodeInpStream = child.exportData(); out.putNextEntry(new TarEntry(TarHeader.createHeader( parent + child.getUri().getNodePath().getNodeName(), child.getNodeInfo().getSize(), child.getNodeInfo().getMtime().getTime() / 1000, false))); IOUtils.copyLarge(nodeInpStream, out); nodeInpStream.close(); } catch (Exception ex) { } } } }
From source file:autoupdater.FileDAO.java
/** * Writes a stream to disk./*from ww w. j ava 2 s . c o m*/ * * @param in the stream to write to disk * @param name the name the file that will be created * @param outputLocationFolder the location to write to * @return the written file * @throws FileNotFoundException * @throws IOException */ public File writeStreamToDisk(InputStream in, String name, File outputLocationFolder) throws FileNotFoundException, IOException { if (!outputLocationFolder.exists()) { if (!outputLocationFolder.mkdirs()) { throw new IOException("could not create the folders to write stream to disk"); } } File outputFile = new File(outputLocationFolder, name); FileOutputStream out = null; try { out = new FileOutputStream(outputFile); IOUtils.copyLarge(in, out); } finally { if (out != null) { out.close(); } } in.close(); return outputFile; }