List of usage examples for java.util.zip DeflaterOutputStream DeflaterOutputStream
public DeflaterOutputStream(OutputStream out)
From source file:org.apache.hadoop.hdfs.TestDFSShell.java
private void textTest(Path root, Configuration conf) throws Exception { PrintStream bak = null;/*from ww w . j a va2 s . co m*/ try { final FileSystem fs = root.getFileSystem(conf); fs.mkdirs(root); // Test the gzip type of files. Magic detection. OutputStream zout = new GZIPOutputStream(fs.create(new Path(root, "file.gz"))); Random r = new Random(); bak = System.out; ByteArrayOutputStream file = new ByteArrayOutputStream(); for (int i = 0; i < 1024; ++i) { char c = Character.forDigit(r.nextInt(26) + 10, 36); file.write(c); zout.write(c); } zout.close(); ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); String[] argv = new String[2]; argv[0] = "-text"; argv[1] = new Path(root, "file.gz").toString(); int ret = ToolRunner.run(new FsShell(conf), argv); assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret); assertTrue("Output doesn't match input", Arrays.equals(file.toByteArray(), out.toByteArray())); // Create a sequence file with a gz extension, to test proper // container detection. Magic detection. SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(new Path(root, "file.gz")), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(Text.class)); writer.append(new Text("Foo"), new Text("Bar")); writer.close(); out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); argv = new String[2]; argv[0] = "-text"; argv[1] = new Path(root, "file.gz").toString(); ret = ToolRunner.run(new FsShell(conf), argv); assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret); assertTrue("Output doesn't match input", Arrays.equals("Foo\tBar\n".getBytes(), out.toByteArray())); out.reset(); // Test deflate. Extension-based detection. OutputStream dout = new DeflaterOutputStream(fs.create(new Path(root, "file.deflate"))); byte[] outbytes = "foo".getBytes(); dout.write(outbytes); dout.close(); out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); argv = new String[2]; argv[0] = "-text"; argv[1] = new Path(root, "file.deflate").toString(); ret = ToolRunner.run(new FsShell(conf), argv); assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret); assertTrue("Output doesn't match input", Arrays.equals(outbytes, out.toByteArray())); out.reset(); // Test a simple codec. Extension based detection. We use // Bzip2 cause its non-native. CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(BZip2Codec.class, conf); String extension = codec.getDefaultExtension(); Path p = new Path(root, "file." + extension); OutputStream fout = new DataOutputStream(codec.createOutputStream(fs.create(p, true))); byte[] writebytes = "foo".getBytes(); fout.write(writebytes); fout.close(); out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); argv = new String[2]; argv[0] = "-text"; argv[1] = new Path(root, p).toString(); ret = ToolRunner.run(new FsShell(conf), argv); assertEquals("'-text " + argv[1] + " returned " + ret, 0, ret); assertTrue("Output doesn't match input", Arrays.equals(writebytes, out.toByteArray())); out.reset(); } finally { if (null != bak) { System.setOut(bak); } } }
From source file:nl.nn.adapterframework.util.JdbcUtil.java
public static void putStringAsBlob(IDbmsSupport dbmsSupport, final ResultSet rs, int columnIndex, String content, String charset, boolean compressBlob) throws IOException, JdbcException, SQLException { if (content != null) { Object blobHandle = dbmsSupport.getBlobUpdateHandle(rs, columnIndex); OutputStream out = dbmsSupport.getBlobOutputStream(rs, columnIndex, blobHandle); if (charset == null) { charset = Misc.DEFAULT_INPUT_STREAM_ENCODING; }/*from w ww . ja v a 2s. c om*/ if (compressBlob) { DeflaterOutputStream dos = new DeflaterOutputStream(out); dos.write(content.getBytes(charset)); dos.close(); } else { out.write(content.getBytes(charset)); } out.close(); dbmsSupport.updateBlob(rs, columnIndex, blobHandle); } else { log.warn("content to store in blob was null"); } }
From source file:org.apache.flink.api.common.io.GenericCsvInputFormatTest.java
private FileInputSplit createTempDeflateFile(String content) throws IOException { File tempFile = File.createTempFile("test_contents", "tmp.deflate"); tempFile.deleteOnExit();/*from ww w . java 2 s .c o m*/ DataOutputStream dos = new DataOutputStream(new DeflaterOutputStream(new FileOutputStream(tempFile))); dos.writeBytes(content); dos.close(); return new FileInputSplit(0, new Path(tempFile.toURI().toString()), 0, tempFile.length(), new String[] { "localhost" }); }
From source file:nl.nn.adapterframework.util.JdbcUtil.java
public static void putByteArrayAsBlob(IDbmsSupport dbmsSupport, final ResultSet rs, int columnIndex, byte content[], boolean compressBlob) throws IOException, JdbcException, SQLException { if (content != null) { Object blobHandle = dbmsSupport.getBlobUpdateHandle(rs, columnIndex); OutputStream out = dbmsSupport.getBlobOutputStream(rs, columnIndex, blobHandle); if (compressBlob) { DeflaterOutputStream dos = new DeflaterOutputStream(out); dos.write(content);/* w w w . j a v a 2 s .c o m*/ dos.close(); } else { out.write(content); } out.close(); dbmsSupport.updateBlob(rs, columnIndex, blobHandle); } else { log.warn("content to store in blob was null"); } }
From source file:de.burlov.amazon.s3.dirsync.DirSync.java
private void uploadObject(String s3key, byte[] encKey, Serializable obj) throws S3ServiceException { ByteArrayOutputStream bout = new ByteArrayOutputStream(); try {/*from www . j a va2s.c o m*/ ObjectOutputStream oout = new ObjectOutputStream( new DeflaterOutputStream(new CryptOutputStream(bout, cipher, encKey))); oout.writeObject(obj); oout.close(); } catch (IOException e) { /* * sollte eigentlich nie vorkommen */ throw new RuntimeException(e); } byte[] data = bout.toByteArray(); upload(new ByteArrayInputStream(data), s3key, data.length); transferredData += data.length; }
From source file:github.daneren2005.dsub.util.FileUtil.java
public static <T extends Serializable> boolean serializeCompressed(Context context, T obj, String fileName) { Output out = null;//from w ww .j a v a2 s . c o m try { RandomAccessFile file = new RandomAccessFile(context.getCacheDir() + "/" + fileName, "rw"); out = new Output(new DeflaterOutputStream(new FileOutputStream(file.getFD()))); synchronized (kryo) { kryo.writeObject(out, obj); } return true; } catch (Throwable x) { Log.w(TAG, "Failed to serialize compressed object to " + fileName); return false; } finally { Util.close(out); } }
From source file:weave.servlets.GenericServlet.java
protected void serializeCompressedAmf3(Object objToSerialize, ServletOutputStream servletOutputStream) { try {/*from www . j a v a2 s . c om*/ SerializationContext context = getSerializationContext(); DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(servletOutputStream); Amf3Output amf3Output = new Amf3Output(context); amf3Output.setOutputStream(deflaterOutputStream); // compress amf3Output.writeObject(objToSerialize); amf3Output.flush(); deflaterOutputStream.close(); // this is necessary to finish the compression /* * Do not call amf3Output.close() because that will * send a 'reset' packet and cause the response to fail. * * http://viveklakhanpal.wordpress.com/2010/07/01/error-2032ioerror/ */ } catch (Exception e) { e.printStackTrace(); } }
From source file:nl.nn.adapterframework.jdbc.JdbcTransactionalStorage.java
protected String storeMessageInDatabase(Connection conn, String messageId, String correlationId, Timestamp receivedDateTime, String comments, String label, Serializable message) throws IOException, SQLException, JdbcException, SenderException { PreparedStatement stmt = null; try {/*from w w w . jav a2 s . c o m*/ IDbmsSupport dbmsSupport = getDbmsSupport(); if (log.isDebugEnabled()) log.debug("preparing insert statement [" + insertQuery + "]"); if (!dbmsSupport.mustInsertEmptyBlobBeforeData()) { stmt = conn.prepareStatement(insertQuery, Statement.RETURN_GENERATED_KEYS); } else { stmt = conn.prepareStatement(insertQuery); } stmt.clearParameters(); int parPos = 0; if (StringUtils.isNotEmpty(getTypeField())) { stmt.setString(++parPos, type); } if (StringUtils.isNotEmpty(getSlotId())) { stmt.setString(++parPos, getSlotId()); } if (StringUtils.isNotEmpty(getHostField())) { stmt.setString(++parPos, host); } if (StringUtils.isNotEmpty(getLabelField())) { stmt.setString(++parPos, label); } stmt.setString(++parPos, messageId); stmt.setString(++parPos, correlationId); stmt.setTimestamp(++parPos, receivedDateTime); stmt.setString(++parPos, comments); if (type.equalsIgnoreCase(TYPE_MESSAGELOG_PIPE) || type.equalsIgnoreCase(TYPE_MESSAGELOG_RECEIVER)) { if (getRetention() < 0) { stmt.setTimestamp(++parPos, null); } else { Date date = new Date(); Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.add(Calendar.DAY_OF_MONTH, getRetention()); stmt.setTimestamp(++parPos, new Timestamp(cal.getTime().getTime())); } } else { stmt.setTimestamp(++parPos, null); } if (!isStoreFullMessage()) { if (isOnlyStoreWhenMessageIdUnique()) { stmt.setString(++parPos, messageId); stmt.setString(++parPos, slotId); } stmt.execute(); return null; } if (!dbmsSupport.mustInsertEmptyBlobBeforeData()) { ByteArrayOutputStream out = new ByteArrayOutputStream(); if (isBlobsCompressed()) { DeflaterOutputStream dos = new DeflaterOutputStream(out); ObjectOutputStream oos = new ObjectOutputStream(dos); oos.writeObject(message); dos.close(); } else { ObjectOutputStream oos = new ObjectOutputStream(out); oos.writeObject(message); } stmt.setBytes(++parPos, out.toByteArray()); if (isOnlyStoreWhenMessageIdUnique()) { stmt.setString(++parPos, messageId); stmt.setString(++parPos, slotId); } stmt.execute(); ResultSet rs = stmt.getGeneratedKeys(); if (rs.next()) { return rs.getString(1); } else { return null; } } if (isOnlyStoreWhenMessageIdUnique()) { stmt.setString(++parPos, messageId); stmt.setString(++parPos, slotId); } stmt.execute(); int updateCount = stmt.getUpdateCount(); if (log.isDebugEnabled()) log.debug("update count for insert statement: " + updateCount); if (updateCount > 0) { if (log.isDebugEnabled()) log.debug("preparing select statement [" + selectKeyQuery + "]"); stmt = conn.prepareStatement(selectKeyQuery); ResultSet rs = null; try { // retrieve the key rs = stmt.executeQuery(); if (!rs.next()) { throw new SenderException("could not retrieve key of stored message"); } String newKey = rs.getString(1); rs.close(); // and update the blob if (log.isDebugEnabled()) log.debug("preparing update statement [" + updateBlobQuery + "]"); stmt = conn.prepareStatement(updateBlobQuery); stmt.clearParameters(); stmt.setString(1, newKey); rs = stmt.executeQuery(); if (!rs.next()) { throw new SenderException("could not retrieve row for stored message [" + messageId + "]"); } // String newKey = rs.getString(1); // BLOB blob = (BLOB)rs.getBlob(2); Object blobHandle = dbmsSupport.getBlobUpdateHandle(rs, 1); OutputStream out = dbmsSupport.getBlobOutputStream(rs, 1, blobHandle); // OutputStream out = JdbcUtil.getBlobUpdateOutputStream(rs,1); if (isBlobsCompressed()) { DeflaterOutputStream dos = new DeflaterOutputStream(out); ObjectOutputStream oos = new ObjectOutputStream(dos); oos.writeObject(message); oos.close(); dos.close(); } else { ObjectOutputStream oos = new ObjectOutputStream(out); oos.writeObject(message); oos.close(); } out.close(); dbmsSupport.updateBlob(rs, 1, blobHandle); return newKey; } finally { if (rs != null) { rs.close(); } } } else { if (isOnlyStoreWhenMessageIdUnique()) { return "already there"; } else { throw new SenderException( "update count for update statement not greater than 0 [" + updateCount + "]"); } } } finally { if (stmt != null) { stmt.close(); } } }
From source file:org.georchestra.security.Proxy.java
/** * For certain requests (OGC Web services mainly), the charset is absolutely * required. So for certain content types (xml-based normally) this method * is called to detect the charset of the data. This method is a slow way of * transferring data, so data of any significant size should not enter this * method./*from ww w . ja va 2 s . c o m*/ */ private void doHandleRequestCharsetRequired(HttpServletRequest orignalRequest, HttpServletResponse finalResponse, RequestType requestType, HttpResponse proxiedResponse, String contentType) { InputStream streamFromServer = null; OutputStream streamToClient = null; try { /* * Here comes the tricky part because some host send files without * the charset in the header, therefore we do not know how they are * text encoded. It can result in serious issues on IE browsers when * parsing those files. There is a workaround which consists to read * the encoding within the file. It is made possible because this * proxy mainly forwards xml files. They all have the encoding * attribute in the first xml node. * * This is implemented as follows: * * A. The content type provides a charset: Nothing special, just * send back the stream to the client B. There is no charset * provided: The encoding has to be extracted from the file. The * file is read in ASCII, which is common to many charsets, like * that the encoding located in the first not can be retrieved. Once * the charset is found, the content-type header is overridden and * the charset is appended. * * /!\ Special case: whenever data are compressed in gzip/deflate * the stream has to be uncompressed and re-compressed */ boolean isCharsetKnown = proxiedResponse.getEntity().getContentType().getValue().toLowerCase() .contains("charset"); // String contentEncoding = // getContentEncoding(proxiedResponse.getAllHeaders()); String contentEncoding = getContentEncoding(proxiedResponse.getHeaders("Content-Encoding")); if (logger.isDebugEnabled()) { String cskString = "\tisCharSetKnown=" + isCharsetKnown; String cEString = "\tcontentEncoding=" + contentEncoding; logger.debug("Charset is required so verifying that it has been added to the headers\n" + cskString + "\n" + cEString); } if (contentEncoding == null || isCharsetKnown) { // A simple stream can do the job for data that is not in // content encoded // but also for data content encoded with a known charset streamFromServer = proxiedResponse.getEntity().getContent(); streamToClient = finalResponse.getOutputStream(); } else if (!isCharsetKnown && ("gzip".equalsIgnoreCase(contentEncoding) || "x-gzip".equalsIgnoreCase(contentEncoding))) { // the charset is unknown and the data are compressed in gzip // we add the gzip wrapper to be able to read/write the stream // content streamFromServer = new GZIPInputStream(proxiedResponse.getEntity().getContent()); streamToClient = new GZIPOutputStream(finalResponse.getOutputStream()); } else if ("deflate".equalsIgnoreCase(contentEncoding) && !isCharsetKnown) { // same but with deflate streamFromServer = new DeflaterInputStream(proxiedResponse.getEntity().getContent()); streamToClient = new DeflaterOutputStream(finalResponse.getOutputStream()); } else { doHandleRequest(orignalRequest, finalResponse, requestType, proxiedResponse); return; } byte[] buf = new byte[1024]; // read maximum 1024 bytes int len; // number of bytes read from the stream boolean first = true; // helps to find the encoding once and only // once String s = ""; // piece of file that should contain the encoding while ((len = streamFromServer.read(buf)) > 0) { if (first && !isCharsetKnown) { // charset is unknown try to find it in the file content for (int i = 0; i < len; i++) { s += (char) buf[i]; // get the beginning of the file as // ASCII } // s has to be long enough to contain the encoding if (s.length() > 200) { if (logger.isTraceEnabled()) { logger.trace("attempting to read charset from: " + s); } String charset = getCharset(s); // extract charset if (charset == null) { if (logger.isTraceEnabled()) { logger.trace("unable to find charset from raw ASCII data. Trying to unzip it"); } // the charset cannot be found, IE users must be // warned // that the request cannot be fulfilled, nothing // good would happen otherwise } if (charset == null) { String guessedCharset = null; if (logger.isDebugEnabled()) { logger.debug( "unable to find charset so using the first one from the accept-charset request header"); } String calculateDefaultCharset = calculateDefaultCharset(orignalRequest); if (calculateDefaultCharset != null) { guessedCharset = calculateDefaultCharset; if (logger.isDebugEnabled()) { logger.debug("hopefully the server responded with this charset: " + calculateDefaultCharset); } } else { guessedCharset = defaultCharset; if (logger.isDebugEnabled()) { logger.debug("unable to find charset, so using default:" + defaultCharset); } } String adjustedContentType = proxiedResponse.getEntity().getContentType().getValue() + ";charset=" + guessedCharset; finalResponse.setHeader("Content-Type", adjustedContentType); first = false; // we found the encoding, don't try // to do it again finalResponse.setCharacterEncoding(guessedCharset); } else { if (logger.isDebugEnabled()) { logger.debug("found charset: " + charset); } String adjustedContentType = proxiedResponse.getEntity().getContentType().getValue() + ";charset=" + charset; finalResponse.setHeader("Content-Type", adjustedContentType); first = false; // we found the encoding, don't try // to do it again finalResponse.setCharacterEncoding(charset); } } } // for everyone, the stream is just forwarded to the client streamToClient.write(buf, 0, len); } } catch (IOException e) { // connection problem with the host e.printStackTrace(); } finally { IOException exc = close(streamFromServer); exc = close(streamToClient, exc); if (exc != null) { logger.error("Error closing streams", exc); } } }