List of usage examples for java.util.zip GZIPOutputStream write
public synchronized void write(byte[] buf, int off, int len) throws IOException
From source file:es.mityc.firmaJava.libreria.utilidades.Base64.java
/** * Encodes a byte array into Base64 notation. * <p>//from w w w .j a v a2 s .c o m * Valid options:<pre> * GZIP: gzip-compresses object before encoding it. * DONT_BREAK_LINES: don't break lines at 76 characters * <i>Note: Technically, this makes your encoding non-compliant.</i> * </pre> * <p> * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or * <p> * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code> * * * @param source The data to convert * @param off Offset in array where conversion should begin * @param len Length of data to convert * @param options Specified options * @param options alphabet type is pulled from this (standard, url-safe, ordered) * @see Base64#GZIP * @see Base64#DONT_BREAK_LINES * @since 2.0 */ public static String encodeBytes(byte[] source, int off, int len, int options) { // Isolate options int dontBreakLines = (options & ConstantesXADES.DONT_BREAK_LINES); int gzip = (options & ConstantesXADES.GZIP); // Compress? if (gzip == ConstantesXADES.GZIP) { ByteArrayOutputStream baos = null; GZIPOutputStream gzos = null; Base64.OutputStream b64os = null; try { // GZip -> Base64 -> ByteArray baos = new ByteArrayOutputStream(); b64os = new Base64.OutputStream(baos, ConstantesXADES.ENCODE | options); gzos = new GZIPOutputStream(b64os); gzos.write(source, off, len); //gzos.close(); } // end try catch (IOException e) { log.error(e); return null; } // end catch finally { try { gzos.close(); } catch (Exception e) { log.error(e); } try { b64os.close(); } catch (Exception e) { log.error(e); } try { baos.close(); } catch (Exception e) { log.error(e); } } // end finally // Return value according to relevant encoding. try { return new String(baos.toByteArray(), PREFERRED_ENCODING); } // end try catch (UnsupportedEncodingException uue) { return new String(baos.toByteArray()); } // end catch } // end if: compress // Else, don't compress. Better not to use streams at all then. else { // Convert option to boolean in way that code likes it. boolean breakLines = dontBreakLines == 0; int len43 = len * 4 / 3; byte[] outBuff = new byte[(len43) // Main 4:3 + ((len % 3) > 0 ? 4 : 0) // Account for padding + (breakLines ? (len43 / MAX_LINE_LENGTH) : 0)]; // New lines int d = 0; int e = 0; int len2 = len - 2; int lineLength = 0; int maxLineLength = MAX_LINE_LENGTH; byte newLine = NEW_LINE; // for( ; d < len2; d+=3, e+=4 ) // { // encode3to4( source, d+off, 3, outBuff, e, options ); // // lineLength += 4; // if( breakLines && lineLength == MAX_LINE_LENGTH ) // { // outBuff[e+4] = NEW_LINE; // e++; // lineLength = 0; // } // end if: end of line // } // en dfor: each piece of array while (d < len2) { encode3to4(source, d + off, 3, outBuff, e, options); lineLength += 4; if (breakLines && lineLength == maxLineLength) { outBuff[e + 4] = newLine; e++; lineLength = 0; } // end if: end of line d += 3; e += 4; } if (d < len) { encode3to4(source, d + off, len - d, outBuff, e, options); e += 4; } // end if: some padding needed // Return value according to relevant encoding. try { return new String(outBuff, 0, e, PREFERRED_ENCODING); } // end try catch (UnsupportedEncodingException uue) { return new String(outBuff, 0, e); } // end catch } // end else: don't compress }
From source file:it.drwolf.ridire.session.async.Mapper.java
private void createArchivedResource(File f, CrawledResource cr, EntityManager entityManager) { // System.out.println(System.getProperty("java.io.tmpdir")); String posEnabled = this.em.find(Parameter.class, Parameter.POS_ENABLED.getKey()).getValue(); File resourceDir;//w ww. ja va 2 s. com int status = Parameter.FINISHED; try { resourceDir = new File(FilenameUtils.getFullPath(f.getCanonicalPath().replaceAll("__\\d+", "")) + JobMapperMonitor.RESOURCESDIR); if (!resourceDir.exists()) { FileUtils.forceMkdir(resourceDir); } ArchiveReader reader = ArchiveReaderFactory.get(f); ARCRecord record = (ARCRecord) reader.get(cr.getOffset()); record.skipHttpHeader(); byte[] buf = new byte[Mapper.BUFLENGTH]; int count = 0; String resourceFile = cr.getDigest() + ".gz"; GZIPOutputStream baos = new GZIPOutputStream(new FileOutputStream(new File(resourceDir, resourceFile))); while ((count = record.read(buf)) != -1) { baos.write(buf, 0, count); } baos.finish(); baos.close(); reader.close(); // long t1 = System.currentTimeMillis(); StringWithEncoding cleanText = this.createPlainTextResource(f, cr, entityManager); this.removeGZippedResource(resourceDir, resourceFile); // long t2 = System.currentTimeMillis(); // System.out.println("Creazione plain text: " + (t2 - t1)); String plainTextFileName = cr.getDigest() + ".txt"; if (cleanText != null && cleanText.getString() != null && cleanText.getString().trim().length() > 0 && cleanText.getCleaner() != null && (cleanText.getCleaner().equals(Mapper.ALCHEMY) || cleanText.getCleaner().equals(Mapper.READABILITY))) { cr.setCleaner(cleanText.getCleaner()); File plainTextFile = new File(resourceDir, plainTextFileName); FileUtils.writeStringToFile(plainTextFile, cleanText.getString(), cleanText.getEncoding()); cr.setExtractedTextHash(MD5DigestCreator.getMD5Digest(plainTextFile)); // language detection // t1 = System.currentTimeMillis(); String language = this.detectLanguage(cleanText.getString()); // t2 = System.currentTimeMillis(); // System.out.println("Language detection: " + (t2 - t1)); cr.setLanguage(language); if (language != null && language.equalsIgnoreCase(Mapper.ITALIAN) && posEnabled != null && posEnabled.equalsIgnoreCase("true")) { // PoS tag if it's an italian text // t1 = System.currentTimeMillis(); String posTagResourceFileName = this.createPoSTagResource(plainTextFile, entityManager, cleanText.getEncoding()); // t2 = System.currentTimeMillis(); // System.out.println("PoS tagging: " + (t2 - t1)); if (posTagResourceFileName != null) { Integer wordsNumber = Mapper.countWordsFromPoSTagResource(posTagResourceFileName); cr.setWordsNumber(wordsNumber); } } } } catch (Exception e) { status = Parameter.PROCESSING_ERROR; e.printStackTrace(); } cr.setProcessed(status); }
From source file:org.kriand.warpdrive.processors.bundles.BundleProcessor.java
/** * Creates a set of configured bundles./*from w w w .ja v a2 s . com*/ * * @param bundles The bundles to create. The keys will be the bundle filenames. * The values are comma-separated lists containing the files in each bundle. * @param bundleDir The directory where the bundle will be created. * @throws IOException If the bundles can not be created. */ private void createBundlesInDir(final Map<String, String> bundles, final String bundleDir) throws IOException { assert bundleDir == null : "Bundledir was null"; if (bundles == null || bundles.size() == 0) { getLog().info(String.format("No bundles configured in directory: %s", bundleDir)); return; } for (Map.Entry<String, String> bundleEntry : bundles.entrySet()) { final String filenameWithVersion = FilenameUtils.insertVersion(bundleDir + bundleEntry.getKey(), getMojo().getVersion()); final String filenameWithVersionAndGzipExtension = FilenameUtils .insertVersionAndGzipExtension(bundleDir + bundleEntry.getKey(), getMojo().getVersion()); final File outputFile = new File(getMojo().getWebappTargetDir() + filenameWithVersion); final File gzippedOutputFile = new File( getMojo().getWebappTargetDir() + filenameWithVersionAndGzipExtension); FileOutputStream output = null; GZIPOutputStream zippedOutput = null; try { output = new FileOutputStream(outputFile); zippedOutput = new GZIPOutputStream(new FileOutputStream(gzippedOutputFile)); final String files = bundleEntry.getValue(); for (String file : files.split(org.kriand.warpdrive.Runtime.MULTIVAL_SEPARATOR)) { FileInputStream fis = null; try { file = file.trim(); final String versionedFile = FilenameUtils.insertVersion(bundleDir + file, getMojo().getVersion()); final File f = new File(getMojo().getWebappTargetDir() + versionedFile); fis = new FileInputStream(f); byte[] buf = new byte[WarpDriveMojo.WRITE_BUFFER_SIZE]; int read = 0; while ((read = fis.read(buf)) != -1) { output.write(buf, 0, read); zippedOutput.write(buf, 0, read); } } finally { IOUtils.closeQuietly(fis); } } } finally { IOUtils.closeQuietly(output); IOUtils.closeQuietly(zippedOutput); } } }
From source file:com.ichi2.libanki.sync.BasicHttpSyncer.java
public HttpResponse req(String method, InputStream fobj, int comp, boolean hkey, JSONObject registerData, Connection.CancelCallback cancelCallback) { File tmpFileBuffer = null;/*from www. jav a 2s.c o m*/ try { String bdry = "--" + BOUNDARY; StringWriter buf = new StringWriter(); // compression flag and session key as post vars buf.write(bdry + "\r\n"); buf.write("Content-Disposition: form-data; name=\"c\"\r\n\r\n" + (comp != 0 ? 1 : 0) + "\r\n"); if (hkey) { buf.write(bdry + "\r\n"); buf.write("Content-Disposition: form-data; name=\"k\"\r\n\r\n" + mHKey + "\r\n"); } tmpFileBuffer = File.createTempFile("syncer", ".tmp", new File(AnkiDroidApp.getCacheStorageDirectory())); FileOutputStream fos = new FileOutputStream(tmpFileBuffer); BufferedOutputStream bos = new BufferedOutputStream(fos); GZIPOutputStream tgt; // payload as raw data or json if (fobj != null) { // header buf.write(bdry + "\r\n"); buf.write( "Content-Disposition: form-data; name=\"data\"; filename=\"data\"\r\nContent-Type: application/octet-stream\r\n\r\n"); buf.close(); bos.write(buf.toString().getBytes("UTF-8")); // write file into buffer, optionally compressing int len; BufferedInputStream bfobj = new BufferedInputStream(fobj); byte[] chunk = new byte[65536]; if (comp != 0) { tgt = new GZIPOutputStream(bos); while ((len = bfobj.read(chunk)) >= 0) { tgt.write(chunk, 0, len); } tgt.close(); bos = new BufferedOutputStream(new FileOutputStream(tmpFileBuffer, true)); } else { while ((len = bfobj.read(chunk)) >= 0) { bos.write(chunk, 0, len); } } bos.write(("\r\n" + bdry + "--\r\n").getBytes("UTF-8")); } else { buf.close(); bos.write(buf.toString().getBytes("UTF-8")); } bos.flush(); bos.close(); // connection headers String url = Collection.SYNC_URL; if (method.equals("register")) { url = url + "account/signup" + "?username=" + registerData.getString("u") + "&password=" + registerData.getString("p"); } else if (method.startsWith("upgrade")) { url = url + method; } else { url = url + "sync/" + method; } HttpPost httpPost = new HttpPost(url); HttpEntity entity = new ProgressByteEntity(tmpFileBuffer); // body httpPost.setEntity(entity); httpPost.setHeader("Content-type", "multipart/form-data; boundary=" + BOUNDARY); // HttpParams HttpParams params = new BasicHttpParams(); params.setParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 30); params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRouteBean(30)); params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false); params.setParameter(CoreProtocolPNames.USER_AGENT, "AnkiDroid-" + AnkiDroidApp.getPkgVersion()); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpConnectionParams.setSoTimeout(params, Connection.CONN_TIMEOUT); // Registry SchemeRegistry registry = new SchemeRegistry(); registry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); registry.register(new Scheme("https", new EasySSLSocketFactory(), 443)); ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, registry); if (cancelCallback != null) { cancelCallback.setConnectionManager(cm); } try { HttpClient httpClient = new DefaultHttpClient(cm, params); return httpClient.execute(httpPost); } catch (SSLException e) { Log.e(AnkiDroidApp.TAG, "SSLException while building HttpClient", e); return null; } } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } catch (IOException e) { Log.e(AnkiDroidApp.TAG, "BasicHttpSyncer.sync: IOException", e); return null; } catch (JSONException e) { throw new RuntimeException(e); } finally { if (tmpFileBuffer != null && tmpFileBuffer.exists()) { tmpFileBuffer.delete(); } } }
From source file:com.hichinaschool.flashcards.libanki.sync.BasicHttpSyncer.java
public HttpResponse req(String method, InputStream fobj, int comp, boolean hkey, JSONObject registerData, Connection.CancelCallback cancelCallback) { File tmpFileBuffer = null;/* ww w .java 2 s . c o m*/ try { String bdry = "--" + BOUNDARY; StringWriter buf = new StringWriter(); HashMap<String, Object> vars = new HashMap<String, Object>(); // compression flag and session key as post vars vars.put("c", comp != 0 ? 1 : 0); if (hkey) { vars.put("k", mHKey); vars.put("s", mSKey); } for (String key : vars.keySet()) { buf.write(bdry + "\r\n"); buf.write(String.format(Locale.US, "Content-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n", key, vars.get(key))); } tmpFileBuffer = File.createTempFile("syncer", ".tmp", new File(AnkiDroidApp.getCacheStorageDirectory())); FileOutputStream fos = new FileOutputStream(tmpFileBuffer); BufferedOutputStream bos = new BufferedOutputStream(fos); GZIPOutputStream tgt; // payload as raw data or json if (fobj != null) { // header buf.write(bdry + "\r\n"); buf.write( "Content-Disposition: form-data; name=\"data\"; filename=\"data\"\r\nContent-Type: application/octet-stream\r\n\r\n"); buf.close(); bos.write(buf.toString().getBytes("UTF-8")); // write file into buffer, optionally compressing int len; BufferedInputStream bfobj = new BufferedInputStream(fobj); byte[] chunk = new byte[65536]; if (comp != 0) { tgt = new GZIPOutputStream(bos); while ((len = bfobj.read(chunk)) >= 0) { tgt.write(chunk, 0, len); } tgt.close(); bos = new BufferedOutputStream(new FileOutputStream(tmpFileBuffer, true)); } else { while ((len = bfobj.read(chunk)) >= 0) { bos.write(chunk, 0, len); } } bos.write(("\r\n" + bdry + "--\r\n").getBytes("UTF-8")); } else { buf.close(); bos.write(buf.toString().getBytes("UTF-8")); } bos.flush(); bos.close(); // connection headers String url = Collection.SYNC_URL; if (method.equals("register")) { url = url + "account/signup" + "?username=" + registerData.getString("u") + "&password=" + registerData.getString("p"); } else if (method.startsWith("upgrade")) { url = url + method; } else { url = url + "sync/" + method; } HttpPost httpPost = new HttpPost(url); HttpEntity entity = new ProgressByteEntity(tmpFileBuffer); // body httpPost.setEntity(entity); httpPost.setHeader("Content-type", "multipart/form-data; boundary=" + BOUNDARY); // HttpParams HttpParams params = new BasicHttpParams(); params.setParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 30); params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRouteBean(30)); params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false); params.setParameter(CoreProtocolPNames.USER_AGENT, "AnkiDroid-" + AnkiDroidApp.getPkgVersionName()); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpConnectionParams.setSoTimeout(params, Connection.CONN_TIMEOUT); // Registry SchemeRegistry registry = new SchemeRegistry(); registry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); registry.register(new Scheme("https", new EasySSLSocketFactory(), 443)); ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, registry); if (cancelCallback != null) { cancelCallback.setConnectionManager(cm); } try { HttpClient httpClient = new DefaultHttpClient(cm, params); return httpClient.execute(httpPost); } catch (SSLException e) { Log.e(AnkiDroidApp.TAG, "SSLException while building HttpClient", e); return null; } } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } catch (IOException e) { Log.e(AnkiDroidApp.TAG, "BasicHttpSyncer.sync: IOException", e); return null; } catch (JSONException e) { throw new RuntimeException(e); } finally { if (tmpFileBuffer != null && tmpFileBuffer.exists()) { tmpFileBuffer.delete(); } } }
From source file:com.ichi2.libanki.sync.HttpSyncer.java
public HttpResponse req(String method, InputStream fobj, int comp, JSONObject registerData, Connection.CancelCallback cancelCallback) throws UnknownHttpResponseException { File tmpFileBuffer = null;/* w w w. ja v a 2s . co m*/ try { String bdry = "--" + BOUNDARY; StringWriter buf = new StringWriter(); // post vars mPostVars.put("c", comp != 0 ? 1 : 0); for (String key : mPostVars.keySet()) { buf.write(bdry + "\r\n"); buf.write(String.format(Locale.US, "Content-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n", key, mPostVars.get(key))); } tmpFileBuffer = File.createTempFile("syncer", ".tmp", new File(AnkiDroidApp.getCacheStorageDirectory())); FileOutputStream fos = new FileOutputStream(tmpFileBuffer); BufferedOutputStream bos = new BufferedOutputStream(fos); GZIPOutputStream tgt; // payload as raw data or json if (fobj != null) { // header buf.write(bdry + "\r\n"); buf.write( "Content-Disposition: form-data; name=\"data\"; filename=\"data\"\r\nContent-Type: application/octet-stream\r\n\r\n"); buf.close(); bos.write(buf.toString().getBytes("UTF-8")); // write file into buffer, optionally compressing int len; BufferedInputStream bfobj = new BufferedInputStream(fobj); byte[] chunk = new byte[65536]; if (comp != 0) { tgt = new GZIPOutputStream(bos); while ((len = bfobj.read(chunk)) >= 0) { tgt.write(chunk, 0, len); } tgt.close(); bos = new BufferedOutputStream(new FileOutputStream(tmpFileBuffer, true)); } else { while ((len = bfobj.read(chunk)) >= 0) { bos.write(chunk, 0, len); } } bos.write(("\r\n" + bdry + "--\r\n").getBytes("UTF-8")); } else { buf.close(); bos.write(buf.toString().getBytes("UTF-8")); } bos.flush(); bos.close(); // connection headers String url = Consts.SYNC_BASE; if (method.equals("register")) { url = url + "account/signup" + "?username=" + registerData.getString("u") + "&password=" + registerData.getString("p"); } else if (method.startsWith("upgrade")) { url = url + method; } else { url = syncURL() + method; } HttpPost httpPost = new HttpPost(url); HttpEntity entity = new ProgressByteEntity(tmpFileBuffer); // body httpPost.setEntity(entity); httpPost.setHeader("Content-type", "multipart/form-data; boundary=" + BOUNDARY); // HttpParams HttpParams params = new BasicHttpParams(); params.setParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 30); params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRouteBean(30)); params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false); params.setParameter(CoreProtocolPNames.USER_AGENT, "AnkiDroid-" + VersionUtils.getPkgVersionName()); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpConnectionParams.setSoTimeout(params, Connection.CONN_TIMEOUT); // Registry SchemeRegistry registry = new SchemeRegistry(); registry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); registry.register(new Scheme("https", new EasySSLSocketFactory(), 443)); ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, registry); if (cancelCallback != null) { cancelCallback.setConnectionManager(cm); } try { HttpClient httpClient = new DefaultHttpClient(cm, params); HttpResponse httpResponse = httpClient.execute(httpPost); // we assume badAuthRaises flag from Anki Desktop always False // so just throw new RuntimeException if response code not 200 or 403 assertOk(httpResponse); return httpResponse; } catch (SSLException e) { Timber.e(e, "SSLException while building HttpClient"); throw new RuntimeException("SSLException while building HttpClient"); } } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } catch (IOException e) { Timber.e(e, "BasicHttpSyncer.sync: IOException"); throw new RuntimeException(e); } catch (JSONException e) { throw new RuntimeException(e); } finally { if (tmpFileBuffer != null && tmpFileBuffer.exists()) { tmpFileBuffer.delete(); } } }
From source file:website.openeng.libanki.sync.HttpSyncer.java
public HttpResponse req(String method, InputStream fobj, int comp, JSONObject registerData, Connection.CancelCallback cancelCallback) throws UnknownHttpResponseException { File tmpFileBuffer = null;/* w w w . j a v a 2 s .com*/ try { String bdry = "--" + BOUNDARY; StringWriter buf = new StringWriter(); // post vars mPostVars.put("c", comp != 0 ? 1 : 0); for (String key : mPostVars.keySet()) { buf.write(bdry + "\r\n"); buf.write(String.format(Locale.US, "Content-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n", key, mPostVars.get(key))); } tmpFileBuffer = File.createTempFile("syncer", ".tmp", new File(KanjiDroidApp.getCacheStorageDirectory())); FileOutputStream fos = new FileOutputStream(tmpFileBuffer); BufferedOutputStream bos = new BufferedOutputStream(fos); GZIPOutputStream tgt; // payload as raw data or json if (fobj != null) { // header buf.write(bdry + "\r\n"); buf.write( "Content-Disposition: form-data; name=\"data\"; filename=\"data\"\r\nContent-Type: application/octet-stream\r\n\r\n"); buf.close(); bos.write(buf.toString().getBytes("UTF-8")); // write file into buffer, optionally compressing int len; BufferedInputStream bfobj = new BufferedInputStream(fobj); byte[] chunk = new byte[65536]; if (comp != 0) { tgt = new GZIPOutputStream(bos); while ((len = bfobj.read(chunk)) >= 0) { tgt.write(chunk, 0, len); } tgt.close(); bos = new BufferedOutputStream(new FileOutputStream(tmpFileBuffer, true)); } else { while ((len = bfobj.read(chunk)) >= 0) { bos.write(chunk, 0, len); } } bos.write(("\r\n" + bdry + "--\r\n").getBytes("UTF-8")); } else { buf.close(); bos.write(buf.toString().getBytes("UTF-8")); } bos.flush(); bos.close(); // connection headers String url = Consts.SYNC_BASE; if (method.equals("register")) { url = url + "account/signup" + "?username=" + registerData.getString("u") + "&password=" + registerData.getString("p"); } else if (method.startsWith("upgrade")) { url = url + method; } else { url = syncURL() + method; } HttpPost httpPost = new HttpPost(url); HttpEntity entity = new ProgressByteEntity(tmpFileBuffer); // body httpPost.setEntity(entity); httpPost.setHeader("Content-type", "multipart/form-data; boundary=" + BOUNDARY); // HttpParams HttpParams params = new BasicHttpParams(); params.setParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 30); params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRouteBean(30)); params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false); params.setParameter(CoreProtocolPNames.USER_AGENT, "KanjiDroid-" + VersionUtils.getPkgVersionName()); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpConnectionParams.setSoTimeout(params, Connection.CONN_TIMEOUT); // Registry SchemeRegistry registry = new SchemeRegistry(); registry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); registry.register(new Scheme("https", new EasySSLSocketFactory(), 443)); ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, registry); if (cancelCallback != null) { cancelCallback.setConnectionManager(cm); } try { HttpClient httpClient = new DefaultHttpClient(cm, params); HttpResponse httpResponse = httpClient.execute(httpPost); // we assume badAuthRaises flag from Anki Desktop always False // so just throw new RuntimeException if response code not 200 or 403 assertOk(httpResponse); return httpResponse; } catch (SSLException e) { Timber.e(e, "SSLException while building HttpClient"); throw new RuntimeException("SSLException while building HttpClient"); } } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } catch (IOException e) { Timber.e(e, "BasicHttpSyncer.sync: IOException"); throw new RuntimeException(e); } catch (JSONException e) { throw new RuntimeException(e); } finally { if (tmpFileBuffer != null && tmpFileBuffer.exists()) { tmpFileBuffer.delete(); } } }
From source file:com.funambol.transport.http.server.Sync4jServlet.java
/** * Sets the content of HTTP response.// w w w. jav a 2s . c o m * * Compresses the response if the Accept-Encoding is gzip or deflate. * Sets the Content-Encoding according to the encoding used. * Sets the Content-Length with the length of the compressed response. * Sets the Uncompressed-Content-Length with the length of the uncompressed * response. * The response will be compressed only if the length of the uncompressed * response is greater than the give sizeThreashold. * * @param httpResponse the HttpServletResponse * @param requestAcceptEncoding the <code>Accept-Encoding</code> specified * in the request * @param sizeThreshold if the response is smaller of this value, it * should not be compressed * @param resp the SyncResponse object contains the response message * @param requestTime the time in which the request is arrived to servlet * @param sessionId the session identifier * @throws java.io.IOException if an error occurs * */ private void setResponseContent(HttpServletResponse httpResponse, String requestAcceptEncoding, String sizeThreshold, SyncResponse resp, long requestTime, String sessionId) throws IOException { byte[] responseContent = null; OutputStream out = null; try { out = httpResponse.getOutputStream(); responseContent = resp.getMessage(); int uncompressedContentLength = responseContent.length; if (supportedEncoding != null && !"".equals(supportedEncoding) && enableCompression) { if (log.isTraceEnabled()) { log.trace("Setting Accept-Encoding to " + supportedEncoding); } httpResponse.setHeader(HEADER_ACCEPT_ENCODING, supportedEncoding); } String encodingToUse = null; if (requestAcceptEncoding != null) { if (requestAcceptEncoding.indexOf(COMPRESSION_TYPE_GZIP) != -1 && requestAcceptEncoding.indexOf(COMPRESSION_TYPE_DEFLATE) != -1) { encodingToUse = preferredEncoding; } else if (requestAcceptEncoding.indexOf(COMPRESSION_TYPE_DEFLATE) != -1) { encodingToUse = COMPRESSION_TYPE_DEFLATE; } else if (requestAcceptEncoding.indexOf(COMPRESSION_TYPE_GZIP) != -1) { encodingToUse = COMPRESSION_TYPE_GZIP; } } int threshold = 0; try { if (sizeThreshold != null && sizeThreshold.length() != 0) { threshold = Integer.parseInt(sizeThreshold); } } catch (NumberFormatException ex) { // // Ignoring the specified value // if (log.isTraceEnabled()) { log.trace("The size threshold specified by the client (" + sizeThreshold + ") is not valid."); } } // // If the encodingToUse is null or the // uncompressed response length is less than // sizeThreshold, the response will not be compressed. // if (encodingToUse == null || uncompressedContentLength < threshold) { if (log.isTraceEnabled()) { if (enableCompression) { if (requestAcceptEncoding == null) { log.trace( "The client doesn't support any encoding. " + "The response is not compressed"); } else if (encodingToUse == null) { log.trace("The specified Accept-Encoding (" + requestAcceptEncoding + ") is not recognized. The response is not compressed"); } else if (uncompressedContentLength < threshold) { log.trace("The response is not compressed because smaller than " + threshold); } } } if (log.isTraceEnabled()) { log.trace("Setting Content-Length to: " + uncompressedContentLength); } httpResponse.setContentLength(uncompressedContentLength); out.write(responseContent); out.flush(); return; } if (encodingToUse != null) { if (log.isTraceEnabled()) { log.trace("Compressing the response using: " + encodingToUse); log.trace("Setting Uncompressed-Content-Length to: " + uncompressedContentLength); } httpResponse.setHeader(HEADER_UNCOMPRESSED_CONTENT_LENGTH, String.valueOf(uncompressedContentLength)); if (encodingToUse.equals(COMPRESSION_TYPE_GZIP)) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); GZIPOutputStream outTmp = new GZIPOutputStream(bos); outTmp.write(responseContent, 0, uncompressedContentLength); outTmp.flush(); outTmp.close(); // // Get the compressed data // responseContent = bos.toByteArray(); int compressedLength = responseContent.length; if (log.isTraceEnabled()) { log.trace("Setting Content-Length to: " + compressedLength); log.trace("Setting Content-Encoding to: " + COMPRESSION_TYPE_GZIP); } httpResponse.setContentLength(compressedLength); httpResponse.setHeader(HEADER_CONTENT_ENCODING, COMPRESSION_TYPE_GZIP); out.write(responseContent); out.flush(); } else if (encodingToUse.equals(COMPRESSION_TYPE_DEFLATE)) { // // Create the compressor with specificated level of compression // Deflater compressor = new Deflater(); compressor.setLevel(compressionLevel); compressor.setInput(responseContent); compressor.finish(); // // Create an expandable byte array to hold the compressed data. // You cannot use an array that's the same size as the orginal because // there is no guarantee that the compressed data will be smaller than // the uncompressed data. // ByteArrayOutputStream bos = new ByteArrayOutputStream(uncompressedContentLength); // // Compress the response // byte[] buf = new byte[SIZE_INPUT_BUFFER]; while (!compressor.finished()) { int count = compressor.deflate(buf); bos.write(buf, 0, count); } // // Get the compressed data // responseContent = bos.toByteArray(); int compressedLength = responseContent.length; if (log.isTraceEnabled()) { log.trace("Setting Content-Length to: " + compressedLength); log.trace("Setting Content-Encoding to: " + COMPRESSION_TYPE_DEFLATE); } httpResponse.setContentLength(compressedLength); httpResponse.setHeader(HEADER_CONTENT_ENCODING, COMPRESSION_TYPE_DEFLATE); out.write(responseContent); out.flush(); } } } finally { if (out != null) { out.close(); } if (logMessages) { logResponse(responseContent, requestTime, sessionId); } } }
From source file:com.eucalyptus.blockstorage.S3SnapshotTransfer.java
/** * Compresses the snapshot and uploads it to a bucket in objectstorage gateway as a single or multipart upload based on the configuration in * {@link StorageInfo}. Bucket name should be configured before invoking this method. It can be looked up and initialized by {@link #prepareForUpload()} or * explicitly set using {@link #setBucketName(String)} * //from w ww.j a v a 2 s. c o m * @param sourceFileName * absolute path to the snapshot on the file system */ @Override public void upload(String sourceFileName) throws SnapshotTransferException { validateInput(); // Validate input loadTransferConfig(); // Load the transfer configuration parameters from database SnapshotProgressCallback progressCallback = new SnapshotProgressCallback(snapshotId); // Setup the progress callback Boolean error = Boolean.FALSE; ArrayBlockingQueue<SnapshotPart> partQueue = null; SnapshotPart part = null; SnapshotUploadInfo snapUploadInfo = null; Future<List<PartETag>> uploadPartsFuture = null; Future<String> completeUploadFuture = null; byte[] buffer = new byte[READ_BUFFER_SIZE]; Long readOffset = 0L; Long bytesRead = 0L; Long bytesWritten = 0L; int len; int partNumber = 1; try { // Get the uncompressed file size for uploading as metadata Long uncompressedSize = getFileSize(sourceFileName); // Setup the snapshot and part entities. snapUploadInfo = SnapshotUploadInfo.create(snapshotId, bucketName, keyName); Path zipFilePath = Files.createTempFile(keyName + '-', '-' + String.valueOf(partNumber)); part = SnapshotPart.createPart(snapUploadInfo, zipFilePath.toString(), partNumber, readOffset); FileInputStream inputStream = new FileInputStream(sourceFileName); ByteArrayOutputStream baos = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream(baos); FileOutputStream outputStream = new FileOutputStream(zipFilePath.toString()); try { LOG.debug("Reading snapshot " + snapshotId + " and compressing it to disk in chunks of size " + partSize + " bytes or greater"); while ((len = inputStream.read(buffer)) > 0) { bytesRead += len; gzipStream.write(buffer, 0, len); if ((bytesWritten + baos.size()) < partSize) { baos.writeTo(outputStream); bytesWritten += baos.size(); baos.reset(); } else { gzipStream.close(); baos.writeTo(outputStream); // Order is important. Closing the gzip stream flushes stuff bytesWritten += baos.size(); baos.reset(); outputStream.close(); if (partNumber > 1) {// Update the part status part = part.updateStateCreated(bytesWritten, bytesRead, Boolean.FALSE); } else {// Initialize multipart upload only once after the first part is created LOG.info("Uploading snapshot " + snapshotId + " to objectstorage using multipart upload"); progressCallback.setUploadSize(uncompressedSize); uploadId = initiateMulitpartUpload(uncompressedSize); snapUploadInfo = snapUploadInfo.updateUploadId(uploadId); part = part.updateStateCreated(uploadId, bytesWritten, bytesRead, Boolean.FALSE); partQueue = new ArrayBlockingQueue<SnapshotPart>(queueSize); uploadPartsFuture = Threads.enqueue(serviceConfig, UploadPartTask.class, poolSize, new UploadPartTask(partQueue, progressCallback)); } // Check for the future task before adding part to the queue. if (uploadPartsFuture != null && uploadPartsFuture.isDone()) { // This task shouldn't be done until the last part is added. If it is done at this point, then something might have gone wrong throw new SnapshotUploadPartException( "Error uploading parts, aborting part creation process. Check previous log messages for the exact error"); } // Add part to the queue partQueue.put(part); // Prep the metadata for the next part readOffset += bytesRead; bytesRead = 0L; bytesWritten = 0L; // Setup the part entity for next part zipFilePath = Files.createTempFile(keyName + '-', '-' + String.valueOf((++partNumber))); part = SnapshotPart.createPart(snapUploadInfo, zipFilePath.toString(), partNumber, readOffset); gzipStream = new GZIPOutputStream(baos); outputStream = new FileOutputStream(zipFilePath.toString()); } } gzipStream.close(); baos.writeTo(outputStream); bytesWritten += baos.size(); baos.reset(); outputStream.close(); inputStream.close(); // Update the part status part = part.updateStateCreated(bytesWritten, bytesRead, Boolean.TRUE); // Update the snapshot upload info status snapUploadInfo = snapUploadInfo.updateStateCreatedParts(partNumber); } catch (Exception e) { LOG.error("Failed to upload " + snapshotId + " due to: ", e); error = Boolean.TRUE; throw new SnapshotTransferException("Failed to upload " + snapshotId + " due to: ", e); } finally { if (inputStream != null) { inputStream.close(); } if (gzipStream != null) { gzipStream.close(); } if (outputStream != null) { outputStream.close(); } baos.reset(); } if (partNumber > 1) { // Check for the future task before adding the last part to the queue. if (uploadPartsFuture != null && uploadPartsFuture.isDone()) { // This task shouldn't be done until the last part is added. If it is done at this point, then something might have gone wrong throw new SnapshotUploadPartException( "Error uploading parts, aborting part upload process. Check previous log messages for the exact error"); } // Add the last part to the queue partQueue.put(part); // Kick off the completion task completeUploadFuture = Threads.enqueue(serviceConfig, CompleteMpuTask.class, poolSize, new CompleteMpuTask(uploadPartsFuture, snapUploadInfo, partNumber)); } else { try { LOG.info("Uploading snapshot " + snapshotId + " to objectstorage as a single object. Compressed size of snapshot (" + bytesWritten + " bytes) is less than minimum part size (" + partSize + " bytes) for multipart upload"); PutObjectResult putResult = uploadSnapshotAsSingleObject(zipFilePath.toString(), bytesWritten, uncompressedSize, progressCallback); markSnapshotAvailable(); try { part = part.updateStateUploaded(putResult.getETag()); snapUploadInfo = snapUploadInfo.updateStateUploaded(putResult.getETag()); } catch (Exception e) { LOG.debug("Failed to update status in DB for " + snapUploadInfo); } LOG.info("Uploaded snapshot " + snapshotId + " to objectstorage"); } catch (Exception e) { error = Boolean.TRUE; LOG.error("Failed to upload snapshot " + snapshotId + " due to: ", e); throw new SnapshotTransferException("Failed to upload snapshot " + snapshotId + " due to: ", e); } finally { deleteFile(zipFilePath); } } } catch (SnapshotTransferException e) { error = Boolean.TRUE; throw e; } catch (Exception e) { error = Boolean.TRUE; LOG.error("Failed to upload snapshot " + snapshotId + " due to: ", e); throw new SnapshotTransferException("Failed to upload snapshot " + snapshotId + " due to: ", e); } finally { if (error) { abortUpload(snapUploadInfo); if (uploadPartsFuture != null && !uploadPartsFuture.isDone()) { uploadPartsFuture.cancel(true); } if (completeUploadFuture != null && !completeUploadFuture.isDone()) { completeUploadFuture.cancel(true); } } } }