Example usage for java.util.zip GZIPOutputStream GZIPOutputStream

List of usage examples for java.util.zip GZIPOutputStream GZIPOutputStream

  1. HOME
  2. Java
  3. java
  4. java.util.zip.*
  5. GZIPOutputStream
  6. GZIPOutputStream

Introduction

In this page you can find the example usage for java.util.zip GZIPOutputStream GZIPOutputStream.

Prototype

public GZIPOutputStream(OutputStream out) throws IOException

Source Link

Document

Creates a new output stream with a default buffer size.

Usage

From source file:cc.twittertools.download.AsyncEmbeddedJsonStatusBlockCrawler.java

public void fetch() throws IOException {
    long start = System.currentTimeMillis();
    LOG.info("Processing " + file);

    int cnt = 0;/*from   w  w  w. ja  v a  2  s.  c  om*/
    BufferedReader data = null;
    try {
        data = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
        String line;
        while ((line = data.readLine()) != null) {
            try {
                String[] arr = line.split("\t");
                long id = Long.parseLong(arr[0]);
                String username = (arr.length > 1) ? arr[1] : "a";
                String url = getUrl(id, username);

                connections.incrementAndGet();
                crawlURL(url, new TweetFetcherHandler(id, username, url, 0, !this.noFollow, line));

                cnt++;

                if (cnt % TWEET_BLOCK_SIZE == 0) {
                    LOG.info(cnt + " requests submitted");
                }
            } catch (NumberFormatException e) { // parseLong
                continue;
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        data.close();
    }

    // Wait for the last requests to complete.
    LOG.info("Waiting for remaining requests (" + connections.get() + ") to finish!");
    for (int i = 0; i < 10; i++) {
        if (connections.get() == 0) {
            break;
        }
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    asyncHttpClient.close();

    long end = System.currentTimeMillis();
    long duration = end - start;
    LOG.info("Total request submitted: " + cnt);
    LOG.info(crawl.size() + " tweets fetched in " + duration + "ms");

    LOG.info("Writing tweets...");
    int written = 0;

    OutputStreamWriter out = new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(output)),
            "UTF-8");
    for (Map.Entry<Long, String> entry : crawl.entrySet()) {
        written++;
        out.write(entry.getValue() + "\n");
    }
    out.close();

    LOG.info(written + " statuses written.");

    if (this.repair != null) {
        LOG.info("Writing repair data file...");
        written = 0;
        out = new OutputStreamWriter(new FileOutputStream(repair), "UTF-8");
        for (Map.Entry<Long, String> entry : crawl_repair.entrySet()) {
            written++;
            out.write(entry.getValue() + "\n");
        }
        out.close();

        LOG.info(written + " statuses need repair.");
    }

    LOG.info("Done!");
}

From source file:adams.core.io.GzipUtils.java

/**
 * Compresses the specified bytes using gzip.
 *
 * @param input   the bytes to compress/*from   w  w w . j  av a 2s . c  o  m*/
 * @return      the compressed bytes, null in case of error
 */
public static byte[] compress(byte[] input) {
    ByteArrayInputStream bis;
    ByteArrayOutputStream bos;
    GZIPOutputStream gos;
    int i;

    try {
        bis = new ByteArrayInputStream(input);
        bos = new ByteArrayOutputStream();
        gos = new GZIPOutputStream(bos);
        while ((i = bis.read()) != -1)
            gos.write(i);
        gos.finish();
        return bos.toByteArray();
    } catch (Exception e) {
        System.err.println("Failed to compress bytes!");
        e.printStackTrace();
        return null;
    }
}

From source file:gov.nih.nci.caintegrator.external.caarray.CaArrayFacadeTest.java

/**
 * Sets up objects necessary for unit testing.
 * @throws Exception on error//  w ww .  j av  a2s  .  c om
 */
@SuppressWarnings("unchecked")
@Before
public void setUp() throws Exception {
    caArrayFacade = new CaArrayFacadeImpl();

    dataService = mock(DataService.class);
    when(dataService.getDataSet(any(DataSetRequest.class))).thenAnswer(new Answer<DataSet>() {
        @SuppressWarnings("unused")
        @Override
        public DataSet answer(InvocationOnMock invocation) throws Throwable {
            DataSetRequest r = (DataSetRequest) invocation.getArguments()[0];
            DataSet result = new DataSet();
            result.getDesignElements().add(probeSet1);
            result.getDesignElements().add(probeSet2);
            for (CaArrayEntityReference ref : r.getHybridizations()) {
                ArrayDesign design = new ArrayDesign();
                design.setName("test design");
                Hybridization hybridization = new Hybridization();
                hybridization.setArrayDesign(design);

                HybridizationData data = new HybridizationData();
                data.setHybridization(hybridization);
                result.getDatas().add(data);
                FloatColumn column = new FloatColumn();
                column.setValues(new float[] { 1.1f, 2.2f });
                data.getDataColumns().add(column);
            }
            return result;
        }
    });
    when(dataService.streamFileContents(any(CaArrayEntityReference.class), anyBoolean()))
            .thenAnswer(new Answer<FileStreamableContents>() {

                @Override
                public FileStreamableContents answer(InvocationOnMock invocation) throws Throwable {
                    StringBuffer dataFile = new StringBuffer();
                    dataFile.append("ID\tChromosome\tPhysical.Position\tlogratio\n");
                    dataFile.append("A_probeSet1\t1\t123456\t0.05\n");
                    dataFile.append("DarkCorner\t1\t56789\t-0.0034\n");
                    dataFile.append("*\n");
                    dataFile.append("A_probeSet2\t1\t98765\t-0.1234\n");

                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    GZIPOutputStream gzos = new GZIPOutputStream(baos);
                    IOUtils.write(dataFile.toString().getBytes(), gzos);

                    byte[] gzippedBytes = baos.toByteArray();
                    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(gzippedBytes);
                    FileStreamableContents contents = new FileStreamableContents();
                    contents.setContentStream(new DirectRemoteInputStream(byteArrayInputStream, false));
                    return contents;
                }
            });

    searchService = mock(SearchService.class);
    when(searchService.searchForExperiments(any(ExperimentSearchCriteria.class), any(LimitOffset.class)))
            .thenAnswer(new Answer<SearchResult<Experiment>>() {

                @Override
                public SearchResult<Experiment> answer(InvocationOnMock invocation) throws Throwable {
                    SearchResult<Experiment> result = new SearchResult<Experiment>();
                    ExperimentSearchCriteria crit = (ExperimentSearchCriteria) invocation.getArguments()[0];
                    if (!StringUtils.equals("no-experiment", crit.getPublicIdentifier())) {
                        Experiment experiment = new Experiment();
                        experiment.setPublicIdentifier(crit.getPublicIdentifier());
                        experiment.setId(crit.getPublicIdentifier());
                        experiment.setLastDataModificationDate(DateUtils.addDays(new Date(), -1));
                        result.getResults().add(experiment);
                    }
                    return result;
                }
            });
    when(searchService.searchForBiomaterials(any(BiomaterialSearchCriteria.class), any(LimitOffset.class)))
            .thenAnswer(new Answer<SearchResult<Biomaterial>>() {
                @Override
                public SearchResult<Biomaterial> answer(InvocationOnMock invocation) throws Throwable {
                    Biomaterial sample = new Biomaterial();
                    sample.setName("sample");
                    sample.setLastModifiedDataTime(DateUtils.addDays(new Date(), -1));
                    SearchResult<Biomaterial> result = new SearchResult<Biomaterial>();
                    result.getResults().add(sample);
                    result.setMaxAllowedResults(-1);
                    return result;
                }
            });
    when(searchService.searchForFiles(any(FileSearchCriteria.class), any(LimitOffset.class)))
            .then(new Answer<SearchResult<File>>() {
                @Override
                public SearchResult<File> answer(InvocationOnMock invocation) throws Throwable {
                    SearchResult<File> result = new SearchResult<File>();
                    File file = new File();
                    file.setMetadata(new FileMetadata());
                    file.getMetadata().setName("filename");
                    result.getResults().add(file);
                    return result;
                }
            });
    when(searchService.searchForHybridizations(any(HybridizationSearchCriteria.class), any(LimitOffset.class)))
            .thenAnswer(new Answer<SearchResult<Hybridization>>() {
                @Override
                public SearchResult<Hybridization> answer(InvocationOnMock invocation) throws Throwable {
                    ArrayDesign design = new ArrayDesign();
                    design.setName("test design");

                    Hybridization hybridization = new Hybridization();
                    hybridization.setArrayDesign(design);

                    SearchResult<Hybridization> result = new SearchResult<Hybridization>();
                    result.getResults().add(hybridization);
                    return result;
                }
            });
    when(searchService.searchByExample(any(ExampleSearchCriteria.class), any(LimitOffset.class)))
            .then(new Answer<SearchResult<AbstractCaArrayEntity>>() {
                @Override
                public SearchResult<AbstractCaArrayEntity> answer(InvocationOnMock invocation)
                        throws Throwable {
                    ExampleSearchCriteria<AbstractCaArrayEntity> crit = (ExampleSearchCriteria<AbstractCaArrayEntity>) invocation
                            .getArguments()[0];
                    SearchResult<AbstractCaArrayEntity> result = new SearchResult<AbstractCaArrayEntity>();
                    result.getResults().add(crit.getExample());
                    return result;
                }
            });
    when(searchService.searchForQuantitationTypes(any(QuantitationTypeSearchCriteria.class)))
            .thenAnswer(new Answer<List<QuantitationType>>() {
                @Override
                public List<QuantitationType> answer(InvocationOnMock invocation) throws Throwable {
                    List<QuantitationType> results = new ArrayList<QuantitationType>();
                    QuantitationType quantitationType = new QuantitationType();
                    quantitationType.setName("DataMatrixCopyNumber.Log2Ratio");
                    results.add(quantitationType);
                    return results;
                }
            });

    CaArrayServiceFactory factory = mock(CaArrayServiceFactory.class);
    when(factory.createDataService(any(ServerConnectionProfile.class))).thenReturn(dataService);
    when(factory.createSearchService(any(ServerConnectionProfile.class))).thenReturn(searchService);
    caArrayFacade.setServiceFactory(factory);

    CaIntegrator2Dao dao = mock(CaIntegrator2Dao.class);
    when(dao.getPlatform(anyString())).thenReturn(createTestPlatform());
    caArrayFacade.setDao(dao);
}

From source file:com.ocp.picasa.GDataClient.java

private ByteArrayEntity getCompressedEntity(byte[] data) throws IOException {
    ByteArrayEntity entity;//  w  w w  .  jav a  2s  .c  om
    if (data.length >= MIN_GZIP_SIZE) {
        ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(data.length / 2);
        GZIPOutputStream gzipOutput = new GZIPOutputStream(byteOutput);
        gzipOutput.write(data);
        gzipOutput.close();
        entity = new ByteArrayEntity(byteOutput.toByteArray());
    } else {
        entity = new ByteArrayEntity(data);
    }
    return entity;
}

From source file:com.mbrlabs.mundus.assets.EditorAssetManager.java

public void saveTerrainAssets() throws IOException {
    for (TerrainAsset terrain : getTerrainAssets()) {

        // save .terra file
        DataOutputStream outputStream = new DataOutputStream(
                new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(terrain.getFile().file()))));
        for (float f : terrain.getData()) {
            outputStream.writeFloat(f);//from   w w  w  . j a va  2  s  .  c om
        }
        outputStream.flush();
        outputStream.close();

        // save splatmap
        PixmapTextureAsset splatmap = terrain.getSplatmap();
        if (splatmap != null) {
            PixmapIO.writePNG(splatmap.getFile(), splatmap.getPixmap());
        }

        // save meta file
        terrain.getMeta().save();
    }
}

From source file:com.googlecode.jsonplugin.JSONUtil.java

public static void writeJSONToResponse(SerializationParams serializationParams) throws IOException {
    StringBuilder stringBuilder = new StringBuilder();
    if (TextUtils.stringSet(serializationParams.getSerializedJSON()))
        stringBuilder.append(serializationParams.getSerializedJSON());

    if (TextUtils.stringSet(serializationParams.getWrapPrefix()))
        stringBuilder.insert(0, serializationParams.getWrapPrefix());
    else if (serializationParams.isWrapWithComments()) {
        stringBuilder.insert(0, "/* ");
        stringBuilder.append(" */");
    } else if (serializationParams.isPrefix())
        stringBuilder.insert(0, "{}&& ");

    if (TextUtils.stringSet(serializationParams.getWrapSuffix()))
        stringBuilder.append(serializationParams.getWrapSuffix());

    String json = stringBuilder.toString();

    if (log.isDebugEnabled()) {
        log.debug("[JSON]" + json);
    }//from   w w  w .  ja  v a  2  s. c  o  m

    HttpServletResponse response = serializationParams.getResponse();

    //status or error code
    if (serializationParams.getStatusCode() > 0)
        response.setStatus(serializationParams.getStatusCode());
    else if (serializationParams.getErrorCode() > 0)
        response.sendError(serializationParams.getErrorCode());

    //content type
    if (serializationParams.isSmd())
        response.setContentType("application/json-rpc;charset=" + serializationParams.getEncoding());
    else
        response.setContentType(
                serializationParams.getContentType() + ";charset=" + serializationParams.getEncoding());

    if (serializationParams.isNoCache()) {
        response.setHeader("Cache-Control", "no-cache");
        response.setHeader("Expires", "0");
        response.setHeader("Pragma", "No-cache");
    }

    if (serializationParams.isGzip()) {
        response.addHeader("Content-Encoding", "gzip");
        GZIPOutputStream out = null;
        InputStream in = null;
        try {
            out = new GZIPOutputStream(response.getOutputStream());
            in = new ByteArrayInputStream(json.getBytes());
            byte[] buf = new byte[1024];
            int len;
            while ((len = in.read(buf)) > 0) {
                out.write(buf, 0, len);
            }
        } finally {
            if (in != null)
                in.close();
            if (out != null) {
                out.finish();
                out.close();
            }
        }

    } else {
        response.setContentLength(json.getBytes(serializationParams.getEncoding()).length);
        PrintWriter out = response.getWriter();
        out.print(json);
    }
}

From source file:edu.dfci.cccb.mev.dataset.rest.controllers.WorkspaceController.java

private void zipAnnotations(String name, String dimension, ZipOutputStream zout)
        throws DatasetNotFoundException, IOException {
    Dataset dataset = workspace.get(name);
    long projectId = projectManager.getProjectID(dataset.name() + dimension);
    if (projectId > -1) {
        File annotations = new TemporaryFile();
        GZIPOutputStream gos = new GZIPOutputStream(new FileOutputStream(annotations));
        TarOutputStream tos = new TarOutputStream(gos);
        projectManager.exportProject(projectId, tos);
        tos.flush();//  w  w  w. j a va  2s  . co m
        tos.close();
        zout.putNextEntry(new ZipEntry(String.format("annotations_%s.tar.gz", dimension)));
        IOUtils.copy(new FileInputStream(annotations), zout);
        zout.closeEntry();
    }
}

From source file:gdt.data.entity.ArchiveHandler.java

private static void compressGzipFile(String tarFile$, String gzipFile$) {
    try {//from   w ww  .j  a v a  2  s.  c o  m
        FileInputStream fis = new FileInputStream(tarFile$);
        FileOutputStream fos = new FileOutputStream(gzipFile$);
        GZIPOutputStream gzipOS = new GZIPOutputStream(fos);
        byte[] buffer = new byte[1024];
        int len;
        while ((len = fis.read(buffer)) != -1) {
            gzipOS.write(buffer, 0, len);
        }
        gzipOS.close();
        fos.close();
        fis.close();
    } catch (Exception e) {
        Logger.getLogger(ArchiveHandler.class.getName()).severe(e.toString());
    }

}

From source file:com.google.cloud.dataflow.sdk.io.TextIOTest.java

License:asdf

private static File writeToFile(String[] lines, String filename, CompressionType compression)
        throws IOException {
    File file = tempFolder.resolve(filename).toFile();
    OutputStream output = new FileOutputStream(file);
    switch (compression) {
    case UNCOMPRESSED:
        break;/* w  w  w  .j  av a 2 s .c  o m*/
    case GZIP:
        output = new GZIPOutputStream(output);
        break;
    case BZIP2:
        output = new BZip2CompressorOutputStream(output);
        break;
    default:
        throw new UnsupportedOperationException(compression.toString());
    }
    writeToStreamAndClose(lines, output);
    return file;
}

From source file:net.oddsoftware.android.html.HttpCache.java

private void download(CacheItem cacheItem) {
    try {//from  w w w .  ja v  a2 s  .  co m
        // check to see if file exist, if so check etag and last-modified
        if (cacheItem.mFilename.length() > 0) {
            File f = new File(cacheItem.mFilename);

            try {
                InputStream is = new FileInputStream(f);
                is.close();
            } catch (IOException exc) {
                // no file, nuke the cache stats
                cacheItem.mETag = "";
                cacheItem.mLastModified = 0;
            }
        } else {
            cacheItem.mFilename = mCacheDirectory + File.separator + UUID.randomUUID().toString() + ".html.gz";
        }

        HttpContext httpContext = new BasicHttpContext();
        HttpClient client = createHttpClient();
        HttpUriRequest request = createHttpRequest(cacheItem.mUrl, cacheItem.mETag, cacheItem.mLastModified);

        if (request == null || request.getURI() == null || request.getURI().getHost() == null
                || request.getURI().getHost().length() == 0) {
            if (Globals.LOGGING)
                Log.e(Globals.LOG_TAG, "unable to create http request for url " + cacheItem.mUrl);
            return; // sadness
        }

        HttpResponse response = client.execute(request, httpContext);

        StatusLine status = response.getStatusLine();
        HttpEntity entity = response.getEntity();

        if (status.getStatusCode() == 304) {
            if (Globals.LOGGING)
                Log.d(Globals.LOG_TAG, "received 304 not modified");

            cacheItem.mHitTime = new Date().getTime();

            cacheItem.update(mContentResolver);

            return;
        }

        if (status.getStatusCode() == 200) {
            InputStream inputStream = null;

            if (entity != null) {
                inputStream = entity.getContent();
            } else {
                return;
            }

            long contentLength = entity.getContentLength();

            if (contentLength > MAX_CONTENT_LENGTH) {
                if (Globals.LOGGING)
                    Log.w(Globals.LOG_TAG, "HttpCache.download item " + cacheItem.mUrl
                            + " content length is too big " + contentLength);
                return;
            }

            Header encodingHeader = entity.getContentEncoding();
            boolean encoded = false;

            if (encodingHeader != null) {
                if (encodingHeader.getValue().equalsIgnoreCase("gzip")) {
                    inputStream = new GZIPInputStream(inputStream);
                    encoded = true;
                } else if (encodingHeader.getValue().equalsIgnoreCase("deflate")) {
                    inputStream = new InflaterInputStream(inputStream);
                    encoded = true;
                }
            }

            File tmpFile = File.createTempFile("httpcache", ".html.gz.tmp", mCacheDirectory);
            OutputStream os = new GZIPOutputStream(new FileOutputStream(tmpFile));

            byte[] buffer = new byte[4096];
            int count = 0;
            long fileSize = 0;
            while ((count = inputStream.read(buffer)) != -1) {
                os.write(buffer, 0, count);
                fileSize += count;
            }
            inputStream.close();
            os.close();

            if (!encoded && contentLength > 0 && fileSize != contentLength) {
                Log.e(Globals.LOG_TAG, "HttpCache.download: content-length: " + contentLength
                        + " but file size: " + fileSize + " aborting");
                tmpFile.delete();
                return;
            }

            tmpFile.renameTo(new File(cacheItem.mFilename));

            // if the parse was ok, update these attributes
            // ETag: "6050003-78e5-4981d775e87c0"
            Header etagHeader = response.getFirstHeader("ETag");
            if (etagHeader != null) {
                if (etagHeader.getValue().length() < MAX_ETAG_LENGTH) {
                    cacheItem.mETag = etagHeader.getValue();
                } else {
                    if (Globals.LOGGING)
                        Log.e(Globals.LOG_TAG, "etag length was too big: " + etagHeader.getValue().length());
                }
            }

            // Last-Modified: Fri, 24 Dec 2010 00:57:11 GMT
            Header lastModifiedHeader = response.getFirstHeader("Last-Modified");
            if (lastModifiedHeader != null) {
                try {
                    cacheItem.mLastModified = FeedManager.parseRFC822Date(lastModifiedHeader.getValue())
                            .getTime();
                } catch (ParseException exc) {
                    if (Globals.LOGGING)
                        Log.e(Globals.LOG_TAG, "unable to parse date", exc);
                }
            }

            // Expires: Thu, 01 Dec 1994 16:00:00 GMT
            Header expiresHeader = response.getFirstHeader("Expires");
            if (expiresHeader != null) {
                try {
                    cacheItem.mExpiresAt = FeedManager.parseRFC822Date(expiresHeader.getValue()).getTime();
                } catch (ParseException exc) {
                    if (Globals.LOGGING)
                        Log.e(Globals.LOG_TAG, "unable to parse expires", exc);
                }
            }

            long now = new Date().getTime() + DEFAULT_EXPIRES;
            if (cacheItem.mExpiresAt < now) {
                cacheItem.mExpiresAt = now;
            }

            HttpUriRequest currentReq = (HttpUriRequest) httpContext
                    .getAttribute(ExecutionContext.HTTP_REQUEST);
            HttpHost currentHost = (HttpHost) httpContext.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
            String currentUrl = currentHost.toURI() + currentReq.getURI();

            if (Globals.LOGGING)
                Log.w(Globals.LOG_TAG,
                        "loaded redirect from " + request.getURI().toString() + " to " + currentUrl);

            cacheItem.mLastUrl = currentUrl;

            cacheItem.mHitTime = new Date().getTime();

            cacheItem.update(mContentResolver);
        }
    } catch (IOException exc) {
        if (Globals.LOGGING) {
            Log.e(Globals.LOG_TAG, "error downloading file to cache", exc);
        }
    }
}