Example usage for java.util.zip GZIPInputStream GZIPInputStream

Introduction

In this page you can find the example usage for java.util.zip GZIPInputStream GZIPInputStream.

Prototype

public GZIPInputStream(InputStream in) throws IOException 

Source Link

Document

Creates a new input stream with a default buffer size.

Usage

From source file:eu.delving.metadata.Hasher.java

public void update(File inputFile) throws IOException {
    try {/*from  w  w  w.  ja  va2s . co  m*/
        InputStream inputStream = new FileInputStream(inputFile);
        if (inputFile.getName().endsWith(".gz")) {
            inputStream = new GZIPInputStream(inputStream);
        }
        byte[] buffer = new byte[BLOCK_SIZE];
        int bytesRead;
        while (-1 != (bytesRead = inputStream.read(buffer))) {
            update(buffer, bytesRead);
        }
        inputStream.close();
    } catch (Exception e) {
        throw new IOException("Unable to get hash of " + inputFile.getAbsolutePath(), e);
    }
}

---

From source file:edu.gslis.ts.ChunkToFile.java

/**
 * @param thriftFile//from ww  w. j av  a  2 s.  c  o  m
 */
public void filter(File infile, Map<Integer, FeatureVector> queries, IndexWrapper index, Stopper stopper,
        String outputPath) {
    try {
        InputStream in = null;

        if (infile.getName().endsWith(".gz"))
            in = new GZIPInputStream(new FileInputStream(infile));
        else if (infile.getName().endsWith("xz"))
            in = new XZInputStream(new FileInputStream(infile));
        else {
            System.err.println("Regular FileInputStream");
            in = new FileInputStream(infile);
        }

        TTransport inTransport = new TIOStreamTransport(new BufferedInputStream(in));
        TBinaryProtocol inProtocol = new TBinaryProtocol(inTransport);
        inTransport.open();
        Pairtree ptree = new Pairtree();

        try {
            // Run through items in the chunk file
            while (true) {
                final StreamItem item = new StreamItem();
                item.read(inProtocol);

                FeatureVector dv = new FeatureVector(item.body.clean_visible, stopper);
                double maxScore = Double.NEGATIVE_INFINITY;
                int qid = -1;
                for (int id : queries.keySet()) {
                    FeatureVector qv = queries.get(id);
                    double score = kl(dv, qv, index, MU);
                    if (score > maxScore) {
                        qid = id;
                        maxScore = score;
                    }
                }

                String streamId = item.stream_id;
                System.out.println(streamId + "=" + qid);
                //System.out.println(streamId);
                String ppath = ptree.mapToPPath(streamId.replace("-", ""));
                //System.out.println(streamId + "=>" + ppath);

                File dir = new File(outputPath + File.separator + qid + File.separator + ppath);
                dir.mkdirs();
                XZOutputStream xos = new XZOutputStream(
                        new FileOutputStream(dir.getAbsolutePath() + File.separator + streamId + ".xz"),
                        new LZMA2Options());

                TTransport outTransport = new TIOStreamTransport(xos);
                TBinaryProtocol outProtocol = new TBinaryProtocol(outTransport);
                outTransport.open();
                item.write(outProtocol);
                outTransport.close();

            }
        } catch (TTransportException te) {
            if (te.getType() == TTransportException.END_OF_FILE) {
            } else {
                throw te;
            }
        }

        inTransport.close();

    } catch (Exception e) {
        System.err.println("Error processing " + infile.getAbsolutePath() + " " + infile.getName());
        e.printStackTrace();
    }
}

---

From source file:metadata.etl.lineage.AzJobChecker.java

/**
 * Read the blob from "flow_data", do a topological sort on the nodes. Give them the sort id.
 * @param startTimeStamp the begin timestamp in milli seconds
 * @param endTimeStamp the end timestamp in milli seconds
 * @return/*  w ww .j  av  a  2  s. c  om*/
 */
public List<AzkabanJobExecRecord> getRecentFinishedJobFromFlow(long startTimeStamp, long endTimeStamp)
        throws SQLException, IOException {

    logger.info("Get the jobs from time : {} to time : {}", startTimeStamp, endTimeStamp);
    List<AzkabanJobExecRecord> results = new ArrayList<>();
    Statement stmt = conn.createStatement();
    final String cmd = "select exec_id, flow_id, status, submit_user, flow_data from execution_flows where end_time > "
            + startTimeStamp + " and end_time < " + endTimeStamp;
    logger.info("Get recent flow sql : " + cmd);
    final ResultSet rs = stmt.executeQuery(cmd); // this sql take 3 second to execute

    while (rs.next()) {
        long execId = rs.getLong("exec_id");
        Blob flowBlob = rs.getBlob("flow_data");
        GZIPInputStream gzip = new GZIPInputStream(flowBlob.getBinaryStream());
        String flowJson = IOUtils.toString(gzip, "UTF-8");
        results.addAll(parseJson(flowJson, execId));
    }
    return results;
}

---

From source file:net.andylizi.colormotd.anim.updater.Updater.java

private boolean checkUpdate() {
    if (new File(Bukkit.getUpdateFolderFile(), fileName).exists()) {
        cancel();//from   w  w  w  .j  a v a 2  s  . co m
    }
    URL url;
    try {
        url = new URL(UPDATE_URL);
    } catch (MalformedURLException ex) {
        if (DEBUG) {
            ex.printStackTrace();
        }
        return false;
    }
    try {
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setUseCaches(false);
        conn.setRequestMethod("GET");
        conn.addRequestProperty("User-Agent", USER_AGENT);
        if (etag != null) {
            conn.addRequestProperty("If-None-Match", etag);
        }
        conn.addRequestProperty("Accept", "application/json,text/plain,*/*;charset=utf-8");
        conn.addRequestProperty("Accept-Encoding", "gzip");
        conn.addRequestProperty("Cache-Control", "no-cache");
        conn.addRequestProperty("Date", new Date().toString());
        conn.addRequestProperty("Connection", "close");

        conn.setDoInput(true);
        conn.setDoOutput(false);
        conn.setConnectTimeout(2000);
        conn.setReadTimeout(2000);

        conn.connect();

        if (conn.getHeaderField("ETag") != null) {
            etag = conn.getHeaderField("ETag");
        }
        if (DEBUG) {
            logger.log(Level.INFO, "DEBUG ResponseCode: {0}", conn.getResponseCode());
            logger.log(Level.INFO, "DEBUG ETag: {0}", etag);
        }
        if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
            return false;
        } else if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
            return false;
        }

        BufferedReader input = null;
        if (conn.getContentEncoding() != null && conn.getContentEncoding().contains("gzip")) {
            input = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(conn.getInputStream()), "UTF-8"), 1);
        } else {
            input = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"), 1);
        }

        StringBuilder builder = new StringBuilder();
        String buffer = null;
        while ((buffer = input.readLine()) != null) {
            builder.append(buffer);
        }
        try {
            JSONObject jsonObj = (JSONObject) new JSONParser().parse(builder.toString());
            int build = ((Number) jsonObj.get("build")).intValue();
            if (build <= buildVersion) {
                return false;
            }
            String version = (String) jsonObj.get("version");
            String msg = (String) jsonObj.get("msg");
            String download_url = (String) jsonObj.get("url");
            newVersion = new NewVersion(version, build, msg, download_url);
            return true;
        } catch (ParseException ex) {
            if (DEBUG) {
                ex.printStackTrace();
            }
            exception = ex;
            return false;
        }
    } catch (SocketTimeoutException ex) {
        return false;
    } catch (IOException ex) {
        if (DEBUG) {
            ex.printStackTrace();
        }
        exception = ex;
        return false;
    }
}

---

From source file:com.metamx.druid.loading.S3SegmentPuller.java

@Override
public File getSegmentFiles(DataSegment segment) throws StorageAdapterLoadingException {
    Map<String, Object> loadSpec = segment.getLoadSpec();
    String s3Bucket = MapUtils.getString(loadSpec, "bucket");
    String s3Path = MapUtils.getString(loadSpec, "key");

    log.info("Loading index at path[s3://%s/%s]", s3Bucket, s3Path);

    S3Object s3Obj = null;/*from  ww  w .  j  a va 2  s .  c  om*/
    File tmpFile = null;
    try {
        if (!s3Client.isObjectInBucket(s3Bucket, s3Path)) {
            throw new StorageAdapterLoadingException("IndexFile[s3://%s/%s] does not exist.", s3Bucket, s3Path);
        }

        File cacheFile = new File(config.getCacheDirectory(), computeCacheFilePath(s3Bucket, s3Path));

        if (cacheFile.exists()) {
            S3Object objDetails = s3Client.getObjectDetails(new S3Bucket(s3Bucket), s3Path);
            DateTime cacheFileLastModified = new DateTime(cacheFile.lastModified());
            DateTime s3ObjLastModified = new DateTime(objDetails.getLastModifiedDate().getTime());
            if (cacheFileLastModified.isAfter(s3ObjLastModified)) {
                log.info("Found cacheFile[%s] with modified[%s], which is after s3Obj[%s].  Using.", cacheFile,
                        cacheFileLastModified, s3ObjLastModified);
                return cacheFile.getParentFile();
            }
            FileUtils.deleteDirectory(cacheFile.getParentFile());
        }

        long currTime = System.currentTimeMillis();

        tmpFile = File.createTempFile(s3Bucket, new DateTime().toString());
        log.info("Downloading file[s3://%s/%s] to local tmpFile[%s] for cacheFile[%s]", s3Bucket, s3Path,
                tmpFile, cacheFile);

        s3Obj = s3Client.getObject(new S3Bucket(s3Bucket), s3Path);
        StreamUtils.copyToFileAndClose(s3Obj.getDataInputStream(), tmpFile, DEFAULT_TIMEOUT);
        final long downloadEndTime = System.currentTimeMillis();
        log.info("Download of file[%s] completed in %,d millis", cacheFile, downloadEndTime - currTime);

        if (!cacheFile.getParentFile().mkdirs()) {
            log.info("Unable to make parent file[%s]", cacheFile.getParentFile());
        }
        cacheFile.delete();

        if (s3Path.endsWith("gz")) {
            log.info("Decompressing file[%s] to [%s]", tmpFile, cacheFile);
            StreamUtils.copyToFileAndClose(new GZIPInputStream(new FileInputStream(tmpFile)), cacheFile);
            if (!tmpFile.delete()) {
                log.error("Could not delete tmpFile[%s].", tmpFile);
            }
        } else {
            log.info("Rename tmpFile[%s] to cacheFile[%s]", tmpFile, cacheFile);
            if (!tmpFile.renameTo(cacheFile)) {
                log.warn("Error renaming tmpFile[%s] to cacheFile[%s].  Copying instead.", tmpFile, cacheFile);

                StreamUtils.copyToFileAndClose(new FileInputStream(tmpFile), cacheFile);
                if (!tmpFile.delete()) {
                    log.error("Could not delete tmpFile[%s].", tmpFile);
                }
            }
        }

        long endTime = System.currentTimeMillis();
        log.info("Local processing of file[%s] done in %,d millis", cacheFile, endTime - downloadEndTime);

        return cacheFile.getParentFile();
    } catch (Exception e) {
        throw new StorageAdapterLoadingException(e, e.getMessage());
    } finally {
        S3Utils.closeStreamsQuietly(s3Obj);
        if (tmpFile != null && tmpFile.exists()) {
            log.warn("Deleting tmpFile[%s] in finally block.  Why?", tmpFile);
            tmpFile.delete();
        }
    }
}

---

From source file:com.knowbout.epg.processor.Parser.java

public void parse() throws IOException {
    HibernateUtil.openSession();//  w  w w .  jav  a  2 s .  c  om
    try {
        TransactionManager.beginTransaction();
        log.debug("Begun transaction for headend");
        GZIPInputStream uis = new GZIPInputStream(new FileInputStream(headendFile));
        log.debug("About to parse Headends");
        parseHeadend(uis);
        TransactionManager.commitTransaction();

        TransactionManager.beginTransaction();
        log.debug("Begun transaction for station");
        log.debug("About to parse Stations");
        uis = new GZIPInputStream(new FileInputStream(stationFile));
        parseStation(uis);
        TransactionManager.commitTransaction();

        TransactionManager.beginTransaction();
        log.debug("Begun transaction for lineup");
        log.debug("About to parse Lineups/Channels");
        uis = new GZIPInputStream(new FileInputStream(lineupFile));
        parseLineup(uis);
        TransactionManager.commitTransaction();

        TransactionManager.beginTransaction();
        log.debug("Begun transaction for programs");
        log.debug("About to parse Programs");
        uis = new GZIPInputStream(new FileInputStream(programFile));
        parseProgram(uis);
        log.debug("About to commit last transaction for programs");
        if (TransactionManager.currentTransaction().isActive()) {
            log.debug("it is an active transaction");
        } else {
            log.debug("it is NOT an active transaction");
        }
        TransactionManager.commitTransaction();

        uis = new GZIPInputStream(new FileInputStream(scheduleFile));
        log.debug("About to parse Schedules");
        ScheduleParser parser = new ScheduleParser();
        parser.parseSchedule(uis, config);
    } catch (IOException e) {
        TransactionManager.rollbackTransaction();
        throw e;
    } finally {
        HibernateUtil.closeSession();
    }

}

---

From source file:de.robbers.dashclock.stackextension.StackExtension.java

private String performHttpRequest(String uri) {
    Log.i(TAG, "URI: " + uri);
    DefaultHttpClient client = new DefaultHttpClient();
    HttpGet get = new HttpGet(uri);
    get.addHeader("Accept-Encoding", "gzip");
    try {//from   ww  w.ja v  a2 s  . c o  m
        // get JSON from Stack Exchange API
        HttpResponse response = client.execute(get);
        InputStream inputStream = response.getEntity().getContent();
        GZIPInputStream zis = new GZIPInputStream(new BufferedInputStream(inputStream));
        InputStreamReader reader = new InputStreamReader(zis);
        BufferedReader in = new BufferedReader(reader);
        StringBuilder builder = new StringBuilder();
        String line;
        while ((line = in.readLine()) != null) {
            builder.append(line);
        }
        String json = builder.toString();
        zis.close();
        return json;
    } catch (ClientProtocolException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}

---

From source file:com.cip.crane.agent.utils.FileExtractUtils.java

/**
 * Ungzip an input file into an output file.
 * <p>/*  w  w w.  java2s .  co  m*/
 * The output file is created in the output folder, having the same name
 * as the input file, minus the '.gz' extension. 
 * 
 * @param inputFile     the input .gz file
 * @param outputDir     the output directory file. 
 * @throws IOException 
 * @throws FileNotFoundException
 *  
 * @return  The {@File} with the ungzipped content.
 */
public static File unGzip(final File inputFile, final File outputDir)
        throws FileNotFoundException, IOException {

    LOG.debug(String.format("Ungzipping %s to dir %s.", inputFile.getAbsolutePath(),
            outputDir.getAbsolutePath()));

    final File outputFile = new File(outputDir,
            inputFile.getName().substring(0, inputFile.getName().length() - 3));

    final GZIPInputStream in = new GZIPInputStream(new FileInputStream(inputFile));
    final FileOutputStream out = new FileOutputStream(outputFile);

    IOUtils.copy(in, out);
    in.close();
    out.close();

    return outputFile;
}

---

From source file:org.opendatakit.common.utils.WebUtils.java

/**
 * Decode a safeEncode() string.//from   ww w. ja v  a 2  s . co m
 * 
 * @param encodedWebsafeString
 * @return rawString
 */
public static String safeDecode(String encodedWebsafeString) {
    if (encodedWebsafeString == null || encodedWebsafeString.length() == 0) {
        return encodedWebsafeString;
    }

    try {
        ByteArrayInputStream in = new ByteArrayInputStream(
                Base64.decodeBase64(encodedWebsafeString.getBytes(CharEncoding.UTF_8)));
        GZIPInputStream gzip = new GZIPInputStream(in);
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        int ch = gzip.read();
        while (ch >= 0) {
            out.write(ch);
            ch = gzip.read();
        }
        gzip.close();
        out.flush();
        out.close();
        return new String(out.toByteArray(), CharEncoding.UTF_8);
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
        throw new IllegalArgumentException("Unexpected failure: " + e.toString());
    } catch (IOException e) {
        e.printStackTrace();
        throw new IllegalArgumentException("Unexpected failure: " + e.toString());
    }
}

---

From source file:de.huxhorn.lilith.log4j.xml.Log4jImportCallable.java

public Long call() throws Exception {
    if (!inputFile.isFile()) {
        throw new IllegalArgumentException("'" + inputFile.getAbsolutePath() + "' is not a file!");
    }// w  w  w.ja  v a2 s.c o m
    if (!inputFile.canRead()) {
        throw new IllegalArgumentException("'" + inputFile.getAbsolutePath() + "' is not a readable!");
    }
    long fileSize = inputFile.length();
    setNumberOfSteps(fileSize);
    FileInputStream fis = new FileInputStream(inputFile);
    CountingInputStream cis = new CountingInputStream(fis);

    String fileName = inputFile.getName().toLowerCase(Locale.US);
    BufferedReader br;
    if (fileName.endsWith(".gz")) {
        br = new BufferedReader(new InputStreamReader(new GZIPInputStream(cis), StandardCharsets.UTF_8));
    } else {
        br = new BufferedReader(new InputStreamReader(cis, StandardCharsets.UTF_8));
    }

    StringBuilder builder = new StringBuilder();

    result = 0;
    for (;;) {
        String line = br.readLine();
        setCurrentStep(cis.getByteCount());
        if (line == null) {
            evaluate(builder.toString());
            break;
        }
        for (;;) {
            int closeIndex = line.indexOf(CLOSING_LOG4J_EVENT_TAG);
            if (closeIndex >= 0) {
                int endIndex = closeIndex + CLOSING_LOG4J_EVENT_TAG.length();
                builder.append(line.subSequence(0, endIndex));
                evaluate(builder.toString());
                builder.setLength(0);
                line = line.substring(endIndex);
            } else {
                builder.append(line);
                builder.append("\n");
                break;
            }
        }
    }
    return result;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• »