Example usage for java.util.zip GZIPInputStream GZIPInputStream

Introduction

In this page you can find the example usage for java.util.zip GZIPInputStream GZIPInputStream.

Prototype

public GZIPInputStream(InputStream in) throws IOException 

Source Link

Document

Creates a new input stream with a default buffer size.

Usage

From source file:de.qaware.chronix.importer.csv.FileImporter.java

/**
 * Reads the given file / folder and calls the bi consumer with the extracted points
 *
 * @param points/*from w  ww. j  a va2s.  c om*/
 * @param folder
 * @param databases
 * @return
 */
public Pair<Integer, Integer> importPoints(Map<Attributes, Pair<Instant, Instant>> points, File folder,
        BiConsumer<List<ImportPoint>, Attributes>... databases) {

    final AtomicInteger pointCounter = new AtomicInteger(0);
    final AtomicInteger tsCounter = new AtomicInteger(0);
    final File metricsFile = new File(METRICS_FILE_PATH);

    LOGGER.info("Writing imported metrics to {}", metricsFile);
    LOGGER.info("Import supports csv files as well as gz compressed csv files.");

    try {
        final FileWriter metricsFileWriter = new FileWriter(metricsFile);

        Collection<File> files = new ArrayList<>();
        if (folder.isFile()) {
            files.add(folder);
        } else {
            files.addAll(FileUtils.listFiles(folder, new String[] { "gz", "csv" }, true));
        }

        AtomicInteger counter = new AtomicInteger(0);

        files.parallelStream().forEach(file -> {
            SimpleDateFormat sdf = new SimpleDateFormat(dateFormat);
            NumberFormat nf = DecimalFormat.getInstance(numberLocal);

            InputStream inputStream = null;
            BufferedReader reader = null;
            try {
                inputStream = new FileInputStream(file);

                if (file.getName().endsWith("gz")) {
                    inputStream = new GZIPInputStream(inputStream);
                }
                reader = new BufferedReader(new InputStreamReader(inputStream));

                //Read the first line
                String headerLine = reader.readLine();

                if (headerLine == null || headerLine.isEmpty()) {
                    boolean deleted = deleteFile(file, inputStream, reader);
                    LOGGER.debug("File is empty {}. File {} removed {}", file.getName(), deleted);
                    return;
                }

                //Extract the attributes from the file name
                //E.g. first_second_third_attribute.csv
                String[] fileNameMetaData = file.getName().split("_");

                String[] metrics = headerLine.split(csvDelimiter);

                Map<Integer, Attributes> attributesPerTimeSeries = new HashMap<>(metrics.length);

                for (int i = 1; i < metrics.length; i++) {
                    String metric = metrics[i];
                    String metricOnlyAscii = Normalizer.normalize(metric, Normalizer.Form.NFD);
                    metricOnlyAscii = metric.replaceAll("[^\\x00-\\x7F]", "");
                    Attributes attributes = new Attributes(metricOnlyAscii, fileNameMetaData);

                    //Check if meta data is completely set
                    if (isEmpty(attributes)) {
                        boolean deleted = deleteFile(file, inputStream, reader);
                        LOGGER.info("Attributes contains empty values {}. File {} deleted {}", attributes,
                                file.getName(), deleted);
                        continue;
                    }

                    if (attributes.getMetric().equals(".*")) {
                        boolean deleted = deleteFile(file, inputStream, reader);
                        LOGGER.info("Attributes metric{}. File {} deleted {}", attributes.getMetric(),
                                file.getName(), deleted);
                        continue;
                    }
                    attributesPerTimeSeries.put(i, attributes);
                    tsCounter.incrementAndGet();

                }

                Map<Integer, List<ImportPoint>> dataPoints = new HashMap<>();

                String line;
                while ((line = reader.readLine()) != null) {
                    String[] splits = line.split(csvDelimiter);
                    String date = splits[0];

                    Instant dateObject;
                    if (instantDate) {
                        dateObject = Instant.parse(date);
                    } else if (sdfDate) {
                        dateObject = sdf.parse(date).toInstant();
                    } else {
                        dateObject = Instant.ofEpochMilli(Long.valueOf(date));
                    }

                    for (int column = 1; column < splits.length; column++) {

                        String value = splits[column];
                        double numericValue = nf.parse(value).doubleValue();

                        ImportPoint point = new ImportPoint(dateObject, numericValue);

                        if (!dataPoints.containsKey(column)) {
                            dataPoints.put(column, new ArrayList<>());
                        }
                        dataPoints.get(column).add(point);
                        pointCounter.incrementAndGet();
                    }

                }

                dataPoints.values().forEach(Collections::sort);

                IOUtils.closeQuietly(reader);
                IOUtils.closeQuietly(inputStream);

                dataPoints.forEach((key, importPoints) -> {
                    for (BiConsumer<List<ImportPoint>, Attributes> database : databases) {
                        database.accept(importPoints, attributesPerTimeSeries.get(key));
                    }
                    points.put(attributesPerTimeSeries.get(key), Pair.of(importPoints.get(0).getDate(),
                            importPoints.get(importPoints.size() - 1).getDate()));
                    //write the stats to the file
                    Instant start = importPoints.get(0).getDate();
                    Instant end = importPoints.get(importPoints.size() - 1).getDate();

                    try {
                        writeStatsLine(metricsFileWriter, attributesPerTimeSeries.get(key), start, end);
                    } catch (IOException e) {
                        LOGGER.error("Could not write stats line", e);
                    }
                    LOGGER.info("{} of {} time series imported", counter.incrementAndGet(), tsCounter.get());
                });

            } catch (Exception e) {
                LOGGER.info("Exception while reading points.", e);
            } finally {
                //close all streams
                IOUtils.closeQuietly(reader);
                IOUtils.closeQuietly(inputStream);
            }

        });
    } catch (Exception e) {
        LOGGER.error("Exception occurred during reading points.");
    }
    return Pair.of(tsCounter.get(), pointCounter.get());
}

---

From source file:com.nebhale.cyclinglibrary.web.GzipFilterTest.java

private String gunzipContent(byte[] content) throws IOException {
    InputStream in = null;/* w  w  w.j av a 2  s . c  om*/
    ByteArrayOutputStream out = null;

    try {
        in = new GZIPInputStream(new ByteArrayInputStream(content));
        out = new ByteArrayOutputStream();

        copy(in, out);
        return out.toString("UTF-8");
    } finally {
        closeQuietly(in, out);
    }
}

---

From source file:org.immopoly.android.helper.WebHelper.java

public static JSONObject getHttpData(URL url, boolean signed, Context context) throws JSONException {
    JSONObject obj = null;/* w  ww .  j a  v a 2 s  .  co  m*/
    if (Settings.isOnline(context)) {
        HttpURLConnection request;
        try {

            request = (HttpURLConnection) url.openConnection();

            request.addRequestProperty("User-Agent",
                    "immopoly android client " + ImmopolyActivity.getStaticVersionInfo());
            request.addRequestProperty("Accept-Encoding", "gzip");
            if (signed)
                OAuthData.getInstance(context).consumer.sign(request);
            request.setConnectTimeout(SOCKET_TIMEOUT);

            request.connect();
            String encoding = request.getContentEncoding();

            InputStream in;
            if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
                in = new GZIPInputStream(request.getInputStream());
            } else {
                in = new BufferedInputStream(request.getInputStream());
            }
            String s = readInputStream(in);
            JSONTokener tokener = new JSONTokener(s);
            obj = new JSONObject(tokener);

        } catch (MalformedURLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (OAuthMessageSignerException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (OAuthExpectationFailedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (OAuthCommunicationException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
    return obj;
}

---

From source file:com.simiacryptus.mindseye.test.data.MNIST.java

private static Stream<byte[]> binaryStream(@Nonnull final String name, final int skip, final int recordSize)
        throws IOException {
    @Nullable/* w  w w . ja v a 2s  .  com*/
    InputStream stream = null;
    try {
        stream = Util.cacheStream(TestUtil.S3_ROOT.resolve(name));
    } catch (@Nonnull NoSuchAlgorithmException | KeyManagementException e) {
        throw new RuntimeException(e);
    }
    final byte[] fileData = IOUtils
            .toByteArray(new BufferedInputStream(new GZIPInputStream(new BufferedInputStream(stream))));
    @Nonnull
    final DataInputStream in = new DataInputStream(new ByteArrayInputStream(fileData));
    in.skip(skip);
    return MNIST.toIterator(new BinaryChunkIterator(in, recordSize));
}

---

From source file:og.android.tether.system.WebserviceTask.java

public static boolean downloadBluetoothModule(String downloadFileUrl, String destinationFilename) {
    if (android.os.Environment.getExternalStorageState()
            .equals(android.os.Environment.MEDIA_MOUNTED) == false) {
        return false;
    }/*from  ww  w .j  av a2  s  .  c o m*/
    File bluetoothDir = new File(BLUETOOTH_FILEPATH);
    if (bluetoothDir.exists() == false) {
        bluetoothDir.mkdirs();
    }
    if (downloadFile(downloadFileUrl, "", destinationFilename) == true) {
        try {
            FileOutputStream out = new FileOutputStream(new File(destinationFilename.replace(".gz", "")));
            FileInputStream fis = new FileInputStream(destinationFilename);
            GZIPInputStream gzin = new GZIPInputStream(new BufferedInputStream(fis));
            int count;
            byte buf[] = new byte[8192];
            while ((count = gzin.read(buf, 0, 8192)) != -1) {
                //System.out.write(x);
                out.write(buf, 0, count);
            }
            out.flush();
            out.close();
            gzin.close();
            File inputFile = new File(destinationFilename);
            inputFile.delete();
        } catch (IOException e) {
            return false;
        }
        return true;
    } else
        return false;
}

---

From source file:net.andylizi.colormotd.Updater.java

private boolean checkUpdate() {
    if (file != null && file.exists()) {
        cancel();//from w w  w .  j ava2  s .  c  o  m
    }
    URL url;
    try {
        url = new URL(UPDATE_URL);
    } catch (MalformedURLException ex) {
        if (DEBUG) {
            ex.printStackTrace();
        }
        return false;
    }
    try {
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setUseCaches(false);
        conn.setRequestMethod("GET");
        conn.addRequestProperty("User-Agent", userAgent);
        if (etag != null) {
            conn.addRequestProperty("If-None-Match", etag);
        }
        conn.addRequestProperty("Accept", "application/json,text/plain,*/*;charset=utf-8");
        conn.addRequestProperty("Accept-Encoding", "gzip");
        conn.addRequestProperty("Cache-Control", "no-cache");
        conn.addRequestProperty("Date", new Date().toString());
        conn.addRequestProperty("Connection", "close");

        conn.setDoInput(true);
        conn.setDoOutput(false);
        conn.setConnectTimeout(2000);
        conn.setReadTimeout(2000);

        conn.connect();

        if (conn.getHeaderField("ETag") != null) {
            etag = conn.getHeaderField("ETag");
        }
        if (DEBUG) {
            logger.log(Level.INFO, "DEBUG ResponseCode: {0}", conn.getResponseCode());
            logger.log(Level.INFO, "DEBUG ETag: {0}", etag);
        }
        if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
            return false;
        } else if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
            return false;
        }

        BufferedReader input = null;
        if (conn.getContentEncoding() != null && conn.getContentEncoding().contains("gzip")) {
            input = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(conn.getInputStream()), "UTF-8"), 1);
        } else {
            input = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"), 1);
        }

        StringBuilder builder = new StringBuilder();
        String buffer = null;
        while ((buffer = input.readLine()) != null) {
            builder.append(buffer);
        }
        try {
            JSONObject jsonObj = (JSONObject) new JSONParser().parse(builder.toString());
            int build = ((Number) jsonObj.get("build")).intValue();
            if (build <= ColorMOTD.buildVersion) {
                return false;
            }
            String version = (String) jsonObj.get("version");
            String msg = (String) jsonObj.get("msg");
            String download_url = (String) jsonObj.get("url");
            newVersion = new NewVersion(version, build, msg, download_url);
            return true;
        } catch (ParseException ex) {
            if (DEBUG) {
                ex.printStackTrace();
            }
            exception = ex;
            return false;
        }
    } catch (SocketTimeoutException ex) {
        return false;
    } catch (IOException ex) {
        if (DEBUG) {
            ex.printStackTrace();
        }
        exception = ex;
        return false;
    }
}

---

From source file:net.orzo.lib.Files.java

public FileIterator<Object> gzipFileReader(final String path, final String encoding) throws IOException {
    try {// w  w w .  j a v  a2 s .  co m
        final GZIPInputStream gis = new GZIPInputStream(new FileInputStream(path));
        final Reader reader = new InputStreamReader(gis, encoding);
        return new FileIterator<Object>() {

            private final BufferedReader br = new BufferedReader(reader);
            private String currLine = br.readLine();

            @Override
            public boolean hasNext() {
                return this.currLine != null;
            }

            @Override
            public Object next() {
                String ans;

                if (this.currLine != null) {
                    ans = this.currLine;
                    try {
                        this.currLine = this.br.readLine();

                    } catch (IOException e) {
                        this.currLine = null;
                        throw new IllegalStateException(e);
                    }
                    return ans;

                } else {
                    throw new NoSuchElementException();
                }
            }

            @Override
            public void close() {
                try {
                    this.currLine = null;
                    this.br.close();

                } catch (IOException e) {
                    throw new IllegalStateException(e);
                }
            }

            @Override
            public String getPath() {
                return path;
            }
        };

    } catch (EOFException ex) {
        return new EmptyFileIterator();
    }
}

---

From source file:com.mde.potdroid.helpers.WebsiteInteraction.java

/**
 * Get a xml document from the mods.de api
 * @throws NoConnectionException //from  w  w w  .j  av a2s.c  o  m
 */
public Document getDocument(String url) throws NoConnectionException {
    Document document;

    // no internet connection...
    if (getConnectionType(mContext) == 0) {
        throw new NoConnectionException();
    }

    // our xml parser
    SAXBuilder parser = new SAXBuilder();

    try {
        // get the input stream from fetchContent().
        // return null if the fetching of the document failed
        HttpGet request = new HttpGet(url);
        request.addHeader("Accept-Encoding", "gzip");

        HttpResponse response = mHttpClient.execute(request);
        HttpEntity entity = response.getEntity();

        if ((entity == null) || !entity.isStreaming()) {
            throw new NoConnectionException();
        }

        // get the content input stream and take care of gzip encoding
        InputStream instream = entity.getContent();
        Header contentEncoding = response.getFirstHeader("Content-Encoding");
        if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase("gzip")) {
            instream = new GZIPInputStream(instream);
        }

        // build the xml document object
        document = parser.build(instream);
    } catch (Exception e) {
        e.printStackTrace();
        throw new NoConnectionException();
    }
    return document;
}

---

From source file:com.metamx.druid.loading.S3SegmentGetter.java

@Override
public File getSegmentFiles(Map<String, Object> loadSpec) throws StorageAdapterLoadingException {
    String s3Bucket = MapUtils.getString(loadSpec, "bucket");
    String s3Path = MapUtils.getString(loadSpec, "key");

    log.info("Loading index at path[s3://%s/%s]", s3Bucket, s3Path);

    S3Object s3Obj = null;//w  w  w.  j  a v a2  s  .c  o  m
    File tmpFile = null;
    try {
        if (!s3Client.isObjectInBucket(s3Bucket, s3Path)) {
            throw new StorageAdapterLoadingException("IndexFile[s3://%s/%s] does not exist.", s3Bucket, s3Path);
        }

        File cacheFile = new File(config.getCacheDirectory(), computeCacheFilePath(s3Bucket, s3Path));

        if (cacheFile.exists()) {
            S3Object objDetails = s3Client.getObjectDetails(new S3Bucket(s3Bucket), s3Path);
            DateTime cacheFileLastModified = new DateTime(cacheFile.lastModified());
            DateTime s3ObjLastModified = new DateTime(objDetails.getLastModifiedDate().getTime());
            if (cacheFileLastModified.isAfter(s3ObjLastModified)) {
                log.info("Found cacheFile[%s] with modified[%s], which is after s3Obj[%s].  Using.", cacheFile,
                        cacheFileLastModified, s3ObjLastModified);
                return cacheFile.getParentFile();
            }
            FileUtils.deleteDirectory(cacheFile.getParentFile());
        }

        long currTime = System.currentTimeMillis();

        tmpFile = File.createTempFile(s3Bucket, new DateTime().toString());
        log.info("Downloading file[s3://%s/%s] to local tmpFile[%s] for cacheFile[%s]", s3Bucket, s3Path,
                tmpFile, cacheFile);

        s3Obj = s3Client.getObject(new S3Bucket(s3Bucket), s3Path);
        StreamUtils.copyToFileAndClose(s3Obj.getDataInputStream(), tmpFile, DEFAULT_TIMEOUT);
        final long downloadEndTime = System.currentTimeMillis();
        log.info("Download of file[%s] completed in %,d millis", cacheFile, downloadEndTime - currTime);

        if (!cacheFile.getParentFile().mkdirs()) {
            log.info("Unable to make parent file[%s]", cacheFile.getParentFile());
        }
        cacheFile.delete();

        if (s3Path.endsWith("gz")) {
            log.info("Decompressing file[%s] to [%s]", tmpFile, cacheFile);
            StreamUtils.copyToFileAndClose(new GZIPInputStream(new FileInputStream(tmpFile)), cacheFile);
            if (!tmpFile.delete()) {
                log.error("Could not delete tmpFile[%s].", tmpFile);
            }
        } else {
            log.info("Rename tmpFile[%s] to cacheFile[%s]", tmpFile, cacheFile);
            if (!tmpFile.renameTo(cacheFile)) {
                log.warn("Error renaming tmpFile[%s] to cacheFile[%s].  Copying instead.", tmpFile, cacheFile);

                StreamUtils.copyToFileAndClose(new FileInputStream(tmpFile), cacheFile);
                if (!tmpFile.delete()) {
                    log.error("Could not delete tmpFile[%s].", tmpFile);
                }
            }
        }

        long endTime = System.currentTimeMillis();
        log.info("Local processing of file[%s] done in %,d millis", cacheFile, endTime - downloadEndTime);

        return cacheFile.getParentFile();
    } catch (Exception e) {
        throw new StorageAdapterLoadingException(e, e.getMessage());
    } finally {
        S3Utils.closeStreamsQuietly(s3Obj);
        if (tmpFile != null && tmpFile.exists()) {
            log.warn("Deleting tmpFile[%s] in finally block.  Why?", tmpFile);
            tmpFile.delete();
        }
    }
}

---

From source file:ca.uhn.fhir.rest.server.servlet.ServletRequestDetails.java

@Override
protected byte[] getByteStreamRequestContents() {
    /*// w  ww.jav a2s  .c  o  m
     * This is weird, but this class is used both in clients and in servers, and we want to avoid needing to depend on
     * servlet-api in clients since there is no point. So we dynamically load a class that does the servlet processing
     * in servers. Down the road it may make sense to just split the method binding classes into server and client
     * versions, but this isn't actually a huge deal I don't think.
     */
    IRequestReader reader = ourRequestReader;
    if (reader == null) {
        try {
            Class.forName("javax.servlet.ServletInputStream");
            String className = BaseMethodBinding.class.getName() + "$" + "ActiveRequestReader";
            try {
                reader = (IRequestReader) Class.forName(className).newInstance();
            } catch (Exception e1) {
                throw new ConfigurationException("Failed to instantiate class " + className, e1);
            }
        } catch (ClassNotFoundException e) {
            String className = BaseMethodBinding.class.getName() + "$" + "InactiveRequestReader";
            try {
                reader = (IRequestReader) Class.forName(className).newInstance();
            } catch (Exception e1) {
                throw new ConfigurationException("Failed to instantiate class " + className, e1);
            }
        }
        ourRequestReader = reader;
    }

    try {
        InputStream inputStream = reader.getInputStream(this);
        requestContents = IOUtils.toByteArray(inputStream);

        if (myServer.isUncompressIncomingContents()) {
            String contentEncoding = myServletRequest.getHeader(Constants.HEADER_CONTENT_ENCODING);
            if ("gzip".equals(contentEncoding)) {
                ourLog.debug("Uncompressing (GZip) incoming content");
                if (requestContents.length > 0) {
                    GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(requestContents));
                    requestContents = IOUtils.toByteArray(gis);
                }
            }
        }

        return requestContents;
    } catch (IOException e) {
        ourLog.error("Could not load request resource", e);
        throw new InvalidRequestException(String.format("Could not load request resource: %s", e.getMessage()));
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• »