List of usage examples for java.util.zip GZIPInputStream read
public int read(byte[] buf, int off, int len) throws IOException
From source file:co.cask.hydrator.transforms.CSVParser2.java
/** * UnCompress payload using GZIP Algorithm * * @param body byte array// w ww . ja v a2 s. co m * @return decompressed bytes. * @throws IOException */ private byte[] ungzip(byte[] body) throws IOException { Inflater inf = new Inflater(); ByteArrayInputStream bytein = new ByteArrayInputStream(body); GZIPInputStream gzin = new GZIPInputStream(bytein); ByteArrayOutputStream byteout = new ByteArrayOutputStream(); int res = 0; byte buf[] = new byte[1024]; while (res >= 0) { res = gzin.read(buf, 0, buf.length); if (res > 0) { byteout.write(buf, 0, res); } } byte uncompressed[] = byteout.toByteArray(); return uncompressed; }
From source file:org.apache.nutch.tools.arc.ArcRecordReader.java
/** * <p>Returns true if the next record in the split is read into the key and * value pair. The key will be the arc record header and the values will be * the raw content bytes of the arc record.</p> * /*w w w . j a va 2s .c o m*/ * @param key The record key * @param value The record value * * @return True if the next record is read. * * @throws IOException If an error occurs while reading the record value. */ public boolean next(Text key, BytesWritable value) throws IOException { try { // get the starting position on the input stream long startRead = in.getPos(); byte[] magicBuffer = null; // we need this loop to handle false positives in reading of gzip records while (true) { // while we haven't passed the end of the split if (startRead >= splitEnd) { return false; } // scanning for the gzip header boolean foundStart = false; while (!foundStart) { // start at the current file position and scan for 1K at time, break // if there is no more to read startRead = in.getPos(); magicBuffer = new byte[1024]; int read = in.read(magicBuffer); if (read < 0) { break; } // scan the byte array for the gzip header magic number. This happens // byte by byte for (int i = 0; i < read - 1; i++) { byte[] testMagic = new byte[2]; System.arraycopy(magicBuffer, i, testMagic, 0, 2); if (isMagic(testMagic)) { // set the next start to the current gzip header startRead += i; foundStart = true; break; } } } // seek to the start of the gzip header in.seek(startRead); ByteArrayOutputStream baos = null; int totalRead = 0; try { // read 4K of the gzip at a time putting into a byte array byte[] buffer = new byte[4096]; GZIPInputStream zin = new GZIPInputStream(in); int gzipRead = -1; baos = new ByteArrayOutputStream(); while ((gzipRead = zin.read(buffer, 0, buffer.length)) != -1) { baos.write(buffer, 0, gzipRead); totalRead += gzipRead; } } catch (Exception e) { // there are times we get false positives where the gzip header exists // but it is not an actual gzip record, so we ignore it and start // over seeking System.out.println("Ignoring position: " + (startRead)); if (startRead + 1 < fileLen) { in.seek(startRead + 1); } continue; } // change the output stream to a byte array byte[] content = baos.toByteArray(); // the first line of the raw content in arc files is the header int eol = 0; for (int i = 0; i < content.length; i++) { if (i > 0 && content[i] == '\n') { eol = i; break; } } // create the header and the raw content minus the header String header = new String(content, 0, eol).trim(); byte[] raw = new byte[(content.length - eol) - 1]; System.arraycopy(content, eol + 1, raw, 0, raw.length); // populate key and values with the header and raw content. Text keyText = (Text) key; keyText.set(header); BytesWritable valueBytes = (BytesWritable) value; valueBytes.set(raw, 0, raw.length); // TODO: It would be best to start at the end of the gzip read but // the bytes read in gzip don't match raw bytes in the file so we // overshoot the next header. With this current method you get // some false positives but don't miss records. if (startRead + 1 < fileLen) { in.seek(startRead + 1); } // populated the record, now return return true; } } catch (Exception e) { LOG.equals(StringUtils.stringifyException(e)); } // couldn't populate the record or there is no next record to read return false; }
From source file:org.okj.im.core.service.QQHttpService.java
/** * gzip?//from ww w. ja va2s . co m * @param conn * @return */ protected String getGzipResponseString(HttpURLConnection conn) { InputStream ins = null; GZIPInputStream gzip = null; ByteArrayOutputStream baos = null; try { ins = conn.getInputStream(); gzip = new GZIPInputStream(ins); baos = new ByteArrayOutputStream(); byte[] buf = new byte[1024 * 8]; int num = -1; while ((num = gzip.read(buf, 0, buf.length)) != -1) { baos.write(buf, 0, num); } byte[] b = baos.toByteArray(); baos.flush(); return new String(b).trim(); } catch (IOException ex) { LogUtils.error(LOGGER, "?[gzip]", ex); } finally { IOUtils.closeQuietly(gzip); IOUtils.closeQuietly(ins); IOUtils.closeQuietly(baos); } return null; }
From source file:net.sf.ehcache.constructs.web.PageInfoTest.java
private byte[] ungzip2(final byte[] gzip) throws IOException { final GZIPInputStream inputStream = new GZIPInputStream(new ByteArrayInputStream(gzip)); final byte[] buffer = new byte[1500000]; int bytesRead = 0; int counter = 0; while (bytesRead != -1) { bytesRead = inputStream.read(buffer, counter, 4096); counter += bytesRead;// w ww .j a va 2s . co m } //Revert the last -1 when the end of stream was reached counter++; byte[] unzipped = new byte[counter]; System.arraycopy(buffer, 0, unzipped, 0, counter); return unzipped; }
From source file:model.Modele.java
/** * Methode qui tlcharger le fichier des donnes du site directement * * @param jour jour du fichier telecharger * @param mois mois du fichier telecharger * @param annee annee du fichier telecharger * @param heure heure du fichier telecharger * @return nom du fichier tlcharger/*from w w w .ja v a2 s. c o m*/ * @throws MalformedURLException * @throws IOException */ public String Download_File(String jour, String mois, String annee, String heure) throws MalformedURLException, IOException { if (jour.equals("NULL") && heure.equals("NULL")) { URL url = new URL("https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop." + annee + mois + ".csv.gz"); File file = new File("synop." + annee + mois + ".csv.gz"); FileUtils.copyURLToFile(url, file); FileInputStream in = new FileInputStream("synop." + annee + mois + ".csv.gz"); GZIPInputStream zipin = new GZIPInputStream(in); byte[] buffer = new byte[8192]; FileOutputStream out = new FileOutputStream("synop." + annee + mois + ".csv.gz.txt"); int length; while ((length = zipin.read(buffer, 0, 8192)) != -1) { out.write(buffer, 0, length); } out.close(); zipin.close(); File f = new File("synop." + annee + mois + ".csv.gz.txt"); if (f.exists()) { nom_fichier = Save_data(f); } else { JOptionPane.showMessageDialog(null, "Fichier inexistant ou n'a pas pu tre tlcharg !"); } } else { if (annee.equals("2017")) { URL url = new URL("https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop." + annee + mois + jour + heure + ".csv.gz"); File file = new File("synop." + annee + mois + jour + heure + ".csv"); FileUtils.copyURLToFile(url, file); File f = new File("synop." + annee + mois + jour + heure + ".csv"); if (f.exists()) { nom_fichier = Save_data(f); } else { JOptionPane.showMessageDialog(null, "Fichier inexistant ou n'a pas pu tre tlcharg !"); } } else { JOptionPane.showMessageDialog(null, "Seulement l'anne 2017 disponible pour cette option !"); } } return nom_fichier; }
From source file:org.ocsinventoryng.android.actions.OCSProtocol.java
public String sendmethod(File pFile, String server, boolean gziped) throws OCSProtocolException { OCSLog ocslog = OCSLog.getInstance(); OCSSettings ocssettings = OCSSettings.getInstance(); ocslog.debug("Start send method"); String retour;//from www . j a va 2 s . c om HttpPost httppost; try { httppost = new HttpPost(server); } catch (IllegalArgumentException e) { ocslog.error(e.getMessage()); throw new OCSProtocolException("Incorect serveur URL"); } File fileToPost; if (gziped) { ocslog.debug("Start compression"); fileToPost = ocsfile.getGzipedFile(pFile); if (fileToPost == null) { throw new OCSProtocolException("Error during temp file creation"); } ocslog.debug("Compression done"); } else { fileToPost = pFile; } FileEntity fileEntity = new FileEntity(fileToPost, "text/plain; charset=\"UTF-8\""); httppost.setEntity(fileEntity); httppost.setHeader("User-Agent", http_agent); if (gziped) { httppost.setHeader("Content-Encoding", "gzip"); } DefaultHttpClient httpClient = getNewHttpClient(OCSSettings.getInstance().isSSLStrict()); if (ocssettings.isProxy()) { ocslog.debug("Use proxy : " + ocssettings.getProxyAdr()); HttpHost proxy = new HttpHost(ocssettings.getProxyAdr(), ocssettings.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); } if (ocssettings.isAuth()) { ocslog.debug("Use AUTH : " + ocssettings.getLogin() + "/*****"); UsernamePasswordCredentials creds = new UsernamePasswordCredentials(ocssettings.getLogin(), ocssettings.getPasswd()); ocslog.debug(creds.toString()); httpClient.getCredentialsProvider() .setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), creds); } ocslog.debug("Call : " + server); HttpResponse localHttpResponse; try { localHttpResponse = httpClient.execute(httppost); ocslog.debug("Message sent"); } catch (ClientProtocolException e) { ocslog.error("ClientProtocolException" + e.getMessage()); throw new OCSProtocolException(e.getMessage()); } catch (IOException e) { String msg = appCtx.getString(R.string.err_cant_connect) + " " + e.getMessage(); ocslog.error(msg); throw new OCSProtocolException(msg); } try { int httpCode = localHttpResponse.getStatusLine().getStatusCode(); ocslog.debug("Response status code : " + String.valueOf(httpCode)); if (httpCode == 200) { if (gziped) { InputStream is = localHttpResponse.getEntity().getContent(); GZIPInputStream gzis = new GZIPInputStream(is); ByteArrayOutputStream baos = new ByteArrayOutputStream(); byte[] buff = new byte[128]; int n; while ((n = gzis.read(buff, 0, buff.length)) > 0) { baos.write(buff, 0, n); } retour = baos.toString(); } else { retour = EntityUtils.toString(localHttpResponse.getEntity()); } } else if (httpCode == 400) { throw new OCSProtocolException("Error http 400 may be wrong agent version"); } else { ocslog.error("***Server communication error: "); throw new OCSProtocolException("Http communication error code " + String.valueOf(httpCode)); } ocslog.debug("Finnish send method"); } catch (IOException localIOException) { String msg = localIOException.getMessage(); ocslog.error(msg); throw new OCSProtocolException(msg); } return retour; }
From source file:net.sf.ehcache.constructs.web.PageInfoTest.java
/** * A high performance implementation, although not as fast as gunzip3. * gunzips 100000 of ungzipped content in 9ms on the reference machine. * It does not use a fixed size buffer and is therefore suitable for arbitrary * length arrays./*from www.j a v a 2s . c om*/ * @param gzipped * @return * @throws IOException */ public byte[] ungzip1(final byte[] gzipped) throws IOException { final GZIPInputStream inputStream = new GZIPInputStream(new ByteArrayInputStream(gzipped)); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(gzipped.length); final byte[] buffer = new byte[4096]; int bytesRead = 0; while (bytesRead != -1) { bytesRead = inputStream.read(buffer, 0, 4096); if (bytesRead != -1) { byteArrayOutputStream.write(buffer, 0, bytesRead); } } byte[] ungzipped = byteArrayOutputStream.toByteArray(); inputStream.close(); byteArrayOutputStream.close(); return ungzipped; }
From source file:grails.plugin.cache.web.PageInfo.java
/** * A highly performant ungzip implementation. Do not refactor this without * taking new timings. See ElementTest for timings * * @param gzipped the gzipped content// w ww . j av a2s.c o m * @return an ungzipped byte[] * @throws IOException */ protected byte[] ungzip(final byte[] gzipped) throws IOException { GZIPInputStream inputStream = new GZIPInputStream(new ByteArrayInputStream(gzipped)); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(gzipped.length); byte[] buffer = new byte[FOUR_KB]; int bytesRead = 0; while (bytesRead != -1) { bytesRead = inputStream.read(buffer, 0, FOUR_KB); if (bytesRead != -1) { byteArrayOutputStream.write(buffer, 0, bytesRead); } } byte[] ungzipped = byteArrayOutputStream.toByteArray(); inputStream.close(); byteArrayOutputStream.close(); return ungzipped; }
From source file:org.smartfrog.avalanche.client.sf.compress.ZipUtils.java
public static boolean gunzip(File gzipFile, File outputFile) throws IOException { GZIPInputStream gzipInstream; String gzFileName = gzipFile.getName(); if (!gzipFile.exists()) { log.error("The file " + gzFileName + " does not exist"); return false; }// w ww . java 2s . c o m if (!gzipFile.isFile()) { log.error("The file " + gzFileName + " is not a file"); return false; } if (outputFile.exists()) { // overwrite } if (outputFile.isDirectory()) { log.error("The given path " + outputFile + " is a directory"); return false; } if (gzipFile.getAbsolutePath().equals(outputFile.getAbsolutePath())) { log.error("Source and destination paths for un-zipping the file are same."); return false; } String file = gzipFile.getPath(); FileInputStream in = new FileInputStream(file); BufferedInputStream src = new BufferedInputStream(in); gzipInstream = new GZIPInputStream(src); byte[] buffer = new byte[chunkSize]; int len = 0; FileOutputStream out = new FileOutputStream(outputFile); BufferedOutputStream output = new BufferedOutputStream(out, chunkSize); /* * Read from gzip stream which will uncompress and write to output * stream */ while ((len = gzipInstream.read(buffer, 0, chunkSize)) != -1) { output.write(buffer, 0, len); } output.flush(); out.close(); gzipInstream.close(); log.info("Uncompress completed for file - " + gzFileName); return true; }
From source file:edu.dfci.cccb.mev.dataset.domain.gzip.GzipTsvInput.java
@Override public InputStream input() throws IOException { if (log.isDebugEnabled()) log.debug(url);//from ww w .ja va 2s.c o m GZIPInputStream zis = null; BufferedOutputStream dest = null; BufferedInputStream bis = null; File unzipped = new TemporaryFile(); try { URLConnection urlc = url.openConnection(); zis = new GZIPInputStream(urlc.getInputStream()); int count; byte data[] = new byte[BUFFER]; // write the files to the disk FileOutputStream fos = new FileOutputStream(unzipped); if (log.isDebugEnabled()) log.debug("Unzipping SOFT file to " + unzipped.getAbsolutePath()); dest = new BufferedOutputStream(fos, BUFFER); while ((count = zis.read(data, 0, BUFFER)) != -1) { dest.write(data, 0, count); } dest.flush(); dest.close(); return new FileInputStream(unzipped); } finally { if (bis != null) { try { bis.close(); } catch (IOException ioe) { } } if (dest != null) { try { dest.close(); } catch (IOException ioe) { } } if (zis != null) { try { zis.close(); } catch (IOException ioe) { } } } }