Example usage for java.util.zip GZIPInputStream GZIPInputStream

Introduction

In this page you can find the example usage for java.util.zip GZIPInputStream GZIPInputStream.

Prototype

public GZIPInputStream(InputStream in) throws IOException

Source Link

Document

Creates a new input stream with a default buffer size.

Usage

From source file:com.datatorrent.lib.io.FtpInputOperator.java

@Override
public void setup(OperatorContext arg0) {
    ftp = new FTPClient();
    if (ftpServer == null) {
        throw new RuntimeException("The ftp server can't be null");
    }//from  w w w.  j av  a  2s . com
    if (filePath == null) {
        throw new RuntimeException("The file Path can't be null");
    }
    try {
        if (port != 0) {
            ftp.connect(ftpServer, port);
        } else {
            ftp.connect(ftpServer);
        }
        if (localPassiveMode) {
            ftp.enterLocalPassiveMode();
        }
        ftp.login(getUserName(), getPassword());
        InputStream is = ftp.retrieveFileStream(filePath);
        InputStreamReader reader;
        if (isGzip) {
            GZIPInputStream gzis = new GZIPInputStream(is);
            reader = new InputStreamReader(gzis);
        } else {
            reader = new InputStreamReader(is);
        }
        in = new BufferedReader(reader);
    } catch (Exception e) {
        throw new RuntimeException(e);

    }
}

From source file:de.unidue.ltl.flextag.core.reports.adapter.TtWekaKnownUnknownWordAccuracyReport.java

@Override
protected List<String> extractVocab(File train) throws Exception {
    List<String> training = new ArrayList<String>();
    InputStreamReader streamReader = new InputStreamReader(new GZIPInputStream(new FileInputStream(train)),
            "UTF-8");
    BufferedReader br = new BufferedReader(streamReader);

    String next = null;//  w ww.j  a  v  a 2 s.  c  o  m
    while ((next = br.readLine()) != null) {

        if (next.startsWith("@")) {
            continue;
        }
        if (next.isEmpty()) {
            continue;
        }

        String word = extractUnit(next);
        training.add(word);
    }

    br.close();
    return training;
}

From source file:net.minecrell.ice.launch.transformers.DeobfuscationTransformer.java

public DeobfuscationTransformer() throws Exception {
    Path path = (Path) Launch.blackboard.get("ice.deobf-srg");
    String name = path.getFileName().toString();
    boolean gzip = name.endsWith(".gz");

    ImmutableBiMap.Builder<String, String> classes = ImmutableBiMap.builder();
    ImmutableTable.Builder<String, String, String> fields = ImmutableTable.builder();
    ImmutableTable.Builder<String, String, String> methods = ImmutableTable.builder();

    try (BufferedReader reader = new BufferedReader(new InputStreamReader(
            gzip ? new GZIPInputStream(Files.newInputStream(path)) : Files.newInputStream(path),
            StandardCharsets.UTF_8))) {
        String line;//from  ww  w.  j a  va 2  s . co m
        while ((line = reader.readLine()) != null) {
            if ((line = line.trim()).isEmpty())
                continue;

            String[] parts = StringUtils.split(line, ' ');
            if (parts.length < 3) {
                System.out.println("Invalid line: " + line);
                continue;
            }

            MappingType type = MappingType.of(parts[0]);
            if (type == null) {
                System.out.println("Invalid mapping: " + line);
                continue;
            }

            String[] source, dest;
            switch (type) {
            case CLASS:
                classes.put(parts[1], parts[2]);
                break;
            case FIELD:
                source = getSignature(parts[1]);
                dest = getSignature(parts[2]);
                String fieldType = getFieldType(source[0], source[1]);
                fields.put(source[0], source[1] + ':' + fieldType, dest[1]);
                if (fieldType != null)
                    fields.put(source[0], source[1] + ":null", dest[1]);
                break;
            case METHOD:
                source = getSignature(parts[1]);
                dest = getSignature(parts[3]);
                methods.put(source[0], source[1] + parts[2], dest[1]);
                break;
            }
        }
    }

    this.classes = classes.build();
    this.rawFields = fields.build();
    this.rawMethods = methods.build();

    this.fields = Maps.newHashMapWithExpectedSize(rawFields.size());
    this.methods = Maps.newHashMapWithExpectedSize(rawMethods.size());
}

From source file:uk.ac.ebi.eva.pipeline.jobs.steps.VariantNormalizerStepTest.java

@Test
public void normalizerStepShouldTransformAllVariants() throws Exception {
    Config.setOpenCGAHome(opencgaHome);/*from  ww w .j a  v  a2s.  c  o m*/

    String inputFile = VariantNormalizerStepTest.class.getResource(input).getFile();
    jobOptions.getPipelineOptions().put("input.vcf", inputFile);

    String outputFilename = getTransformedOutputPath(Paths.get(input).getFileName(), ".gz", "/tmp");

    File file = new File(outputFilename);
    if (file.exists())
        file.delete();
    assertFalse(file.exists());

    // When the execute method in variantsTransform is executed
    JobExecution jobExecution = jobLauncherTestUtils.launchStep(GenotypedVcfJob.NORMALIZE_VARIANTS);

    //Then variantsTransform should complete correctly
    assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus());
    assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus());

    // And the transformed file should contain the same number of line of the Vcf input file
    Assert.assertEquals(300, getLines(new GZIPInputStream(new FileInputStream(outputFilename))));

    file.delete();
    new File(outputDir, "small20.vcf.gz.file.json.gz").delete();
}

From source file:jcurl.core.io.SetupSaxDeSer.java

public static SetupBuilder parse(final URL file) throws SAXException, IOException {
    if (file.getFile().endsWith("z"))
        return parse(new GZIPInputStream(file.openStream()));
    return parse(file.openStream());
}

From source file:europarl.PhraseTranslation.java

public boolean getFromGz(String fileName, String targetWord, int limit) {
    String strLine;/*w w w  .java 2 s . c  o m*/
    ArrayList<String> line_triple = new ArrayList<String>();

    BufferedReader gzipReader;
    Pattern word_align = Pattern.compile("(\\w+) \\(\\{(.*?)\\}\\) ");

    Bag<String> words_list = new Bag<String>(); //Set of ALL words: it will be the list of attributes
    ArrayList<PhraseTranslation> translations = new ArrayList<PhraseTranslation>();
    try {
        gzipReader = new BufferedReader(
                new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));

        while ((strLine = gzipReader.readLine()) != null) //read-everything
        {
            line_triple.add(strLine);
            if (line_triple.size() == 3) //triple finished
            {
                //TODO: match only complete words
                //TODO: stem it before doing this

                Matcher matcher = word_align.matcher(line_triple.get(2));
                String[] foreign_words = line_triple.get(1).split(" ");
                line_triple.clear();
                if (!strLine.contains(targetWord)) //skip it
                    continue;

                ArrayList<String> e_phrase = new ArrayList<String>();
                String translation = "";
                while (matcher.find()) //each iteration is word +alignment
                {
                    assert matcher.groupCount() == 2;
                    String e_word = matcher.group(1).trim();
                    if (e_word.equals("NULL"))
                        e_word = "";
                    if (stopwordsList.contains(e_word))
                        continue;
                    if (stemmer != null)
                        e_word = stemmer.stem(e_word);

                    e_phrase.add(e_word);
                    words_list.add(e_word);

                    //we don't care about the alignment of non-target words
                    if (!e_word.equals(targetWord))
                        continue;

                    //parse the { x y z } alignment part
                    ArrayList<Integer> f_words = new ArrayList<Integer>();
                    translation = "";
                    //for each number between curly brackets
                    for (String number : matcher.group(2).split(" ")) {
                        if (!number.isEmpty()) {
                            int n_word = Integer.parseInt(number) - 1;
                            f_words.add(n_word);
                            translation += foreign_words[n_word] + " ";
                        }
                    } // end of curly brackets for

                } //end of word+alignment while
                if (!translation.isEmpty()) {
                    PhraseTranslation trans = new PhraseTranslation(e_phrase, translation);
                    translations.add(trans);
                }
                line_triple.clear();
            } //end of triple-finished if
            if (translations.size() == limit)
                break; //stop collecting!
        } //end of the read-everything while
    } catch (Exception e) {
        log.error("Error: " + e);
        e.printStackTrace();
        return false;
    }

    //what we NOW have: a set of attributes in HashSet<String>words_list
    //a ArrayList<PhraseTranslation> translations      
    log.info("Collected " + translations.size() + " phrases and " + words_list.size() + " words");

    postProcessData(translations, words_list);

    //now convert the data we collected to Weka data
    //we needed to do "double passing" because we need to initialize
    //the dataset with the complete list of attributes

    //this will convert word to attributes: they are all "boolean"
    ArrayList<Attribute> attrs = new ArrayList<Attribute>();
    HashMap<String, Attribute> attrs_map = new HashMap<String, Attribute>();
    Attribute att;
    for (String word : words_list) {
        att = new Attribute(word);
        attrs.add(att);
        attrs_map.put(word, att);
    }

    //now we need to manage class.
    //each translation is a class, so we need to get all of them
    HashMap<String, Integer> class_map = new HashMap<String, Integer>();
    ArrayList<String> classes = new ArrayList<String>();
    for (PhraseTranslation phraseTranslation : translations) {
        if (!class_map.containsKey(phraseTranslation.getTranslatedWord())) {
            class_map.put(phraseTranslation.getTranslatedWord(), classes.size());
            classes.add(phraseTranslation.getTranslatedWord());
        }
    }

    log.info(targetWord + " has " + classes.size() + " translations:");
    if (log.isInfoEnabled())
        for (String translation : classes)
            System.out.println(translation);
    att = new Attribute("%class", classes);
    attrs.add(att);
    attrs_map.put("%class", att);
    dataSet = new Instances("dataset", attrs, 0);
    for (PhraseTranslation phraseTranslation : translations) {
        SparseInstance inst = new SparseInstance(attrs.size());
        //set everything to 0
        for (int i = 0; i < attrs.size(); i++)
            inst.setValue(i, 0);
        //set present word to 1
        for (String word : phraseTranslation.getPhraseWords())
            inst.setValue(attrs_map.get(word), 1);
        //set class of instance
        inst.setValue(attrs_map.get("%class"), class_map.get(phraseTranslation.getTranslatedWord()));
        dataSet.add(inst);
    }

    return true;
}

From source file:net.nikr.eve.jeveasset.io.online.CitadelGetter.java

private void updateCache(UpdateTask updateTask) {
    LOG.info("Citadels updating:");
    if (citadelSettings.getNextUpdate().after(new Date()) && !Settings.get().isForceUpdate()
            && !Program.isForceUpdate()) { //Check if we can update now
        if (updateTask != null) {
            updateTask.addError(DialoguesUpdate.get().citadel(), "Not allowed yet.\r\n(Fix: Just wait a bit)");
        }//from  w  w w .j  a va2  s  .  c om
        LOG.info("   Citadels failed to update (NOT ALLOWED YET)");
        return;
    }
    //Update citadel
    InputStream in = null;
    try { //Update from API
        ObjectMapper mapper = new ObjectMapper(); //create once, reuse
        URL url = new URL(URL);
        HttpURLConnection con = (HttpURLConnection) url.openConnection();
        con.setRequestProperty("Accept-Encoding", "gzip");

        long contentLength = con.getContentLengthLong();
        String contentEncoding = con.getContentEncoding();
        InputStream inputStream = new UpdateTaskInputStream(con.getInputStream(), contentLength, updateTask);
        if ("gzip".equals(contentEncoding)) {
            in = new GZIPInputStream(inputStream);
        } else {
            in = inputStream;
        }
        Map<Long, Citadel> results = mapper.readValue(in, new TypeReference<Map<Long, Citadel>>() {
        });
        if (results != null) { //Updated OK
            for (Map.Entry<Long, Citadel> entry : results.entrySet()) {
                citadelSettings.put(entry.getKey(), entry.getValue());
            }
        }
        citadelSettings.setNextUpdate();
        saveXml();
        LOG.info("   Updated citadels for jEveAssets");
    } catch (IOException ex) {
        if (updateTask != null) {
            updateTask.addError(DialoguesUpdate.get().citadel(), ex.getMessage());
        }
        LOG.error("   Citadels failed to update", ex);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (IOException ex) {
                //No problem...
            }
        }
    }
}

From source file:org.droidparts.http.wrapper.DefaultHttpClientWrapper.java

public static InputStream getUnpackedInputStream(HttpEntity entity) throws HTTPException {
    try {//from   ww  w . j a v  a  2s  .  co m
        InputStream is = entity.getContent();
        Header contentEncodingHeader = entity.getContentEncoding();
        L.d(contentEncodingHeader);
        if (contentEncodingHeader != null) {
            String contentEncoding = contentEncodingHeader.getValue();
            if (!isEmpty(contentEncoding)) {
                contentEncoding = contentEncoding.toLowerCase();
                if (contentEncoding.contains("gzip")) {
                    return new GZIPInputStream(is);
                } else if (contentEncoding.contains("deflate")) {
                    return new InflaterInputStream(is);
                }
            }
        }
        return is;
    } catch (Exception e) {
        throw new HTTPException(e);
    }
}

From source file:org.calrissian.accumulorecipes.blobstore.impl.AccumuloBlobStoreTest.java

@Test
public void testSaveAndQueryComplex() throws Exception {
    AccumuloBlobStore blobStore = new AccumuloBlobStore(getConnector(), CHUNK_SIZE);

    Collection<String> testValues = new ArrayList<String>(10);
    for (int i = 0; i < CHUNK_SIZE; i++)
        testValues.add(randomUUID().toString());

    //Store json in a Gzipped format
    OutputStream storageStream = blobStore.store("test", "1", currentTimeMillis(), "");
    mapper.writeValue(new GZIPOutputStream(storageStream), testValues);

    //reassemble the json after unzipping the stream.
    InputStream retrievalStream = blobStore.get("test", "1", Auths.EMPTY);
    Collection<String> actualValues = mapper.readValue(new GZIPInputStream(retrievalStream), strColRef);

    //if there were no errors, then verify that the two collections are equal.
    assertThat(actualValues, is(equalTo(testValues)));
}

From source file:com.servoy.extensions.plugins.http.Response.java

/**
 * Get the content of response as binary data. It also supports gzip-ed content.
 *
 * @sample//from ww  w .  j a v  a 2  s  .  c  o m
 * var mediaData = response.getMediaData();
 */
public byte[] js_getMediaData() {
    if (response_body == null) {
        try {
            ByteArrayOutputStream sb = new ByteArrayOutputStream();
            InputStream is = null;
            Header contentEncoding = res.getFirstHeader("Content-Encoding");
            boolean gziped = contentEncoding == null ? false
                    : "gzip".equalsIgnoreCase(contentEncoding.getValue());
            is = res.getEntity().getContent();
            if (gziped) {
                is = new GZIPInputStream(is);
            }
            BufferedInputStream bis = new BufferedInputStream(is);
            Utils.streamCopy(bis, sb);
            bis.close();
            is.close();
            response_body = sb.toByteArray();
        } catch (IOException e) {
            Debug.error(e);
        }
    }
    return response_body instanceof byte[] ? (byte[]) response_body : null;
}