Example usage for org.apache.commons.io FileUtils copyURLToFile

Introduction

In this page you can find the example usage for org.apache.commons.io FileUtils copyURLToFile.

Prototype

public static void copyURLToFile(URL source, File destination) throws IOException 

Source Link

Document

Copies bytes from the URL source to a file destination.

Usage

From source file:org.deeplearning4j.datasets.loader.ReutersNewsGroupsLoader.java

private void getIfNotExists() throws Exception {
    String home = System.getProperty("user.home");
    String rootDir = home + File.separator + "reuters";
    reutersRootDir = new File(rootDir);
    if (!reutersRootDir.exists())
        reutersRootDir.mkdir();/*from  w w  w. j  a  v a  2 s .  c o  m*/
    else if (reutersRootDir.exists())
        return;

    File rootTarFile = new File(reutersRootDir, "20news-18828.tar.gz");
    if (rootTarFile.exists())
        rootTarFile.delete();
    rootTarFile.createNewFile();

    FileUtils.copyURLToFile(new URL(NEWSGROUP_URL), rootTarFile);
    ArchiveUtils.unzipFileTo(rootTarFile.getAbsolutePath(), reutersRootDir.getAbsolutePath());
    rootTarFile.delete();
    FileUtils.copyDirectory(new File(reutersRootDir, "20news-18828"), reutersRootDir);
    FileUtils.deleteDirectory(new File(reutersRootDir, "20news-18828"));
    if (reutersRootDir.listFiles() == null)
        throw new IllegalStateException("No files found!");

}

---

From source file:org.deeplearning4j.examples.modelimport.keras.ImportDeepMoji.java

public static void main(String[] args) throws Exception {

    // First, register the Keras layer wrapped around our custom SameDiff attention layer
    KerasLayer.registerCustomLayer("AttentionWeightedAverage", KerasDeepMojiAttention.class);

    // Then, download the model from azure (check if it's cached)
    File directory = new File(DATA_PATH);
    if (!directory.exists())
        directory.mkdir();//from  w  w  w.j a va 2s  .c o  m

    String modelUrl = "http://blob.deeplearning4j.org/models/deepmoji.h5";
    String downloadPath = DATA_PATH + "deepmoji_model.h5";
    File cachedKerasFile = new File(downloadPath);

    if (!cachedKerasFile.exists()) {
        System.out.println("Downloading model to " + cachedKerasFile.toString());
        FileUtils.copyURLToFile(new URL(modelUrl), cachedKerasFile);
        System.out.println("Download complete");
        cachedKerasFile.deleteOnExit();
    }

    // Finally, import the model and test on artificial input data.
    ComputationGraph graph = KerasModelImport.importKerasModelAndWeights(cachedKerasFile.getAbsolutePath());
    ;
    INDArray input = Nd4j.create(new int[] { 10, 30 });
    graph.output(input);
    System.out.println("Example completed.");
}

---

From source file:org.deeplearning4j.examples.multigpu.rnn.LSTMCharModellingExample.java

/** Downloads Shakespeare training data and stores it locally (temp directory). Then set up and return a simple
 * DataSetIterator that does vectorization based on the text.
 * @param miniBatchSize Number of text segments in each training mini-batch
 * @param sequenceLength Number of characters in each text segment.
 *//*  www  .  j  a v a 2  s . c om*/
public static CharacterIterator getShakespeareIterator(int miniBatchSize, int sequenceLength) throws Exception {
    //The Complete Works of William Shakespeare
    //5.3MB file in UTF-8 Encoding, ~5.4 million characters
    //https://www.gutenberg.org/ebooks/100
    String url = "https://s3.amazonaws.com/dl4j-distribution/pg100.txt";
    String tempDir = System.getProperty("java.io.tmpdir");
    String fileLocation = tempDir + "/Shakespeare.txt"; //Storage location from downloaded file
    File f = new File(fileLocation);
    if (!f.exists()) {
        FileUtils.copyURLToFile(new URL(url), f);
        System.out.println("File downloaded to " + f.getAbsolutePath());
    } else {
        System.out.println("Using existing text file at " + f.getAbsolutePath());
    }

    if (!f.exists())
        throw new IOException("File does not exist: " + fileLocation); //Download problem?

    char[] validCharacters = CharacterIterator.getMinimalCharacterSet(); //Which characters are allowed? Others will be removed
    return new CharacterIterator(fileLocation, Charset.forName("UTF-8"), miniBatchSize, sequenceLength,
            validCharacters, new Random(12345));
}

---

From source file:org.deeplearning4j.examples.multigpu.vgg16.dataHelpers.FlowerDataSetIterator.java

public static void downloadAndUntar() throws IOException {
    File rootFile = new File(DATA_DIR);
    if (!rootFile.exists()) {
        rootFile.mkdir();/*from   ww w  .  ja  v a2  s. c  o m*/
    }
    File tarFile = new File(DATA_DIR, "flower_photos.tgz");
    if (!tarFile.isFile()) {
        log.info("Downloading the flower dataset from " + DATA_URL + "...");
        FileUtils.copyURLToFile(new URL(DATA_URL), tarFile);
    }
    ArchiveUtils.unzipFileTo(tarFile.getAbsolutePath(), rootFile.getAbsolutePath());
}

---

From source file:org.deeplearning4j.examples.multigpu.w2vsentiment.DataSetsBuilder.java

private static void downloadData() throws Exception {
    //Create directory if required
    File directory = new File(DATA_PATH);
    if (!directory.exists())
        directory.mkdir();//www. j av  a2  s.  c o  m

    //Download file:
    String archizePath = DATA_PATH + "aclImdb_v1.tar.gz";
    File archiveFile = new File(archizePath);
    String extractedPath = DATA_PATH + "aclImdb";
    File extractedFile = new File(extractedPath);

    if (!archiveFile.exists()) {
        System.out.println("Starting data download (80MB)...");
        FileUtils.copyURLToFile(new URL(DATA_URL), archiveFile);
        System.out.println("Data (.tar.gz file) downloaded to " + archiveFile.getAbsolutePath());
        //Extract tar.gz file to output directory
        extractTarGz(archizePath, DATA_PATH);
    } else {
        //Assume if archive (.tar.gz) exists, then data has already been extracted
        System.out.println("Data (.tar.gz file) already exists at " + archiveFile.getAbsolutePath());
        if (!extractedFile.exists()) {
            //Extract tar.gz file to output directory
            extractTarGz(archizePath, DATA_PATH);
        } else {
            System.out.println("Data (extracted) already exists at " + extractedFile.getAbsolutePath());
        }
    }
}

---

From source file:org.deeplearning4j.examples.recurrent.character.melodl4j.MelodyModelingExample.java

public static void makeSureFileIsInTmpDir(String filename) {
    final File f = new File(tmpDir + "/" + filename);
    if (!f.exists()) {
        URL url = null;/*from  w  ww .  j a va  2 s. co m*/
        try {
            url = new URL("http://truthsite.org/music/" + filename);
            FileUtils.copyURLToFile(url, f);
        } catch (Exception exc) {
            System.err.println("Error copying " + url + " to " + f);
            throw new RuntimeException(exc);
        }
        if (!f.exists()) {
            throw new RuntimeException(f.getAbsolutePath() + " does not exist");
        }
        System.out.println("File downloaded to " + f.getAbsolutePath());
    } else {
        System.out.println("Using existing text file at " + f.getAbsolutePath());
    }
}

---

From source file:org.deeplearning4j.legacyExamples.rnn.SparkLSTMCharacterExample.java

private static List<String> getShakespeareAsList(int sequenceLength) throws IOException {
    //The Complete Works of William Shakespeare
    //5.3MB file in UTF-8 Encoding, ~5.4 million characters
    //https://www.gutenberg.org/ebooks/100
    String url = "https://s3.amazonaws.com/dl4j-distribution/pg100.txt";
    String tempDir = System.getProperty("java.io.tmpdir");
    String fileLocation = tempDir + "/Shakespeare.txt"; //Storage location from downloaded file
    File f = new File(fileLocation);
    if (!f.exists()) {
        FileUtils.copyURLToFile(new URL(url), f);
        System.out.println("File downloaded to " + f.getAbsolutePath());
    } else {//from w w w.jav a  2  s . c om
        System.out.println("Using existing text file at " + f.getAbsolutePath());
    }

    if (!f.exists())
        throw new IOException("File does not exist: " + fileLocation); //Download problem?

    String allData = getDataAsString(fileLocation);

    List<String> list = new ArrayList<>();
    int length = allData.length();
    int currIdx = 0;
    while (currIdx + sequenceLength < length) {
        int end = currIdx + sequenceLength;
        String substr = allData.substring(currIdx, end);
        currIdx = end;
        list.add(substr);
    }
    return list;
}

---

From source file:org.deeplearning4j.nn.modelimport.keras.e2e.KerasCustomLayerTest.java

public void testCustomLayerImport() throws Exception {
    // file paths
    String kerasWeightsAndConfigUrl = "http://blob.deeplearning4j.org/models/googlenet_keras_weightsandconfig.h5";
    File cachedKerasFile = new File(System.getProperty("java.io.tmpdir"),
            "googlenet_keras_weightsandconfig.h5");
    String outputPath = System.getProperty("java.io.tmpdir") + "/googlenet_dl4j_inference.zip";

    KerasLayer.registerCustomLayer("PoolHelper", KerasPoolHelper.class);
    KerasLayer.registerCustomLayer("LRN", KerasLRN.class);

    // download file
    if (!cachedKerasFile.exists()) {
        log.info("Downloading model to " + cachedKerasFile.toString());
        FileUtils.copyURLToFile(new URL(kerasWeightsAndConfigUrl), cachedKerasFile);
        cachedKerasFile.deleteOnExit();/*  w ww. j  a va  2 s.co m*/
    }

    org.deeplearning4j.nn.api.Model importedModel = KerasModelImport
            .importKerasModelAndWeights(cachedKerasFile.getAbsolutePath());
    ModelSerializer.writeModel(importedModel, outputPath, false);

    ComputationGraph serializedModel = ModelSerializer.restoreComputationGraph(outputPath);
    log.info(serializedModel.summary());
}

---

From source file:org.deeplearning4j.nn.modelimport.keras.e2e.KerasModelEndToEndTest.java

/**
 * Inception V4//w  w w  .  ja  va2 s . com
 */
@Test
@Ignore
// Model and weights have about 170mb, too large for test resources and also too excessive to enable as unit test
public void importInceptionV4() throws Exception {
    String modelUrl = DL4JResources.getURLString("models/inceptionv4_keras_imagenet_weightsandconfig.h5");
    File kerasFile = testDir.newFile("inceptionv4_keras_imagenet_weightsandconfig.h5");

    if (!kerasFile.exists()) {
        FileUtils.copyURLToFile(new URL(modelUrl), kerasFile);
        kerasFile.deleteOnExit();
    }

    int[] inputShape = new int[] { 299, 299, 3 };
    ComputationGraph graph = importFunctionalModelH5Test(kerasFile.getAbsolutePath(), inputShape, false);

    // System.out.println(graph.summary());

}

---

From source file:org.deeplearning4j.ui.nearestneighbors.NearestNeighborsResource.java

@POST
@Path("/update")
@Produces(MediaType.APPLICATION_JSON)/*from www .java 2s. co  m*/
public Response updateFilePath(UrlResource resource) {
    if (!resource.getUrl().startsWith("http")) {
        this.localFile = new File(".", resource.getUrl());
        handleUpload(localFile);
    } else {
        File dl = new File(filePath, UUID.randomUUID().toString());
        try {
            FileUtils.copyURLToFile(new URL(resource.getUrl()), dl);
        } catch (Exception e) {
            e.printStackTrace();
        }

        handleUpload(dl);

    }

    return Response.ok(Collections.singletonMap("message", "Uploaded file")).build();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• »