Example usage for org.apache.commons.io FileUtils copyURLToFile

List of usage examples for org.apache.commons.io FileUtils copyURLToFile

Introduction

In this page you can find the example usage for org.apache.commons.io FileUtils copyURLToFile.

Prototype

public static void copyURLToFile(URL source, File destination) throws IOException 

Source Link

Document

Copies bytes from the URL source to a file destination.

Usage

From source file:com.sonar.it.scanner.msbuild.CppTest.java

@Test
public void testCppWithSharedFiles() throws Exception {
    String projectKey = "cpp-shared";
    String fileKey = "cpp-shared:cpp-shared:90BD7FAF-0B72-4D37-9610-D7C92B217BB0:Project1.cpp";

    ORCHESTRATOR.getServer().restoreProfile(FileLocation.of("src/test/resources/TestQualityProfileCpp.xml"));
    ORCHESTRATOR.getServer().provisionProject(projectKey, "Cpp");
    ORCHESTRATOR.getServer().associateProjectToQualityProfile(projectKey, "cpp", "ProfileForTestCpp");

    Path projectDir = TestUtils.projectDir(temp, "CppSharedFiles");
    File wrapperOutDir = new File(projectDir.toFile(), "out");

    ORCHESTRATOR.executeBuild(TestUtils.newScanner(ORCHESTRATOR, projectDir).addArgument("begin")
            .setProjectKey(projectKey).setProjectName("Cpp").setProjectVersion("1.0")
            .setProperty("sonar.cfamily.build-wrapper-output", wrapperOutDir.toString()));
    File buildWrapper = temp.newFile();
    File buildWrapperDir = temp.newFolder();
    FileUtils.copyURLToFile(
            new URL(ORCHESTRATOR.getServer().getUrl() + "/static/cpp/build-wrapper-win-x86.zip"), buildWrapper);
    ZipUtils.unzip(buildWrapper, buildWrapperDir);

    TestUtils.runMSBuildWithBuildWrapper(ORCHESTRATOR, projectDir,
            new File(buildWrapperDir, "build-wrapper-win-x86/build-wrapper-win-x86-64.exe"), wrapperOutDir,
            "/t:Rebuild");

    BuildResult result = ORCHESTRATOR// www.  ja va  2  s.  co m
            .executeBuild(TestUtils.newScanner(ORCHESTRATOR, projectDir).addArgument("end"));
    assertThat(result.getLogs()).doesNotContain("Invalid character encountered in file");

    List<Issue> issues = TestUtils.allIssues(ORCHESTRATOR);

    List<String> keys = issues.stream().map(i -> i.getRule()).collect(Collectors.toList());
    assertThat(keys).containsAll(Arrays.asList("cpp:S106"));

    assertThat(getMeasureAsInteger(projectKey, "ncloc")).isEqualTo(22);
    assertThat(getMeasureAsInteger(fileKey, "ncloc")).isEqualTo(8);
}

From source file:com.carrotgarden.maven.scr.CarrotOsgiScrGenerate.java

/**
 * Attach empty place holder DS component descriptor to the final jar.
 *//* www. j  a v a2s  .c  om*/
protected void includeEmptyDescriptor() throws Exception {

    if (isContextIncremental()) {
        logDebug("skip including empty descriptor for incremental build");
        return;
    }

    final URL source = getClass().getResource(NULL_XML);

    final File target = absolute(new File(outputDirectorySCR(), NULL_XML));

    logDebug("");
    logDebug("including empty descriptor = " + target);

    FileUtils.copyURLToFile(source, target);

}

From source file:com.ssn.listener.SSNInstagramSelectionListener.java

private void createComponents(SSNHomeForm ssnHomeForm, List<InstagramMedia> completePhotoList) {
    SSNHelper.toggleDeleteAndShareImages(false, ssnHomeForm);

    try {//  www  .  j  a  v  a  2  s.com
        List<File> listOfFiles = new ArrayList<File>();

        File instagramPhotosDir = new File(SSNHelper.getInstagramPhotosDirPath());
        if (!instagramPhotosDir.exists()) {
            instagramPhotosDir.mkdir();
        }

        for (InstagramMedia photo : completePhotoList) {
            String imageUrl = photo.getImageUrl();

            URL url = new URL(imageUrl);
            File file = new File(SSNHelper.getInstagramPhotosDirPath() + photo.getId() + ".jpg");
            if (!file.exists()) {
                try {
                    FileUtils.copyURLToFile(url, file);
                    listOfFiles.add(file);
                } catch (Exception e) {
                    // e.printStackTrace();
                }
            } else {
                listOfFiles.add(file);
            }
        }

        File[] fileArray = listOfFiles.toArray(new File[0]);
        SSNGalleryHelper contentPane = new SSNGalleryHelper(fileArray, ssnHomeForm);
        contentPane.setBackground(SSNConstants.SSN_BLACK_BACKGROUND_COLOR);

        ssnHomeForm.getSsnHomeCenterPanel().removeAll();
        ssnHomeForm.getSsnHomeCenterMainPanel().removeAll();

        ssnHomeForm.getSsnHomeCenterPanel().add(ssnHomeForm.getScrollPane(contentPane,
                SSNHelper.getAlbumNameFromPath(ssnHomeForm.ssnFileExplorer.m_display.getText())));
        ssnHomeForm
                .getSsnHomeCenterMainPanel().add(
                        ssnHomeForm.getSortPanel("Date", false,
                                SSNHelper
                                        .getAlbumNameFromPath(ssnHomeForm.ssnFileExplorer.m_display.getText())),
                        BorderLayout.NORTH);
        ssnHomeForm.getSsnHomeCenterMainPanel().add(ssnHomeForm.getSsnHomeCenterPanel(), BorderLayout.CENTER);
        ssnHomeForm.getHomeModel().getSSNMediaFolderProperties(
                ssnHomeForm.getHomeModel().getHomeForm().ssnFileExplorer.m_display.getText(), fileArray);
        //ssnHomeForm.add(ssnHomeForm.getSsnHomeCenterMainPanel());

        //             ssnHomeForm.getSplitPane().setLeftComponent(ssnHomeForm.getSsnHomeLeftMainPanel());
        //             ssnHomeForm.getSplitPane().setRightComponent(ssnHomeForm.getSsnHomeCenterMainPanel());
        //                //ssnHomeForm.getSplitPane().setDividerLocation(200);
        //             ssnHomeForm.getSplitPane().revalidate();
        //             ssnHomeForm.getSplitPane().repaint();
        ssnHomeForm.revalidate();
        ssnHomeForm.repaint();
        ssnHomeForm.setCursor(new Cursor(Cursor.DEFAULT_CURSOR));

    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:ca.weblite.xmlvm.XMLVMTest.java

@Test
public void testGetRequiredClasses() throws ParserConfigurationException, SAXException, IOException {
    org.apache.tools.ant.Project proj = new org.apache.tools.ant.Project();
    File basedir = new File("/Volumes/Windows VMS/NetbeansProjects/HelloWorldCN1");
    proj.setBasedir(basedir.getAbsolutePath());
    XMLVM xmlvm = new XMLVM();
    xmlvm.setProject(proj);/*from w  ww .j  av  a  2 s. c  o  m*/

    File input = File.createTempFile("input", ".xmlvm");

    FileUtils.copyURLToFile(this.getClass().getResource("resources/com_codename1_cloud_CloudObject.xmlvm"),
            input);
    String[] classes = xmlvm.getRequiredClasses(input, false);
    assertEquals(2, classes.length);
    assertEquals(21, xmlvm.getRequiredClasses(input, true).length);
}

From source file:de.fabianonline.telegram_backup.exporter.HTMLExporter.java

public void export(UserManager user) {
    try {//from   w  w w  . ja v  a 2  s  . c om
        Database db = new Database(user, null);

        // Create base dir
        logger.debug("Creating base dir");
        String base = user.getFileBase() + "files" + File.separatorChar;
        new File(base).mkdirs();
        new File(base + "dialogs").mkdirs();

        logger.debug("Fetching dialogs");
        LinkedList<Database.Dialog> dialogs = db.getListOfDialogsForExport();
        logger.trace("Got {} dialogs", dialogs.size());
        logger.debug("Fetching chats");
        LinkedList<Database.Chat> chats = db.getListOfChatsForExport();
        logger.trace("Got {} chats", chats.size());

        logger.debug("Generating index.html");
        HashMap<String, Object> scope = new HashMap<String, Object>();
        scope.put("user", user);
        scope.put("dialogs", dialogs);
        scope.put("chats", chats);

        // Collect stats data
        scope.put("count.chats", chats.size());
        scope.put("count.dialogs", dialogs.size());

        int count_messages_chats = 0;
        int count_messages_dialogs = 0;
        for (Database.Chat c : chats)
            count_messages_chats += c.count;
        for (Database.Dialog d : dialogs)
            count_messages_dialogs += d.count;

        scope.put("count.messages", count_messages_chats + count_messages_dialogs);
        scope.put("count.messages.chats", count_messages_chats);
        scope.put("count.messages.dialogs", count_messages_dialogs);

        scope.put("count.messages.from_me", db.getMessagesFromUserCount());

        scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix()));

        scope.putAll(db.getMessageAuthorsWithCount());
        scope.putAll(db.getMessageTypesWithCount());
        scope.putAll(db.getMessageMediaTypesWithCount());

        MustacheFactory mf = new DefaultMustacheFactory();
        Mustache mustache = mf.compile("templates/html/index.mustache");
        OutputStreamWriter w = getWriter(base + "index.html");
        mustache.execute(w, scope);
        w.close();

        mustache = mf.compile("templates/html/chat.mustache");

        int i = 0;
        logger.debug("Generating {} dialog pages", dialogs.size());
        for (Database.Dialog d : dialogs) {
            i++;
            logger.trace("Dialog {}/{}: {}", i, dialogs.size(), Utils.anonymize("" + d.id));
            LinkedList<HashMap<String, Object>> messages = db.getMessagesForExport(d);
            scope.clear();
            scope.put("user", user);
            scope.put("dialog", d);
            scope.put("messages", messages);

            scope.putAll(db.getMessageAuthorsWithCount(d));
            scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(d)));
            scope.putAll(db.getMessageTypesWithCount(d));
            scope.putAll(db.getMessageMediaTypesWithCount(d));

            w = getWriter(base + "dialogs" + File.separatorChar + "user_" + d.id + ".html");
            mustache.execute(w, scope);
            w.close();
        }

        i = 0;
        logger.debug("Generating {} chat pages", chats.size());
        for (Database.Chat c : chats) {
            i++;
            logger.trace("Chat {}/{}: {}", i, chats.size(), Utils.anonymize("" + c.id));
            LinkedList<HashMap<String, Object>> messages = db.getMessagesForExport(c);
            scope.clear();
            scope.put("user", user);
            scope.put("chat", c);
            scope.put("messages", messages);

            scope.putAll(db.getMessageAuthorsWithCount(c));
            scope.put("heatmap_data", intArrayToString(db.getMessageTimesMatrix(c)));
            scope.putAll(db.getMessageTypesWithCount(c));
            scope.putAll(db.getMessageMediaTypesWithCount(c));

            w = getWriter(base + "dialogs" + File.separatorChar + "chat_" + c.id + ".html");
            mustache.execute(w, scope);
            w.close();
        }

        logger.debug("Generating additional files");
        // Copy CSS
        URL cssFile = getClass().getResource("/templates/html/style.css");
        File dest = new File(base + "style.css");
        FileUtils.copyURLToFile(cssFile, dest);
        logger.debug("Done exporting.");
    } catch (IOException e) {
        e.printStackTrace();
        throw new RuntimeException("Exception above!");
    }
}

From source file:eu.sisob.uma.extractors.adhoc.websearchers_cv.WebSearchersCVExtractor.java

/**
 *
 * @param input_file/*from   w w  w  . ja  v  a2  s .  com*/
 * @param results_dir
 * @param zip_output_file
 * @param output_file_2
 * @param error_sw
 */
public static void download_files(File input_file, File results_dir, File zip_output_file, File output_file_2,
        StringWriter error_sw) {
    CSVReader reader = null;
    try {
        reader = new CSVReader(new FileReader(input_file), CSV_SEPARATOR);
    } catch (FileNotFoundException ex) {
        Logger.getRootLogger().error("Error reading " + input_file.getName() + " - " + ex.toString());
        return;
    }

    int idStaffIdentifier = -1;
    int idName = -1;
    int idFirstName = -1;
    int idLastName = -1;
    int idInitials = -1;
    int idUnitOfAssessment_Description = -1;
    int idInstitutionName = -1;
    int idWebAddress = -1;
    int idResearchGroupDescription = -1;
    int idResearcherWebAddress = -1;
    int idResearcherWebAddressType = -1;
    int idResearcherWebAddressExt = -1;
    int idScoreUrl = -1;

    String[] nextLine;
    try {
        if ((nextLine = reader.readNext()) != null) {
            //Locate indexes            
            //Locate indexes                        
            for (int i = 0; i < nextLine.length; i++) {
                String column_name = nextLine[i];
                if (column_name.equals(FileFormatConversor.CSV_COL_ID))
                    idStaffIdentifier = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_NAME))
                    idName = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_FIRSTNAME))
                    idFirstName = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_LASTNAME))
                    idLastName = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_INITIALS))
                    idInitials = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_SUBJECT))
                    idUnitOfAssessment_Description = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_INSTITUTION_NAME))
                    idInstitutionName = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_INSTITUTION_URL))
                    idWebAddress = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_RESEARCHER_PAGE_URL))
                    idResearcherWebAddress = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_RESEARCHER_PAGE_TYPE))
                    idResearcherWebAddressType = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_RESEARCHER_PAGE_EXT))
                    idResearcherWebAddressExt = i;
                else if (column_name.equals(FileFormatConversor.CSV_COL_SCORE_URL))
                    idScoreUrl = i;
            }

        }
    } catch (Exception ex) {
        String error_msg = "Error reading headers of " + input_file.getName();
        Logger.getRootLogger().error(error_msg + " - " + ex.toString());
        if (error_sw != null)
            error_sw.append(error_msg + "\r\n");

        return;
    }

    if (idResearcherWebAddress != -1 && idResearcherWebAddressType != -1 && idResearcherWebAddressExt != -1) {
        Logger.getRootLogger().info("Going to downloads results files");

        try {
            for (int i = 0; i < nextLine.length; i++)
                nextLine[i] = "\"" + nextLine[i] + "\"";

            FileUtils.write(output_file_2, StringUtil.join(Arrays.asList(nextLine), ";") + "\r\n", "UTF-8",
                    false);
        } catch (IOException ex) {
            Logger.getLogger("root").error(ex.toString());
        }

        if (!results_dir.exists())
            results_dir.mkdirs();
        try {
            while ((nextLine = reader.readNext()) != null) {
                String url = nextLine[idResearcherWebAddress];
                String ext = nextLine[idResearcherWebAddressExt];
                String id = nextLine[idStaffIdentifier];

                try {
                    Logger.getRootLogger().info("Downloading " + url);
                    String filename = id + "." + ext;
                    FileUtils.copyURLToFile(new URL(url), new File(results_dir, filename));

                    nextLine[idResearcherWebAddress] = filename;
                    try {
                        for (int i = 0; i < nextLine.length; i++) {
                            nextLine[i] = "\"" + nextLine[i] + "\"";
                        }
                        FileUtils.write(output_file_2, StringUtil.join(Arrays.asList(nextLine), ";") + "\r\n",
                                "UTF-8", true);
                    } catch (IOException ex) {
                        Logger.getLogger("root").error(ex.toString());
                    }

                } catch (IOException ex) {
                    Logger.getRootLogger().error("Error downloading " + url);
                }
            }
        } catch (IOException ex) {
            Logger.getRootLogger().error("Error reading " + input_file.getName() + " " + ex.getMessage());
        }

        ZipFile zf;
        try {
            zip_output_file.delete();
            zf = new ZipFile(zip_output_file);
            zf.createZipFileFromFolder(results_dir, new ZipParameters(), false, 0);
        } catch (ZipException ex) {
            Logger.getRootLogger().error("Error zipping results from " + input_file.getName());
        }

    } else {
        Logger.getRootLogger().error("Headers incorrect " + input_file.getName());
    }
}

From source file:deincraftlauncher.start.StartMinecraft.java

public static void fixCaches(String dir) {

    try {//  w  ww  . jav  a 2 s  .c o m
        //create guava 16.0 file
        FileUtils.deleteDirectory(new File(dir + "/libs/com/google/guava/guava/"));
        String targetPath = dir + "/libs/com/google/guava/guava/16.0/";

        new File(targetPath).mkdirs();
        URL inputUrl = DownloadHandler.getInstance().getClass()
                .getResource("/deincraftlauncher/Images/guava-16.0.jar");
        File dest = new File(targetPath + "guava-16.0.jar");
        FileUtils.copyURLToFile(inputUrl, dest);

        //fix Forge Multipart
        System.out.println("fixing forge multipart if existing");
        String targetMod = dir.replace("Cache", "mods") + File.separator
                + "ForgeMultipart-1.7.10-1.2.0.345-universal.jar";
        if (new File(targetMod).exists()) {
            String targetModPath = dir.replace("Cache", "mods") + File.separator + "1.7.10" + File.separator;
            String targetModLocation = targetModPath + "ForgeMultipart-1.7.10-1.2.0.345-universal.jar";
            new File(targetModPath).mkdirs();
            new File(targetMod).renameTo(new File(targetModLocation));
        }

    } catch (IOException ex) {
        Logger.getLogger(StartMinecraft.class.getName()).log(Level.SEVERE, null, ex);
    }
}

From source file:com.anritsu.mcreleaseportal.tcintegration.ProcessTCRequest.java

public Result validateChangesXML(String changesXMLFileName) {
    result = new Result();
    try {/*from   w  w w .j a  va2s.co  m*/
        File dir = new File(Configuration.getInstance().getSavePath());
        if (!dir.isDirectory()) {
            new File(Configuration.getInstance().getSavePath()).mkdirs();
        }
        String fileName = buildName + "_" + buildNumber + "_" + System.currentTimeMillis();
        File file = new File(dir, fileName);
        file.createNewFile();

        FileUtils.copyURLToFile(new URL(artifactsLocation + changesXMLFileName), file);

        pXML = new ProcessChangesXML(file.getAbsolutePath());
        // Check if xml is succesfully validated. If validateXmlXsd() returns an non empty string result code is 1
        result.setResultMessage(pXML.validateXmlXsd());
        if (!result.getResultMessage().replace("Success", "").equalsIgnoreCase("")) {
            result.setResultCode("1");
        }
        file.delete();

    } catch (IOException ex) {
        Logger.getLogger(ProcessTCRequest.class.getName()).log(Level.SEVERE, null, ex);
        result.setResultCode("1");
        result.setResultMessage(ex.getMessage());
    }
    return result;
}

From source file:de.shadowhunt.subversion.internal.AbstractHelper.java

public void pullCurrentDumpData() throws Exception {
    final File zip = new File(base, "dump.zip");
    if (zip.exists()) {
        final String localMD5 = calculateMd5(zip);
        final String hashContent = copyUrlToString(md5Uri.toURL());
        final String remoteMD5 = hashContent.substring(0, 32);
        if (localMD5.equals(remoteMD5)) {
            return;
        }/*ww  w  .j  a v a2 s.  c  o  m*/
    }
    FileUtils.deleteQuietly(base);

    base.mkdirs();
    FileUtils.copyURLToFile(dumpUri.toURL(), zip);
    extractArchive(zip, base);
}

From source file:com.thesmartweb.swebrank.LinksParseAnalysis.java

/**
 * Method that exports the content from the urls provided and stores it in the ElasticSearch cluster of ours in a specific index
 * and calls the Semantic Analysis algorithm selected. Until now the method exports content from: 
 * -html/*from   w w w.  j a  va2  s. c  o m*/
 * -youtube videos
 * -pdf files
 * @param total_links It contains all the links that we are going to analyze
 * @param domain The domain that we analyze
 * @param engine The search engine that we analyze the results from
 * @param example_dir It contains the directory where to save the results of the analysis
 * @param quer It contains the query for which the urls were the results for (it is used for the creation of the id in elasticsearch)
 * @param nTopics The number of topics for Latent Dirichlet Allocation
 * @param alpha The alpha value of LDA
 * @param beta The beta value of LDA
 * @param niters The number of iterations of LDA
 * @param top_words The amount of top words per topic to keep for LDA
 * @param LDAflag Flag if LDA is used
 * @param TFIDFflag Flag if TFIDF is used
 * @param config_path the path that contains the configuration files
 * @return the parsed output for each url provided
 */
public String[] perform(String[] total_links, String domain, String engine, String example_dir, String quer,
        int nTopics, double alpha, double beta, int niters, int top_words, boolean LDAflag, boolean TFIDFflag,
        String config_path) {
    String[] parse_output = new String[total_links.length];
    try {
        System.gc();
        WebParser web = new WebParser();//our web parser
        APIconn apicon = new APIconn();//our instance to check the connection to a url
        int counter_LDA_documents = 0;
        Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", "lshrankldacluster")
                .build();
        Client client = new TransportClient(settings)
                .addTransportAddress(new InetSocketTransportAddress("localhost", 9300));

        //Node node = nodeBuilder().client(true).clusterName("lshrankldacluster").node();//our elasticsearch node builder
        //Client client = node.client();//the client for elasticsearch node
        for (int i = 0; i < (total_links.length); i++) {
            parse_output[i] = "";
            if (total_links[i] != null) {
                System.out.println("Link: " + total_links[i] + "\n");
                DataManipulation dm = new DataManipulation();
                boolean structuredFiled = dm.StructuredFileCheck(total_links[i]);//we check if the url contains a structured document file type

                if (!apicon.check_conn(total_links[i]).contains("ok-conn") || structuredFiled
                        || total_links[i].contains("http://www.youtube.com/watch?")) {
                    if (total_links[i].contains("http://www.youtube.com/watch?")) {//if the link is a youtube link we have to treat its JSON differently
                        String ventry = total_links[i].substring(31);
                        JSONparsing ypr = new JSONparsing();
                        url_check = total_links[i];
                        File current_url = new File(example_dir + engine + "/" + i + "/" + "current_url.txt");
                        FileUtils.writeStringToFile(current_url, url_check);
                        parse_output[i] = ypr.GetYoutubeDetails(ventry).replace("\n", "").replace("r", "");
                        System.gc();
                        if (parse_output[i] != null) {
                            counter_LDA_documents++;
                            String directory = example_dir + engine + "/" + i + "/";
                            File file_content_lda = new File(directory + "youtube_content.txt");
                            FileUtils.writeStringToFile(file_content_lda, parse_output[i]);
                        }
                    }
                    if (total_links[i].contains(".pdf")) {//if the link has a pdf we use Snowtide Pdf reader
                        url_check = total_links[i];
                        File current_url = new File(example_dir + engine + "/" + i + "/" + "current_url.txt");
                        FileUtils.writeStringToFile(current_url, url_check);
                        File current_pdf = new File(example_dir + engine + "/" + i + "/" + "current_pdf.txt");
                        URL URLlink = new URL(url_check);
                        FileUtils.copyURLToFile(URLlink, current_pdf);
                        Document pdf = PDF.open(current_pdf);
                        StringWriter buffer = new StringWriter();
                        pdf.pipe(new OutputTarget(buffer));
                        pdf.close();
                        parse_output[i] = buffer.toString().replace("\n", "").replace("\r", "");
                        Stopwords stopwords = new Stopwords();
                        parse_output[i] = stopwords.stop(parse_output[i]);
                        System.gc();
                        boolean deleteQuietly = FileUtils.deleteQuietly(current_pdf);//we delete the file after we read it
                        if (parse_output[i] != null) {
                            counter_LDA_documents++;
                            String directory = example_dir + engine + "/" + i + "/";
                            File file_content_lda = new File(directory + "pdf_content.txt");
                            FileUtils.writeStringToFile(file_content_lda, parse_output[i]);
                        }
                    }
                } else {//if the link does not follow to the cases above, we parse it using WebParser
                    int number = i;
                    String directory = example_dir + engine + "/" + number + "/";
                    System.out.println("Link:" + total_links[i] + "\n");
                    url_check = total_links[i];
                    File current_url = new File(directory + "current_url.txt");
                    FileUtils.writeStringToFile(current_url, url_check);
                    System.gc();
                    parse_output[i] = web.Parse(url_check);//we call the parser
                    System.gc();
                    if (parse_output[i] != null) {
                        counter_LDA_documents++;//we count the amount of documents, as it is needed for JGibbLDA as seen in http://jgibblda.sourceforge.net/#2.3._Input_Data_Format
                        directory = example_dir + engine + "/" + i + "/";
                        //write the output from the html parsing
                        File file_content_lda = new File(directory + "html_parse_content.txt");
                        FileUtils.writeStringToFile(file_content_lda, parse_output[i]);
                    }
                }
                JSONObject obj = new JSONObject();//an object to save the parsed content in elasticsearch
                obj.put("ParsedContent", parse_output[i]);
                String id = domain + "/" + quer + "/" + engine + "/" + total_links[i];
                ReadInput ri = new ReadInput();
                List<String> elasticIndexes = ri.GetKeyFile(config_path, "elasticSearchIndexes");
                IndexRequest indexReq = new IndexRequest(elasticIndexes.get(3), "content", id);
                indexReq.source(obj);
                IndexResponse indexRes = client.index(indexReq).actionGet();
            }
        }
        //node.close();
        client.close();
        String output_string_content = Integer.toString(counter_LDA_documents);
        TwitterAnalysis tw = new TwitterAnalysis();//we are going gather info from Twitter using Twitter4j
        String twitter_txt = tw.perform(quer, config_path);
        for (int i = 0; i < parse_output.length; i++) {//we combine all the parsed content into one document
            if (parse_output[i] != null) {
                output_string_content = output_string_content + "\n" + parse_output[i];
            }
        }
        if (!(twitter_txt.equalsIgnoreCase("fail"))) {
            output_string_content = output_string_content + "\n" + twitter_txt;//we add also the twitter content
        }
        String directory = example_dir + engine + "/";
        //call LDA
        File file_content_lda = new File(directory + "content_for_analysis.txt");//we are going to save the content also in txt format for backup and usage for LDA
        FileUtils.writeStringToFile(file_content_lda, output_string_content);
        if (LDAflag) {
            LDAcall ld = new LDAcall();//we call lda
            ld.call(nTopics, alpha, beta, niters, top_words, directory);
        } else if (TFIDFflag) {
            TFIDF tf = new TFIDF();//we call TFIDF
            topWordsTFIDF = tf.compute(parse_output, top_words, example_dir);
        }
        return parse_output;
    } catch (IOException | ElasticsearchException | ArrayIndexOutOfBoundsException ex) {
        Logger.getLogger(LinksParseAnalysis.class.getName()).log(Level.SEVERE, null, ex);
        return parse_output;
    }
}