Example usage for java.net URL toExternalForm

Introduction

In this page you can find the example usage for java.net URL toExternalForm.

Prototype

public String toExternalForm() 

Source Link

Document

Constructs a string representation of this URL .

Usage

From source file:easyshop.downloadhelper.HttpPageGetter.java

public HttpPage getDHttpPage(PageRef url, String charSet) {
    count++;/*  w  ww  . j a  va 2 s  .c o  m*/
    log.debug("getURL(" + count + ")");

    if (url.getUrlStr() == null) {
        ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0);
        return new OriHttpPage(-1, null, null, null, conRes, null);
    }
    URL requestedURL = null;
    try {
        requestedURL = new URL(url.getUrlStr());
    } catch (MalformedURLException e1) {
        // TODO Auto-generated catch block
        log.error("wrong urlstr" + url.getUrlStr());
        ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0);
        return new OriHttpPage(-1, null, null, null, conRes, null);
    }
    ;
    //        System.out.println(""+requestedURL.toExternalForm());
    URL referer = null;
    try {
        log.debug("Creating HTTP connection to " + requestedURL);
        HttpURLConnection conn = (HttpURLConnection) requestedURL.openConnection();
        if (referer != null) {
            log.debug("Setting Referer header to " + referer);
            conn.setRequestProperty("Referer", referer.toExternalForm());
        }

        if (userAgent != null) {
            log.debug("Setting User-Agent to " + userAgent);
            conn.setRequestProperty("User-Agent", userAgent);
        }

        //            DateFormat dateFormat=DateFormat.getDateInstance();
        //            conn.setRequestProperty("If-Modlfied-Since",dateFormat.parse("2005-08-15 20:18:30").toGMTString());
        conn.setUseCaches(false);

        //            conn.setRequestProperty("connection","keep-alive");
        for (Iterator it = conn.getRequestProperties().keySet().iterator(); it.hasNext();) {
            String key = (String) it.next();
            if (key == null) {
                break;
            }
            String value = conn.getHeaderField(key);
            //                System.out.println("Request header " + key + ": " + value);
        }
        log.debug("Opening URL");
        long startTime = System.currentTimeMillis();
        conn.connect();

        String resp = conn.getResponseMessage();
        log.debug("Remote server response: " + resp);

        int code = conn.getResponseCode();

        if (code != 200) {
            log.error("Could not get connection for code=" + code);
            System.err.println("Could not get connection for code=" + code);
            ConnResponse conRes = new ConnResponse(null, null, 0, 0, code);
            return new HttpPage(requestedURL.toExternalForm(), null, conRes, null);
        }

        //            if (conn.getContentLength()<=0||conn.getContentLength()>10000000){
        //               log.error("Content length==0");
        //               System.err.println("Content length==0");
        //               ConnResponse conRes=new ConnResponse(null,null,null,0,0,-100);
        //               return new URLObject(-1,requestedURL, null,null,conRes);
        //            }

        String respStr = conn.getHeaderField(0);
        long serverDate = conn.getDate();
        //            log.info("Server response: " + respStr);

        for (int i = 1; i < conn.getHeaderFields().size(); i++) {
            String key = conn.getHeaderFieldKey(i);
            if (key == null) {
                break;
            }
            String value = conn.getHeaderField(key);
            //                System.out.println("Received header " + key + ": " + value);
            //               log.debug("Received header " + key + ": " + value);
        }

        //            log.debug("Getting buffered input stream from remote connection");
        log.debug("start download(" + count + ")");
        BufferedInputStream remoteBIS = new BufferedInputStream(conn.getInputStream());
        ByteArrayOutputStream baos = new ByteArrayOutputStream(10240);
        byte[] buf = new byte[1024];
        int bytesRead = 0;
        while (bytesRead >= 0) {
            baos.write(buf, 0, bytesRead);
            bytesRead = remoteBIS.read(buf);
        }
        //            baos.write(remoteBIS.read(new byte[conn.getContentLength()]));
        //            remoteBIS.close();
        byte[] content = baos.toByteArray();
        long timeTaken = System.currentTimeMillis() - startTime;
        if (timeTaken < 100)
            timeTaken = 500;

        int bytesPerSec = (int) ((double) content.length / ((double) timeTaken / 1000.0));
        //            log.info("Downloaded " + content.length + " bytes, " + bytesPerSec + " bytes/sec");
        if (content.length < conn.getContentLength()) {
            log.warn("Didn't download full content for URL: " + url);
            //                failureCount++;
            ConnResponse conRes = new ConnResponse(conn.getContentType(), null, content.length, serverDate,
                    code);
            return new HttpPage(requestedURL.toExternalForm(), null, conRes, conn.getContentType());
        }

        log.debug("download(" + count + ")");

        ConnResponse conRes = new ConnResponse(conn.getContentType(), null, conn.getContentLength(), serverDate,
                code);
        String c = charSet;
        if (c == null)
            c = conRes.getCharSet();
        HttpPage obj = new HttpPage(requestedURL.toExternalForm(), content, conRes, c);
        return obj;
    } catch (IOException ioe) {
        log.warn("Caught IO Exception: " + ioe.getMessage(), ioe);
        failureCount++;
        ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0);
        return new HttpPage(requestedURL.toExternalForm(), null, conRes, null);
    } catch (Exception e) {
        log.warn("Caught Exception: " + e.getMessage(), e);
        failureCount++;
        ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0);
        return new HttpPage(requestedURL.toExternalForm(), null, conRes, null);
    }
}

---

From source file:com.gargoylesoftware.htmlunit.WebRequest.java

/**
 * Sets the target URL. The URL may be simplified if needed (for instance eliminating
 * irrelevant path portions like "/./").
 * @param url the target URL/*from  ww w.j  av a2 s .  c  o  m*/
 */
public void setUrl(URL url) {
    if (url != null) {
        final String path = url.getPath();
        if (path.isEmpty()) {
            url = buildUrlWithNewFile(url, "/" + url.getFile());
        } else if (path.contains("/.")) {
            final String query = (url.getQuery() != null) ? "?" + url.getQuery() : "";
            url = buildUrlWithNewFile(url, removeDots(path) + query);
        }
        url_ = url.toExternalForm();

        // http://john.smith:secret@localhost
        final String userInfo = url.getUserInfo();
        if (userInfo != null) {
            final int splitPos = userInfo.indexOf(':');
            if (splitPos == -1) {
                urlCredentials_ = new UsernamePasswordCredentials(userInfo, "");
            } else {
                final String username = userInfo.substring(0, splitPos);
                final String password = userInfo.substring(splitPos + 1);
                urlCredentials_ = new UsernamePasswordCredentials(username, password);
            }
        }
    } else {
        url_ = null;
    }
}

---

From source file:org.azkfw.crawler.downloader.engine.SimpleDownloadEngine.java

@Override
protected final DownloadEngineResult doDownload(final DownloadEngineCondition condition) {
    DownloadEngineResult result = new DownloadEngineResult();

    URL targetUrl = condition.getContentURL();
    File destFile = condition.getDestFile();

    if (ObjectUtility.isNull(targetUrl)) {
        throw new NullPointerException("TargetUrl");
    }/* w ww  .j a v a  2  s.  c  om*/
    if (ObjectUtility.isNull(destFile)) {
        throw new NullPointerException("DestFile");
    }

    HttpGet httpGet = null;
    InputStream reader = null;
    FileOutputStream writer = null;
    try {
        doBefore(httpClient, condition);

        long startTime = System.currentTimeMillis();
        info(String.format("URL : %s", targetUrl.toExternalForm()));

        httpGet = new HttpGet(targetUrl.toExternalForm());
        HttpResponse response = httpClient.execute(httpGet);

        int statusCode = response.getStatusLine().getStatusCode();
        result.setStatusCode(statusCode);

        Header[] headers = response.getAllHeaders();
        for (Header header : headers) {
            result.addHeader(header);
            info(String.format("%s : %s", header.getName(), header.getValue()));
        }

        if (200 == statusCode) {
            HttpEntity httpEntity = response.getEntity();
            reader = httpEntity.getContent();

            writer = new FileOutputStream(destFile);
            byte[] buffer = new byte[1024 * 10];
            int len;
            long length = 0;
            while (-1 != (len = reader.read(buffer, 0, 1024 * 10))) {
                if (0 == len) {
                    info("0");
                    continue;
                }
                writer.write(buffer, 0, len);
                length += len;
            }
            result.setLength(length);
        }
        long endTime = System.currentTimeMillis();
        info(String.format("<<<<< END [%d] (%.2fs)", statusCode, (float) (endTime - startTime) / 1000.f));

        doAfter(httpClient, condition);

        result.setResult(true);

    } catch (ClientProtocolException ex) {
        fatal(ex);
    } catch (IOException ex) {
        fatal(ex);
    } finally {
        if (null != writer) {
            try {
                writer.flush();
                writer.close();
            } catch (IOException ex) {
                ex.printStackTrace();
            }
        }
        if (null != reader) {
            try {
                reader.close();
            } catch (IOException ex) {
                ex.printStackTrace();
            }
        }
        if (null != httpGet) {
            httpGet.abort();
        }
        // httpClient.getConnectionManager().shutdown();
    }

    return result;
}

---

From source file:com.testmax.util.FileDownLoader.java

public String downloader(WebElement element, String attribute) throws Exception {
    //Assuming that getAttribute does some magic to return a fully qualified URL
    String downloadLocation = element.getAttribute(attribute);
    if (downloadLocation.trim().equals("")) {
        throw new Exception("The element you have specified does not link to anything!");
    }/*  w w w. ja  v  a2 s .c o  m*/
    URL downloadURL = new URL(downloadLocation);
    HttpClient client = new HttpClient();
    client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    client.setHostConfiguration(mimicHostConfiguration(downloadURL.getHost(), downloadURL.getPort()));
    client.setState(mimicCookieState(driver.manage().getCookies()));
    HttpMethod getRequest = new GetMethod(downloadURL.getPath());
    String file_path = downloadPath + downloadURL.getFile().replaceFirst("/|\\\\", "");
    FileWriter downloadedFile = new FileWriter(file_path, true);
    try {
        int status = client.executeMethod(getRequest);
        WmLog.getCoreLogger().info("HTTP Status {} when getting '{}'" + status + downloadURL.toExternalForm());
        BufferedInputStream in = new BufferedInputStream(getRequest.getResponseBodyAsStream());
        int offset = 0;
        int len = 4096;
        int bytes = 0;
        byte[] block = new byte[len];
        while ((bytes = in.read(block, offset, len)) > -1) {
            downloadedFile.write(bytes);

        }
        downloadedFile.close();
        in.close();
        WmLog.getCoreLogger().info("File downloaded to '{}'" + file_path);
    } catch (Exception Ex) {
        WmLog.getCoreLogger().error("Download failed: {}", Ex);
        throw new Exception("Download failed!");
    } finally {
        getRequest.releaseConnection();
    }
    return file_path;
}

---

From source file:com.sshtools.appframework.ui.SshToolsApplication.java

public void openURL(URL url) throws IOException {
    BrowserLauncher.openURL(url.toExternalForm());
}

---

From source file:com.gargoylesoftware.htmlunit.javascript.host.xml.XMLHttpRequest3Test.java

/**
 * Asynchronous callback should be called in "main" js thread and not parallel to other js execution.
 * See http://sourceforge.net/p/htmlunit/bugs/360/.
 * @throws Exception if the test fails/*from   w  w w.  j  a va 2  s .  c o  m*/
 */
@Test
public void noParallelJSExecutionInPage() throws Exception {
    final String content = "<html><head><script>\n" + "var j = 0;\n" + "function test() {\n" + "  req = "
            + XMLHttpRequest2Test.XHRInstantiation_ + ";\n" + "  req.onreadystatechange = handler;\n"
            + "  req.open('post', 'foo.xml', true);\n" + "  req.send('');\n" + "  alert('before long loop');\n"
            + "  for (var i = 0; i < 5000; i++) {\n" + "     j = j + 1;\n" + "  }\n"
            + "  alert('after long loop');\n" + "}\n" + "function handler() {\n"
            + "  if (req.readyState == 4) {\n" + "    alert('ready state handler, content loaded: j=' + j);\n"
            + "  }\n" + "}\n" + "</script></head>\n" + "<body onload='test()'></body></html>";

    final WebClient client = getWebClient();
    final List<String> collectedAlerts = Collections.synchronizedList(new ArrayList<String>());
    client.setAlertHandler(new CollectingAlertHandler(collectedAlerts));
    final MockWebConnection conn = new MockWebConnection() {
        @Override
        public WebResponse getResponse(final WebRequest webRequest) throws IOException {
            collectedAlerts.add(webRequest.getUrl().toExternalForm());
            return super.getResponse(webRequest);
        }
    };
    conn.setResponse(URL_FIRST, content);
    final URL urlPage2 = new URL(URL_FIRST + "foo.xml");
    conn.setResponse(urlPage2, "<foo/>\n", "text/xml");
    client.setWebConnection(conn);
    client.getPage(URL_FIRST);

    assertEquals(0, client.waitForBackgroundJavaScriptStartingBefore(1000));

    final String[] alerts = { URL_FIRST.toExternalForm(), "before long loop", "after long loop",
            urlPage2.toExternalForm(), "ready state handler, content loaded: j=5000" };
    assertEquals(alerts, collectedAlerts);
}

---

From source file:org.sonatype.nexus.proxy.storage.remote.httpclient.HttpClientRemoteStorage.java

/**
 * Appends repository configured additional query string to provided URL.
 *
 * @param url        to append to/*from w  w w  . j ava2 s  . c om*/
 * @param repository that may contain additional query string
 * @return URL with appended query string or original URL if repository does not have an configured query string
 * @throws RemoteStorageException if query string could not be appended (resulted in an Malformed URL exception)
 */
private URL appendQueryString(final URL url, final ProxyRepository repository) throws RemoteStorageException {
    final RemoteStorageContext ctx = getRemoteStorageContext(repository);

    final String queryString = ctx.getRemoteConnectionSettings().getQueryString();
    if (StringUtils.isNotBlank(queryString)) {
        try {
            if (StringUtils.isBlank(url.getQuery())) {
                return new URL(url.toExternalForm() + "?" + queryString);
            } else {
                return new URL(url.toExternalForm() + "&" + queryString);
            }
        } catch (MalformedURLException e) {
            throw new RemoteStorageException(
                    "Could not append query string \"" + queryString + "\" to url \"" + url + "\"", e);
        }
    }
    return url;
}

---

From source file:com.trunghoang.teammedical.utils.FileDownloader.java

public File urlDownloader(String downloadLocation) throws Exception {
    URL downloadURL = new URL(downloadLocation);
    HttpClient client = new HttpClient();
    client.getParams().setCookiePolicy(CookiePolicy.RFC_2965);
    client.setHostConfiguration(mimicHostConfiguration(downloadURL.getHost(), downloadURL.getPort()));
    client.getParams().makeLenient();//  w ww  .ja v  a  2 s.com
    client.setState(mimicCookieState(driver.manage().getCookies()));
    HttpMethod getRequest = new GetMethod(downloadLocation);
    getRequest.setFollowRedirects(true);
    FileHandler downloadedFile = new FileHandler(
            downloadPath + downloadURL.getFile().replaceFirst("/|\\\\", ""), true);
    try {
        int status = client.executeMethod(getRequest);
        log.info("HTTP Status {} when getting '{}'", status, downloadURL.toExternalForm());
        downloadedFile.getWritableFileOutputStream().write(getRequest.getResponseBody());
        downloadedFile.close();
        log.info("File downloaded to '{}'", downloadedFile.getAbsoluteFile());
    } catch (Exception Ex) {
        log.error("Download failed: {}", Ex);
        throw new Exception("Download failed!");
    } finally {
        getRequest.releaseConnection();
    }
    return downloadedFile.getFile();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »