List of usage examples for java.net URL toExternalForm
public String toExternalForm()
From source file:com.lazerycode.ebselen.customhandlers.FileDownloader.java
public String downloader(WebElement element, String attribute) throws Exception { //Assuming that getAttribute does some magic to return a fully qualified URL String downloadLocation = element.getAttribute(attribute); if (downloadLocation.trim().equals("")) { throw new Exception("The element you have specified does not link to anything!"); }/* w w w . j av a 2 s. com*/ URL downloadURL = new URL(downloadLocation); HttpClient client = new HttpClient(); client.getParams().setCookiePolicy(CookiePolicy.RFC_2965); client.setHostConfiguration(mimicHostConfiguration(downloadURL.getHost(), downloadURL.getPort())); client.setState(mimicCookieState(driver.manage().getCookies())); HttpMethod getRequest = new GetMethod(downloadURL.getPath()); FileHandler downloadedFile = new FileHandler( downloadPath + downloadURL.getFile().replaceFirst("/|\\\\", ""), true); try { int status = client.executeMethod(getRequest); LOGGER.info("HTTP Status {} when getting '{}'", status, downloadURL.toExternalForm()); BufferedInputStream in = new BufferedInputStream(getRequest.getResponseBodyAsStream()); int offset = 0; int len = 4096; int bytes = 0; byte[] block = new byte[len]; while ((bytes = in.read(block, offset, len)) > -1) { downloadedFile.getWritableFileOutputStream().write(block, 0, bytes); } downloadedFile.close(); in.close(); LOGGER.info("File downloaded to '{}'", downloadedFile.getAbsoluteFile()); } catch (Exception Ex) { LOGGER.error("Download failed: {}", Ex); throw new Exception("Download failed!"); } finally { getRequest.releaseConnection(); } return downloadedFile.getAbsoluteFile(); }
From source file:com.gargoylesoftware.htmlunit.javascript.host.Location.java
/** * Returns the location URL./*from w w w.j a v a 2s . c o m*/ * @return the location URL * @see <a href="http://msdn.microsoft.com/en-us/library/ms533867.aspx">MSDN Documentation</a> */ @JsxGetter public String getHref() { final Page page = window_.getWebWindow().getEnclosedPage(); if (page == null) { return UNKNOWN; } try { URL url = page.getUrl(); final boolean encodeHash = getBrowserVersion().hasFeature(JS_LOCATION_HASH_IS_ENCODED); final String hash = getHash(encodeHash); if (hash != null) { url = UrlUtils.getUrlWithNewRef(url, hash); } String s = url.toExternalForm(); if (s.startsWith("file:/") && !s.startsWith("file:///")) { // Java (sometimes?) returns file URLs with a single slash; however, browsers return // three slashes. See http://www.cyanwerks.com/file-url-formats.html for more info. s = "file:///" + s.substring("file:/".length()); } return s; } catch (final MalformedURLException e) { LOG.error(e.getMessage(), e); return page.getUrl().toExternalForm(); } }
From source file:easyshop.downloadhelper.HttpPageGetter.java
public HttpPage getURLOnly(WebLink url) { log.debug("getURL(" + url + ")"); if (url.getUrlStr() == null) { ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new HttpPage(null, null, conRes, null); }/*from w ww . j a v a 2s.c om*/ URL requestedURL = null; URL referer = null; try { requestedURL = new URL(url.getUrlStr()); log.debug("Creating HTTP connection to " + requestedURL); HttpURLConnection conn = (HttpURLConnection) requestedURL.openConnection(); if (referer != null) { log.debug("Setting Referer header to " + referer); conn.setRequestProperty("Referer", referer.toExternalForm()); } if (userAgent != null) { log.debug("Setting User-Agent to " + userAgent); conn.setRequestProperty("User-Agent", userAgent); } conn.setUseCaches(false); log.debug("Opening URL"); long startTime = System.currentTimeMillis(); conn.connect(); String resp = conn.getResponseMessage(); log.debug("Remote server response: " + resp); int code = conn.getResponseCode(); if (code != 200) { log.error("Could not get connection for code=" + code); System.err.println("Could not get connection for code=" + code); } ConnResponse conRes = new ConnResponse(conn.getContentType(), null, 0, 0, code); conn.disconnect(); return new HttpPage(requestedURL.toExternalForm(), null, conRes, conRes.getCharSet()); } catch (IOException ioe) { log.warn("Caught IO Exception: " + ioe.getMessage(), ioe); failureCount++; ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new HttpPage(requestedURL.toExternalForm(), null, conRes, null); } catch (Exception e) { log.warn("Caught Exception: " + e.getMessage(), e); failureCount++; ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new HttpPage(requestedURL.toExternalForm(), null, conRes, null); } }
From source file:com.digitalpebble.stormcrawler.bolt.FetcherBolt.java
private void handleOutlink(Tuple t, String sourceUrl, String newUrl, Metadata sourceMetadata, boolean markAsSitemap) { // build an absolute URL URL sURL;// w w w . j ava 2s .c o m try { sURL = new URL(sourceUrl); URL tmpURL = URLUtil.resolveURL(sURL, newUrl); newUrl = tmpURL.toExternalForm(); } catch (MalformedURLException e) { LOG.debug("MalformedURLException on {} or {}: {}", sourceUrl, newUrl, e); return; } // apply URL filters newUrl = this.urlFilters.filter(sURL, sourceMetadata, newUrl); // filtered if (newUrl == null) { return; } Metadata metadata = metadataTransfer.getMetaForOutlink(newUrl, sourceUrl, sourceMetadata); if (markAsSitemap) { metadata.setValue(SiteMapParserBolt.isSitemapKey, "true"); } _collector.emit(Constants.StatusStreamName, t, new Values(newUrl, metadata, Status.DISCOVERED)); }
From source file:com.googlecode.fightinglayoutbugs.DetectInvalidImageUrls.java
private void checkImageUrlAsync(URL completeUrl, final String errorDescriptionPrefix) throws MalformedURLException { final String completeUrlAsString = completeUrl.toExternalForm(); String error = _checkedImageUrls.putIfAbsent(completeUrlAsString, ""); if (error == null) { _mockBrowser.downloadAsync(completeUrl, new DownloadCallback() { @Override//from w w w .ja v a 2 s . co m public void onSuccess(GetMethod getMethod) { if (getMethod.getStatusCode() >= 400) { if (getMethod.getStatusCode() == 401) { LOG.info("Ignoring HTTP response status code 401 (" + getMethod.getStatusText() + ") for image URL " + completeUrlAsString); } else { handleError("HTTP server responded with: " + getMethod.getStatusCode() + " " + getMethod.getStatusText()); } } else { final Header contentTypeHeader = getMethod.getResponseHeader("Content-Type"); if (contentTypeHeader == null) { handleError("HTTP response did not contain Content-Type header."); } else { final String contentType = contentTypeHeader.getValue(); if (!contentType.startsWith("image/")) { handleError("Content-Type HTTP response header \"" + contentType + "\" does not start with \"image/\"."); } else { // TODO: check if the response body is a valid image } } } } @Override public void onFailure(IOException e) { handleError("HTTP GET failed: " + e.getMessage()); } private void handleError(String error) { _checkedImageUrls.put(completeUrlAsString, error); addLayoutBugIfNotPresent(errorDescriptionPrefix + " -- " + error); } }); } else if (error.length() > 0) { addLayoutBugIfNotPresent(errorDescriptionPrefix + " -- " + error); } }
From source file:com.gargoylesoftware.htmlunit.WebTestCase.java
/** * Facility to test external form of urls. Comparing external form of URLs is * really faster than URL.equals() as the host doesn't need to be resolved. * @param message the message to display if assertion fails * @param expectedUrl the string representation of the expected URL * @param actualUrl the URL to test/* w w w . ja va2 s .com*/ */ protected void assertEquals(final String message, final URL expectedUrl, final URL actualUrl) { Assert.assertEquals(message, expectedUrl.toExternalForm(), actualUrl.toExternalForm()); }
From source file:ca.oson.json.gson.functional.DefaultTypeAdaptersTest.java
public void testUrlDeserialization() { String urlValue = "http://google.com/"; String json = "'http:\\/\\/google.com\\/'"; URL target = gson.fromJson(json, URL.class); assertEquals(urlValue, target.toExternalForm()); gson.fromJson('"' + urlValue + '"', URL.class); assertEquals(urlValue, target.toExternalForm()); target = oson.fromJson(json, URL.class); assertEquals(urlValue, target.toExternalForm()); target = oson.fromJson('"' + urlValue + '"', URL.class); assertEquals(urlValue, target.toExternalForm()); }
From source file:be.fedict.eid.idp.webapp.IdentityProviderServletContextListener.java
/** * Checks the deployment descriptor or {@link @WebServiceProvider} * annotation to see if it points to any WSDL. If so, returns the * {@link com.sun.xml.ws.api.server.SDDocumentSource}. * /* w w w . j av a 2 s. co m*/ * @param implementorClass * WS impl class name * @return The pointed WSDL, if any. Otherwise null. */ private SDDocumentSource getPrimaryWSDL(Class<?> implementorClass) { String wsdlFile = EndpointFactory.getWsdlLocation(implementorClass); if (wsdlFile != null) { if (!wsdlFile.startsWith(JAXWS_WSDL_DD_DIR)) { LOG.warn("Ignoring wrong wsdl=" + wsdlFile + ". It should start with " + JAXWS_WSDL_DD_DIR + ". Going to generate and publish a new WSDL."); return null; } URL wsdl; try { wsdl = loader.getResource('/' + wsdlFile); } catch (MalformedURLException e) { LOG.error(e); throw new RuntimeException(e); } if (wsdl == null) { LOG.error("No WSDL found"); throw new RuntimeException("No WSDL found"); } SDDocumentSource docInfo = docs.get(wsdl.toExternalForm()); assert docInfo != null; return docInfo; } return null; }
From source file:org.apache.karaf.cave.server.storage.CaveRepositoryImpl.java
/** * Proxy an URL (by adding repository.xml OBR information) in the Cave repository. * * @param url the URL to proxyFilesystem. the URL to proxyFilesystem. * @param filter regex filter. Only artifacts URL matching the filter will be considered. * @throws Exception/* w w w . j ava 2 s .c o m*/ */ public void proxy(URL url, String filter) throws Exception { if (url.getProtocol().equals("file")) { // filesystem proxyFilesystem (to another folder) File proxyFolder = new File(url.toURI()); this.proxyFilesystem(proxyFolder, filter); } if (url.getProtocol().equals("http")) { // HTTP proxyFilesystem this.proxyHttp(url.toExternalForm(), filter); } this.generateRepositoryXml(); }
From source file:spring.osgi.io.OsgiBundleResource.java
/** * Used internally to get all the URLs matching a certain location. The * method is required to extract the folder from the given location as well * the file.//from w w w . j a va 2s.co m * * @param location location to look for * @return an array of URLs * @throws java.io.IOException */ ContextResource[] getAllUrlsFromBundleSpace(String location) throws IOException { if (bundle == null) throw new IllegalArgumentException( "cannot locate items in bundle-space w/o a bundle; specify one when creating this resolver"); Assert.notNull(location); Set<ContextResource> resources = new LinkedHashSet<>(5); location = StringUtils.cleanPath(location); location = OsgiResourceUtils.stripPrefix(location); if (!StringUtils.hasText(location)) location = OsgiResourceUtils.FOLDER_DELIMITER; // the root folder is requested (special case) if (OsgiResourceUtils.FOLDER_DELIMITER.equals(location)) { // there is no way to determine the URL to the root directly // through findEntries so we'll have to use another way // getEntry can't be used since it doesn't consider fragments // so we have to rely on findEntries // we could ask for a known entry (such as META-INF) // but not all jars have a dedicated entry for it // so we'll just ask for whatever is present in the root Enumeration candidates = bundle.findEntries("/", null, false); // since there can be multiple root paths (when fragments are present) // iterate on all candidates while (candidates != null && candidates.hasMoreElements()) { URL url = (URL) candidates.nextElement(); // determined the root path // we'll have to parse the string since some implementations // do not normalize the resulting URL resulting in mismatches String rootPath = OsgiResourceUtils.findUpperFolder(url.toExternalForm()); resources.add(new UrlContextResource(rootPath)); } } else { // remove leading and trailing / if any if (location.startsWith(OsgiResourceUtils.FOLDER_DELIMITER)) location = location.substring(1); if (location.endsWith(OsgiResourceUtils.FOLDER_DELIMITER)) location = location.substring(0, location.length() - 1); // do we have at least on folder or is this just a file boolean hasFolder = (location.contains(OsgiResourceUtils.FOLDER_DELIMITER)); String path = (hasFolder ? location : OsgiResourceUtils.FOLDER_DELIMITER); String file = (hasFolder ? null : location); // find the file and path int separatorIndex = location.lastIndexOf(OsgiResourceUtils.FOLDER_DELIMITER); if (separatorIndex > -1 && separatorIndex + 1 < location.length()) { // update the path path = location.substring(0, separatorIndex); // determine file (if there is any) if (separatorIndex + 1 < location.length()) file = location.substring(separatorIndex + 1); } Enumeration candidates = bundle.findEntries(path, file, false); // add the leading / to be consistent String contextPath = OsgiResourceUtils.FOLDER_DELIMITER + location; while (candidates != null && candidates.hasMoreElements()) { resources.add(new UrlContextResource((URL) candidates.nextElement(), contextPath)); } } return resources.toArray(new ContextResource[resources.size()]); }