List of usage examples for java.net URLConnection setRequestProperty
public void setRequestProperty(String key, String value)
From source file:org.cloudifysource.dsl.internal.tools.download.ResourceDownloader.java
private InputStream openConnectionInputStream(final URL url) throws ResourceDownloadException { final DefaultHttpClient httpClient = new DefaultHttpClient(); final HttpHead httpMethod = new HttpHead(url.toString()); HttpResponse response;/* w w w .ja v a 2s. c o m*/ try { logger.fine("validating url"); response = httpClient.execute(httpMethod); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { logger.warning("Failed to validate Resource URL: " + url.toString()); throw new ResourceDownloadException("Invalid resource URL: " + url.toString()); } final URLConnection connection = url.openConnection(); if (this.userName != null || this.password != null) { logger.fine("Setting connection credentials"); String up = this.userName + ":" + this.password; String encoding = new String(Base64.encodeBase64(up.getBytes())); connection.setRequestProperty("Authorization", "Basic " + encoding); } return connection.getInputStream(); } catch (ClientProtocolException e) { throw new ResourceDownloadException("Invalid connection protocol " + url.toString(), e); } catch (IOException e) { throw new ResourceDownloadException("Invalid resource URL: " + url.toString(), e); } }
From source file:com.enonic.vertical.engine.PresentationEngine.java
private Document getURL(String address, String encoding, int timeoutMs) { InputStream in = null;/*from w w w .j av a 2s. c o m*/ BufferedReader reader = null; Document result; try { URL url = new URL(address); URLConnection urlConn = url.openConnection(); urlConn.setConnectTimeout(timeoutMs > 0 ? timeoutMs : DEFAULT_CONNECTION_TIMEOUT); urlConn.setRequestProperty("User-Agent", VerticalProperties.getVerticalProperties().getDataSourceUserAgent()); String userInfo = url.getUserInfo(); if (StringUtils.isNotBlank(userInfo)) { String userInfoBase64Encoded = new String(Base64.encodeBase64(userInfo.getBytes())); urlConn.setRequestProperty("Authorization", "Basic " + userInfoBase64Encoded); } in = urlConn.getInputStream(); // encoding == null: XML file if (encoding == null) { result = XMLTool.domparse(in); } else { StringBuffer sb = new StringBuffer(1024); reader = new BufferedReader(new InputStreamReader(in, encoding)); char[] line = new char[1024]; int charCount = reader.read(line); while (charCount > 0) { sb.append(line, 0, charCount); charCount = reader.read(line); } result = XMLTool.createDocument("urlresult"); Element root = result.getDocumentElement(); XMLTool.createCDATASection(result, root, sb.toString()); } } catch (SocketTimeoutException ste) { String message = "Socket timeout when trying to get url: " + address; VerticalEngineLogger.warn(this.getClass(), 0, message, null); result = null; } catch (IOException ioe) { String message = "Failed to get URL: %t"; VerticalEngineLogger.warn(this.getClass(), 0, message, ioe); result = null; } catch (RuntimeException re) { String message = "Failed to get URL: %t"; VerticalEngineLogger.warn(this.getClass(), 0, message, re); result = null; } finally { try { if (reader != null) { reader.close(); } else if (in != null) { in.close(); } } catch (IOException ioe) { String message = "Failed to close URL connection: %t"; VerticalEngineLogger.warn(this.getClass(), 0, message, ioe); } } if (result == null) { result = XMLTool.createDocument("noresult"); } return result; }
From source file:com.allblacks.utils.web.HttpUtil.java
/** * Gets data from URL as char[] throws {@link RuntimeException} If anything * goes wrong//from ww w. ja va 2 s . c om * * @return The content of the URL as a char[] * @throws java.io.IOException */ public byte[] postDataAsByteArray(String url, String contentType, String contentName, byte[] content) throws IOException { URLConnection urlc = null; OutputStream os = null; InputStream is = null; ByteArrayOutputStream bis = null; byte[] dat = null; final String boundary = "" + new Date().getTime(); try { urlc = new URL(url).openConnection(); urlc.setDoOutput(true); urlc.setRequestProperty(HttpUtil.CONTENT_TYPE, "multipart/form-data; boundary=---------------------------" + boundary); os = urlc.getOutputStream(); String message1 = "-----------------------------" + boundary + HttpUtil.BNL; message1 += "Content-Disposition: form-data; name=\"nzbfile\"; filename=\"" + contentName + "\"" + HttpUtil.BNL; message1 += "Content-Type: " + contentType + HttpUtil.BNL; message1 += HttpUtil.BNL; String message2 = HttpUtil.BNL + "-----------------------------" + boundary + "--" + HttpUtil.BNL; os.write(message1.getBytes()); os.write(content); os.write(message2.getBytes()); os.flush(); if (urlc.getContentEncoding() != null && urlc.getContentEncoding().equalsIgnoreCase(HttpUtil.GZIP)) { is = new GZIPInputStream(urlc.getInputStream()); } else { is = urlc.getInputStream(); } bis = new ByteArrayOutputStream(); int ch; while ((ch = is.read()) != -1) { bis.write(ch); } dat = bis.toByteArray(); } catch (IOException exception) { throw exception; } finally { try { bis.close(); os.close(); is.close(); } catch (Exception e) { // we do not care about this } } return dat; }
From source file:com.googlecode.jmxtrans.model.output.StackdriverWriter.java
/** * Use a Cloud provider local metadata endpoint to determine the instance ID that this code is running on. * Useful if you don't want to configure the instance ID manually. * Pass detectInstance param with a cloud provider ID (AWS|GCE) to have this run in your configuration. * //from w ww . j av a2 s. co m * @return String containing an instance id, or null if none is found */ private String getLocalInstanceId(final String cloudProvider, final String metadataEndpoint, final Map<String, String> headers) { String detectedInstanceId = null; try { final URL metadataUrl = new URL(metadataEndpoint); URLConnection metadataConnection = metadataUrl.openConnection(); // add any additional headers passed in if (headers != null) { for (Map.Entry<String, String> header : headers.entrySet()) { metadataConnection.setRequestProperty(header.getKey(), header.getValue()); } } BufferedReader in = new BufferedReader( new InputStreamReader(metadataConnection.getInputStream(), "UTF-8")); String inputLine; while ((inputLine = in.readLine()) != null) { detectedInstanceId = inputLine; } in.close(); } catch (Exception e) { logger.warn("unable to determine " + cloudProvider + " instance ID", e); } return detectedInstanceId; }
From source file:com.allblacks.utils.web.HttpUtil.java
/** * Gets data from URL as char[] throws {@link RuntimeException} If anything * goes wrong/*from ww w . ja v a 2 s . c om*/ * * @return The content of the URL as a char[] * @throws java.io.IOException */ public char[] postDataAsCharArray(String url, String contentType, String contentName, char[] content) throws IOException { URLConnection urlc = null; OutputStream os = null; InputStream is = null; CharArrayWriter dat = null; BufferedReader reader = null; String boundary = "" + new Date().getTime(); try { urlc = new URL(url).openConnection(); urlc.setDoOutput(true); urlc.setRequestProperty(HttpUtil.CONTENT_TYPE, "multipart/form-data; boundary=---------------------------" + boundary); String message1 = "-----------------------------" + boundary + HttpUtil.BNL; message1 += "Content-Disposition: form-data; name=\"nzbfile\"; filename=\"" + contentName + "\"" + HttpUtil.BNL; message1 += "Content-Type: " + contentType + HttpUtil.BNL; message1 += HttpUtil.BNL; String message2 = HttpUtil.BNL + "-----------------------------" + boundary + "--" + HttpUtil.BNL; os = urlc.getOutputStream(); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, Charset.forName(HttpUtil.UTF_8))); writer.write(message1); writer.write(content); writer.write(message2); writer.flush(); dat = new CharArrayWriter(); if (urlc.getContentEncoding() != null && urlc.getContentEncoding().equalsIgnoreCase(HttpUtil.GZIP)) { is = new GZIPInputStream(urlc.getInputStream()); } else { is = urlc.getInputStream(); } reader = new BufferedReader(new InputStreamReader(is, Charset.forName(HttpUtil.UTF_8))); int c; while ((c = reader.read()) != -1) { dat.append((char) c); } } catch (IOException exception) { throw exception; } finally { try { reader.close(); os.close(); is.close(); } catch (Exception e) { // we do not care about this } } return dat.toCharArray(); }
From source file:captureplugin.drivers.dreambox.connector.DreamboxConnector.java
private InputStream openStreamForLocalUrl(final String localUrl) throws MalformedURLException, IOException { URL url = new URL("http://" + mConfig.getDreamboxAddress() + localUrl); URLConnection connection = url.openConnection(); // set user and password String userpassword = mConfig.getUserName() + ":" + mConfig.getPassword(); String encoded = new String(Base64.encodeBase64(userpassword.getBytes())); connection.setRequestProperty("Authorization", "Basic " + encoded); // set timeout connection.setConnectTimeout(mConfig.getTimeout()); InputStream stream = connection.getInputStream(); return stream; }
From source file:youtube.transcription.Video.java
public InputStreamReader readURL(String s) throws MalformedURLException, IOException { URL url;/* w w w .j a v a 2 s .com*/ InputStreamReader isr; String appName, appVersion; URLConnection urlconn; appName = java.util.ResourceBundle.getBundle("Bundle").getString("app.name"); appVersion = java.util.ResourceBundle.getBundle("Bundle").getString("app.version"); url = new URL(s); if (proxy != null) { urlconn = url.openConnection(proxy); } else { urlconn = url.openConnection(); } urlconn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); urlconn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"); urlconn.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; " + appName + "/" + appVersion + ")"); urlconn.connect(); isr = new InputStreamReader(urlconn.getInputStream(), "UTF-8"); return isr; }
From source file:org.apache.cocoon.components.crawler.SimpleCocoonCrawlerImpl.java
/** * Compute list of links from the url./*w w w . j a va 2 s . c o m*/ * <p> * Check for include, exclude pattern, content-type, and if url * has been craweled already. * </p> * * @param url Crawl this URL * @return List of URLs, which are links from url, asserting the conditions. * @since */ private List getLinks(URL url) { ArrayList url_links = null; String sURL = url.toString(); if (!isIncludedURL(sURL) || isExcludedURL(sURL)) { return null; } // don't try to get links for url which has been crawled already if (crawled.contains(sURL)) { return null; } // mark it as crawled crawled.add(sURL); // get links of url if (getLogger().isDebugEnabled()) { getLogger().debug("Getting links of URL " + sURL); } BufferedReader br = null; try { sURL = url.getFile(); URL links = new URL(url, sURL + ((sURL.indexOf("?") == -1) ? "?" : "&") + linkViewQuery); URLConnection links_url_connection = links.openConnection(); links_url_connection.setRequestProperty("Accept", accept); links_url_connection.setRequestProperty("User-Agent", userAgent); links_url_connection.connect(); InputStream is = links_url_connection.getInputStream(); br = new BufferedReader(new InputStreamReader(is)); String contentType = links_url_connection.getContentType(); if (contentType == null) { if (getLogger().isDebugEnabled()) { getLogger().debug("Ignoring " + sURL + " (no content type)"); } // there is a check on null in the calling method return null; } int index = contentType.indexOf(';'); if (index != -1) { contentType = contentType.substring(0, index); } if (getLogger().isDebugEnabled()) { getLogger().debug("Content-type: " + contentType); } if (contentType.equals(linkContentType)) { url_links = new ArrayList(); // content is supposed to be a list of links, // relative to current URL String line; while ((line = br.readLine()) != null) { final URL newUrl = new URL(url, line); final String sNewUrl = newUrl.toString(); boolean add_url = true; // don't add new_url twice if (add_url) { add_url &= !url_links.contains(sNewUrl); } // don't add new_url if it has been crawled already if (add_url) { add_url &= !crawled.contains(sNewUrl); } // don't add if is not matched by existing include definition if (add_url) { add_url &= isIncludedURL(sNewUrl); } // don't add if is matched by existing exclude definition if (add_url) { add_url &= !isExcludedURL(sNewUrl); } if (add_url) { if (getLogger().isDebugEnabled()) { getLogger().debug("Add URL: " + sNewUrl); } url_links.add(newUrl); } } // now we have a list of URL which should be examined } } catch (IOException ioe) { getLogger().warn("Problems get links of " + url, ioe); } finally { if (br != null) { try { br.close(); br = null; } catch (IOException ignored) { } } } return url_links; }
From source file:dk.dma.ais.downloader.QueryService.java
/** * Asynchronously loads the given file/*from w w w.j ava 2s . co m*/ * @param url the URL to load * @param path the path to save the file to */ private Future<Path> asyncLoadFile(final String url, final Path path) { Callable<Path> job = () -> { long t0 = System.currentTimeMillis(); // For the resulting file, drop the ".download" suffix String name = path.getFileName().toString(); name = name.substring(0, name.length() - DOWNLOAD_SUFFIX.length()); try { // Set up a few timeouts and fetch the attachment URLConnection con = new URL(url).openConnection(); con.setConnectTimeout(60 * 1000); // 1 minute con.setReadTimeout(60 * 60 * 1000); // 1 hour if (!StringUtils.isEmpty(authHeader)) { con.setRequestProperty("Authorization", authHeader); } try (ReadableByteChannel rbc = Channels.newChannel(con.getInputStream()); FileOutputStream fos = new FileOutputStream(path.toFile())) { fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); } log.info(String.format("Copied %s -> %s in %d ms", url, path, System.currentTimeMillis() - t0)); } catch (Exception e) { log.log(Level.SEVERE, "Failed downloading " + url + ": " + e.getMessage()); // Delete the old file if (Files.exists(path)) { try { Files.delete(path); } catch (IOException e1) { log.finer("Failed deleting old file " + path); } } // Save an error file Path errorFile = path.getParent().resolve(name + ".err.txt"); try (PrintStream err = new PrintStream(new FileOutputStream(errorFile.toFile()))) { e.printStackTrace(err); } catch (IOException ex) { log.finer("Failed generating error file " + errorFile); } return errorFile; } Path resultPath = path.getParent().resolve(name); try { Files.move(path, resultPath); } catch (IOException e) { log.log(Level.SEVERE, "Failed renaming path " + path + ": " + e.getMessage()); } return resultPath; }; log.info("Submitting new job: " + url); return processPool.submit(job); }
From source file:org.wymiwyg.wrhapi.test.BaseTests.java
/** * Test whether the getPort method returns the actual request port and not the one * included in the host-header// w w w.j a v a2 s .co m * @throws Exception */ @Test public void testPort() throws Exception { WebServer webServer = createServer().startNewWebServer(new Handler() { public void handle(Request request, Response response) throws HandlerException { assertEquals(serverBinding.getPort(), request.getPort()); } }, serverBinding); try { URL serverURL = new URL("http://" + serverBinding.getInetAddress().getHostAddress() + ":" + serverBinding.getPort() + "/"); URLConnection connection = serverURL.openConnection(); connection.setRequestProperty("host", "foo:88"); connection.setRequestProperty("FOO", "bar"); connection.connect(); // for the handler to be invoked, something of the response has to // be asked //assertEquals(headerValue, connection.getHeaderField("Cookie")); connection.getContentLength(); } catch (MalformedURLException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } finally { webServer.stop(); } }