List of usage examples for java.net URLConnection getContentEncoding
public String getContentEncoding()
From source file:org.apache.hadoop.yarn.client.cli.TopCLI.java
private JSONObject getJSONObject(URLConnection conn) throws IOException, JSONException { InputStream in = conn.getInputStream(); String encoding = conn.getContentEncoding(); encoding = encoding == null ? "UTF-8" : encoding; String body = IOUtils.toString(in, encoding); JSONObject obj = new JSONObject(body); JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); return clusterInfo; }
From source file:com.apache.ivy.BasicURLHandler.java
public void download(URL src, File dest, CopyProgressListener l) throws IOException { // Install the IvyAuthenticator if ("http".equals(src.getProtocol()) || "https".equals(src.getProtocol())) { IvyAuthenticator.install();//from w w w . j av a2 s. c om } URLConnection srcConn = null; try { src = normalizeToURL(src); srcConn = src.openConnection(); srcConn.setRequestProperty("User-Agent", "Apache Ivy/1.0");// + Ivy.getIvyVersion()); srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate"); if (srcConn instanceof HttpURLConnection) { HttpURLConnection httpCon = (HttpURLConnection) srcConn; if (!checkStatusCode(src, httpCon)) { throw new IOException("The HTTP response code for " + src + " did not indicate a success." + " See log for more detail."); } } // do the download InputStream inStream = getDecodingInputStream(srcConn.getContentEncoding(), srcConn.getInputStream()); FileUtil.copy(inStream, dest, l); // check content length only if content was not encoded if (srcConn.getContentEncoding() == null) { int contentLength = srcConn.getContentLength(); if (contentLength != -1 && dest.length() != contentLength) { dest.delete(); throw new IOException("Downloaded file size doesn't match expected Content Length for " + src + ". Please retry."); } } // update modification date long lastModified = srcConn.getLastModified(); if (lastModified > 0) { dest.setLastModified(lastModified); } } finally { disconnect(srcConn); } }
From source file:org.transitime.avl.PollUrlAvlModule.java
/** * Actually reads data from feed and processes it by opening up a URL * specified by getUrl() and then reading the contents. Calls the abstract * method processData() to actually process the input stream. * <p>/*from www. ja va 2s. c om*/ * This method needs to be overwritten if not real data from a URL * * @throws Exception * Throws a generic exception since the processing is done in * the abstract method processData() and it could throw any type * of exception since we don't really know how the AVL feed will * be processed. */ protected void getAndProcessData() throws Exception { // For logging IntervalTimer timer = new IntervalTimer(); // Get from the AVL feed subclass the URL to use for this feed String fullUrl = getUrl(); // Log what is happening logger.info("Getting data from feed using url=" + fullUrl); // Create the connection URL url = new URL(fullUrl); URLConnection con = url.openConnection(); // Set the timeout so don't wait forever int timeoutMsec = AvlConfig.getAvlFeedTimeoutInMSecs(); con.setConnectTimeout(timeoutMsec); con.setReadTimeout(timeoutMsec); // Request compressed data to reduce bandwidth used if (useCompression) con.setRequestProperty("Accept-Encoding", "gzip,deflate"); // If authentication being used then set user and password if (authenticationUser.getValue() != null && authenticationPassword.getValue() != null) { String authString = authenticationUser.getValue() + ":" + authenticationPassword.getValue(); byte[] authEncBytes = Base64.encodeBase64(authString.getBytes()); String authStringEnc = new String(authEncBytes); con.setRequestProperty("Authorization", "Basic " + authStringEnc); } // Set any additional AVL feed specific request headers setRequestHeaders(con); // Create appropriate input stream depending on whether content is // compressed or not InputStream in = con.getInputStream(); if ("gzip".equals(con.getContentEncoding())) { in = new GZIPInputStream(in); logger.debug("Returned data is compressed"); } else { logger.debug("Returned data is NOT compressed"); } // For debugging logger.debug("Time to access inputstream {} msec", timer.elapsedMsec()); // Call the abstract method to actually process the data timer.resetTimer(); Collection<AvlReport> avlReportsReadIn = processData(in); in.close(); logger.debug("Time to parse document {} msec", timer.elapsedMsec()); // Process all the reports read in if (shouldProcessAvl.getValue()) processAvlReports(avlReportsReadIn); }
From source file:com.allblacks.utils.web.HttpUtil.java
/** * Gets data from URL as char[] throws {@link RuntimeException} If anything * goes wrong// w ww . ja v a 2s .c o m * * @return The content of the URL as a char[] * @throws java.io.IOException */ public byte[] postDataAsByteArray(String url, String contentType, String contentName, byte[] content) throws IOException { URLConnection urlc = null; OutputStream os = null; InputStream is = null; ByteArrayOutputStream bis = null; byte[] dat = null; final String boundary = "" + new Date().getTime(); try { urlc = new URL(url).openConnection(); urlc.setDoOutput(true); urlc.setRequestProperty(HttpUtil.CONTENT_TYPE, "multipart/form-data; boundary=---------------------------" + boundary); os = urlc.getOutputStream(); String message1 = "-----------------------------" + boundary + HttpUtil.BNL; message1 += "Content-Disposition: form-data; name=\"nzbfile\"; filename=\"" + contentName + "\"" + HttpUtil.BNL; message1 += "Content-Type: " + contentType + HttpUtil.BNL; message1 += HttpUtil.BNL; String message2 = HttpUtil.BNL + "-----------------------------" + boundary + "--" + HttpUtil.BNL; os.write(message1.getBytes()); os.write(content); os.write(message2.getBytes()); os.flush(); if (urlc.getContentEncoding() != null && urlc.getContentEncoding().equalsIgnoreCase(HttpUtil.GZIP)) { is = new GZIPInputStream(urlc.getInputStream()); } else { is = urlc.getInputStream(); } bis = new ByteArrayOutputStream(); int ch; while ((ch = is.read()) != -1) { bis.write(ch); } dat = bis.toByteArray(); } catch (IOException exception) { throw exception; } finally { try { bis.close(); os.close(); is.close(); } catch (Exception e) { // we do not care about this } } return dat; }
From source file:com.allblacks.utils.web.HttpUtil.java
/** * Gets data from URL as char[] throws {@link RuntimeException} If anything * goes wrong// w w w . j a v a 2 s . c om * * @return The content of the URL as a char[] * @throws java.io.IOException */ public char[] postDataAsCharArray(String url, String contentType, String contentName, char[] content) throws IOException { URLConnection urlc = null; OutputStream os = null; InputStream is = null; CharArrayWriter dat = null; BufferedReader reader = null; String boundary = "" + new Date().getTime(); try { urlc = new URL(url).openConnection(); urlc.setDoOutput(true); urlc.setRequestProperty(HttpUtil.CONTENT_TYPE, "multipart/form-data; boundary=---------------------------" + boundary); String message1 = "-----------------------------" + boundary + HttpUtil.BNL; message1 += "Content-Disposition: form-data; name=\"nzbfile\"; filename=\"" + contentName + "\"" + HttpUtil.BNL; message1 += "Content-Type: " + contentType + HttpUtil.BNL; message1 += HttpUtil.BNL; String message2 = HttpUtil.BNL + "-----------------------------" + boundary + "--" + HttpUtil.BNL; os = urlc.getOutputStream(); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, Charset.forName(HttpUtil.UTF_8))); writer.write(message1); writer.write(content); writer.write(message2); writer.flush(); dat = new CharArrayWriter(); if (urlc.getContentEncoding() != null && urlc.getContentEncoding().equalsIgnoreCase(HttpUtil.GZIP)) { is = new GZIPInputStream(urlc.getInputStream()); } else { is = urlc.getInputStream(); } reader = new BufferedReader(new InputStreamReader(is, Charset.forName(HttpUtil.UTF_8))); int c; while ((c = reader.read()) != -1) { dat.append((char) c); } } catch (IOException exception) { throw exception; } finally { try { reader.close(); os.close(); is.close(); } catch (Exception e) { // we do not care about this } } return dat.toCharArray(); }
From source file:com.apache.ivy.BasicURLHandler.java
public InputStream openStreamPost(URL url, ArrayList<NameValuePair> postData) throws IOException { // Install the IvyAuthenticator if ("http".equals(url.getProtocol()) || "https".equals(url.getProtocol())) { IvyAuthenticator.install();// www . ja v a 2s .c o m } URLConnection conn = null; try { url = normalizeToURL(url); conn = url.openConnection(); conn.setRequestProperty("User-Agent", "Apache Ivy/1.0");// + Ivy.getIvyVersion()); conn.setRequestProperty("Accept-Encoding", "gzip,deflate"); if (conn instanceof HttpURLConnection) { HttpURLConnection httpCon = (HttpURLConnection) conn; httpCon.setRequestMethod("POST"); httpCon.setDoInput(true); httpCon.setDoOutput(true); byte[] postDataBytes = getQuery(postData).getBytes(Charset.forName("UTF-8")); String contentLength = String.valueOf(postDataBytes.length); httpCon.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); httpCon.setRequestProperty("Content-Length", contentLength); httpCon.setDoOutput(true); httpCon.connect(); httpCon.getOutputStream().write(postDataBytes); conn.getOutputStream().flush(); if (!checkStatusCode(url, httpCon)) { throw new IOException("The HTTP response code for " + url + " did not indicate a success." + " See log for more detail."); } } // read the output InputStream inStream = getDecodingInputStream(conn.getContentEncoding(), conn.getInputStream()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); byte[] buffer = new byte[BUFFER_SIZE]; int len; while ((len = inStream.read(buffer)) > 0) { outStream.write(buffer, 0, len); } return new ByteArrayInputStream(outStream.toByteArray()); } finally { disconnect(conn); } }
From source file:iracing.webapi.IracingWebApi.java
private String getResponseText(URLConnection conn) throws IOException { byte[] bytes; // NOTE: The response is read into memory first before unzipping // because it resulted in a 200% performance increase // as opposed to unzipping directly from the connection inputstream // Read the response in to memory (GZipped or not) InputStream is = conn.getInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try {/* w w w . j a v a 2 s.com*/ byte[] b = new byte[1024]; int bytesRead; while ((bytesRead = is.read(b)) != -1) { baos.write(b, 0, bytesRead); } bytes = baos.toByteArray(); } finally { baos.close(); is.close(); } // System.err.println(conn.getURL().toString()); this.bytesReceived += bytes.length; // Unzip the response if necessary if ("gzip".equals(conn.getContentEncoding())) { // System.err.println("compressed : " + bytesReceived.length + " bytes"); GZIPInputStream is2 = null; try { is2 = new GZIPInputStream(new ByteArrayInputStream(bytes)); baos = new ByteArrayOutputStream(); byte[] b = new byte[1024]; int bytesRead; while ((bytesRead = is2.read(b)) != -1) { baos.write(b, 0, bytesRead); } bytes = baos.toByteArray(); } finally { baos.close(); is2.close(); } } // System.err.println("uncompressed : " + bytesReceived.length + " bytes"); return new String(bytes); }
From source file:tufts.vue.URLResource.java
private Properties scrapeHTMLmetaData(URLConnection connection, int maxSearchBytes) throws java.io.IOException { Properties metaData = new Properties(); InputStream byteStream = connection.getInputStream(); if (DEBUG.DND && DEBUG.META) { System.err.println("Getting headers from " + connection); System.err.println("Headers: " + connection.getHeaderFields()); }//from w ww . j a v a 2s . c o m // note: be sure to call getContentType and don't rely on getting it from the HeaderFields map, // as sometimes it's set by the OS for a file:/// URL when there are no header fields (no http server) // (actually, this is set by java via a mime type table based on file extension, or a guess based on the stream) if (DEBUG.DND) System.err.println("*** getting contentType & encoding..."); final String contentType = connection.getContentType(); final String contentEncoding = connection.getContentEncoding(); final int contentLength = connection.getContentLength(); if (DEBUG.DND) System.err.println("*** contentType [" + contentType + "]"); if (DEBUG.DND) System.err.println("*** contentEncoding [" + contentEncoding + "]"); if (DEBUG.DND) System.err.println("*** contentLength [" + contentLength + "]"); setProperty("url.contentType", contentType); setProperty("url.contentEncoding", contentEncoding); if (contentLength >= 0) setProperty("url.contentLength", contentLength); //if (contentType.toLowerCase().startsWith("text/html") == false) { if (!isHTML()) { // we only currently handle HTML if (DEBUG.Enabled) System.err.println("*** contentType [" + contentType + "] not HTML; skipping title extraction"); return metaData; } if (DEBUG.DND) System.err.println("*** scanning for HTML meta-data..."); try { final BufferedInputStream bufStream = new BufferedInputStream(byteStream, maxSearchBytes); bufStream.mark(maxSearchBytes); final byte[] byteBuffer = new byte[maxSearchBytes]; int bytesRead = 0; int len = 0; // BufferedInputStream still won't read thru a block, so we need to allow // a few reads here to get thru a couple of blocks, so we can get up to // our maxbytes (e.g., a common return chunk count is 1448 bytes, presumably related to the MTU) do { int max = maxSearchBytes - bytesRead; len = bufStream.read(byteBuffer, bytesRead, max); System.out.println("*** read " + len); if (len > 0) bytesRead += len; else if (len < 0) break; } while (len > 0 && bytesRead < maxSearchBytes); if (DEBUG.DND) System.out.println("*** Got total chars: " + bytesRead); String html = new String(byteBuffer, 0, bytesRead); if (DEBUG.DND && DEBUG.META) System.out.println("*** HTML-STRING[" + html + "]"); // first, look for a content encoding, so we can search for and get the title // on a properly encoded character stream String charset = null; Matcher cm = Content_Charset_Regex.matcher(html); if (cm.lookingAt()) { charset = cm.group(1); if (DEBUG.DND) System.err.println("*** found HTML specified charset [" + charset + "]"); setProperty("charset", charset); } if (charset == null && contentEncoding != null) { if (DEBUG.DND || true) System.err.println("*** no charset found: using contentEncoding charset " + contentEncoding); charset = contentEncoding; } final String decodedHTML; if (charset != null) { bufStream.reset(); InputStreamReader decodedStream = new InputStreamReader(bufStream, charset); //InputStreamReader decodedStream = new InputStreamReader(new ByteArrayInputStream(byteBuffer), charset); if (true || DEBUG.DND) System.out.println("*** decoding bytes into characters with official encoding " + decodedStream.getEncoding()); setProperty("contentEncoding", decodedStream.getEncoding()); char[] decoded = new char[bytesRead]; int decodedChars = decodedStream.read(decoded); decodedStream.close(); if (true || DEBUG.DND) System.err.println("*** " + decodedChars + " characters decoded using " + charset); decodedHTML = new String(decoded, 0, decodedChars); } else decodedHTML = html; // we'll just have to go with the default platform charset... // these needed to be left open till the decodedStream was done, which // although it should never need to read beyond what's already buffered, // some internal java code has checks that make sure the underlying stream // isn't closed, even it it isn't used. byteStream.close(); bufStream.close(); Matcher m = HTML_Title_Regex.matcher(decodedHTML); if (m.lookingAt()) { String title = m.group(1); if (true || DEBUG.DND) System.err.println("*** found title [" + title + "]"); metaData.put("title", title.trim()); } } catch (Throwable e) { System.err.println("scrapeHTMLmetaData: " + e); if (DEBUG.DND) e.printStackTrace(); } if (DEBUG.DND || DEBUG.Enabled) System.err.println("*** scrapeHTMLmetaData returning [" + metaData + "]"); return metaData; }
From source file:ch.unifr.pai.twice.widgets.mpproxy.server.JettyProxy.java
public ProcessResult loadFromProxy(HttpServletRequest request, HttpServletResponse response, String uri, String servletPath, String proxyPath) throws ServletException, IOException { //System.out.println("LOAD "+uri); //System.out.println("LOAD "+proxyPath); if ("CONNECT".equalsIgnoreCase(request.getMethod())) { handleConnect(request, response); } else {/* w w w . j av a2 s .com*/ URL url = new URL(uri); URLConnection connection = url.openConnection(); connection.setAllowUserInteraction(false); // Set method HttpURLConnection http = null; if (connection instanceof HttpURLConnection) { http = (HttpURLConnection) connection; http.setRequestMethod(request.getMethod()); http.setInstanceFollowRedirects(false); } // check connection header String connectionHdr = request.getHeader("Connection"); if (connectionHdr != null) { connectionHdr = connectionHdr.toLowerCase(); if (connectionHdr.equals("keep-alive") || connectionHdr.equals("close")) connectionHdr = null; } // copy headers boolean xForwardedFor = false; boolean hasContent = false; Enumeration enm = request.getHeaderNames(); while (enm.hasMoreElements()) { // TODO could be better than this! String hdr = (String) enm.nextElement(); String lhdr = hdr.toLowerCase(); if (_DontProxyHeaders.contains(lhdr)) continue; if (connectionHdr != null && connectionHdr.indexOf(lhdr) >= 0) continue; if ("content-type".equals(lhdr)) hasContent = true; Enumeration vals = request.getHeaders(hdr); while (vals.hasMoreElements()) { String val = (String) vals.nextElement(); if (val != null) { connection.addRequestProperty(hdr, val); xForwardedFor |= "X-Forwarded-For".equalsIgnoreCase(hdr); } } } // Proxy headers connection.setRequestProperty("Via", "1.1 (jetty)"); if (!xForwardedFor) connection.addRequestProperty("X-Forwarded-For", request.getRemoteAddr()); // a little bit of cache control String cache_control = request.getHeader("Cache-Control"); if (cache_control != null && (cache_control.indexOf("no-cache") >= 0 || cache_control.indexOf("no-store") >= 0)) connection.setUseCaches(false); // customize Connection try { connection.setDoInput(true); // do input thang! InputStream in = request.getInputStream(); if (hasContent) { connection.setDoOutput(true); IOUtils.copy(in, connection.getOutputStream()); } // Connect connection.connect(); } catch (Exception e) { e.printStackTrace(); } InputStream proxy_in = null; // handler status codes etc. int code = 500; if (http != null) { proxy_in = http.getErrorStream(); code = http.getResponseCode(); response.setStatus(code, http.getResponseMessage()); } if (proxy_in == null) { try { proxy_in = connection.getInputStream(); } catch (Exception e) { e.printStackTrace(); proxy_in = http.getErrorStream(); } } // clear response defaults. response.setHeader("Date", null); response.setHeader("Server", null); // set response headers int h = 0; String hdr = connection.getHeaderFieldKey(h); String val = connection.getHeaderField(h); while (hdr != null || val != null) { String lhdr = hdr != null ? hdr.toLowerCase() : null; if (hdr != null && val != null && !_DontProxyHeaders.contains(lhdr)) { if (hdr.equalsIgnoreCase("Location")) { val = Rewriter.translateCleanUrl(val, servletPath, proxyPath); } response.addHeader(hdr, val); } h++; hdr = connection.getHeaderFieldKey(h); val = connection.getHeaderField(h); } boolean isGzipped = connection.getContentEncoding() != null && connection.getContentEncoding().contains("gzip"); response.addHeader("Via", "1.1 (jetty)"); // boolean process = connection.getContentType() == null // || connection.getContentType().isEmpty() // || connection.getContentType().contains("html"); boolean process = connection.getContentType() != null && connection.getContentType().contains("text"); if (proxy_in != null) { if (!process) { IOUtils.copy(proxy_in, response.getOutputStream()); proxy_in.close(); } else { InputStream in; if (isGzipped && proxy_in != null && proxy_in.available() > 0) { in = new GZIPInputStream(proxy_in); } else { in = proxy_in; } ByteArrayOutputStream byteArrOS = new ByteArrayOutputStream(); IOUtils.copy(in, byteArrOS); in.close(); if (in != proxy_in) proxy_in.close(); String charset = response.getCharacterEncoding(); if (charset == null || charset.isEmpty()) { charset = "ISO-8859-1"; } String originalContent = new String(byteArrOS.toByteArray(), charset); byteArrOS.close(); return new ProcessResult(originalContent, connection.getContentType(), charset, isGzipped); } } } return null; }