List of usage examples for java.net HttpURLConnection getContentEncoding
public String getContentEncoding()
From source file:com.cgxlib.xq.rebind.JsniBundleGenerator.java
/** * Get the content of a javascript source. It supports remote sources hosted in CDN's. *///from w w w .ja v a 2s.c om private String getContent(TreeLogger logger, String path, String src) throws UnableToCompleteException { HttpURLConnection connection = null; InputStream in = null; try { if (!src.matches("(?i)https?://.*")) { String file = path + "/" + src; logger.log(TreeLogger.INFO, getClass().getSimpleName() + " - importing external javascript: " + file); in = this.getClass().getClassLoader().getResourceAsStream(file); if (in == null) { logger.log(TreeLogger.ERROR, "Unable to read javascript file: " + file); } } else { logger.log(TreeLogger.INFO, getClass().getSimpleName() + " - downloading external javascript: " + src); URL url = new URL(src); connection = (HttpURLConnection) url.openConnection(); connection.setRequestProperty("Accept-Encoding", "gzip, deflate"); connection.setRequestProperty("Host", url.getHost()); connection.setConnectTimeout(3000); connection.setReadTimeout(3000); int status = connection.getResponseCode(); if (status != HttpURLConnection.HTTP_OK) { logger.log(TreeLogger.ERROR, "Server Error: " + status + " " + connection.getResponseMessage()); throw new UnableToCompleteException(); } String encoding = connection.getContentEncoding(); in = connection.getInputStream(); if ("gzip".equalsIgnoreCase(encoding)) { in = new GZIPInputStream(in); } else if ("deflate".equalsIgnoreCase(encoding)) { in = new InflaterInputStream(in); } } return inputStreamToString(in); } catch (IOException e) { logger.log(TreeLogger.ERROR, "Error: " + e.getMessage()); throw new UnableToCompleteException(); } finally { if (connection != null) { connection.disconnect(); } } }
From source file:org.kawanfw.commons.client.http.HttpTransferOne.java
/** * Get and analyze the response.//from w w w .j a v a2 s. c o m * * @param conn * the URL connection in use * * @throws UnknownHostException * @throws ConnectException * @throws RemoteException * @throws IOException */ private void getAndAnalyzeResponse(HttpURLConnection conn) throws UnknownHostException, ConnectException, RemoteException, IOException { BufferedReader reader = null; File contentFile = null; try { // Analyze the error after request execution statusCode = conn.getResponseCode(); if (statusCode != HttpURLConnection.HTTP_OK) { // The server is up, but the servlet is not accessible throw new ConnectException( url + ": Servlet failed: " + conn.getResponseMessage() + " status: " + statusCode); } // it's ok to use a buffered stream with SSL with HttpUrlConnection // Check the server sent us back a compressed content if ("gzip".equals(conn.getContentEncoding())) { reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(conn.getInputStream()))); } else { reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); } // line 1: Contains the request status - line 2: Contains the datas String responseStatus = reader.readLine(); // debug("responseStatus : " + responseStatus); if (doReceiveInFile) { // Content is saved back into a file, minus the first line // status receiveFile = HttpTransferUtil.createKawansoftTempFile(); copyResponseIntoFile(reader, receiveFile); } else { copyResponseIntoString(reader); } // Analyze applicative response header // "SEND_OK" // "SEND_FAILED" if (responseStatus.startsWith(TransferStatus.SEND_OK)) { return; // OK! } else if (responseStatus.startsWith(TransferStatus.SEND_FAILED)) { BufferedReader bufferedReaderException = null; debug(TransferStatus.SEND_FAILED); try { // We must throw the remote exception if (doReceiveInFile) { bufferedReaderException = new BufferedReader(new FileReader(receiveFile)); HttpTransferUtil.throwTheRemoteException(bufferedReaderException); } else { bufferedReaderException = new BufferedReader(new StringReader(m_responseBody)); HttpTransferUtil.throwTheRemoteException(bufferedReaderException); } } finally { IOUtils.closeQuietly(bufferedReaderException); if (doReceiveInFile && !DEBUG) { receiveFile.delete(); } } } else { String message = "The Server response does not start with awaited SEND_OK or SEND_FAILED." + CR_LF + "This could be a configuration failure with an URL that does not correspond to a Kawansoft Servlet." + CR_LF + "URL: " + url + CR_LF + "This could also be a communication failure. Content of server response: " + CR_LF; message += FileUtils.readFileToString(contentFile); throw new IOException(message); } } finally { IOUtils.closeQuietly(reader); if (!DEBUG) { FileUtils.deleteQuietly(contentFile); } } }
From source file:net.andylizi.colormotd.Updater.java
private boolean checkUpdate() { if (file != null && file.exists()) { cancel();/*from w w w.ja v a 2 s. c o m*/ } URL url; try { url = new URL(UPDATE_URL); } catch (MalformedURLException ex) { if (DEBUG) { ex.printStackTrace(); } return false; } try { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setUseCaches(false); conn.setRequestMethod("GET"); conn.addRequestProperty("User-Agent", userAgent); if (etag != null) { conn.addRequestProperty("If-None-Match", etag); } conn.addRequestProperty("Accept", "application/json,text/plain,*/*;charset=utf-8"); conn.addRequestProperty("Accept-Encoding", "gzip"); conn.addRequestProperty("Cache-Control", "no-cache"); conn.addRequestProperty("Date", new Date().toString()); conn.addRequestProperty("Connection", "close"); conn.setDoInput(true); conn.setDoOutput(false); conn.setConnectTimeout(2000); conn.setReadTimeout(2000); conn.connect(); if (conn.getHeaderField("ETag") != null) { etag = conn.getHeaderField("ETag"); } if (DEBUG) { logger.log(Level.INFO, "DEBUG ResponseCode: {0}", conn.getResponseCode()); logger.log(Level.INFO, "DEBUG ETag: {0}", etag); } if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { return false; } else if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { return false; } BufferedReader input = null; if (conn.getContentEncoding() != null && conn.getContentEncoding().contains("gzip")) { input = new BufferedReader( new InputStreamReader(new GZIPInputStream(conn.getInputStream()), "UTF-8"), 1); } else { input = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"), 1); } StringBuilder builder = new StringBuilder(); String buffer = null; while ((buffer = input.readLine()) != null) { builder.append(buffer); } try { JSONObject jsonObj = (JSONObject) new JSONParser().parse(builder.toString()); int build = ((Number) jsonObj.get("build")).intValue(); if (build <= ColorMOTD.buildVersion) { return false; } String version = (String) jsonObj.get("version"); String msg = (String) jsonObj.get("msg"); String download_url = (String) jsonObj.get("url"); newVersion = new NewVersion(version, build, msg, download_url); return true; } catch (ParseException ex) { if (DEBUG) { ex.printStackTrace(); } exception = ex; return false; } } catch (SocketTimeoutException ex) { return false; } catch (IOException ex) { if (DEBUG) { ex.printStackTrace(); } exception = ex; return false; } }
From source file:net.andylizi.colormotd.anim.updater.Updater.java
private boolean checkUpdate() { if (new File(Bukkit.getUpdateFolderFile(), fileName).exists()) { cancel();/*from www. j av a 2s. c o m*/ } URL url; try { url = new URL(UPDATE_URL); } catch (MalformedURLException ex) { if (DEBUG) { ex.printStackTrace(); } return false; } try { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setUseCaches(false); conn.setRequestMethod("GET"); conn.addRequestProperty("User-Agent", USER_AGENT); if (etag != null) { conn.addRequestProperty("If-None-Match", etag); } conn.addRequestProperty("Accept", "application/json,text/plain,*/*;charset=utf-8"); conn.addRequestProperty("Accept-Encoding", "gzip"); conn.addRequestProperty("Cache-Control", "no-cache"); conn.addRequestProperty("Date", new Date().toString()); conn.addRequestProperty("Connection", "close"); conn.setDoInput(true); conn.setDoOutput(false); conn.setConnectTimeout(2000); conn.setReadTimeout(2000); conn.connect(); if (conn.getHeaderField("ETag") != null) { etag = conn.getHeaderField("ETag"); } if (DEBUG) { logger.log(Level.INFO, "DEBUG ResponseCode: {0}", conn.getResponseCode()); logger.log(Level.INFO, "DEBUG ETag: {0}", etag); } if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { return false; } else if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { return false; } BufferedReader input = null; if (conn.getContentEncoding() != null && conn.getContentEncoding().contains("gzip")) { input = new BufferedReader( new InputStreamReader(new GZIPInputStream(conn.getInputStream()), "UTF-8"), 1); } else { input = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"), 1); } StringBuilder builder = new StringBuilder(); String buffer = null; while ((buffer = input.readLine()) != null) { builder.append(buffer); } try { JSONObject jsonObj = (JSONObject) new JSONParser().parse(builder.toString()); int build = ((Number) jsonObj.get("build")).intValue(); if (build <= buildVersion) { return false; } String version = (String) jsonObj.get("version"); String msg = (String) jsonObj.get("msg"); String download_url = (String) jsonObj.get("url"); newVersion = new NewVersion(version, build, msg, download_url); return true; } catch (ParseException ex) { if (DEBUG) { ex.printStackTrace(); } exception = ex; return false; } } catch (SocketTimeoutException ex) { return false; } catch (IOException ex) { if (DEBUG) { ex.printStackTrace(); } exception = ex; return false; } }
From source file:org.broad.igv.util.HttpUtils.java
public InputStream openConnectionStream(URL url, Map<String, String> requestProperties) throws IOException { HttpURLConnection conn = openConnection(url, requestProperties); if (conn == null) { return null; }//from ww w .j av a 2 s .c o m boolean rangeRequestedNotReceived = isExpectedRangeMissing(conn, requestProperties); if (rangeRequestedNotReceived) { String msg = "Byte range requested, but no Content-Range header in response"; log.error(msg); // if(Globals.isTesting()){ // throw new IOException(msg); // } } InputStream input = conn.getInputStream(); if ("gzip".equals(conn.getContentEncoding())) { input = new GZIPInputStream(input); } return input; }
From source file:com.nxt.zyl.data.volley.toolbox.HurlStack.java
@Override public HttpResponse performRequest(Request<?> request, Map<String, String> additionalHeaders) throws IOException, AuthFailureError { String url = request.getUrl(); HashMap<String, String> map = new HashMap<String, String>(); // chenbo add gzip support,new user-agent if (request.isShouldGzip()) { map.put(HEADER_ACCEPT_ENCODING, ENCODING_GZIP); }// w w w. ja v a 2 s. c o m map.put(USER_AGENT, mUserAgent); // end map.putAll(request.getHeaders()); map.putAll(additionalHeaders); if (mUrlRewriter != null) { String rewritten = mUrlRewriter.rewriteUrl(url); if (rewritten == null) { throw new IOException("URL blocked by rewriter: " + url); } url = rewritten; } URL parsedUrl = new URL(url); HttpURLConnection connection = openConnection(parsedUrl, request); for (String headerName : map.keySet()) { connection.addRequestProperty(headerName, map.get(headerName)); } // if (request instanceof MultiPartRequest) { // setConnectionParametersForMultipartRequest(connection, request); // } else { // } setConnectionParametersForRequest(connection, request); // Initialize HttpResponse with data from the HttpURLConnection. ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1); int responseCode = connection.getResponseCode(); if (responseCode == -1) { // -1 is returned by getResponseCode() if the response code could not be retrieved. // Signal to the caller that something was wrong with the connection. throw new IOException("Could not retrieve response code from HttpUrlConnection."); } StatusLine responseStatus = new BasicStatusLine(protocolVersion, connection.getResponseCode(), connection.getResponseMessage()); BasicHttpResponse response = new BasicHttpResponse(responseStatus); response.setEntity(entityFromConnection(connection)); for (Entry<String, List<String>> header : connection.getHeaderFields().entrySet()) { if (header.getKey() != null) { Header h = new BasicHeader(header.getKey(), header.getValue().get(0)); response.addHeader(h); } } if (ENCODING_GZIP.equalsIgnoreCase(connection.getContentEncoding())) { response.setEntity(new InflatingEntity(response.getEntity())); } return response; }
From source file:uk.co.jassoft.network.Network.java
public InputStream read(String httpUrl, String method, boolean cache) throws IOException { HttpURLConnection conn = null; try {//w w w. j ava 2 s. co m URL base, next; String location; while (true) { // inputing the keywords to google search engine URL url = new URL(httpUrl); // if(cache) { // //Proxy instance, proxy ip = 10.0.0.1 with port 8080 // Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("cache", 3128)); // // makking connection to the internet // conn = (HttpURLConnection) url.openConnection(proxy); // } // else { conn = (HttpURLConnection) url.openConnection(); // } conn.setConnectTimeout(connectTimeout); conn.setReadTimeout(readTimeout); conn.setRequestMethod(method); conn.setRequestProperty("User-Agent", "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 ( .NET CLR 3.5.30729)"); conn.setRequestProperty("Accept-Encoding", "gzip"); switch (conn.getResponseCode()) { case HttpURLConnection.HTTP_MOVED_PERM: case HttpURLConnection.HTTP_MOVED_TEMP: location = conn.getHeaderField("Location"); base = new URL(httpUrl); next = new URL(base, location); // Deal with relative URLs httpUrl = next.toExternalForm(); continue; } break; } if (!conn.getContentType().startsWith("text/") || conn.getContentType().equals("text/xml")) { throw new IOException(String.format("Content of story is not Text. Returned content type is [%s]", conn.getContentType())); } if ("gzip".equals(conn.getContentEncoding())) { return new GZIPInputStream(conn.getInputStream()); } // getting the input stream of page html into bufferedreader return conn.getInputStream(); } catch (Exception exception) { throw new IOException(exception); } finally { if (conn != null && conn.getErrorStream() != null) { conn.getErrorStream().close(); } } }
From source file:org.geotools.data.sfs.SFSDataStore.java
/** * This method requests the response from the URL either through GET/POST * and returns the response as a stream/* w w w . j av a2 s . co m*/ * The request is going to be a POST one if postData is not null, a GET otherwise * @param is * @return String * @throws IOException */ InputStream resourceToStream(String resource, String postData) throws MalformedURLException, IOException, ProtocolException { boolean doPost = false; URL url; if (postData != null && (baseURL.length() + resource.length() + postData.length() < 2048)) { url = new URL(baseURL + resource + "?" + postData); } else { url = new URL(baseURL + resource); } // TODO: use commons http client + persistent connections instead HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); urlConnection.setRequestProperty("Accept-Encoding", "gzip"); // if we have username/password use them to setup basic auth if (user != null && password != null) { String combined = nullToEmpty(user) + ":" + nullToEmpty(password); byte[] authBytes = combined.getBytes("US-ASCII"); String encoded = new String(Base64.encodeBase64(authBytes)); urlConnection.setRequestProperty("Authorization", "Basic " + encoded); } // enforce the timeout if we have one if (timeout > 0) { urlConnection.setConnectTimeout(timeout); urlConnection.setReadTimeout(timeout); } urlConnection.setDoInput(true); if (doPost && postData != null && !"".equals(postData.trim())) { urlConnection.setRequestMethod("POST"); urlConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); urlConnection.setDoOutput(true); /*write the data to server*/ OutputStreamWriter wr = null; try { wr = new OutputStreamWriter(urlConnection.getOutputStream()); wr.write(postData); wr.flush(); } finally { if (wr != null) { wr.close(); } } } else { urlConnection.setRequestMethod("GET"); } /* Get the response from the server*/ if (urlConnection.getResponseCode() != 200) { throw new IOException("Server reported and error with code " + urlConnection.getResponseCode() + ": " + urlConnection.getResponseMessage() + " accessing resource " + url.toExternalForm()); } else { InputStream is = urlConnection.getInputStream(); String encoding = urlConnection.getContentEncoding(); if ("gzip".equalsIgnoreCase(encoding)) { is = new GZIPInputStream(is); } return is; } }
From source file:tr.edu.gsu.nerwip.retrieval.reader.wikipedia.WikipediaReader.java
/** * Reads the source code of the web page at the specified * URL./*from ww w . j a va 2s . c om*/ * * @param url * Address of the web page to be read. * @return * String containing the read HTML source code. * * @throws IOException * Problem while accessing the specified URL. */ private String manuallyReadUrl(URL url) throws IOException { boolean trad = false; BufferedReader br = null; // open page the traditional way if (trad) { InputStream is = url.openStream(); InputStreamReader isr = new InputStreamReader(is); br = new BufferedReader(isr); } // open with more options else { // setup connection HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); connection.setDoOutput(true); connection.setReadTimeout(2000); connection.setChunkedStreamingMode(0); connection.setRequestProperty("Content-Length", "0"); // connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2"); connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36"); connection.connect(); // setup input stream // part retrieved from http://stackoverflow.com/questions/538999/java-util-scanner-and-wikipedia // original author: Marco Beggio InputStream is = null; String encoding = connection.getContentEncoding(); if (connection.getContentEncoding() != null && encoding.equals("gzip")) { is = new GZIPInputStream(connection.getInputStream()); } else if (encoding != null && encoding.equals("deflate")) { is = new InflaterInputStream(connection.getInputStream(), new Inflater(true)); } else { is = connection.getInputStream(); } // alternative to spot error details // InputStream is; // if (connection.getResponseCode() != 200) // is = connection.getErrorStream(); // else // is = connection.getInputStream(); InputStreamReader isr = new InputStreamReader(is); br = new BufferedReader(isr); } // read page StringBuffer sourceCode = new StringBuffer(); String line = br.readLine(); while (line != null) { sourceCode.append(line + "\n"); line = br.readLine(); } String result = sourceCode.toString(); br.close(); return result; }