List of usage examples for java.net HttpURLConnection getContentEncoding
public String getContentEncoding()
From source file:org.lightjason.agentspeak.action.buildin.rest.IBaseRest.java
/** * creates a HTTP connection and reads the data * * @param p_url url/*w w w. j a v a2 s. c o m*/ * @return url data * @throws IOException is thrown on connection errors */ private static String httpdata(final String p_url) throws IOException { final HttpURLConnection l_connection = (HttpURLConnection) new URL(p_url).openConnection(); // follow HTTP redirects l_connection.setInstanceFollowRedirects(true); // set a HTTP-User-Agent if not exists l_connection.setRequestProperty("User-Agent", (System.getProperty("http.agent") == null) || (System.getProperty("http.agent").isEmpty()) ? CCommon.PACKAGEROOT + CCommon.configuration().getString("version") : System.getProperty("http.agent")); // read stream data final InputStream l_stream = l_connection.getInputStream(); final String l_return = CharStreams.toString(new InputStreamReader(l_stream, (l_connection.getContentEncoding() == null) || (l_connection.getContentEncoding().isEmpty()) ? Charsets.UTF_8 : Charset.forName(l_connection.getContentEncoding()))); Closeables.closeQuietly(l_stream); return l_return; }
From source file:org.lightjason.agentspeak.action.builtin.rest.IBaseRest.java
/** * creates a HTTP connection and reads the data * * @param p_url url//from w w w . ja v a2s. c o m * @return url data * * @throws IOException is thrown on connection errors */ @Nonnull private static String httpdata(@Nonnull final String p_url) throws IOException { final HttpURLConnection l_connection = (HttpURLConnection) new URL(p_url).openConnection(); // follow HTTP redirects l_connection.setInstanceFollowRedirects(true); // set a HTTP-User-Agent if not exists l_connection.setRequestProperty("User-Agent", (System.getProperty("http.agent") == null) || (System.getProperty("http.agent").isEmpty()) ? CCommon.PACKAGEROOT + CCommon.configuration().getString("version") : System.getProperty("http.agent")); // read stream data final InputStream l_stream = l_connection.getInputStream(); final String l_return = CharStreams.toString(new InputStreamReader(l_stream, (l_connection.getContentEncoding() == null) || (l_connection.getContentEncoding().isEmpty()) ? Charsets.UTF_8 : Charset.forName(l_connection.getContentEncoding()))); Closeables.closeQuietly(l_stream); return l_return; }
From source file:net.fenyo.mail4hotspot.service.Browser.java
public static String getHtml(final String target_url, final Cookie[] cookies) throws IOException { // log.debug("RETRIEVING_URL=" + target_url); final URL url = new URL(target_url); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); //Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("10.69.60.6", 3128)); // final HttpURLConnection conn = (HttpURLConnection) url.openConnection(proxy); // HttpURLConnection.setFollowRedirects(true); // conn.setRequestProperty("User-agent", "my agent name"); conn.setRequestProperty("Accept-Language", "en-US"); // conn.setRequestProperty(key, value); // allow both GZip and Deflate (ZLib) encodings conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); final String encoding = conn.getContentEncoding(); InputStream is = null;/* w ww .ja va 2 s . c om*/ // create the appropriate stream wrapper based on the encoding type if (encoding != null && encoding.equalsIgnoreCase("gzip")) is = new GZIPInputStream(conn.getInputStream()); else if (encoding != null && encoding.equalsIgnoreCase("deflate")) is = new InflaterInputStream(conn.getInputStream(), new Inflater(true)); else is = conn.getInputStream(); final InputStreamReader reader = new InputStreamReader(new BufferedInputStream(is)); final CharBuffer cb = CharBuffer.allocate(1024 * 1024); int ret; do { ret = reader.read(cb); } while (ret > 0); cb.flip(); return cb.toString(); }
From source file:ja.ohac.wallet.ExchangeRatesProvider.java
private static Map<String, ExchangeRate> requestExchangeRates(final URL url, final String userAgent, final String source, final String... fields) { final long start = System.currentTimeMillis(); HttpURLConnection connection = null; Reader reader = null;//from w w w .ja v a 2 s .c o m try { connection = (HttpURLConnection) url.openConnection(); connection.setInstanceFollowRedirects(false); connection.setConnectTimeout(Constants.HTTP_TIMEOUT_MS); connection.setReadTimeout(Constants.HTTP_TIMEOUT_MS); connection.addRequestProperty("User-Agent", userAgent); connection.addRequestProperty("Accept-Encoding", "gzip"); connection.connect(); final int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { final String contentEncoding = connection.getContentEncoding(); InputStream is = new BufferedInputStream(connection.getInputStream(), 1024); if ("gzip".equalsIgnoreCase(contentEncoding)) is = new GZIPInputStream(is); reader = new InputStreamReader(is, Charsets.UTF_8); final StringBuilder content = new StringBuilder(); final long length = Io.copy(reader, content); final Map<String, ExchangeRate> rates = new TreeMap<String, ExchangeRate>(); final JSONObject head = new JSONObject(content.toString()); for (final Iterator<String> i = head.keys(); i.hasNext();) { final String currencyCode = i.next(); if (!"timestamp".equals(currencyCode)) { final JSONObject o = head.getJSONObject(currencyCode); for (final String field : fields) { final String rateStr = o.optString(field, null); if (rateStr != null) { try { final BigInteger rate = GenericUtils.parseCoin(rateStr, 0); if (rate.signum() > 0) { rates.put(currencyCode, new ExchangeRate(currencyCode, rate, source)); break; } } catch (final ArithmeticException x) { log.warn("problem fetching {} exchange rate from {} ({}): {}", currencyCode, url, contentEncoding, x.getMessage()); } } } } } log.info("fetched exchange rates from {} ({}), {} chars, took {} ms", url, contentEncoding, length, System.currentTimeMillis() - start); return rates; } else { log.warn("http status {} when fetching exchange rates from {}", responseCode, url); } } catch (final Exception x) { log.warn("problem fetching exchange rates from " + url, x); } finally { if (reader != null) { try { reader.close(); } catch (final IOException x) { // swallow } } if (connection != null) connection.disconnect(); } return null; }
From source file:de.schildbach.wallet.ExchangeRatesProvider.java
private static Map<String, ExchangeRate> requestExchangeRates(final URL url, final String userAgent, final String source, final String... fields) { final long start = System.currentTimeMillis(); HttpURLConnection connection = null; Reader reader = null;//from www .ja va2s . c o m try { connection = (HttpURLConnection) url.openConnection(); connection.setInstanceFollowRedirects(false); connection.setConnectTimeout(Constants.HTTP_TIMEOUT_MS); connection.setReadTimeout(Constants.HTTP_TIMEOUT_MS); connection.addRequestProperty("User-Agent", userAgent); connection.addRequestProperty("Accept-Encoding", "gzip"); connection.connect(); final int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { final String contentEncoding = connection.getContentEncoding(); InputStream is = new BufferedInputStream(connection.getInputStream(), 1024); if ("gzip".equalsIgnoreCase(contentEncoding)) is = new GZIPInputStream(is); reader = new InputStreamReader(is, Charsets.UTF_8); final StringBuilder content = new StringBuilder(); final long length = Io.copy(reader, content); final Map<String, ExchangeRate> rates = new TreeMap<String, ExchangeRate>(); final JSONObject head = new JSONObject(content.toString()); for (final Iterator<String> i = head.keys(); i.hasNext();) { final String currencyCode = i.next(); if (!"timestamp".equals(currencyCode)) { final JSONObject o = head.getJSONObject(currencyCode); for (final String field : fields) { final String rateStr = o.optString(field, null); if (rateStr != null) { try { final Fiat rate = Fiat.parseFiat(currencyCode, rateStr); if (rate.signum() > 0) { rates.put(currencyCode, new ExchangeRate( new org.bitcoinj.utils.ExchangeRate(rate), source)); break; } } catch (final NumberFormatException x) { log.warn("problem fetching {} exchange rate from {} ({}): {}", currencyCode, url, contentEncoding, x.getMessage()); } } } } } log.info("fetched exchange rates from {} ({}), {} chars, took {} ms", url, contentEncoding, length, System.currentTimeMillis() - start); return rates; } else { log.warn("http status {} when fetching exchange rates from {}", responseCode, url); } } catch (final Exception x) { log.warn("problem fetching exchange rates from " + url, x); } finally { if (reader != null) { try { reader.close(); } catch (final IOException x) { // swallow } } if (connection != null) connection.disconnect(); } return null; }
From source file:org.apache.flink.test.util.TestBaseUtils.java
public static String getFromHTTP(String url) throws Exception { URL u = new URL(url); LOG.info("Accessing URL " + url + " as URL: " + u); HttpURLConnection connection = (HttpURLConnection) u.openConnection(); connection.setConnectTimeout(100000); connection.connect();// w w w .j a v a 2s. c o m InputStream is; if (connection.getResponseCode() >= 400) { // error! LOG.warn("HTTP Response code when connecting to {} was {}", url, connection.getResponseCode()); is = connection.getErrorStream(); } else { is = connection.getInputStream(); } return IOUtils.toString(is, connection.getContentEncoding() != null ? connection.getContentEncoding() : "UTF-8"); }
From source file:de.jdellay.wallet.ExchangeRatesProvider.java
private static Map<String, ExchangeRate> requestExchangeRates(final URL url, float ccnBtcConversion, final String userAgent, final String source, final String... fields) { final long start = System.currentTimeMillis(); HttpURLConnection connection = null; Reader reader = null;//www. j a va 2 s . c o m try { connection = (HttpURLConnection) url.openConnection(); connection.setInstanceFollowRedirects(false); connection.setConnectTimeout(Constants.HTTP_TIMEOUT_MS); connection.setReadTimeout(Constants.HTTP_TIMEOUT_MS); connection.addRequestProperty("User-Agent", userAgent); connection.addRequestProperty("Accept-Encoding", "gzip"); connection.connect(); final int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { final String contentEncoding = connection.getContentEncoding(); InputStream is = new BufferedInputStream(connection.getInputStream(), 1024); if ("gzip".equalsIgnoreCase(contentEncoding)) is = new GZIPInputStream(is); reader = new InputStreamReader(is, Constants.UTF_8); final StringBuilder content = new StringBuilder(); final long length = Io.copy(reader, content); final Map<String, ExchangeRate> rates = new TreeMap<String, ExchangeRate>(); final JSONObject head = new JSONObject(content.toString()); for (final Iterator<String> i = head.keys(); i.hasNext();) { final String currencyCode = i.next(); if (!"timestamp".equals(currencyCode)) { final JSONObject o = head.getJSONObject(currencyCode); for (final String field : fields) { final String rate = o.optString(field, null); if (rate != null) { try { BigDecimal btcRate = new BigDecimal(GenericUtils.toNanoCoins(rate, 0)); BigInteger ccnRate = btcRate.multiply(BigDecimal.valueOf(ccnBtcConversion)) .toBigInteger(); if (ccnRate.signum() > 0) { rates.put(currencyCode, new ExchangeRate(currencyCode, ccnRate, source)); break; } } catch (final ArithmeticException x) { log.warn("problem fetching {} exchange rate from {} ({}): {}", currencyCode, url, contentEncoding, x.getMessage()); } } } } } log.info("fetched exchange rates from {} ({}), {} chars, took {} ms", url, contentEncoding, length, System.currentTimeMillis() - start); return rates; } else { log.warn("http status {} when fetching {}", responseCode, url); } } catch (final Exception x) { log.warn("problem fetching exchange rates from " + url, x); } finally { if (reader != null) { try { reader.close(); } catch (final IOException x) { // swallow } } if (connection != null) connection.disconnect(); } return null; }
From source file:org.geowebcache.proxy.ProxyDispatcher.java
protected ModelAndView handleRequestInternal(HttpServletRequest request, HttpServletResponse response) throws Exception { String methStr = request.getMethod(); if (!methStr.equalsIgnoreCase("POST")) { throw new ServletException("Illegal method " + methStr + " for this proxy."); }// w w w . j a v a2 s . c o m String urlStr = request.getParameter("url"); if (urlStr == null || urlStr.length() == 0 || !urlStr.startsWith("http://")) { throw new ServletException("Expected url parameter."); } synchronized (this) { long time = System.currentTimeMillis(); if (time - lastRequest < 1000) { throw new ServletException("Only one request per second please."); } else { lastRequest = time; } } log.info("Proxying request for " + request.getRemoteAddr() + " to " + " " + urlStr); String charEnc = request.getCharacterEncoding(); if (charEnc == null) { charEnc = "UTF-8"; } String decodedUrl = URLDecoder.decode(urlStr, charEnc); URL url = new URL(decodedUrl); HttpURLConnection wmsBackendCon = (HttpURLConnection) url.openConnection(); if (wmsBackendCon.getContentEncoding() != null) { response.setCharacterEncoding(wmsBackendCon.getContentEncoding()); } response.setContentType(wmsBackendCon.getContentType()); int read = 0; byte[] data = new byte[1024]; while (read > -1) { read = wmsBackendCon.getInputStream().read(data); if (read > -1) { response.getOutputStream().write(data, 0, read); } } return null; }
From source file:com.gmt2001.HttpRequest.java
public static HttpResponse getData(RequestType type, String url, String post, HashMap<String, String> headers) { Thread.setDefaultUncaughtExceptionHandler(com.gmt2001.UncaughtExceptionHandler.instance()); HttpResponse r = new HttpResponse(); r.type = type;// w ww . ja va2 s. co m r.url = url; r.post = post; r.headers = headers; try { URL u = new URL(url); HttpURLConnection h = (HttpURLConnection) u.openConnection(); for (Entry<String, String> e : headers.entrySet()) { h.addRequestProperty(e.getKey(), e.getValue()); } h.setRequestMethod(type.name()); h.setUseCaches(false); h.setDefaultUseCaches(false); h.setConnectTimeout(timeout); h.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.52 Safari/537.36 PhantomBotJ/2015"); if (!post.isEmpty()) { h.setDoOutput(true); } h.connect(); if (!post.isEmpty()) { BufferedOutputStream stream = new BufferedOutputStream(h.getOutputStream()); stream.write(post.getBytes()); stream.flush(); stream.close(); } if (h.getResponseCode() < 400) { r.content = IOUtils.toString(new BufferedInputStream(h.getInputStream()), h.getContentEncoding()); r.httpCode = h.getResponseCode(); r.success = true; } else { r.content = IOUtils.toString(new BufferedInputStream(h.getErrorStream()), h.getContentEncoding()); r.httpCode = h.getResponseCode(); r.success = false; } } catch (IOException ex) { r.success = false; r.httpCode = 0; r.exception = ex.getMessage(); com.gmt2001.Console.err.printStackTrace(ex); } return r; }
From source file:de.langerhans.wallet.ExchangeRatesProvider.java
private static Map<String, ExchangeRate> requestExchangeRates(final URL url, double dogeBtcConversion, final String userAgent, final String source, final String... fields) { final long start = System.currentTimeMillis(); HttpURLConnection connection = null; Reader reader = null;/*from ww w . ja va 2 s .com*/ try { connection = (HttpURLConnection) url.openConnection(); connection.setInstanceFollowRedirects(false); connection.setConnectTimeout(Constants.HTTP_TIMEOUT_MS); connection.setReadTimeout(Constants.HTTP_TIMEOUT_MS); connection.addRequestProperty("User-Agent", userAgent); connection.addRequestProperty("Accept-Encoding", "gzip"); connection.connect(); final int responseCode = connection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { final String contentEncoding = connection.getContentEncoding(); InputStream is = new BufferedInputStream(connection.getInputStream(), 1024); if ("gzip".equalsIgnoreCase(contentEncoding)) is = new GZIPInputStream(is); reader = new InputStreamReader(is, Charsets.UTF_8); final StringBuilder content = new StringBuilder(); final long length = Io.copy(reader, content); final Map<String, ExchangeRate> rates = new TreeMap<String, ExchangeRate>(); final JSONObject head = new JSONObject(content.toString()); for (final Iterator<String> i = head.keys(); i.hasNext();) { final String currencyCode = i.next(); if (!"timestamp".equals(currencyCode)) { final JSONObject o = head.getJSONObject(currencyCode); for (final String field : fields) { final String rate = o.optString(field, null); if (rate != null) { try { final double btcRate = Double .parseDouble(Fiat.parseFiat(currencyCode, rate).toPlainString()); DecimalFormat df = new DecimalFormat("#.########"); df.setRoundingMode(RoundingMode.HALF_UP); DecimalFormatSymbols dfs = new DecimalFormatSymbols(); dfs.setDecimalSeparator('.'); dfs.setGroupingSeparator(','); df.setDecimalFormatSymbols(dfs); final Fiat dogeRate = Fiat.parseFiat(currencyCode, df.format(btcRate * dogeBtcConversion)); if (dogeRate.signum() > 0) { rates.put(currencyCode, new ExchangeRate( new com.dogecoin.dogecoinj.utils.ExchangeRate(dogeRate), source)); break; } } catch (final NumberFormatException x) { log.warn("problem fetching {} exchange rate from {} ({}): {}", currencyCode, url, contentEncoding, x.getMessage()); } } } } } log.info("fetched exchange rates from {} ({}), {} chars, took {} ms", url, contentEncoding, length, System.currentTimeMillis() - start); return rates; } else { log.warn("http status {} when fetching exchange rates from {}", responseCode, url); } } catch (final Exception x) { log.warn("problem fetching exchange rates from " + url, x); } finally { if (reader != null) { try { reader.close(); } catch (final IOException x) { // swallow } } if (connection != null) connection.disconnect(); } return null; }