List of usage examples for java.net HttpURLConnection getContentEncoding
public String getContentEncoding()
From source file:com.cr_wd.android.network.HttpClient.java
/** * Performs a HTTP GET/POST Request/* w w w . ja va 2 s . c o m*/ * * @return id of the request */ protected int doRequest(final Method method, final String url, HttpHeaders headers, HttpParams params, final HttpHandler handler) { if (headers == null) { headers = new HttpHeaders(); } if (params == null) { params = new HttpParams(); } handler.client = this; final int requestId = incrementRequestId(); class HandlerRunnable extends Handler implements Runnable { private final Method method; private String url; private final HttpHeaders headers; private final HttpParams params; private final HttpHandler handler; private HttpResponse response; private boolean canceled = false; private int retries = 0; protected HandlerRunnable(final Method method, final String url, final HttpHeaders headers, final HttpParams params, final HttpHandler handler) { this.method = method; this.url = url; this.headers = headers; this.params = params; this.handler = handler; } @Override public void run() { execute(); } private void execute() { response = new HttpResponse(requestId, method, url); HttpURLConnection conn = null; try { /* append query string for GET requests */ if (method == Method.GET) { if (!params.urlParams.isEmpty()) { url += ('?' + params.getParamString()); } } /* setup headers for POST requests */ if (method == Method.POST) { headers.addHeader("Accept-Charset", requestOptions.encoding); if (params.hasMultipartParams()) { final SimpleMultipart multipart = params.getMultipart(); headers.addHeader("Content-Type", multipart.getContentType()); } else { headers.addHeader("Content-Type", "application/x-www-form-urlencoded;charset=" + requestOptions.encoding); } } if (canceled) { postCancel(); return; } /* open and configure the connection */ conn = (HttpURLConnection) new URL(url).openConnection(); postStart(); if (method == Method.GET) { conn = (HttpURLConnection) new URL(url).openConnection(); conn.setRequestMethod("GET"); } else if (method == Method.POST) { conn.setRequestMethod("POST"); conn.setDoOutput(true); conn.setDoInput(true); conn.setUseCaches(false); } conn.setAllowUserInteraction(false); conn.setReadTimeout(requestOptions.readTimeout); conn.setConnectTimeout(requestOptions.connectTimeout); /* add headers to the connection */ for (final Map.Entry<String, List<String>> entry : headers.getHeaders().entrySet()) { for (final String value : entry.getValue()) { conn.addRequestProperty(entry.getKey(), value); } } if (canceled) { try { conn.disconnect(); } catch (final Exception e) { } postCancel(); return; } response.requestProperties = conn.getRequestProperties(); /* do post */ if (method == Method.POST) { InputStream is; if (params.hasMultipartParams()) { is = params.getMultipart().getContent(); } else { is = new ByteArrayInputStream(params.getParamString().getBytes()); } final OutputStream os = conn.getOutputStream(); writeStream(os, is); } else { conn.connect(); } if (canceled) { try { conn.disconnect(); } catch (final Exception e) { } postCancel(); return; } response.contentEncoding = conn.getContentEncoding(); response.contentLength = conn.getContentLength(); response.contentType = conn.getContentType(); response.date = conn.getDate(); response.expiration = conn.getExpiration(); response.headerFields = conn.getHeaderFields(); response.ifModifiedSince = conn.getIfModifiedSince(); response.lastModified = conn.getLastModified(); response.responseCode = conn.getResponseCode(); response.responseMessage = conn.getResponseMessage(); /* do get */ if (conn.getResponseCode() < 400) { response.responseBody = readStream(conn.getInputStream()); postSuccess(); } else { response.responseBody = readStream(conn.getErrorStream()); response.throwable = new Exception(response.responseMessage); postError(); } } catch (final Exception e) { if (retries < requestOptions.maxRetries) { retries++; postRetry(); execute(); } else { response.responseBody = e.getMessage(); response.throwable = e; postError(); } } finally { if (conn != null) { conn.disconnect(); } } } private String readStream(final InputStream is) throws IOException { final BufferedInputStream bis = new BufferedInputStream(is); final ByteArrayBuffer baf = new ByteArrayBuffer(50); int read = 0; final byte[] buffer = new byte[8192]; while (true) { if (canceled) { break; } read = bis.read(buffer); if (read == -1) { break; } baf.append(buffer, 0, read); } try { bis.close(); } catch (final IOException e) { } try { is.close(); } catch (final IOException e) { } return new String(baf.toByteArray()); } private void writeStream(final OutputStream os, final InputStream is) throws IOException { final BufferedInputStream bis = new BufferedInputStream(is); int read = 0; final byte[] buffer = new byte[8192]; while (true) { if (canceled) { break; } read = bis.read(buffer); if (read == -1) { break; } os.write(buffer, 0, read); } if (!canceled) { os.flush(); } try { os.close(); } catch (final IOException e) { } try { bis.close(); } catch (final IOException e) { } try { is.close(); } catch (final IOException e) { } } @Override public void handleMessage(final Message msg) { if (msg.what == HttpHandler.MESSAGE_CANCEL) { canceled = true; } } private void postSuccess() { postMessage(HttpHandler.MESSAGE_SUCCESS); } private void postError() { postMessage(HttpHandler.MESSAGE_ERROR); } private void postCancel() { postMessage(HttpHandler.MESSAGE_CANCEL); } private void postStart() { postMessage(HttpHandler.MESSAGE_START); } private void postRetry() { postMessage(HttpHandler.MESSAGE_RETRY); } private void postMessage(final int what) { final Message msg = handler.obtainMessage(); msg.what = what; msg.arg1 = requestId; msg.obj = response; handler.sendMessage(msg); } } ; /* Create a new HandlerRunnable and start it */ final HandlerRunnable hr = new HandlerRunnable(method, url, headers, params, handler); requests.put(requestId, new WeakReference<Handler>(hr)); new Thread(hr).start(); /* Return with the request id */ return requestId; }
From source file:org.apache.cordova.filetransfer.FileTransfer.java
/** * Downloads a file form a given URL and saves it to the specified directory. * * @param source URL of the server to receive the file * @param target Full path of the file on the file system *//*from w w w . j av a 2 s.co m*/ private void download(final String source, final String target, JSONArray args, CallbackContext callbackContext) throws JSONException { Log.d(LOG_TAG, "download " + source + " to " + target); final CordovaResourceApi resourceApi = webView.getResourceApi(); final boolean trustEveryone = args.optBoolean(2); final String objectId = args.getString(3); final JSONObject headers = args.optJSONObject(4); final Uri sourceUri = resourceApi.remapUri(Uri.parse(source)); // Accept a path or a URI for the source. Uri tmpTarget = Uri.parse(target); final Uri targetUri = resourceApi .remapUri(tmpTarget.getScheme() != null ? tmpTarget : Uri.fromFile(new File(target))); int uriType = CordovaResourceApi.getUriType(sourceUri); final boolean useHttps = uriType == CordovaResourceApi.URI_TYPE_HTTPS; final boolean isLocalTransfer = !useHttps && uriType != CordovaResourceApi.URI_TYPE_HTTP; if (uriType == CordovaResourceApi.URI_TYPE_UNKNOWN) { JSONObject error = createFileTransferError(INVALID_URL_ERR, source, target, null, 0, null); Log.e(LOG_TAG, "Unsupported URI: " + sourceUri); callbackContext.sendPluginResult(new PluginResult(PluginResult.Status.IO_EXCEPTION, error)); return; } /* This code exists for compatibility between 3.x and 4.x versions of Cordova. * Previously the CordovaWebView class had a method, getWhitelist, which would * return a Whitelist object. Since the fixed whitelist is removed in Cordova 4.x, * the correct call now is to shouldAllowRequest from the plugin manager. */ Boolean shouldAllowRequest = null; if (isLocalTransfer) { shouldAllowRequest = true; } if (shouldAllowRequest == null) { try { Method gwl = webView.getClass().getMethod("getWhitelist"); Whitelist whitelist = (Whitelist) gwl.invoke(webView); shouldAllowRequest = whitelist.isUrlWhiteListed(source); } catch (NoSuchMethodException e) { } catch (IllegalAccessException e) { } catch (InvocationTargetException e) { } } if (shouldAllowRequest == null) { try { Method gpm = webView.getClass().getMethod("getPluginManager"); PluginManager pm = (PluginManager) gpm.invoke(webView); Method san = pm.getClass().getMethod("shouldAllowRequest", String.class); shouldAllowRequest = (Boolean) san.invoke(pm, source); } catch (NoSuchMethodException e) { } catch (IllegalAccessException e) { } catch (InvocationTargetException e) { } } if (!Boolean.TRUE.equals(shouldAllowRequest)) { Log.w(LOG_TAG, "Source URL is not in white list: '" + source + "'"); JSONObject error = createFileTransferError(CONNECTION_ERR, source, target, null, 401, null); callbackContext.sendPluginResult(new PluginResult(PluginResult.Status.IO_EXCEPTION, error)); return; } final RequestContext context = new RequestContext(source, target, callbackContext); synchronized (activeRequests) { activeRequests.put(objectId, context); } cordova.getThreadPool().execute(new Runnable() { public void run() { if (context.aborted) { return; } HttpURLConnection connection = null; HostnameVerifier oldHostnameVerifier = null; SSLSocketFactory oldSocketFactory = null; File file = null; PluginResult result = null; TrackingInputStream inputStream = null; boolean cached = false; OutputStream outputStream = null; try { OpenForReadResult readResult = null; file = resourceApi.mapUriToFile(targetUri); context.targetFile = file; Log.d(LOG_TAG, "Download file:" + sourceUri); FileProgressResult progress = new FileProgressResult(); if (isLocalTransfer) { readResult = resourceApi.openForRead(sourceUri); if (readResult.length != -1) { progress.setLengthComputable(true); progress.setTotal(readResult.length); } inputStream = new SimpleTrackingInputStream(readResult.inputStream); } else { // connect to server // Open a HTTP connection to the URL based on protocol connection = resourceApi.createHttpConnection(sourceUri); if (useHttps && trustEveryone) { // Setup the HTTPS connection class to trust everyone HttpsURLConnection https = (HttpsURLConnection) connection; oldSocketFactory = trustAllHosts(https); // Save the current hostnameVerifier oldHostnameVerifier = https.getHostnameVerifier(); // Setup the connection not to verify hostnames https.setHostnameVerifier(DO_NOT_VERIFY); } connection.setRequestMethod("GET"); // TODO: Make OkHttp use this CookieManager by default. String cookie = getCookies(sourceUri.toString()); if (cookie != null) { connection.setRequestProperty("cookie", cookie); } // This must be explicitly set for gzip progress tracking to work. connection.setRequestProperty("Accept-Encoding", "gzip"); // Handle the other headers if (headers != null) { addHeadersToRequest(connection, headers); } connection.connect(); if (connection.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { cached = true; connection.disconnect(); Log.d(LOG_TAG, "Resource not modified: " + source); JSONObject error = createFileTransferError(NOT_MODIFIED_ERR, source, target, connection, null); result = new PluginResult(PluginResult.Status.ERROR, error); } else { if (connection.getContentEncoding() == null || connection.getContentEncoding().equalsIgnoreCase("gzip")) { // Only trust content-length header if we understand // the encoding -- identity or gzip if (connection.getContentLength() != -1) { progress.setLengthComputable(true); progress.setTotal(connection.getContentLength()); } } inputStream = getInputStream(connection); } } if (!cached) { try { synchronized (context) { if (context.aborted) { return; } context.connection = connection; } // write bytes to file byte[] buffer = new byte[MAX_BUFFER_SIZE]; int bytesRead = 0; outputStream = resourceApi.openOutputStream(targetUri); while ((bytesRead = inputStream.read(buffer)) > 0) { outputStream.write(buffer, 0, bytesRead); // Send a progress event. progress.setLoaded(inputStream.getTotalRawBytesRead()); PluginResult progressResult = new PluginResult(PluginResult.Status.OK, progress.toJSONObject()); progressResult.setKeepCallback(true); context.sendPluginResult(progressResult); } } finally { synchronized (context) { context.connection = null; } safeClose(inputStream); safeClose(outputStream); } Log.d(LOG_TAG, "Saved file: " + target); // create FileEntry object Class webViewClass = webView.getClass(); PluginManager pm = null; try { Method gpm = webViewClass.getMethod("getPluginManager"); pm = (PluginManager) gpm.invoke(webView); } catch (NoSuchMethodException e) { } catch (IllegalAccessException e) { } catch (InvocationTargetException e) { } if (pm == null) { try { Field pmf = webViewClass.getField("pluginManager"); pm = (PluginManager) pmf.get(webView); } catch (NoSuchFieldException e) { } catch (IllegalAccessException e) { } } file = resourceApi.mapUriToFile(targetUri); context.targetFile = file; FileUtils filePlugin = (FileUtils) pm.getPlugin("File"); if (filePlugin != null) { JSONObject fileEntry = filePlugin.getEntryForFile(file); if (fileEntry != null) { result = new PluginResult(PluginResult.Status.OK, fileEntry); } else { JSONObject error = createFileTransferError(CONNECTION_ERR, source, target, connection, null); Log.e(LOG_TAG, "File plugin cannot represent download path"); result = new PluginResult(PluginResult.Status.IO_EXCEPTION, error); } } else { Log.e(LOG_TAG, "File plugin not found; cannot save downloaded file"); result = new PluginResult(PluginResult.Status.ERROR, "File plugin not found; cannot save downloaded file"); } } } catch (FileNotFoundException e) { JSONObject error = createFileTransferError(FILE_NOT_FOUND_ERR, source, target, connection, e); Log.e(LOG_TAG, error.toString(), e); result = new PluginResult(PluginResult.Status.IO_EXCEPTION, error); } catch (IOException e) { JSONObject error = createFileTransferError(CONNECTION_ERR, source, target, connection, e); Log.e(LOG_TAG, error.toString(), e); result = new PluginResult(PluginResult.Status.IO_EXCEPTION, error); } catch (JSONException e) { Log.e(LOG_TAG, e.getMessage(), e); result = new PluginResult(PluginResult.Status.JSON_EXCEPTION); } catch (Throwable e) { JSONObject error = createFileTransferError(CONNECTION_ERR, source, target, connection, e); Log.e(LOG_TAG, error.toString(), e); result = new PluginResult(PluginResult.Status.IO_EXCEPTION, error); } finally { synchronized (activeRequests) { activeRequests.remove(objectId); } if (connection != null) { // Revert back to the proper verifier and socket factories if (trustEveryone && useHttps) { HttpsURLConnection https = (HttpsURLConnection) connection; https.setHostnameVerifier(oldHostnameVerifier); https.setSSLSocketFactory(oldSocketFactory); } } if (result == null) { result = new PluginResult(PluginResult.Status.ERROR, createFileTransferError(CONNECTION_ERR, source, target, connection, null)); } // Remove incomplete download. if (!cached && result.getStatus() != PluginResult.Status.OK.ordinal() && file != null) { file.delete(); } context.sendPluginResult(result); } } }); }
From source file:cgeo.geocaching.cgBase.java
public static void postTweet(cgeoapplication app, cgSettings settings, String status, final Geopoint coords) { if (app == null) { return;/*ww w . ja v a 2 s. c o m*/ } if (settings == null || StringUtils.isBlank(settings.tokenPublic) || StringUtils.isBlank(settings.tokenSecret)) { return; } try { Map<String, String> parameters = new HashMap<String, String>(); parameters.put("status", status); if (coords != null) { parameters.put("lat", String.format("%.6f", coords.getLatitude())); parameters.put("long", String.format("%.6f", coords.getLongitude())); parameters.put("display_coordinates", "true"); } final String paramsDone = cgOAuth.signOAuth("api.twitter.com", "/1/statuses/update.json", "POST", false, parameters, settings.tokenPublic, settings.tokenSecret); HttpURLConnection connection = null; try { final StringBuffer buffer = new StringBuffer(); final URL u = new URL("http://api.twitter.com/1/statuses/update.json"); final URLConnection uc = u.openConnection(); uc.setRequestProperty("Host", "api.twitter.com"); connection = (HttpURLConnection) uc; connection.setReadTimeout(30000); connection.setRequestMethod("POST"); HttpURLConnection.setFollowRedirects(true); connection.setDoInput(true); connection.setDoOutput(true); final OutputStream out = connection.getOutputStream(); final OutputStreamWriter wr = new OutputStreamWriter(out); wr.write(paramsDone); wr.flush(); wr.close(); Log.i(cgSettings.tag, "Twitter.com: " + connection.getResponseCode() + " " + connection.getResponseMessage()); InputStream ins; final String encoding = connection.getContentEncoding(); if (encoding != null && encoding.equalsIgnoreCase("gzip")) { ins = new GZIPInputStream(connection.getInputStream()); } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) { ins = new InflaterInputStream(connection.getInputStream(), new Inflater(true)); } else { ins = connection.getInputStream(); } final InputStreamReader inr = new InputStreamReader(ins); final BufferedReader br = new BufferedReader(inr); readIntoBuffer(br, buffer); br.close(); ins.close(); inr.close(); connection.disconnect(); } catch (IOException e) { Log.e(cgSettings.tag, "cgBase.postTweet.IO: " + connection.getResponseCode() + ": " + connection.getResponseMessage() + " ~ " + e.toString()); final InputStream ins = connection.getErrorStream(); final StringBuffer buffer = new StringBuffer(); final InputStreamReader inr = new InputStreamReader(ins); final BufferedReader br = new BufferedReader(inr); readIntoBuffer(br, buffer); br.close(); ins.close(); inr.close(); } catch (Exception e) { Log.e(cgSettings.tag, "cgBase.postTweet.inner: " + e.toString()); } connection.disconnect(); } catch (Exception e) { Log.e(cgSettings.tag, "cgBase.postTweet: " + e.toString()); } }
From source file:com.google.code.linkedinapi.client.impl.BaseLinkedInApiClient.java
/** * @param apiUrl/*from ww w . j a v a 2s . c o m*/ * @param expected * @param httpHeaders * @return */ protected InputStream callApiMethod(String apiUrl, int expected, List<HttpHeader> httpHeaders) { try { LinkedInOAuthService oAuthService = LinkedInOAuthServiceFactory.getInstance() .createLinkedInOAuthService(apiConsumer.getConsumerKey(), apiConsumer.getConsumerSecret()); if (accessToken.getOauth2Token() != null && !accessToken.getOauth2Token().isEmpty()) { // oauth2 token hack, this seemed like the easiest... place to do it if (apiUrl.contains("?")) { apiUrl = apiUrl + "&oauth2_access_token=" + accessToken.getOauth2Token(); } else { apiUrl = apiUrl + "?oauth2_access_token=" + accessToken.getOauth2Token(); } } if (OUTPUT_RESPONSE) { LOG.info("Calling LinkedIn URL: " + apiUrl); } URL url = new URL(apiUrl); HttpURLConnection request = (HttpURLConnection) url.openConnection(); if (ApplicationConstants.CONNECT_TIMEOUT > -1) { request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT); } if (ApplicationConstants.READ_TIMEOUT > -1) { request.setReadTimeout(ApplicationConstants.READ_TIMEOUT); } for (String headerName : requestHeaders.keySet()) { request.setRequestProperty(headerName, requestHeaders.get(headerName)); } for (HttpHeader header : httpHeaders) { request.setRequestProperty(header.getName(), header.getValue()); } oAuthService.signRequestWithToken(request, accessToken); request.connect(); if (request.getResponseCode() != expected) { Error error = readResponse(Error.class, getWrappedInputStream(request.getErrorStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))); throw createLinkedInApiClientException(error); } else { return getWrappedInputStream(request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding())); } } catch (IOException e) { throw new LinkedInApiClientException(e); } }
From source file:Fetcher.Fetcher.java
@Deprecated @Override//from w w w.java2s.c o m /** * run() is deprecated. Use startFetching() instead. */ public void run() { WebDocument link = null; HttpURLConnection connection; Proxy p; //PreConnfiguration //Configure proxy //TODO Anonymizer is deprecated. Use in following for warning generation. switch (Variables.anonymizerProxyType) { case DIRECT: p = new Proxy(Proxy.Type.DIRECT, new InetSocketAddress(Variables.anonymizerIP, Variables.anonymizerPort)); break; case HTTP: p = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(Variables.anonymizerIP, Variables.anonymizerPort)); break; case SOCKS: p = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress(Variables.anonymizerIP, Variables.anonymizerPort)); break; case NONE: default: p = null; break; } link = Methods.getNextProfileLink(); while (link != null && isWorking) { //Start fetching ... //Check if it should work or not Date currentTime = Methods.getCurrentTime(); if (!currentTime.after(Variables.startTime) || !currentTime.before(Variables.endTime)) { try { synchronized (t) { getThread().wait(60000); //sleep 60 seconds } } catch (InterruptedException ex) { if (Variables.debug) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Time is not between start and end time and thread is in exception!"); } } finally { continue; } } String URL = link.getNextUrl(); String UA = Methods.getRandomUserAgent(); //Use this UA for refererd or single links. //loop for referer for (int i = 0; i <= link.getRefererCount(); URL = link.getNextUrl(), i++) { if (Variables.debug && Variables.vv) { Variables.logger.Log(Fetcher.class, Variables.LogType.Trace, "Fetcher (" + Methods.Colorize(name, Methods.Color.Green) + ") start getting " + URL); } try { //Anonymizer if (Variables.anonymizerProxyType == Variables.AnonymizerProxy.NONE) { connection = (HttpURLConnection) new URL(URL).openConnection(); } else { connection = (HttpURLConnection) new URL(URL).openConnection(p); } connection.setDoOutput(true); connection.setDoInput(true); connection.setRequestProperty("User-Agent", UA); connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); connection.setRequestProperty("Accept-Language", "en-US,en;q=0.5"); connection.setRequestProperty("Accept-Encoding", "gzip, deflated"); String referer = link.getNextReferrer(); if (referer != null) { connection.setRequestProperty("Referer", referer); referer = null; System.gc(); } //Send Cookie using user input if (!(Variables.Cookie == null || Variables.Cookie.equalsIgnoreCase(""))) { connection.setRequestProperty("Cookie", Variables.Cookie); } else if (cookies.getCookieStore().getCookies().size() > 0) { //From referer, there are some cookies connection.setRequestProperty("Cookie", Join(",", cookies.getCookieStore().getCookies())); } connection.setRequestMethod("GET"); connection.connect(); //Get Cookie from response getCookies(connection); if (connection.getResponseCode() == 200) { //Write to file String outputName = Variables.outputDirectory + link.getOutputName().substring(0, link.getOutputName().lastIndexOf(".")) + i + link.getOutputName().substring(link.getOutputName().lastIndexOf(".")); //Check extension if (!(outputName.endsWith("html") || outputName.endsWith("htm"))) { outputName += "html"; } //get content String html = ""; if (connection.getContentEncoding().equalsIgnoreCase("gzip")) { html = IOUtils.toString(new GZIPInputStream(connection.getInputStream())); } else if (connection.getContentEncoding().equalsIgnoreCase("deflate")) { html = IOUtils.toString(new InflaterInputStream(connection.getInputStream())); } FileWriter fw = new FileWriter(outputName); fw.write(html); fw.flush(); fw.close(); } else { //The returned code is not 200. if (Variables.debug) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Fetcher could not download (" + Methods.Colorize(URL, Methods.Color.Red) + ") in " + name); if (Variables.vv) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Server responded (" + Methods.Colorize(connection.getResponseCode() + " - " + connection.getResponseMessage(), Methods.Color.Red) + ") for " + URL); } } } //Close the connection connection.disconnect(); //Report progress Variables.logger.logResult(connection, link); Methods.oneFinished(); if (Variables.debug && Variables.vv) { Variables.logger.Log(Fetcher.class, Variables.LogType.Info, "[+] Done fetching (" + Methods.Colorize(URL, Methods.Color.Red) + "]"); } try { synchronized (t) { t.wait(Methods.getNextRandom() * 1000); } } catch (InterruptedException ex) { if (Variables.debug) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Cannot interrupt thread [" + Methods.Colorize(name, Methods.Color.Red) + "]. Interrupted before!"); } } catch (IllegalArgumentException ex) { if (Variables.debug) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "-1 is returned as random number for thread [" + Methods.Colorize(name, Methods.Color.Red) + "]."); } } } catch (IOException ex) { if (Variables.debug) { if (Variables.vv) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Error in fetching [" + Methods.Colorize(URL, Methods.Color.Red) + "] in fetcher (" + Methods.Colorize(name, Methods.Color.Yellow) + ") for writing in (" + Methods.Colorize(link.getOutputName(), Methods.Color.White) + "). Detail:\r\n" + ex.getMessage()); } else { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Error in fetching [" + Methods.Colorize(URL, Methods.Color.Red) + "]"); } } } catch (NullPointerException ex) { //Thrown sometimes and make the thread as Dead! if (Variables.debug) { if (Variables.vv) { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Null pointer occured. Error in fetching [" + Methods.Colorize(URL, Methods.Color.Red) + "] in fetcher (" + Methods.Colorize(name, Methods.Color.Yellow) + ") for writing in (" + Methods.Colorize(link.getOutputName(), Methods.Color.White) + ")."); } else { Variables.logger.Log(Fetcher.class, Variables.LogType.Error, "Null pointer occured. Error in fetching [" + Methods.Colorize(URL, Methods.Color.Red) + "]"); } } } } //Check size limit and compress ... long size = Methods.getFolderSize(Variables.outputDirectory); if (size >= Variables.outputSizeLimit) { //Deactivate itself by waiting ... Variables.state = Variables.microbotState.Compressing; Variables.threadController.changeActiveThreads(false, t, Variables.microbotState.Compressing); } //Check if user terminated program or not if (isWorking) { link = Methods.getNextProfileLink(); } } //Thread finished. (Normally or by force) Variables.state = Variables.microbotState.Stopping; Variables.threadController.changeActiveThreads(false, t, Variables.microbotState.Stopping); //URLs done. This thread finishes its work. if (Variables.debug) { Variables.logger.Log(Fetcher.class, Variables.LogType.Info, "Fetcher (" + Methods.Colorize(name, Methods.Color.Green) + ") finished its work."); } }
From source file:com.google.code.linkedinapi.client.impl.BaseLinkedInApiClient.java
/** * @param apiUrl//from w w w. j av a2 s . c o m * @param xmlContent * @param contentType * @param method * @param expected * @return */ protected void callApiMethod(String apiUrl, String xmlContent, String contentType, HttpMethod method, int expected) { try { LinkedInOAuthService oAuthService = LinkedInOAuthServiceFactory.getInstance() .createLinkedInOAuthService(apiConsumer.getConsumerKey(), apiConsumer.getConsumerSecret()); if (accessToken.getOauth2Token() != null && !accessToken.getOauth2Token().isEmpty()) { // oauth2 token hack, this seemed like the easiest... place to do it if (apiUrl.contains("?")) { apiUrl = apiUrl + "&oauth2_access_token=" + accessToken.getOauth2Token(); } else { apiUrl = apiUrl + "?oauth2_access_token=" + accessToken.getOauth2Token(); } } if (OUTPUT_RESPONSE) { LOG.info("Calling LinkedIn URL: " + apiUrl); LOG.info("XML Content: " + xmlContent); } URL url = new URL(apiUrl); HttpURLConnection request = (HttpURLConnection) url.openConnection(); if (ApplicationConstants.CONNECT_TIMEOUT > -1) { request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT); } if (ApplicationConstants.READ_TIMEOUT > -1) { request.setReadTimeout(ApplicationConstants.READ_TIMEOUT); } for (String headerName : requestHeaders.keySet()) { request.setRequestProperty(headerName, requestHeaders.get(headerName)); } request.setRequestMethod(method.fieldName()); request.setDoOutput(true); oAuthService.signRequestWithToken(request, accessToken); if (contentType != null) { request.setRequestProperty("Content-Type", contentType); } if (xmlContent != null) { PrintWriter out = new PrintWriter( new OutputStreamWriter(request.getOutputStream(), UTF_8_CHAR_SET)); out.print(xmlContent); out.flush(); out.close(); } request.connect(); if (request.getResponseCode() != expected) { Error error = readResponse(Error.class, getWrappedInputStream(request.getErrorStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))); throw createLinkedInApiClientException(error); } else { // return getWrappedInputStream(request.getInputStream(), // GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding())); InputStream inputStream = getWrappedInputStream(request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding())); String response = convertStreamToString(inputStream); if (OUTPUT_RESPONSE) { LOG.info("Response: " + response); } } } catch (IOException e) { throw new LinkedInApiClientException(e); } }
From source file:com.diablominer.DiabloMiner.NetworkState.JSONRPCNetworkState.java
JsonNode doJSONRPCCall(boolean longPoll, ObjectNode message) throws IOException { HttpURLConnection connection = null; try {//w w w .j av a2 s .c o m URL url; if (longPoll) url = longPollUrl; else url = queryUrl; Proxy proxy = diabloMiner.getProxy(); if (proxy == null) connection = (HttpURLConnection) url.openConnection(); else connection = (HttpURLConnection) url.openConnection(proxy); if (longPoll) { connection.setConnectTimeout(10 * 60 * 1000); connection.setReadTimeout(10 * 60 * 1000); } else { connection.setConnectTimeout(15 * 1000); connection.setReadTimeout(15 * 1000); } connection.setRequestProperty("Authorization", userPass); connection.setRequestProperty("Accept", "application/json"); connection.setRequestProperty("Accept-Encoding", "gzip,deflate"); connection.setRequestProperty("Content-Type", "application/json"); connection.setRequestProperty("Cache-Control", "no-cache"); connection.setRequestProperty("User-Agent", "DiabloMiner"); connection.setRequestProperty("X-Mining-Extensions", "longpoll rollntime switchto"); connection.setDoOutput(true); OutputStream requestStream = connection.getOutputStream(); Writer request = new OutputStreamWriter(requestStream); request.write(message.toString()); request.close(); requestStream.close(); ObjectNode responseMessage = null; InputStream responseStream = null; try { String xLongPolling = connection.getHeaderField("X-Long-Polling"); if (xLongPolling != null && !"".equals(xLongPolling) && longPollAsync == null) { if (xLongPolling.startsWith("http")) longPollUrl = new URL(xLongPolling); else if (xLongPolling.startsWith("/")) longPollUrl = new URL(queryUrl.getProtocol(), queryUrl.getHost(), queryUrl.getPort(), xLongPolling); else longPollUrl = new URL(queryUrl.getProtocol(), queryUrl.getHost(), queryUrl.getPort(), (url.getFile() + "/" + xLongPolling).replace("//", "/")); longPollAsync = new LongPollAsync(); Thread thread = new Thread(longPollAsync, "DiabloMiner JSONRPC LongPollAsync for " + url.getHost()); thread.start(); diabloMiner.addThread(thread); workLifetime = 60000; diabloMiner.debug(queryUrl.getHost() + ": Enabling long poll support"); } String xRollNTime = connection.getHeaderField("X-Roll-NTime"); if (xRollNTime != null && !"".equals(xRollNTime)) { if (!"n".equalsIgnoreCase(xRollNTime) && rollNTime == false) { rollNTime = true; if (xRollNTime.startsWith("expire=")) { try { workLifetime = Integer.parseInt(xRollNTime.substring(7)) * 1000; } catch (NumberFormatException ex) { } } else { workLifetime = 60000; } diabloMiner.debug(queryUrl.getHost() + ": Enabling roll ntime support, expire after " + (workLifetime / 1000) + " seconds"); } else if ("n".equalsIgnoreCase(xRollNTime) && rollNTime == true) { rollNTime = false; if (longPoll) workLifetime = 60000; else workLifetime = diabloMiner.getWorkLifetime(); diabloMiner.debug(queryUrl.getHost() + ": Disabling roll ntime support"); } } String xSwitchTo = connection.getHeaderField("X-Switch-To"); if (xSwitchTo != null && !"".equals(xSwitchTo)) { String oldHost = queryUrl.getHost(); JsonNode newHost = mapper.readTree(xSwitchTo); queryUrl = new URL(queryUrl.getProtocol(), newHost.get("host").asText(), newHost.get("port").getIntValue(), queryUrl.getPath()); if (longPollUrl != null) longPollUrl = new URL(longPollUrl.getProtocol(), newHost.get("host").asText(), newHost.get("port").getIntValue(), longPollUrl.getPath()); diabloMiner.info(oldHost + ": Switched to " + queryUrl.getHost()); } String xRejectReason = connection.getHeaderField("X-Reject-Reason"); if (xRejectReason != null && !"".equals(xRejectReason)) { rejectReason = xRejectReason; } String xIsP2Pool = connection.getHeaderField("X-Is-P2Pool"); if (xIsP2Pool != null && !"".equals(xIsP2Pool)) { if (!noDelay) diabloMiner.info("P2Pool no delay mode enabled"); noDelay = true; } if (connection.getContentEncoding() != null) { if (connection.getContentEncoding().equalsIgnoreCase("gzip")) responseStream = new GZIPInputStream(connection.getInputStream()); else if (connection.getContentEncoding().equalsIgnoreCase("deflate")) responseStream = new InflaterInputStream(connection.getInputStream()); } else { responseStream = connection.getInputStream(); } if (responseStream == null) throw new IOException("Drop to error handler"); Object output = mapper.readTree(responseStream); if (NullNode.class.equals(output.getClass())) { throw new IOException("Bitcoin returned unparsable JSON"); } else { try { responseMessage = (ObjectNode) output; } catch (ClassCastException e) { throw new IOException("Bitcoin returned unparsable JSON"); } } responseStream.close(); } catch (JsonProcessingException e) { throw new IOException("Bitcoin returned unparsable JSON"); } catch (IOException e) { InputStream errorStream = null; IOException e2 = null; if (connection.getErrorStream() == null) throw new IOException("Bitcoin disconnected during response: " + connection.getResponseCode() + " " + connection.getResponseMessage()); if (connection.getContentEncoding() != null) { if (connection.getContentEncoding().equalsIgnoreCase("gzip")) errorStream = new GZIPInputStream(connection.getErrorStream()); else if (connection.getContentEncoding().equalsIgnoreCase("deflate")) errorStream = new InflaterInputStream(connection.getErrorStream()); } else { errorStream = connection.getErrorStream(); } if (errorStream == null) throw new IOException("Bitcoin disconnected during response: " + connection.getResponseCode() + " " + connection.getResponseMessage()); byte[] errorbuf = new byte[8192]; if (errorStream.read(errorbuf) < 1) throw new IOException("Bitcoin returned an error, but with no message"); String error = new String(errorbuf).trim(); if (error.startsWith("{")) { try { Object output = mapper.readTree(error); if (NullNode.class.equals(output.getClass())) throw new IOException("Bitcoin returned an error message: " + error); else try { responseMessage = (ObjectNode) output; } catch (ClassCastException f) { throw new IOException("Bitcoin returned unparsable JSON"); } if (responseMessage.get("error") != null) { if (responseMessage.get("error").get("message") != null && responseMessage.get("error").get("message").asText() != null) { error = responseMessage.get("error").get("message").asText().trim(); e2 = new IOException("Bitcoin returned error message: " + error); } else if (responseMessage.get("error").asText() != null) { error = responseMessage.get("error").asText().trim(); if (!"null".equals(error) && !"".equals(error)) e2 = new IOException("Bitcoin returned an error message: " + error); } } } catch (JsonProcessingException f) { e2 = new IOException("Bitcoin returned unparsable JSON"); } } else { e2 = new IOException("Bitcoin returned an error message: " + error); } errorStream.close(); if (responseStream != null) responseStream.close(); if (e2 == null) e2 = new IOException("Bitcoin returned an error, but with no message"); throw e2; } if (responseMessage.get("error") != null) { if (responseMessage.get("error").get("message") != null && responseMessage.get("error").get("message").asText() != null) { String error = responseMessage.get("error").get("message").asText().trim(); throw new IOException("Bitcoin returned error message: " + error); } else if (responseMessage.get("error").asText() != null) { String error = responseMessage.get("error").asText().trim(); if (!"null".equals(error) && !"".equals(error)) throw new IOException("Bitcoin returned error message: " + error); } } JsonNode result; try { result = responseMessage.get("result"); } catch (Exception e) { throw new IOException("Bitcoin returned unparsable JSON"); } if (result == null) throw new IOException("Bitcoin did not return a result or an error"); return result; } catch (IOException e) { if (connection != null) connection.disconnect(); throw e; } }
From source file:carnero.cgeo.cgBase.java
public void postTweet(cgeoapplication app, cgSettings settings, String status, Double latitude, Double longitude) {//from w ww . j a v a2 s. c om if (app == null) { return; } if (settings == null || settings.tokenPublic == null || settings.tokenPublic.length() == 0 || settings.tokenSecret == null || settings.tokenSecret.length() == 0) { return; } try { HashMap<String, String> parameters = new HashMap<String, String>(); parameters.put("status", status); if (latitude != null && longitude != null) { parameters.put("lat", String.format("%.6f", latitude)); parameters.put("long", String.format("%.6f", longitude)); parameters.put("display_coordinates", "true"); } final String paramsDone = cgOAuth.signOAuth("api.twitter.com", "/1/statuses/update.json", "POST", false, parameters, settings.tokenPublic, settings.tokenSecret); HttpURLConnection connection = null; try { final StringBuffer buffer = new StringBuffer(); final URL u = new URL("http://api.twitter.com/1/statuses/update.json"); final URLConnection uc = u.openConnection(); uc.setRequestProperty("Host", "api.twitter.com"); connection = (HttpURLConnection) uc; connection.setReadTimeout(30000); connection.setRequestMethod("POST"); HttpURLConnection.setFollowRedirects(true); connection.setDoInput(true); connection.setDoOutput(true); final OutputStream out = connection.getOutputStream(); final OutputStreamWriter wr = new OutputStreamWriter(out); wr.write(paramsDone); wr.flush(); wr.close(); Log.i(cgSettings.tag, "Twitter.com: " + connection.getResponseCode() + " " + connection.getResponseMessage()); InputStream ins; final String encoding = connection.getContentEncoding(); if (encoding != null && encoding.equalsIgnoreCase("gzip")) { ins = new GZIPInputStream(connection.getInputStream()); } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) { ins = new InflaterInputStream(connection.getInputStream(), new Inflater(true)); } else { ins = connection.getInputStream(); } final InputStreamReader inr = new InputStreamReader(ins); final BufferedReader br = new BufferedReader(inr); readIntoBuffer(br, buffer); br.close(); ins.close(); inr.close(); connection.disconnect(); } catch (IOException e) { Log.e(cgSettings.tag, "cgBase.postTweet.IO: " + connection.getResponseCode() + ": " + connection.getResponseMessage() + " ~ " + e.toString()); final InputStream ins = connection.getErrorStream(); final StringBuffer buffer = new StringBuffer(); final InputStreamReader inr = new InputStreamReader(ins); final BufferedReader br = new BufferedReader(inr); readIntoBuffer(br, buffer); br.close(); ins.close(); inr.close(); } catch (Exception e) { Log.e(cgSettings.tag, "cgBase.postTweet.inner: " + e.toString()); } connection.disconnect(); } catch (Exception e) { Log.e(cgSettings.tag, "cgBase.postTweet: " + e.toString()); } }
From source file:carnero.cgeo.original.libs.Base.java
public String requestJSON(String scheme, String host, String path, String method, String params) { int httpCode = -1; String httpLocation = null;//from w ww. j a v a 2 s.c om if (method == null) { method = "GET"; } else { method = method.toUpperCase(); } boolean methodPost = false; if (method.equalsIgnoreCase("POST")) { methodPost = true; } URLConnection uc = null; HttpURLConnection connection = null; Integer timeout = 30000; final StringBuffer buffer = new StringBuffer(); for (int i = 0; i < 3; i++) { if (i > 0) { Log.w(Settings.tag, "Failed to download data, retrying. Attempt #" + (i + 1)); } buffer.delete(0, buffer.length()); timeout = 30000 + (i * 15000); try { try { URL u = null; if (methodPost) { u = new URL(scheme + host + path); } else { u = new URL(scheme + host + path + "?" + params); } if (u.getProtocol().toLowerCase().equals("https")) { trustAllHosts(); HttpsURLConnection https = (HttpsURLConnection) u.openConnection(); https.setHostnameVerifier(doNotVerify); uc = https; } else { uc = (HttpURLConnection) u.openConnection(); } uc.setRequestProperty("Host", host); uc.setRequestProperty("Accept", "application/json, text/javascript, */*; q=0.01"); if (methodPost) { uc.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); uc.setRequestProperty("Content-Length", Integer.toString(params.length())); uc.setRequestProperty("X-HTTP-Method-Override", "GET"); } else { uc.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); } uc.setRequestProperty("X-Requested-With", "XMLHttpRequest"); connection = (HttpURLConnection) uc; connection.setReadTimeout(timeout); connection.setRequestMethod(method); HttpURLConnection.setFollowRedirects(false); // TODO: Fix these (FilCab) connection.setDoInput(true); if (methodPost) { connection.setDoOutput(true); final OutputStream out = connection.getOutputStream(); final OutputStreamWriter wr = new OutputStreamWriter(out); wr.write(params); wr.flush(); wr.close(); } else { connection.setDoOutput(false); } final String encoding = connection.getContentEncoding(); InputStream ins; if (encoding != null && encoding.equalsIgnoreCase("gzip")) { ins = new GZIPInputStream(connection.getInputStream()); } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) { ins = new InflaterInputStream(connection.getInputStream(), new Inflater(true)); } else { ins = connection.getInputStream(); } final InputStreamReader inr = new InputStreamReader(ins); final BufferedReader br = new BufferedReader(inr); readIntoBuffer(br, buffer); httpCode = connection.getResponseCode(); final String paramsLog = params.replaceAll(passMatch, "password=***"); Log.i(Settings.tag + " | JSON", "[POST " + (int) (params.length() / 1024) + "k | " + httpCode + " | " + (int) (buffer.length() / 1024) + "k] Downloaded " + "http://" + host + path + "?" + paramsLog); connection.disconnect(); br.close(); ins.close(); inr.close(); } catch (IOException e) { httpCode = connection.getResponseCode(); Log.e(Settings.tag, "cgeoBase.requestJSON.IOException: " + httpCode + ": " + connection.getResponseMessage() + " ~ " + e.toString()); } } catch (Exception e) { Log.e(Settings.tag, "cgeoBase.requestJSON: " + e.toString()); } if (buffer != null && buffer.length() > 0) { break; } if (httpCode == 403) { // we're not allowed to download content, so let's move break; } } String page = null; if (httpCode == 302 && httpLocation != null) { final Uri newLocation = Uri.parse(httpLocation); if (newLocation.isRelative() == true) { page = requestJSONgc(host, path, params); } else { page = requestJSONgc(newLocation.getHost(), newLocation.getPath(), params); } } else { page = replaceWhitespace(buffer); } if (page != null) { return page; } else { return ""; } }