List of usage examples for java.net HttpURLConnection getHeaderField
public String getHeaderField(int n)
From source file:org.apache.openaz.xacml.rest.XACMLPdpRegisterThread.java
/** * This is our thread that runs on startup to tell the PAP server we are up-and-running. *//*from ww w. java 2s.c om*/ @Override public void run() { synchronized (this) { this.isRunning = true; } boolean registered = false; boolean interrupted = false; int seconds; try { seconds = Integer.parseInt(XACMLProperties.getProperty(XACMLRestProperties.PROP_PDP_REGISTER_SLEEP)); } catch (NumberFormatException e) { logger.error("REGISTER_SLEEP: ", e); seconds = 5; } if (seconds < 5) { seconds = 5; } int retries; try { retries = Integer.parseInt(XACMLProperties.getProperty(XACMLRestProperties.PROP_PDP_REGISTER_RETRIES)); } catch (NumberFormatException e) { logger.error("REGISTER_SLEEP: ", e); retries = -1; } while (!registered && !interrupted && this.isRunning()) { HttpURLConnection connection = null; try { // // Get the PAP Servlet URL // URL url = new URL(XACMLProperties.getProperty(XACMLRestProperties.PROP_PAP_URL)); logger.info("Registering with " + url.toString()); boolean finished = false; while (!finished) { // // Open up the connection // connection = (HttpURLConnection) url.openConnection(); // // Setup our method and headers // connection.setRequestMethod("POST"); connection.setRequestProperty("Accept", "text/x-java-properties"); connection.setRequestProperty("Content-Type", "text/x-java-properties"); connection.setRequestProperty(XACMLRestProperties.PROP_PDP_HTTP_HEADER_ID, XACMLProperties.getProperty(XACMLRestProperties.PROP_PDP_ID)); connection.setUseCaches(false); // // Adding this in. It seems the HttpUrlConnection class does NOT // properly forward our headers for POST re-direction. It does so // for a GET re-direction. // // So we need to handle this ourselves. // connection.setInstanceFollowRedirects(false); connection.setDoOutput(true); connection.setDoInput(true); try { // // Send our current policy configuration // String lists = XACMLProperties.PROP_ROOTPOLICIES + "=" + XACMLProperties.getProperty(XACMLProperties.PROP_ROOTPOLICIES); lists = lists + "\n" + XACMLProperties.PROP_REFERENCEDPOLICIES + "=" + XACMLProperties.getProperty(XACMLProperties.PROP_REFERENCEDPOLICIES) + "\n"; try (InputStream listsInputStream = new ByteArrayInputStream(lists.getBytes()); InputStream pipInputStream = Files.newInputStream(XACMLPdpLoader.getPIPConfig()); OutputStream os = connection.getOutputStream()) { IOUtils.copy(listsInputStream, os); // // Send our current PIP configuration // IOUtils.copy(pipInputStream, os); } } catch (Exception e) { logger.error("Failed to send property file", e); } // // Do the connect // connection.connect(); if (connection.getResponseCode() == 204) { logger.info("Success. We are configured correctly."); finished = true; registered = true; } else if (connection.getResponseCode() == 200) { logger.info("Success. We have a new configuration."); Properties properties = new Properties(); properties.load(connection.getInputStream()); logger.info("New properties: " + properties.toString()); // // Queue it // // The incoming properties does NOT include urls PutRequest req = new PutRequest(XACMLProperties.getPolicyProperties(properties, false), XACMLProperties.getPipProperties(properties)); XACMLPdpServlet.queue.offer(req); // // We are now registered // finished = true; registered = true; } else if (connection.getResponseCode() >= 300 && connection.getResponseCode() <= 399) { // // Re-direction // String newLocation = connection.getHeaderField("Location"); if (newLocation == null || newLocation.isEmpty()) { logger.warn("Did not receive a valid re-direction location"); finished = true; } else { logger.info("New Location: " + newLocation); url = new URL(newLocation); } } else { logger.warn("Failed: " + connection.getResponseCode() + " message: " + connection.getResponseMessage()); finished = true; } } } catch (Exception e) { logger.error(e); } finally { // cleanup the connection if (connection != null) { try { // For some reason trying to get the inputStream from the connection // throws an exception rather than returning null when the InputStream does not exist. InputStream is = null; try { is = connection.getInputStream(); } catch (Exception e1) { //NOPMD // ignore this } if (is != null) { is.close(); } } catch (IOException ex) { logger.error("Failed to close connection: " + ex, ex); } connection.disconnect(); } } // // Wait a little while to try again // try { if (!registered) { if (retries > 0) { retries--; } else if (retries == 0) { break; } Thread.sleep(seconds * 1000); } } catch (InterruptedException e) { interrupted = true; this.terminate(); } } synchronized (this) { this.isRunning = false; } logger.info("Thread exiting...(registered=" + registered + ", interrupted=" + interrupted + ", isRunning=" + this.isRunning() + ", retries=" + retries + ")"); }
From source file:org.witness.ssc.xfer.utils.PublishingUtils.java
private String gdataUpload(File file, String uploadUrl, int start, int end, Activity activity) throws IOException { mActivity = activity;// w w w .j a v a2 s .co m int chunk = end - start + 1; int bufferSize = 1024; byte[] buffer = new byte[bufferSize]; FileInputStream fileStream = new FileInputStream(file); HttpURLConnection urlConnection = getGDataUrlConnection(uploadUrl); // some mobile proxies do not support PUT, using X-HTTP-Method-Override // to get around this problem if (isFirstRequest()) { Log.d(TAG, String.format("Uploaded %d bytes so far, using POST method.", (int) totalBytesUploaded)); urlConnection.setRequestMethod("POST"); } else { urlConnection.setRequestMethod("POST"); urlConnection.setRequestProperty("X-HTTP-Method-Override", "PUT"); Log.d(TAG, String.format("Uploaded %d bytes so far, using POST with X-HTTP-Method-Override PUT method.", (int) totalBytesUploaded)); } urlConnection.setDoOutput(true); urlConnection.setFixedLengthStreamingMode(chunk); // /XXX hardcoded video mimetype urlConnection.setRequestProperty("Content-Type", "video/mp4"); urlConnection.setRequestProperty("Content-Range", String.format("bytes %d-%d/%d", start, end, file.length())); Log.d(TAG, urlConnection.getRequestProperty("Content-Range")); OutputStream outStreamWriter = urlConnection.getOutputStream(); fileStream.skip(start); int bytesRead; int totalRead = 0; Thread thread = new Thread() { public void run() { int lastPercent = 0; while (lastPercent < 100) { if (lastPercent != percentUploaded) { ((SSCXferActivity) mActivity).showProgress("uploading...", percentUploaded); lastPercent = percentUploaded; } try { Thread.sleep(1000); } catch (Exception e) { } } } }; thread.start(); while ((bytesRead = fileStream.read(buffer, 0, bufferSize)) != -1) { outStreamWriter.write(buffer, 0, bytesRead); totalRead += bytesRead; this.totalBytesUploaded += bytesRead; percentUploaded = (int) (((float) totalBytesUploaded) / ((float) currentFileSize) * 99f); if (totalRead == (end - start + 1)) { break; } } outStreamWriter.close(); int responseCode = urlConnection.getResponseCode(); Log.d(TAG, "responseCode=" + responseCode); Log.d(TAG, "responseMessage=" + urlConnection.getResponseMessage()); try { if (responseCode == 201) { String videoId = parseVideoId(urlConnection.getInputStream()); Log.i(TAG, "Youtube video submitted - new video id is " + videoId); // 100% finished here. // dialog.setProgress(100); return videoId; } else if (responseCode == 200) { Set<String> keySet = urlConnection.getHeaderFields().keySet(); String keys = urlConnection.getHeaderFields().keySet().toString(); Log.d(TAG, String.format("Headers keys %s.", keys)); for (String key : keySet) { Log.d(TAG, String.format("Header key %s value %s.", key, urlConnection.getHeaderField(key))); } Log.w(TAG, "Received 200 response during resumable uploading"); throw new IOException(String.format("Unexpected response code : responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage())); } else { if ((responseCode + "").startsWith("5")) { String error = String.format("responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage()); Log.w(TAG, error); // TODO - this exception will trigger retry mechanism to // kick in // TODO - even though it should not, consider introducing a // new type so // TODO - resume does not kick in upon 5xx throw new IOException(error); } else if (responseCode == 308) { // OK, the chunk completed successfully Log.d(TAG, String.format("responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage())); } else { // TODO - this case is not handled properly yet Log.w(TAG, String.format("Unexpected return code : %d %s while uploading :%s", responseCode, urlConnection.getResponseMessage(), uploadUrl)); } } } catch (ParserConfigurationException e) { e.printStackTrace(); } catch (SAXException e) { e.printStackTrace(); } return null; }
From source file:com.BeatYourRecord.SubmitActivity.java
private String gdataUpload(File file, String uploadUrl, int start, int end) throws IOException { int chunk = end - start + 1; int bufferSize = 1024; byte[] buffer = new byte[bufferSize]; FileInputStream fileStream = new FileInputStream(file); HttpURLConnection urlConnection = getGDataUrlConnection(uploadUrl); // some mobile proxies do not support PUT, using X-HTTP-Method-Override to get around this problem if (isFirstRequest()) { Log.d(LOG_TAG, String.format("Uploaded %d bytes so far, using POST method.", (int) totalBytesUploaded)); urlConnection.setRequestMethod("POST"); } else {// w ww . j av a2 s. c o m urlConnection.setRequestMethod("POST"); urlConnection.setRequestProperty("X-HTTP-Method-Override", "PUT"); Log.d(LOG_TAG, String.format("Uploaded %d bytes so far, using POST with X-HTTP-Method-Override PUT method.", (int) totalBytesUploaded)); } urlConnection.setDoOutput(true); urlConnection.setFixedLengthStreamingMode(chunk); urlConnection.setRequestProperty("Content-Type", "video/3gpp"); urlConnection.setRequestProperty("Content-Range", String.format("bytes %d-%d/%d", start, end, file.length())); Log.d(LOG_TAG, urlConnection.getRequestProperty("Content-Range")); //////may be //log.v("id man id",urlConnection.getRequestProperty("Content-Range")); OutputStream outStreamWriter = urlConnection.getOutputStream(); fileStream.skip(start); int bytesRead; int totalRead = 0; while ((bytesRead = fileStream.read(buffer, 0, bufferSize)) != -1) { outStreamWriter.write(buffer, 0, bytesRead); totalRead += bytesRead; this.totalBytesUploaded += bytesRead; double percent = (totalBytesUploaded / currentFileSize) * 99; /* Log.d(LOG_TAG, String.format( "fileSize=%f totalBytesUploaded=%f percent=%f", currentFileSize, totalBytesUploaded, percent)); */ dialog.setProgress((int) percent); if (totalRead == (end - start + 1)) { break; } } outStreamWriter.close(); int responseCode = urlConnection.getResponseCode(); Log.d(LOG_TAG, "responseCode=" + responseCode); Log.d(LOG_TAG, "responseMessage=" + urlConnection.getResponseMessage()); try { if (responseCode == 201) { String videoId = parseVideoId(urlConnection.getInputStream()); //log.v("Video ID", videoId); vidId = videoId; String latLng = null; if (this.videoLocation != null) { latLng = String.format("lat=%f lng=%f", this.videoLocation.getLatitude(), this.videoLocation.getLongitude()); } submitToYtdDomain(this.ytdDomain, this.assignmentId, videoId, this.youTubeName, SubmitActivity.this.clientLoginToken, getTitleText(), getDescriptionText(), this.dateTaken, latLng, this.tags); dialog.setProgress(100); //log.v("10video id",videoId); return videoId; } else if (responseCode == 200) { Set<String> keySet = urlConnection.getHeaderFields().keySet(); String keys = urlConnection.getHeaderFields().keySet().toString(); Log.d(LOG_TAG, String.format("Headers keys %s.", keys)); //////////////may be for (String key : keySet) { Log.d(LOG_TAG, String.format("Header key %s value %s.", key, urlConnection.getHeaderField(key))); } Log.w(LOG_TAG, "Received 200 response during resumable uploading"); throw new IOException(String.format("Unexpected response code : responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage())); } else { if ((responseCode + "").startsWith("5")) { String error = String.format("responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage()); Log.w(LOG_TAG, error); // TODO - this exception will trigger retry mechanism to kick in // TODO - even though it should not, consider introducing a new type so // TODO - resume does not kick in upon 5xx throw new IOException(error); } else if (responseCode == 308) { // OK, the chunk completed succesfully Log.d(LOG_TAG, String.format("responseCode=%d responseMessage=%s", responseCode, urlConnection.getResponseMessage())); } else { // TODO - this case is not handled properly yet Log.w(LOG_TAG, String.format("Unexpected return code : %d %s while uploading :%s", responseCode, urlConnection.getResponseMessage(), uploadUrl)); } } } catch (ParserConfigurationException e) { e.printStackTrace(); } catch (SAXException e) { e.printStackTrace(); } return null; }
From source file:com.sckftr.android.utils.net.Network.java
/** * Call the webservice using the given parameters to construct the request and return the * result.//from w w w .j a v a 2s. co m * * * @param context The context to use for this operation. Used to generate the user agent if * needed. * @param urlValue The webservice URL. * @param method The request method to use. * @param handler *@param parameterList The parameters to add to the request. * @param headerMap The headers to add to the request. * @param isGzipEnabled Whether the request will use gzip compression if available on the * server. * @param userAgent The user agent to set in the request. If null, a default Android one will be * created. * @param postText The POSTDATA text to add in the request. * @param credentials The credentials to use for authentication. * @param isSslValidationEnabled Whether the request will validate the SSL certificates. @return The result of the webservice call. */ static <Out> Out execute(Context context, String urlValue, Method method, Executor<BufferedReader, Out> handler, ArrayList<BasicNameValuePair> parameterList, HashMap<String, String> headerMap, boolean isGzipEnabled, String userAgent, String postText, UsernamePasswordCredentials credentials, boolean isSslValidationEnabled) throws NetworkConnectionException { HttpURLConnection connection = null; try { // Prepare the request information if (userAgent == null) { userAgent = UserAgentUtils.get(context); } if (headerMap == null) { headerMap = new HashMap<String, String>(); } headerMap.put(HTTP.USER_AGENT, userAgent); if (isGzipEnabled) { headerMap.put(ACCEPT_ENCODING_HEADER, "gzip"); } headerMap.put(ACCEPT_CHARSET_HEADER, UTF8_CHARSET); if (credentials != null) { headerMap.put(AUTHORIZATION_HEADER, createAuthenticationHeader(credentials)); } StringBuilder paramBuilder = new StringBuilder(); if (parameterList != null && !parameterList.isEmpty()) { for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String name = parameter.getName(); String value = parameter.getValue(); if (TextUtils.isEmpty(name)) { // Empty parameter name. Check the next one. continue; } if (value == null) { value = ""; } paramBuilder.append(URLEncoder.encode(name, UTF8_CHARSET)); paramBuilder.append("="); paramBuilder.append(URLEncoder.encode(value, UTF8_CHARSET)); paramBuilder.append("&"); } } // Log the request //if (Log.canLog(Log.DEBUG)) { Log.d(TAG, method.toString() + ": " + urlValue); if (parameterList != null && !parameterList.isEmpty()) { //Log.d(TAG, "Parameters:"); for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String message = "- \"" + parameter.getName() + "\" = \"" + parameter.getValue() + "\""; Log.d(TAG, message); } //Log.d(TAG, "Parameters String: \"" + paramBuilder.toString() + "\""); } if (postText != null) { Log.d(TAG, "Post data: " + postText); } if (headerMap != null && !headerMap.isEmpty()) { //Log.d(TAG, "Headers:"); for (Entry<String, String> header : headerMap.entrySet()) { //Log.d(TAG, "- " + header.getKey() + " = " + header.getValue()); } } //} // Create the connection object URL url = null; String outputText = null; switch (method) { case GET: case DELETE: String fullUrlValue = urlValue; if (paramBuilder.length() > 0) { fullUrlValue += "?" + paramBuilder.toString(); } url = new URL(fullUrlValue); connection = (HttpURLConnection) url.openConnection(); break; case PUT: case POST: url = new URL(urlValue); connection = (HttpURLConnection) url.openConnection(); connection.setDoOutput(true); if (paramBuilder.length() > 0) { outputText = paramBuilder.toString(); headerMap.put(HTTP.CONTENT_TYPE, "application/x-www-form-urlencoded"); headerMap.put(HTTP.CONTENT_LEN, String.valueOf(outputText.getBytes().length)); } else if (postText != null) { outputText = postText; } break; } // Set the request method connection.setRequestMethod(method.toString()); // If it's an HTTPS request and the SSL Validation is disabled if (url.getProtocol().equals("https") && !isSslValidationEnabled) { HttpsURLConnection httpsConnection = (HttpsURLConnection) connection; httpsConnection.setSSLSocketFactory(getAllHostsValidSocketFactory()); httpsConnection.setHostnameVerifier(getAllHostsValidVerifier()); } // Add the headers if (!headerMap.isEmpty()) { for (Entry<String, String> header : headerMap.entrySet()) { connection.addRequestProperty(header.getKey(), header.getValue()); } } // Set the connection and read timeout connection.setConnectTimeout(OPERATION_TIMEOUT); connection.setReadTimeout(OPERATION_TIMEOUT); // Set the outputStream content for POST and PUT requests if ((method == Method.POST || method == Method.PUT) && outputText != null) { OutputStream output = null; try { output = connection.getOutputStream(); output.write(outputText.getBytes()); } finally { if (output != null) { try { output.close(); } catch (IOException e) { // Already catching the first IOException so nothing to do here. } } } } String contentEncoding = connection.getHeaderField(HTTP.CONTENT_ENCODING); int responseCode = connection.getResponseCode(); boolean isGzip = contentEncoding != null && contentEncoding.equalsIgnoreCase("gzip"); Log.d(TAG, "Response code: " + responseCode); if (responseCode == HttpStatus.SC_MOVED_PERMANENTLY) { String redirectionUrl = connection.getHeaderField(LOCATION_HEADER); throw new NetworkConnectionException("New location : " + redirectionUrl, redirectionUrl); } InputStream errorStream = connection.getErrorStream(); if (errorStream != null) { String err = evaluateStream(errorStream, new StringReaderHandler(), isGzip); throw new NetworkConnectionException(err, responseCode); } return evaluateStream(connection.getInputStream(), handler, isGzip); } catch (IOException e) { Log.e(TAG, "IOException", e); throw new NetworkConnectionException(e); } catch (KeyManagementException e) { Log.e(TAG, "KeyManagementException", e); throw new NetworkConnectionException(e); } catch (NoSuchAlgorithmException e) { Log.e(TAG, "NoSuchAlgorithmException", e); throw new NetworkConnectionException(e); } finally { if (connection != null) { connection.disconnect(); } } }
From source file:org.apache.openaz.xacml.admin.util.RESTfulPAPEngine.java
/** * Send a request to the PAP Servlet and get the response. * /* w w w . j a v a 2 s. c o m*/ * The content is either an InputStream to be copied to the Request OutputStream * OR it is an object that is to be encoded into JSON and pushed into the Request OutputStream. * * The Request parameters may be encoded in multiple "name=value" sets, or parameters may be combined by the caller. * * @param method * @param content - EITHER an InputStream OR an Object to be encoded in JSON * @param collectionTypeClass * @param responseContentClass * @param parameters * @return * @throws Exception */ private Object sendToPAP(String method, Object content, Class collectionTypeClass, Class responseContentClass, String... parameters) throws PAPException { HttpURLConnection connection = null; try { String fullURL = papServletURLString; if (parameters != null && parameters.length > 0) { String queryString = ""; for (String p : parameters) { queryString += "&" + p; } fullURL += "?" + queryString.substring(1); } // special case - Status (actually the detailed status) comes from the PDP directly, not the PAP if (method.equals("GET") && content instanceof PDP && responseContentClass == StdPDPStatus.class) { // Adjust the url and properties appropriately fullURL = ((PDP) content).getId() + "?type=Status"; content = null; } URL url = new URL(fullURL); // // Open up the connection // connection = (HttpURLConnection) url.openConnection(); // // Setup our method and headers // connection.setRequestMethod(method); // connection.setRequestProperty("Accept", "text/x-java-properties"); // connection.setRequestProperty("Content-Type", "text/x-java-properties"); connection.setUseCaches(false); // // Adding this in. It seems the HttpUrlConnection class does NOT // properly forward our headers for POST re-direction. It does so // for a GET re-direction. // // So we need to handle this ourselves. // connection.setInstanceFollowRedirects(false); connection.setDoOutput(true); connection.setDoInput(true); if (content != null) { if (content instanceof InputStream) { try { // // Send our current policy configuration // try (OutputStream os = connection.getOutputStream()) { int count = IOUtils.copy((InputStream) content, os); if (logger.isDebugEnabled()) { logger.debug("copied to output, bytes=" + count); } } } catch (Exception e) { logger.error("Failed to write content in '" + method + "'", e); throw e; } } else { // The content is an object to be encoded in JSON ObjectMapper mapper = new ObjectMapper(); mapper.writeValue(connection.getOutputStream(), content); } } // // Do the connect // connection.connect(); if (connection.getResponseCode() == 204) { logger.info("Success - no content."); return null; } else if (connection.getResponseCode() == 200) { logger.info("Success. We have a return object."); // get the response content into a String String json = null; // read the inputStream into a buffer (trick found online scans entire input looking for end-of-file) java.util.Scanner scanner = new java.util.Scanner(connection.getInputStream()); scanner.useDelimiter("\\A"); json = scanner.hasNext() ? scanner.next() : ""; scanner.close(); logger.info("JSON response from PAP: " + json); // convert Object sent as JSON into local object ObjectMapper mapper = new ObjectMapper(); if (collectionTypeClass != null) { // collection of objects expected final CollectionType javaType = mapper.getTypeFactory() .constructCollectionType(collectionTypeClass, responseContentClass); Object objectFromJSON = mapper.readValue(json, javaType); return objectFromJSON; } else { // single value object expected Object objectFromJSON = mapper.readValue(json, responseContentClass); return objectFromJSON; } } else if (connection.getResponseCode() >= 300 && connection.getResponseCode() <= 399) { // redirection String newURL = connection.getHeaderField("Location"); if (newURL == null) { logger.error( "No Location header to redirect to when response code=" + connection.getResponseCode()); throw new IOException( "No redirect Location header when response code=" + connection.getResponseCode()); } int qIndex = newURL.indexOf("?"); if (qIndex > 0) { newURL = newURL.substring(0, qIndex); } logger.info("Redirect seen. Redirecting " + fullURL + " to " + newURL); return newURL; } else { logger.warn("Unexpected response code: " + connection.getResponseCode() + " message: " + connection.getResponseMessage()); throw new IOException("Server Response: " + connection.getResponseCode() + ": " + connection.getResponseMessage()); } } catch (Exception e) { logger.error("HTTP Request/Response to PAP: " + e, e); throw new PAPException("Request/Response threw :" + e); } finally { // cleanup the connection if (connection != null) { try { // For some reason trying to get the inputStream from the connection // throws an exception rather than returning null when the InputStream does not exist. InputStream is = null; try { is = connection.getInputStream(); } catch (Exception e1) { //NOPMD // ignore this } if (is != null) { is.close(); } } catch (IOException ex) { logger.error("Failed to close connection: " + ex, ex); } connection.disconnect(); } } }
From source file:com.phicomm.account.network.NetworkConnectionImpl.java
/** * Call the webservice using the given parameters to construct the request and return the * result./*from w w w . j ava 2 s . c o m*/ * * @param context The context to use for this operation. Used to generate the user agent if * needed. * @param urlValue The webservice URL. * @param method The request method to use. * @param parameterList The parameters to add to the request. * @param headerMap The headers to add to the request. * @param isGzipEnabled Whether the request will use gzip compression if available on the * server. * @param userAgent The user agent to set in the request. If null, a default Android one will be * created. * @param postText The POSTDATA text to add in the request. * @param credentials The credentials to use for authentication. * @param isSslValidationEnabled Whether the request will validate the SSL certificates. * @return The result of the webservice call. */ public static ConnectionResult execute(Context context, String urlValue, Method method, ArrayList<BasicNameValuePair> parameterList, HashMap<String, String> headerMap, boolean isGzipEnabled, String userAgent, String postText, UsernamePasswordCredentials credentials, boolean isSslValidationEnabled) throws ConnectionException { Thread.dumpStack(); Log.i("ss", "NetworkConnectionImpl_____________________________________execute__urlValue:" + urlValue); HttpURLConnection connection = null; try { // Prepare the request information if (userAgent == null) { userAgent = UserAgentUtils.get(context); } if (headerMap == null) { headerMap = new HashMap<String, String>(); } headerMap.put(HTTP.USER_AGENT, userAgent); if (isGzipEnabled) { headerMap.put(ACCEPT_ENCODING_HEADER, "gzip"); } headerMap.put(ACCEPT_CHARSET_HEADER, UTF8_CHARSET); if (credentials != null) { headerMap.put(AUTHORIZATION_HEADER, createAuthenticationHeader(credentials)); } StringBuilder paramBuilder = new StringBuilder(); if (parameterList != null && !parameterList.isEmpty()) { for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String name = parameter.getName(); String value = parameter.getValue(); if (TextUtils.isEmpty(name)) { // Empty parameter name. Check the next one. continue; } if (value == null) { value = ""; } paramBuilder.append(URLEncoder.encode(name, UTF8_CHARSET)); paramBuilder.append("="); paramBuilder.append(URLEncoder.encode(value, UTF8_CHARSET)); paramBuilder.append("&"); } } // Log the request if (true) { Log.d(TAG, "Request url: " + urlValue); Log.d(TAG, "Method: " + method.toString()); if (parameterList != null && !parameterList.isEmpty()) { Log.d(TAG, "Parameters:"); for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String message = "- \"" + parameter.getName() + "\" = \"" + parameter.getValue() + "\""; Log.d(TAG, message); } Log.d(TAG, "Parameters String: \"" + paramBuilder.toString() + "\""); } if (postText != null) { Log.d(TAG, "Post data: " + postText); } if (headerMap != null && !headerMap.isEmpty()) { Log.d(TAG, "Headers:"); for (Entry<String, String> header : headerMap.entrySet()) { Log.d(TAG, "- " + header.getKey() + " = " + header.getValue()); } } } // Create the connection object URL url = null; String outputText = null; switch (method) { case GET: case DELETE: String fullUrlValue = urlValue; if (paramBuilder.length() > 0) { fullUrlValue += "?" + paramBuilder.toString(); } url = new URL(fullUrlValue); connection = HttpUrlConnectionHelper.openUrlConnection(url); break; case PUT: case POST: url = new URL(urlValue); connection = HttpUrlConnectionHelper.openUrlConnection(url); connection.setDoOutput(true); if (paramBuilder.length() > 0) { outputText = paramBuilder.toString(); headerMap.put(HTTP.CONTENT_TYPE, "application/x-www-form-urlencoded"); headerMap.put(HTTP.CONTENT_LEN, String.valueOf(outputText.getBytes().length)); } else if (postText != null) { outputText = postText; } break; } // Set the request method connection.setRequestMethod(method.toString()); // If it's an HTTPS request and the SSL Validation is disabled if (url.getProtocol().equals("https") && !isSslValidationEnabled) { HttpsURLConnection httpsConnection = (HttpsURLConnection) connection; httpsConnection.setSSLSocketFactory(getAllHostsValidSocketFactory()); httpsConnection.setHostnameVerifier(getAllHostsValidVerifier()); } // Add the headers if (!headerMap.isEmpty()) { for (Entry<String, String> header : headerMap.entrySet()) { connection.addRequestProperty(header.getKey(), header.getValue()); } } // Set the connection and read timeout connection.setConnectTimeout(OPERATION_TIMEOUT); connection.setReadTimeout(OPERATION_TIMEOUT); // Set the outputStream content for POST and PUT requests if ((method == Method.POST || method == Method.PUT) && outputText != null) { OutputStream output = null; try { output = connection.getOutputStream(); output.write(outputText.getBytes()); } finally { if (output != null) { try { output.close(); } catch (IOException e) { // Already catching the first IOException so nothing to do here. } } } } String contentEncoding = connection.getHeaderField(HTTP.CONTENT_ENCODING); int responseCode = connection.getResponseCode(); boolean isGzip = contentEncoding != null && contentEncoding.equalsIgnoreCase("gzip"); Log.d(TAG, "Response code: " + responseCode); if (responseCode == HttpStatus.SC_MOVED_PERMANENTLY) { String redirectionUrl = connection.getHeaderField(LOCATION_HEADER); throw new ConnectionException("New location : " + redirectionUrl, redirectionUrl); } InputStream errorStream = connection.getErrorStream(); if (errorStream != null) { String error = convertStreamToString(errorStream, isGzip); throw new ConnectionException(error, responseCode); } String body = convertStreamToString(connection.getInputStream(), isGzip); if (true) { Log.v(TAG, "Response body: "); int pos = 0; int bodyLength = body.length(); while (pos < bodyLength) { Log.v(TAG, body.substring(pos, Math.min(bodyLength - 1, pos + 200))); pos = pos + 200; } } return new ConnectionResult(connection.getHeaderFields(), body); } catch (IOException e) { Log.e(TAG, "IOException", e); throw new ConnectionException(e); } catch (KeyManagementException e) { Log.e(TAG, "KeyManagementException", e); throw new ConnectionException(e); } catch (NoSuchAlgorithmException e) { Log.e(TAG, "NoSuchAlgorithmException", e); throw new ConnectionException(e); } finally { if (connection != null) { connection.disconnect(); } } }
From source file:com.foxykeep.datadroid.internal.network.NetworkConnectionImpl.java
/** * Call the webservice using the given parameters to construct the request and return the * result.//w w w .j a v a2s . c o m * * @param context The context to use for this operation. Used to generate the user agent if * needed. * @param urlValue The webservice URL. * @param method The request method to use. * @param parameterList The parameters to add to the request. * @param headerMap The headers to add to the request. * @param isGzipEnabled Whether the request will use gzip compression if available on the * server. * @param userAgent The user agent to set in the request. If null, a default Android one will be * created. * @param postText The POSTDATA text to add in the request. * @param credentials The credentials to use for authentication. * @param isSslValidationEnabled Whether the request will validate the SSL certificates. * @return The result of the webservice call. */ public static ConnectionResult execute(Context context, String urlValue, Method method, ArrayList<BasicNameValuePair> parameterList, HashMap<String, String> headerMap, boolean isGzipEnabled, String userAgent, String postText, UsernamePasswordCredentials credentials, boolean isSslValidationEnabled) throws ConnectionException { HttpURLConnection connection = null; try { // Prepare the request information if (userAgent == null) { userAgent = UserAgentUtils.get(context); } if (headerMap == null) { headerMap = new HashMap<String, String>(); } headerMap.put(HTTP.USER_AGENT, userAgent); if (isGzipEnabled) { headerMap.put(ACCEPT_ENCODING_HEADER, "gzip"); } headerMap.put(ACCEPT_CHARSET_HEADER, UTF8_CHARSET); if (credentials != null) { headerMap.put(AUTHORIZATION_HEADER, createAuthenticationHeader(credentials)); } StringBuilder paramBuilder = new StringBuilder(); if (parameterList != null && !parameterList.isEmpty()) { for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String name = parameter.getName(); String value = parameter.getValue(); if (TextUtils.isEmpty(name)) { // Empty parameter name. Check the next one. continue; } if (value == null) { value = ""; } paramBuilder.append(URLEncoder.encode(name, UTF8_CHARSET)); paramBuilder.append("="); paramBuilder.append(URLEncoder.encode(value, UTF8_CHARSET)); paramBuilder.append("&"); } } // Log the request if (DataDroidLog.canLog(Log.DEBUG)) { DataDroidLog.d(TAG, "Request url: " + urlValue); DataDroidLog.d(TAG, "Method: " + method.toString()); if (parameterList != null && !parameterList.isEmpty()) { DataDroidLog.d(TAG, "Parameters:"); for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String message = "- \"" + parameter.getName() + "\" = \"" + parameter.getValue() + "\""; DataDroidLog.d(TAG, message); } DataDroidLog.d(TAG, "Parameters String: \"" + paramBuilder.toString() + "\""); } if (postText != null) { DataDroidLog.d(TAG, "Post data: " + postText); } if (headerMap != null && !headerMap.isEmpty()) { DataDroidLog.d(TAG, "Headers:"); for (Entry<String, String> header : headerMap.entrySet()) { DataDroidLog.d(TAG, "- " + header.getKey() + " = " + header.getValue()); } } } // Create the connection object URL url = null; String outputText = null; switch (method) { case GET: case DELETE: String fullUrlValue = urlValue; if (paramBuilder.length() > 0) { fullUrlValue += "?" + paramBuilder.toString(); } url = new URL(fullUrlValue); connection = HttpUrlConnectionHelper.openUrlConnection(url); break; case PUT: case POST: url = new URL(urlValue); connection = HttpUrlConnectionHelper.openUrlConnection(url); connection.setDoOutput(true); if (paramBuilder.length() > 0) { outputText = paramBuilder.toString(); headerMap.put(HTTP.CONTENT_TYPE, "application/x-www-form-urlencoded"); headerMap.put(HTTP.CONTENT_LEN, String.valueOf(outputText.getBytes().length)); } else if (postText != null) { outputText = postText; } break; } // Set the request method connection.setRequestMethod(method.toString()); // If it's an HTTPS request and the SSL Validation is disabled if (url.getProtocol().equals("https") && !isSslValidationEnabled) { HttpsURLConnection httpsConnection = (HttpsURLConnection) connection; httpsConnection.setSSLSocketFactory(getAllHostsValidSocketFactory()); httpsConnection.setHostnameVerifier(getAllHostsValidVerifier()); } // Add the headers if (!headerMap.isEmpty()) { for (Entry<String, String> header : headerMap.entrySet()) { connection.addRequestProperty(header.getKey(), header.getValue()); } } // Set the connection and read timeout connection.setConnectTimeout(OPERATION_TIMEOUT); connection.setReadTimeout(OPERATION_TIMEOUT); // Set the outputStream content for POST and PUT requests if ((method == Method.POST || method == Method.PUT) && outputText != null) { OutputStream output = null; try { output = connection.getOutputStream(); output.write(outputText.getBytes()); } finally { if (output != null) { try { output.close(); } catch (IOException e) { // Already catching the first IOException so nothing to do here. } } } } String contentEncoding = connection.getHeaderField(HTTP.CONTENT_ENCODING); int responseCode = connection.getResponseCode(); boolean isGzip = contentEncoding != null && contentEncoding.equalsIgnoreCase("gzip"); DataDroidLog.d(TAG, "Response code: " + responseCode); if (responseCode == HttpStatus.SC_MOVED_PERMANENTLY) { String redirectionUrl = connection.getHeaderField(LOCATION_HEADER); throw new ConnectionException("New location : " + redirectionUrl, redirectionUrl); } InputStream errorStream = connection.getErrorStream(); if (errorStream != null) { String error = convertStreamToString(errorStream, isGzip); throw new ConnectionException(error, responseCode); } String body = convertStreamToString(connection.getInputStream(), isGzip); if (DataDroidLog.canLog(Log.VERBOSE)) { DataDroidLog.v(TAG, "Response body: "); int pos = 0; int bodyLength = body.length(); while (pos < bodyLength) { DataDroidLog.v(TAG, body.substring(pos, Math.min(bodyLength - 1, pos + 200))); pos = pos + 200; } } return new ConnectionResult(connection.getHeaderFields(), body); } catch (IOException e) { DataDroidLog.e(TAG, "IOException", e); throw new ConnectionException(e); } catch (KeyManagementException e) { DataDroidLog.e(TAG, "KeyManagementException", e); throw new ConnectionException(e); } catch (NoSuchAlgorithmException e) { DataDroidLog.e(TAG, "NoSuchAlgorithmException", e); throw new ConnectionException(e); } finally { if (connection != null) { connection.disconnect(); } } }
From source file:com.sun.faces.systest.ant.SystestClient.java
/** * Execute the test via use of an HttpURLConnection. * * @throws BuildException if an exception occurs *///from w w w . j a va 2 s . c om protected void executeHttp() throws BuildException { // Construct a summary of the request we will be sending String summary = "[" + method + " " + request + "]"; boolean success = true; String result = null; Throwable throwable = null; HttpURLConnection conn = null; try { // Configure an HttpURLConnection for this request if (log.isDebugEnabled()) { log.debug("Configuring HttpURLConnection for this request"); } URL url = new URL("http", host, port, request); conn = (HttpURLConnection) url.openConnection(); conn.setAllowUserInteraction(false); conn.setDoInput(true); if (inContent != null) { conn.setDoOutput(true); conn.setRequestProperty("Content-Length", "" + inContent.length()); if (log.isTraceEnabled()) { log.trace("INPH: Content-Length: " + inContent.length()); } } else { conn.setDoOutput(false); } // Send the session id cookie (if any) if (joinSession && (sessionId != null)) { conn.setRequestProperty("Cookie", "JSESSIONID=" + sessionId); if (log.isTraceEnabled()) { log.trace("INPH: Cookie: JSESSIONID=" + sessionId); } } if (this.redirect && log.isTraceEnabled()) { log.trace("FLAG: setInstanceFollowRedirects(" + this.redirect + ")"); } conn.setInstanceFollowRedirects(this.redirect); conn.setRequestMethod(method); if (inHeaders != null) { String headers = inHeaders; while (headers.length() > 0) { int delimiter = headers.indexOf("##"); String header = null; if (delimiter < 0) { header = headers; headers = ""; } else { header = headers.substring(0, delimiter); headers = headers.substring(delimiter + 2); } int colon = header.indexOf(":"); if (colon < 0) break; String name = header.substring(0, colon).trim(); String value = header.substring(colon + 1).trim(); conn.setRequestProperty(name, value); if (log.isTraceEnabled()) { log.trace("INPH: " + name + ": " + value); } } } // Connect to the server and send our output if necessary conn.connect(); if (inContent != null) { if (log.isTraceEnabled()) { log.trace("INPD: " + inContent); } OutputStream os = conn.getOutputStream(); for (int i = 0, length = inContent.length(); i < length; i++) os.write(inContent.charAt(i)); os.close(); } // Acquire the response data, if there is any String outData = ""; String outText = ""; boolean eol = false; InputStream is = conn.getInputStream(); int lines = 0; while (true) { String line = read(is); if (line == null) break; if (lines == 0) outData = line; else outText += line + "\r\n"; saveResponse.add(line); lines++; } is.close(); // Dump out the response stuff if (log.isTraceEnabled()) { log.trace("RESP: " + conn.getResponseCode() + " " + conn.getResponseMessage()); } for (int i = 1; i < 1000; i++) { String name = conn.getHeaderFieldKey(i); String value = conn.getHeaderField(i); if ((name == null) || (value == null)) break; if (log.isTraceEnabled()) { log.trace("HEAD: " + name + ": " + value); } save(name, value); if ("Set-Cookie".equals(name)) parseSession(value); } if (log.isTraceEnabled()) { log.trace("DATA: " + outData); if (outText.length() > 2) { log.trace("TEXT: " + outText); } } // Validate the response against our criteria if (success) { result = validateStatus(conn.getResponseCode()); if (result != null) success = false; } if (success) { result = validateMessage(conn.getResponseMessage()); if (result != null) success = false; } if (success) { result = validateHeaders(); if (result != null) success = false; } if (success) { result = validateData(outData); if (result != null) success = false; } if (success) { result = validateGolden(); if (result != null) success = false; } } catch (Throwable t) { if (t instanceof FileNotFoundException) { if (status == 404) { success = true; result = "Not Found"; throwable = null; } else { success = false; try { result = "Status=" + conn.getResponseCode() + ", Message=" + conn.getResponseMessage(); } catch (IOException e) { result = e.toString(); } throwable = null; } } else if (t instanceof ConnectException) { success = false; result = t.getMessage(); throwable = null; } else { success = false; result = t.getMessage(); throwable = t; } } // Log the results of executing this request if (success) { System.out.println("OK " + summary); } else { System.out.println("FAIL " + summary + " " + result); if (throwable != null) throwable.printStackTrace(System.out); if (failonerror) { if (throwable != null) { throw new BuildException("System test failed", throwable); } else { throw new BuildException("System test failed"); } } } }
From source file:com.foxykeep.datadroid.internal.network.NetworkConnectionImplF.java
/** * Call the webservice using the given parameters to construct the request and return the * result./*from w ww . jav a2 s . c om*/ * * @param context The context to use for this operation. Used to generate the user agent if * needed. * @param urlValue The webservice URL. * @param method The request method to use. * @param parameterList The parameters to add to the request. * @param headerMap The headers to add to the request. * @param isGzipEnabled Whether the request will use gzip compression if available on the * server. * @param userAgent The user agent to set in the request. If null, a default Android one will be * created. * @param postText The POSTDATA text to add in the request. * @param credentials The credentials to use for authentication. * @param isSslValidationEnabled Whether the request will validate the SSL certificates. * @return The result of the webservice call. */ public static ConnectionResult execute(Context context, String urlValue, Method method, ArrayList<BasicNameValuePair> parameterList, HashMap<String, String> headerMap, boolean isGzipEnabled, String userAgent, String postText, UsernamePasswordCredentials credentials, boolean isSslValidationEnabled, File file) throws ConnectionException { HttpURLConnection connection = null; try { // Prepare the request information if (userAgent == null) { userAgent = UserAgentUtils.get(context); } if (headerMap == null) { headerMap = new HashMap<String, String>(); } headerMap.put(HTTP.USER_AGENT, userAgent); if (isGzipEnabled) { headerMap.put(ACCEPT_ENCODING_HEADER, "gzip"); } headerMap.put(ACCEPT_CHARSET_HEADER, UTF8_CHARSET); if (credentials != null) { headerMap.put(AUTHORIZATION_HEADER, createAuthenticationHeader(credentials)); } StringBuilder paramBuilder = new StringBuilder(); if (parameterList != null && !parameterList.isEmpty()) { for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String name = parameter.getName(); String value = parameter.getValue(); if (TextUtils.isEmpty(name)) { // Empty parameter name. Check the next one. continue; } if (value == null) { value = ""; } paramBuilder.append(URLEncoder.encode(name, UTF8_CHARSET)); paramBuilder.append("="); paramBuilder.append(URLEncoder.encode(value, UTF8_CHARSET)); paramBuilder.append("&"); } } // Log the request if (DataDroidLog.canLog(Log.DEBUG)) { DataDroidLog.d(TAG, "Request url: " + urlValue); DataDroidLog.d(TAG, "Method: " + method.toString()); if (parameterList != null && !parameterList.isEmpty()) { DataDroidLog.d(TAG, "Parameters:"); for (int i = 0, size = parameterList.size(); i < size; i++) { BasicNameValuePair parameter = parameterList.get(i); String message = "- \"" + parameter.getName() + "\" = \"" + parameter.getValue() + "\""; DataDroidLog.d(TAG, message); } DataDroidLog.d(TAG, "Parameters String: \"" + paramBuilder.toString() + "\""); } if (postText != null) { DataDroidLog.d(TAG, "Post data: " + postText); } if (headerMap != null && !headerMap.isEmpty()) { DataDroidLog.d(TAG, "Headers:"); for (Entry<String, String> header : headerMap.entrySet()) { DataDroidLog.d(TAG, "- " + header.getKey() + " = " + header.getValue()); } } } // Create the connection object URL url = null; String outputText = null; switch (method) { case GET: case DELETE: String fullUrlValue = urlValue; if (paramBuilder.length() > 0) { fullUrlValue += "?" + paramBuilder.toString(); } url = new URL(fullUrlValue); connection = (HttpURLConnection) url.openConnection(); break; case PUT: case POST: url = new URL(urlValue); connection = (HttpURLConnection) url.openConnection(); connection.setDoOutput(true); if (paramBuilder.length() > 0) { outputText = paramBuilder.toString(); headerMap.put(HTTP.CONTENT_TYPE, "application/x-www-form-urlencoded"); headerMap.put(HTTP.CONTENT_LEN, String.valueOf(outputText.getBytes().length)); } else if (postText != null) { outputText = postText; } break; } // Set the request method connection.setRequestMethod(method.toString()); // If it's an HTTPS request and the SSL Validation is disabled if (url.getProtocol().equals("https") && !isSslValidationEnabled) { HttpsURLConnection httpsConnection = (HttpsURLConnection) connection; httpsConnection.setSSLSocketFactory(getAllHostsValidSocketFactory()); httpsConnection.setHostnameVerifier(getAllHostsValidVerifier()); } // Add the headers if (!headerMap.isEmpty()) { for (Entry<String, String> header : headerMap.entrySet()) { connection.addRequestProperty(header.getKey(), header.getValue()); } } // Set the connection and read timeout connection.setConnectTimeout(OPERATION_TIMEOUT); connection.setReadTimeout(READ_OPERATION_TIMEOUT); // Set the outputStream content for POST and PUT requests if ((method == Method.POST || method == Method.PUT) && outputText != null) { OutputStream output = null; try { output = connection.getOutputStream(); output.write(outputText.getBytes()); } finally { if (output != null) { try { output.close(); } catch (IOException e) { // Already catching the first IOException so nothing to do here. } } } } String contentEncoding = connection.getHeaderField(HTTP.CONTENT_ENCODING); int responseCode = connection.getResponseCode(); boolean isGzip = contentEncoding != null && contentEncoding.equalsIgnoreCase("gzip"); DataDroidLog.d(TAG, "Response code: " + responseCode); if (responseCode == HttpStatus.SC_MOVED_PERMANENTLY) { String redirectionUrl = connection.getHeaderField(LOCATION_HEADER); throw new ConnectionException("New location : " + redirectionUrl, redirectionUrl); } InputStream errorStream = connection.getErrorStream(); if (errorStream != null) { throw new ConnectionException("error", responseCode); } String body = convertStreamToString(connection.getInputStream(), isGzip, file, context); return new ConnectionResult(connection.getHeaderFields(), body); } catch (IOException e) { DataDroidLog.e(TAG, "IOException", e); throw new ConnectionException(e); } catch (KeyManagementException e) { DataDroidLog.e(TAG, "KeyManagementException", e); throw new ConnectionException(e); } catch (NoSuchAlgorithmException e) { DataDroidLog.e(TAG, "NoSuchAlgorithmException", e); throw new ConnectionException(e); } finally { if (connection != null) { connection.disconnect(); } } }
From source file:org.apache.streams.urls.LinkResolver.java
public void unwindLink(String url) { Objects.requireNonNull(linkDetails); Objects.requireNonNull(url);/*from w w w. ja va 2s . c om*/ // Check url validity UrlValidator urlValidator = new UrlValidator(); if (!urlValidator.isValid(url)) { linkDetails.setLinkStatus(LinkDetails.LinkStatus.MALFORMED_URL); return; } // Check to see if they wound up in a redirect loop, // IE: 'A' redirects to 'B', then 'B' redirects to 'A' if ((linkDetails.getRedirectCount() != null && linkDetails.getRedirectCount() > 0 && (linkDetails.getOriginalURL().equals(url) || linkDetails.getRedirects().contains(url))) || (linkDetails.getRedirectCount() != null && linkDetails.getRedirectCount() > MAX_ALLOWED_REDIRECTS)) { linkDetails.setLinkStatus(LinkDetails.LinkStatus.LOOP); return; } if (!linkDetails.getOriginalURL().equals(url)) linkDetails.getRedirects().add(url); HttpURLConnection connection = null; // Store where the redirected link will go (if there is one) String reDirectedLink = null; try { // Turn the string into a URL URL thisURL = new URL(url); // Be sensitive to overloading domains STREAMS-77 try { String host = thisURL.getHost().toLowerCase(); if (!domainsSensitiveTo.contains(host)) { domainsSensitiveTo.add(host); long domainWait = LinkResolverHelperFunctions.waitTimeForDomain(thisURL.getHost()); if (domainWait > 0) { LOGGER.debug("Waiting for domain: {}", domainWait); Thread.sleep(domainWait); } } } catch (Exception e) { // noOp } connection = (HttpURLConnection) new URL(url).openConnection(); // now we are going to pretend that we are a browser... // This is the way my mac works. if (!BOTS_ARE_OK.contains(thisURL.getHost())) { connection.addRequestProperty("Host", thisURL.getHost()); // Bots are not 'ok', so we need to spoof the headers for (String k : SPOOF_HTTP_HEADERS.keySet()) connection.addRequestProperty(k, SPOOF_HTTP_HEADERS.get(k)); // the test to seattlemamadoc.com prompted this change. // they auto detect bots by checking the referrer chain and the 'user-agent' // this broke the t.co test. t.co URLs are EXPLICITLY ok with bots // there is a list for URLS that behave this way at the top in BOTS_ARE_OK // smashew 2013-13-2013 if (linkDetails.getRedirectCount() > 0 && BOTS_ARE_OK.contains(thisURL.getHost())) connection.addRequestProperty("Referrer", linkDetails.getOriginalURL()); } connection.setReadTimeout(DEFAULT_HTTP_TIMEOUT); connection.setConnectTimeout(DEFAULT_HTTP_TIMEOUT); // we want to follow this behavior on our own to ensure that we are getting to the // proper place. This is especially true with links that are wounded by special // link winders, // IE: connection.setInstanceFollowRedirects(false); if (linkDetails.getCookies() != null) for (String cookie : linkDetails.getCookies()) connection.addRequestProperty("Cookie", cookie.split(";", 1)[0]); connection.connect(); linkDetails.setFinalResponseCode((long) connection.getResponseCode()); Map<String, List<String>> headers = createCaseInsensitiveMap(connection.getHeaderFields()); /* * If they want us to set cookies, well, then we will set cookies * Example URL: * http://nyti.ms/1bCpesx *****************************************************************/ if (headers.containsKey(SET_COOKIE_IDENTIFIER)) linkDetails.getCookies().add(headers.get(SET_COOKIE_IDENTIFIER).get(0)); switch (linkDetails.getFinalResponseCode().intValue()) { /* * W3C HTTP Response Codes: * http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html */ case 200: // HTTP OK linkDetails.setFinalURL(connection.getURL().toString()); linkDetails.setDomain(new URL(linkDetails.getFinalURL()).getHost()); linkDetails.setLinkStatus(LinkDetails.LinkStatus.SUCCESS); break; case 300: // Multiple choices case 301: // URI has been moved permanently case 302: // Found case 303: // Primarily for a HTTP Post case 304: // Not Modified case 306: // This status code is unused but in the redirect block. case 307: // Temporary re-direct /* * Author: * Smashew * * Date: 2013-11-15 * * Note: * It is possible that we have already found our final URL. In * the event that we have found our final URL, we are going to * save this URL as long as it isn't the original URL. * We are still going to ask the browser to re-direct, but in the * case of yet another redirect, seen with the redbull test * this can be followed by a 304, a browser, by W3C standards would * still render the page with it's content, but for us to assert * a success, we are really hoping for a 304 message. *******************************************************************/ if (!linkDetails.getOriginalURL().toLowerCase() .equals(connection.getURL().toString().toLowerCase())) linkDetails.setFinalURL(connection.getURL().toString()); if (!headers.containsKey(LOCATION_IDENTIFIER)) { LOGGER.info("Headers: {}", headers); linkDetails.setLinkStatus(LinkDetails.LinkStatus.REDIRECT_ERROR); } else { linkDetails.setRedirected(Boolean.TRUE); linkDetails.setRedirectCount(linkDetails.getRedirectCount() + 1); reDirectedLink = connection.getHeaderField(LOCATION_IDENTIFIER); } break; case 305: // User must use the specified proxy (deprecated by W3C) break; case 401: // Unauthorized (nothing we can do here) linkDetails.setLinkStatus(LinkDetails.LinkStatus.UNAUTHORIZED); break; case 403: // HTTP Forbidden (Nothing we can do here) linkDetails.setLinkStatus(LinkDetails.LinkStatus.FORBIDDEN); break; case 404: // Not Found (Page is not found, nothing we can do with a 404) linkDetails.setLinkStatus(LinkDetails.LinkStatus.NOT_FOUND); break; case 500: // Internal Server Error case 501: // Not Implemented case 502: // Bad Gateway case 503: // Service Unavailable case 504: // Gateway Timeout case 505: // Version not supported linkDetails.setLinkStatus(LinkDetails.LinkStatus.HTTP_ERROR_STATUS); break; default: LOGGER.info("Unrecognized HTTP Response Code: {}", linkDetails.getFinalResponseCode()); linkDetails.setLinkStatus(LinkDetails.LinkStatus.NOT_FOUND); break; } } catch (MalformedURLException e) { // the URL is trash, so, it can't load it. linkDetails.setLinkStatus(LinkDetails.LinkStatus.MALFORMED_URL); } catch (IOException ex) { // there was an issue we are going to set to error. linkDetails.setLinkStatus(LinkDetails.LinkStatus.ERROR); } catch (Exception ex) { // there was an unknown issue we are going to set to exception. linkDetails.setLinkStatus(LinkDetails.LinkStatus.EXCEPTION); } finally { // if the connection is not null, then we need to disconnect to close any underlying resources if (connection != null) connection.disconnect(); } // If there was a redirection, then we have to keep going // Placing this code here should help to satisfy ensuring that the connection object // is closed successfully. if (reDirectedLink != null) unwindLink(reDirectedLink); }