Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.streams.urls; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.UrlValidator; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; public class LinkResolver implements Serializable { /** * References: * Some helpful references to demonstrate the different types of browser re-directs that * can happen. If you notice a redirect that was not followed to the proper place please * submit a bug at : * https://issues.apache.org/jira/browse/STREAMS * <p/> * Purpose URL * ------------- ---------------------------------------------------------------- * [Status Codes] http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html * [Test Cases] http://greenbytes.de/tech/tc/httpredirects/ * [t.co behavior] https://dev.twitter.com/docs/tco-redirection-behavior */ private final static Logger LOGGER = LoggerFactory.getLogger(LinkResolver.class); private static final int MAX_ALLOWED_REDIRECTS = 30; // We will only chase the link to it's final destination a max of 30 times. private static final int DEFAULT_HTTP_TIMEOUT = 10000; // We will only wait a max of 10,000 milliseconds (10 seconds) for any HTTP response private static final String LOCATION_IDENTIFIER = "location"; private static final String SET_COOKIE_IDENTIFIER = "set-cookie"; // if Bots are not 'ok' this is the spoof settings that we'll use private static final Map<String, String> SPOOF_HTTP_HEADERS = new HashMap<String, String>() { { put("Connection", "Keep-Alive"); put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36"); put("Accept-Language", "en-US,en;q=0.8,zh;q=0.6"); } }; // These are the known domains that are 'bot' friendly. private static final Collection<String> BOTS_ARE_OK = new ArrayList<String>() { { add("t.co"); } }; // To help canonicalize the URL, these parts are 'known' to be 'ok' to remove private static final Collection<String> URL_TRACKING_TO_REMOVE = new ArrayList<String>() { { /* * Google uses parameters in the URL string to track referrers * on their Google Analytics and promotions. These are the * identified URL patterns. * * URL: * https://support.google.com/analytics/answer/1033867?hl=en *****************************************************************/ // Required. Use utm_source to identify a search engine, newsletter name, or other source. add("([\\?&])utm_source(=)[^&?]*"); // Required. Use utm_medium to identify a medium such as email or cost-per- click. add("([\\?&])utm_medium(=)[^&?]*"); // Used for paid search. Use utm_term to note the keywords for this ad. add("([\\?&])utm_term(=)[^&?]*"); // Used for A/B testing and content-targeted ads. Use utm_content to differentiate ads or links that point to the same add("([\\?&])utm_content(=)[^&?]*"); // Used for keyword analysis. Use utm_campaign to identify a specific product promotion or strategic campaign. add("([\\?&])utm_campaign(=)[^&?]*"); } }; // This element holds all the information about all the re-directs that have taken place // and the steps and HTTP codes that occurred inside of each step. private final LinkDetails linkDetails; private Collection<String> domainsSensitiveTo = new HashSet<>(); /** * Get the link details * * @return Detailed log of every redirection that took place with the browser along with it it's ultimate status code. */ public LinkDetails getLinkDetails() { return linkDetails; } /** * Raw string input of the URL. If the URL is invalid, the response code that is returned will indicate such. * * @param originalURL The URL you wish to unwind represented as a string. */ public LinkResolver(String originalURL) { linkDetails = new LinkDetails(); linkDetails.setOriginalURL(originalURL); } public void run() { Objects.requireNonNull(linkDetails.getOriginalURL()); linkDetails.setStartTime(DateTime.now()); // we are going to try three times just in case we catch a slow server or one that needs // to be warmed up. This tends to happen many times with smaller private servers for (int i = 0; (i < 3) && linkDetails.getFinalURL() == null; i++) if (linkDetails.getLinkStatus() != LinkDetails.LinkStatus.SUCCESS) unwindLink(linkDetails.getOriginalURL()); // because this is a POJO we need to make sure that we set this to false if it was never re-directed if (this.linkDetails.getRedirectCount() == 0 || this.linkDetails.getRedirected() == null) this.linkDetails.setRedirected(false); linkDetails.setFinalURL(cleanURL(linkDetails.getFinalURL())); if (StringUtils.isNotBlank(linkDetails.getFinalURL())) linkDetails.setNormalizedURL(normalizeURL(linkDetails.getFinalURL())); if (StringUtils.isNotBlank(linkDetails.getNormalizedURL())) linkDetails.setUrlParts(tokenizeURL(linkDetails.getNormalizedURL())); this.updateTookInMillis(); } protected void updateTookInMillis() { Objects.requireNonNull(linkDetails.getStartTime()); linkDetails.setTookInMills(DateTime.now().minus(linkDetails.getStartTime().getMillis()).getMillis()); } public void unwindLink(String url) { Objects.requireNonNull(linkDetails); Objects.requireNonNull(url); // Check url validity UrlValidator urlValidator = new UrlValidator(); if (!urlValidator.isValid(url)) { linkDetails.setLinkStatus(LinkDetails.LinkStatus.MALFORMED_URL); return; } // Check to see if they wound up in a redirect loop, // IE: 'A' redirects to 'B', then 'B' redirects to 'A' if ((linkDetails.getRedirectCount() != null && linkDetails.getRedirectCount() > 0 && (linkDetails.getOriginalURL().equals(url) || linkDetails.getRedirects().contains(url))) || (linkDetails.getRedirectCount() != null && linkDetails.getRedirectCount() > MAX_ALLOWED_REDIRECTS)) { linkDetails.setLinkStatus(LinkDetails.LinkStatus.LOOP); return; } if (!linkDetails.getOriginalURL().equals(url)) linkDetails.getRedirects().add(url); HttpURLConnection connection = null; // Store where the redirected link will go (if there is one) String reDirectedLink = null; try { // Turn the string into a URL URL thisURL = new URL(url); // Be sensitive to overloading domains STREAMS-77 try { String host = thisURL.getHost().toLowerCase(); if (!domainsSensitiveTo.contains(host)) { domainsSensitiveTo.add(host); long domainWait = LinkResolverHelperFunctions.waitTimeForDomain(thisURL.getHost()); if (domainWait > 0) { LOGGER.debug("Waiting for domain: {}", domainWait); Thread.sleep(domainWait); } } } catch (Exception e) { // noOp } connection = (HttpURLConnection) new URL(url).openConnection(); // now we are going to pretend that we are a browser... // This is the way my mac works. if (!BOTS_ARE_OK.contains(thisURL.getHost())) { connection.addRequestProperty("Host", thisURL.getHost()); // Bots are not 'ok', so we need to spoof the headers for (String k : SPOOF_HTTP_HEADERS.keySet()) connection.addRequestProperty(k, SPOOF_HTTP_HEADERS.get(k)); // the test to seattlemamadoc.com prompted this change. // they auto detect bots by checking the referrer chain and the 'user-agent' // this broke the t.co test. t.co URLs are EXPLICITLY ok with bots // there is a list for URLS that behave this way at the top in BOTS_ARE_OK // smashew 2013-13-2013 if (linkDetails.getRedirectCount() > 0 && BOTS_ARE_OK.contains(thisURL.getHost())) connection.addRequestProperty("Referrer", linkDetails.getOriginalURL()); } connection.setReadTimeout(DEFAULT_HTTP_TIMEOUT); connection.setConnectTimeout(DEFAULT_HTTP_TIMEOUT); // we want to follow this behavior on our own to ensure that we are getting to the // proper place. This is especially true with links that are wounded by special // link winders, // IE: connection.setInstanceFollowRedirects(false); if (linkDetails.getCookies() != null) for (String cookie : linkDetails.getCookies()) connection.addRequestProperty("Cookie", cookie.split(";", 1)[0]); connection.connect(); linkDetails.setFinalResponseCode((long) connection.getResponseCode()); Map<String, List<String>> headers = createCaseInsensitiveMap(connection.getHeaderFields()); /* * If they want us to set cookies, well, then we will set cookies * Example URL: * http://nyti.ms/1bCpesx *****************************************************************/ if (headers.containsKey(SET_COOKIE_IDENTIFIER)) linkDetails.getCookies().add(headers.get(SET_COOKIE_IDENTIFIER).get(0)); switch (linkDetails.getFinalResponseCode().intValue()) { /* * W3C HTTP Response Codes: * http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html */ case 200: // HTTP OK linkDetails.setFinalURL(connection.getURL().toString()); linkDetails.setDomain(new URL(linkDetails.getFinalURL()).getHost()); linkDetails.setLinkStatus(LinkDetails.LinkStatus.SUCCESS); break; case 300: // Multiple choices case 301: // URI has been moved permanently case 302: // Found case 303: // Primarily for a HTTP Post case 304: // Not Modified case 306: // This status code is unused but in the redirect block. case 307: // Temporary re-direct /* * Author: * Smashew * * Date: 2013-11-15 * * Note: * It is possible that we have already found our final URL. In * the event that we have found our final URL, we are going to * save this URL as long as it isn't the original URL. * We are still going to ask the browser to re-direct, but in the * case of yet another redirect, seen with the redbull test * this can be followed by a 304, a browser, by W3C standards would * still render the page with it's content, but for us to assert * a success, we are really hoping for a 304 message. *******************************************************************/ if (!linkDetails.getOriginalURL().toLowerCase() .equals(connection.getURL().toString().toLowerCase())) linkDetails.setFinalURL(connection.getURL().toString()); if (!headers.containsKey(LOCATION_IDENTIFIER)) { LOGGER.info("Headers: {}", headers); linkDetails.setLinkStatus(LinkDetails.LinkStatus.REDIRECT_ERROR); } else { linkDetails.setRedirected(Boolean.TRUE); linkDetails.setRedirectCount(linkDetails.getRedirectCount() + 1); reDirectedLink = connection.getHeaderField(LOCATION_IDENTIFIER); } break; case 305: // User must use the specified proxy (deprecated by W3C) break; case 401: // Unauthorized (nothing we can do here) linkDetails.setLinkStatus(LinkDetails.LinkStatus.UNAUTHORIZED); break; case 403: // HTTP Forbidden (Nothing we can do here) linkDetails.setLinkStatus(LinkDetails.LinkStatus.FORBIDDEN); break; case 404: // Not Found (Page is not found, nothing we can do with a 404) linkDetails.setLinkStatus(LinkDetails.LinkStatus.NOT_FOUND); break; case 500: // Internal Server Error case 501: // Not Implemented case 502: // Bad Gateway case 503: // Service Unavailable case 504: // Gateway Timeout case 505: // Version not supported linkDetails.setLinkStatus(LinkDetails.LinkStatus.HTTP_ERROR_STATUS); break; default: LOGGER.info("Unrecognized HTTP Response Code: {}", linkDetails.getFinalResponseCode()); linkDetails.setLinkStatus(LinkDetails.LinkStatus.NOT_FOUND); break; } } catch (MalformedURLException e) { // the URL is trash, so, it can't load it. linkDetails.setLinkStatus(LinkDetails.LinkStatus.MALFORMED_URL); } catch (IOException ex) { // there was an issue we are going to set to error. linkDetails.setLinkStatus(LinkDetails.LinkStatus.ERROR); } catch (Exception ex) { // there was an unknown issue we are going to set to exception. linkDetails.setLinkStatus(LinkDetails.LinkStatus.EXCEPTION); } finally { // if the connection is not null, then we need to disconnect to close any underlying resources if (connection != null) connection.disconnect(); } // If there was a redirection, then we have to keep going // Placing this code here should help to satisfy ensuring that the connection object // is closed successfully. if (reDirectedLink != null) unwindLink(reDirectedLink); } private Map<String, List<String>> createCaseInsensitiveMap(Map<String, List<String>> input) { Map<String, List<String>> toReturn = new HashMap<>(); for (String k : input.keySet()) if (k != null && input.get(k) != null) toReturn.put(k.toLowerCase(), input.get(k)); return toReturn; } private String cleanURL(String url) { // If they pass us a null URL then we are going to pass that right back to them. if (url == null) return null; // remember how big the URL was at the start int startLength = url.length(); // Iterate through all the known URL parameters of tracking URLs for (String pattern : URL_TRACKING_TO_REMOVE) url = url.replaceAll(pattern, ""); // If the URL is smaller than when it came in. Then it had tracking information if (url.length() < startLength) linkDetails.setTracked(Boolean.TRUE); // return our url. return url; } /** * Removes the protocol, if it exists, from the front and * removes any random encoding characters * Extend this to do other url cleaning/pre-processing * * @param url - The String URL to normalize * @return normalizedUrl - The String URL that has no junk or surprises */ public static String normalizeURL(String url) { // Decode URL to remove any %20 type stuff String normalizedUrl = url; try { // Replaced URLDecode with commons-codec b/c of failing tests URLCodec codec = new URLCodec(); normalizedUrl = codec.decode(url); // Remove the protocol, http:// ftp:// or similar from the front if (normalizedUrl.contains("://")) normalizedUrl = normalizedUrl.split(":/{2}")[1]; } catch (NullPointerException npe) { System.err.println("NPE Decoding URL. Decoding skipped."); npe.printStackTrace(); } catch (Throwable e) { System.err.println("Misc error Decoding URL. Decoding skipped."); e.printStackTrace(); } // Room here to do more pre-processing return normalizedUrl; } /** * Goal is to get the different parts of the URL path. This can be used * in a classifier to help us determine if we are working with * <p/> * Reference: * http://stackoverflow.com/questions/10046178/pattern-matching-for-url-classification * * @param url - Url to be tokenized * @return tokens - A String array of all the tokens */ public static List<String> tokenizeURL(String url) { url = normalizeURL(url); // I assume that we're going to use the whole URL to find tokens in // If you want to just look in the GET parameters, or you want to ignore the domain // or you want to use the domain as a token itself, that would have to be // processed above the next line, and only the remaining parts split List<String> toReturn = new ArrayList<>(); // Split the URL by forward slashes. Most modern browsers will accept a URL // this malformed such as http://www.smashew.com/hello//how////are/you // hence the '+' in the regular expression. for (String part : url.split("/+")) toReturn.add(part.toLowerCase()); // return our object. return toReturn; // One could alternatively use a more complex regex to remove more invalid matches // but this is subject to your (?:in)?ability to actually write the regex you want // These next two get rid of tokens that are too short, also. // Destroys anything that's not alphanumeric and things that are // alphanumeric but only 1 character long //String[] tokens = url.split("(?:[\\W_]+\\w)*[\\W_]+"); // Destroys anything that's not alphanumeric and things that are // alphanumeric but only 1 or 2 characters long //String[] tokens = url.split("(?:[\\W_]+\\w{1,2})*[\\W_]+"); } }