Java tutorial
/* * Copyright (C) 2015 Stratio (http://stratio.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.stratio.morphlines.refererparser; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; import com.stratio.morphlines.refererparser.exception.RefererParserException; public class Parser { private static final String REFERERS_YAML_PATH = "/referers.yml"; public static final String HTTP_LOCALHOST = "http://localhost?"; private Map<String, RefererLookup> referers; /** * Holds the structure of each referer * in our lookup Map. */ private static class RefererLookup { public Medium medium; public String source; public List<String> parameters; public RefererLookup(Medium medium, String source, List<String> parameters) { this.medium = medium; this.source = source; this.parameters = parameters; } } /** * Construct our com.stratio.ingestion.morphline.refererparser.Parser object using the * bundled referers.yml */ public Parser() throws IOException, RefererParserException { this(Parser.class.getResourceAsStream(REFERERS_YAML_PATH)); } /** * Construct our com.stratio.ingestion.morphline.refererparser.Parser object using a * InputStream (in YAML format) * * @param referersYaml The referers YAML * to load into our com.stratio.ingestion.morphline.refererparser.Parser, in * InputStream format */ public Parser(InputStream referersYaml) throws RefererParserException { referers = loadReferers(referersYaml); } /** * Construct our com.stratio.ingestion.morphline.refererparser.Parser object using a * custom resource String * * @param referersResource The resource pointing * to the referers YAML file to load */ public Parser(String referersResource) throws IOException, RefererParserException { this(Parser.class.getResourceAsStream(referersResource)); } public Referer parse(URI refererUri, URI pageUri) { return parse(refererUri, pageUri.getHost()); } public Referer parse(String refererUri, URI pageUri) throws URISyntaxException { return parse(refererUri, pageUri.getHost()); } public Referer parse(String refererUri, String pageHost) throws URISyntaxException { if (StringUtils.isBlank(refererUri)) { return null; } final URI uri = new URI(refererUri); return parse(uri, pageHost); } public Referer parse(URI refererUri, String pageHost) { return parse(refererUri, pageHost, Collections.<String>emptyList()); } public Referer parse(URI refererUri, String pageHost, List<String> internalDomains) { if (refererUri == null) { return null; } return parse(refererUri.getScheme(), refererUri.getHost(), refererUri.getPath(), refererUri.getRawQuery(), pageHost, internalDomains); } public Referer parse(URL refererUrl, String pageHost) { if (refererUrl == null) { return null; } return parse(refererUrl.getProtocol(), refererUrl.getHost(), refererUrl.getPath(), refererUrl.getQuery(), pageHost); } private Referer parse(String scheme, String host, String path, String query, String pageHost) { return parse(scheme, host, path, query, pageHost, Collections.<String>emptyList()); } private Referer parse(String scheme, String host, String path, String query, String pageHost, List<String> internalDomains) { if (scheme == null || (!scheme.equals("http") && !scheme.equals("https"))) { return null; } // Internal link if hosts match exactly // TODO: would also be nice to: // 1. Support a list of other hosts which count as internal // 2. Have an algo for stripping subdomains before checking match if (host == null) { return null; // Not a valid URL } if (host.equals(pageHost)) { return new Referer(Medium.INTERNAL, null, null, null, null); } for (String s : internalDomains) { if (s.trim().equals(host)) { return new Referer(Medium.INTERNAL, null, null, null, null); } } // Try to lookup our referer. First check with paths, then without. // This is the safest way of handling lookups RefererLookup referer = lookupReferer(host, path, true); if (referer == null) { referer = lookupReferer(host, path, false); } if (referer == null) { final Referer refererWithUtm = parseUtmParameters(query); if (refererWithUtm == null) { return new Referer(Medium.UNKNOWN, null, null, null, null); // Unknown referer, nothing more to do } else { return refererWithUtm; } } else { // Potentially add a search term final String term = (referer.medium == Medium.SEARCH) ? extractSearchTerm(query, referer.parameters) : null; return new Referer(referer.medium, referer.source, term, null, null); } } private Referer parseUtmParameters(String query) { if (query != null) { Map<String, String> paramsMap = buildParamsMap(query); String source = paramsMap.get("utm_source"); String medium = paramsMap.get("utm_medium"); String term = paramsMap.get("utm_term"); String campaign = paramsMap.get("utm_campaign"); String content = paramsMap.get("utm_content"); return new Referer(medium, source, term, campaign, content); } else { return null; } } private Map<String, String> buildParamsMap(String query) { List<NameValuePair> params = null; try { params = URLEncodedUtils.parse(new URI(HTTP_LOCALHOST + query), "UTF-8"); } catch (URISyntaxException e) { e.printStackTrace(); } Map<String, String> paramsMap = new HashMap<String, String>(); for (NameValuePair param : params) { paramsMap.put(param.getName(), param.getValue()); } return paramsMap; } /** * Recursive function to lookup a host (or partial host) * in our referers map. * <p/> * First check the host, then the host+full path, then the host+ * one-level path. * <p/> * If not found, remove one subdomain-level off the front * of the host and try again. * * @param refererHost The host of the current page * @param refererPath The path to the current page * @param includePath Whether to include the path in the lookup * @return a RefererLookup object populated with the given * referer, or null if not found */ private RefererLookup lookupReferer(String refererHost, String refererPath, Boolean includePath) { // Check if domain+full path matches, e.g. for apollo.lv/portal/search/ RefererLookup referer = (includePath) ? referers.get(refererHost + refererPath) : referers.get(refererHost); // Check if domain+one-level path matches, e.g. for orange.fr/webmail/fr_FR/read.html (in our YAML it's orange.fr/webmail) if (includePath && referer == null) { final String[] pathElements = refererPath.split("/"); if (pathElements.length > 1) { referer = referers.get(refererHost + "/" + pathElements[1]); } } if (referer == null) { final int idx = refererHost.indexOf('.'); if (idx == -1) { return null; // No "."? Let's quit. } else { return lookupReferer(refererHost.substring(idx + 1), refererPath, includePath); // Recurse } } else { return referer; } } private String extractSearchTerm(String query, List<String> possibleParameters) { List<NameValuePair> params; try { params = URLEncodedUtils.parse(new URI("http://localhost?" + query), "UTF-8"); // params = URLEncodedUtils.parse(query, Charset.forName("UTF-8")); because https://github.com/snowplow/referer-parser/issues/76 } catch (IllegalArgumentException iae) { return null; } catch (URISyntaxException use) { // For new URI return null; } for (NameValuePair pair : params) { final String name = pair.getName(); final String value = pair.getValue(); if (possibleParameters.contains(name)) { return value; } } return null; } /** * Builds the map of hosts to referers from the * input YAML file. * * @param referersYaml An InputStream containing the * referers database in YAML format. * @return a Map where the key is the hostname of each * referer and the value (RefererLookup) * contains all known info about this referer */ private Map<String, RefererLookup> loadReferers(InputStream referersYaml) throws RefererParserException { Yaml yaml = new Yaml(new SafeConstructor()); Map<String, Map<String, Map>> rawReferers = (Map<String, Map<String, Map>>) yaml.load(referersYaml); // This will store all of our referers Map<String, RefererLookup> referers = new HashMap<String, RefererLookup>(); // Outer loop is all referers under a given medium for (Map.Entry<String, Map<String, Map>> mediumReferers : rawReferers.entrySet()) { Medium medium = Medium.fromString(mediumReferers.getKey()); // Inner loop is individual referers for (Map.Entry<String, Map> referer : mediumReferers.getValue().entrySet()) { String sourceName = referer.getKey(); Map<String, List<String>> refererMap = referer.getValue(); // Validate List<String> parameters = refererMap.get("parameters"); if (medium == Medium.SEARCH) { if (parameters == null) { throw new RefererParserException( "No parameters found for search referer '" + sourceName + "'"); } } else { if (parameters != null) { throw new RefererParserException( "Parameters not supported for non-search referer '" + sourceName + "'"); } } List<String> domains = refererMap.get("domains"); if (domains == null) { throw new RefererParserException("No domains found for referer '" + sourceName + "'"); } // Our hash needs referer domain as the // key, so let's expand for (String domain : domains) { if (referers.containsValue(domain)) { throw new RefererParserException("Duplicate of domain '" + domain + "' found"); } referers.put(domain, new RefererLookup(medium, sourceName, parameters)); } } } return referers; } }