Java tutorial
/** * Copyright (c) 2016 Modus Operandi, Inc. * * This is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, either version 3 of the License, or any later version. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more * details. A copy of the GNU Lesser General Public License is distributed along * with this program and can be found at * <http://www.gnu.org/licenses/lgpl.html>. */ /* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ package com.modusoperandi.dragonfly.widgets.map; import com.spatial4j.core.context.SpatialContext; import com.spatial4j.core.io.GeohashUtils; import com.spatial4j.core.shape.Point; import java.io.IOException; import java.math.BigDecimal; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang.StringEscapeUtils; import org.apache.wicket.Component; import org.apache.wicket.ajax.AbstractDefaultAjaxBehavior; import org.apache.wicket.ajax.AjaxRequestTarget; import org.apache.wicket.markup.head.IHeaderResponse; import org.apache.wicket.markup.head.JavaScriptHeaderItem; import org.apache.wicket.markup.html.WebPage; import org.apache.wicket.request.cycle.RequestCycle; import org.apache.wicket.request.mapper.parameter.INamedParameters; import org.apache.wicket.request.mapper.parameter.PageParameters; import org.apache.wicket.util.time.Time; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; /** * * @author dgriffith */ public class MapWidgetBMPage extends WebPage { private transient static Client esClient = null; private static final Logger LOGGER = Logger.getLogger(MapWidgetBMPage.class.getName()); private static final int MAX_GEOHASH_GRID = 20000; private static final int MAX_HITS_TO_SHOW = 2000; private static final int MAX_POPUP_ROW_LENGTH = 50; private static final int ONE_DAY_IN_SECONDS = 86400; private static final long serialVersionUID = 1L; public static void onDestroy() { if (esClient != null) { esClient.close(); esClient = null; } } private static String addGeojsonToMap(final String indexName, final String geojsonFieldName) { final StringBuffer js = new StringBuffer(); js.append("geoJsonLayer.clearLayers();"); final SearchRequestBuilder search = getEsClient().prepareSearch(indexName); search.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); search.setQuery(QueryBuilders.matchAllQuery()); search.setSize(MAX_HITS_TO_SHOW); final SearchResponse results = search.execute().actionGet(); for (final SearchHit hit : results.getHits()) { final String valueStr = (String) hit.getSource().get(geojsonFieldName); if (valueStr != null) { js.append("geoJsonLayer.addData("); js.append(valueStr); js.append(");"); } } return js.toString(); } private static String addWmsToMap(final String tileUrl, final String layers, final String format, final String attribution, final String name) { final StringBuffer tileParams = new StringBuffer(); tileParams.append("{"); tileParams.append("layers: '").append(layers).append("',"); tileParams.append("format: '").append(format).append("',"); tileParams.append("attribution: '").append(attribution).append("',"); tileParams.append("transparent: true"); tileParams.append("}"); final StringBuffer js = new StringBuffer(); js.append("var wms = L.tileLayer.wms('").append(tileUrl).append("',").append(tileParams.toString()) .append(");"); js.append("controls.addOverlay(wms,'").append(name).append("');"); js.append("wms.addTo(map);"); return (js.toString()); } /** * Retrieves the Elasticsearch index mapping for the specified index * * @param indexName * the index name * @return the index mapping */ private static ImmutableOpenMap<String, MappingMetaData> esGetMappings(final String indexName) { final ClusterStateResponse clusterStateResponse = getEsClient().admin().cluster().prepareState().execute() .actionGet(); return clusterStateResponse.getState().getMetaData().index(indexName).getMappings(); } private static void generateHeatmapJs(final SearchResponse results, final StringBuffer heatmapJs) { heatmapJs.append("heatmapLayer.setData({data:["); long min = Long.MAX_VALUE; long max = 2; // The minimum max boolean firstTime = true; final GeoHashGrid geoGrid = results.getAggregations().get("geohashgrid"); for (final GeoHashGrid.Bucket cell : geoGrid.getBuckets()) { final String geohash = cell.getKeyAsString(); final long bucketCount = cell.getDocCount(); final Point hashCenter = GeohashUtils.decode(geohash, SpatialContext.GEO); if (!firstTime) { heatmapJs.append(","); } firstTime = false; heatmapJs.append("{lat: "); heatmapJs.append(hashCenter.getY()); heatmapJs.append(", lng: "); heatmapJs.append(hashCenter.getX()); heatmapJs.append(", count: "); heatmapJs.append(Long.toString(bucketCount)); heatmapJs.append("}"); if (bucketCount > max) { max = bucketCount; } if (bucketCount < min) { min = bucketCount; } } heatmapJs.append("],"); heatmapJs.append("max: "); heatmapJs.append(Long.toString(max)); heatmapJs.append(","); heatmapJs.append("min: "); heatmapJs.append(Long.toString(min - 1)); heatmapJs.append(","); heatmapJs.append("});"); } private static Client getEsClient() { // //////////// // Remote if (esClient == null) { try { final Settings settings = Settings.settingsBuilder() .put("node.name", "DRAGONFLY_" + Long.toString(Thread.currentThread().getId())) .put("cluster.name", "DRAGONFLY").build(); esClient = new TransportClient.Builder().settings(settings).build().addTransportAddress( new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300)); } catch (UnknownHostException ex) { LOGGER.log(Level.SEVERE, null, ex); } } return esClient; } /** * Checks if the specified field is a link based on the Elasticsearch index mapping metadata * * @param field * the field to check * @param meta * the index mapping metadata * @return true, if is link */ private static boolean isLink(final String fieldName, final Map<String, Object> meta) { String fieldInterface = ""; if (meta != null) { for (final Map.Entry<String, Object> entry : meta.entrySet()) { if (entry.getKey().equals("Fields")) { // This is a field descriptor try { @SuppressWarnings("unchecked") final ArrayList<Map<String, Object>> descriptors = (ArrayList<Map<String, Object>>) entry .getValue(); for (final Map<String, Object> field : descriptors) { if (field.get("Name").equals(fieldName)) { fieldInterface = (String) field.get("Extended Type"); } } } catch (final Exception e) { // The meta data is a JSON object created by a query widget converted to a map of maps. // If the conventions were not followed, parsing will fail in many possible ways. // If parsing fails, this field will be treated as a normal field LOGGER.log(Level.SEVERE, e.getMessage(), e); } } } } return fieldInterface.equals("URL"); } private static String wrapAjaxJs(final String location, final String js) { final StringBuffer wrappedJs = new StringBuffer(); wrappedJs.append("try {"); wrappedJs.append(js); wrappedJs.append("} catch(err) { alert('ERROR at ").append(location).append(": ' + err); }"); return wrappedJs.toString(); } private final String defaultTileServerConnection = "baseLayer = L.tileLayer('http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {\n" + "attribution: '© <a href=\"http://openstreetmap.org\">OpenStreetMap</a> contributors, <a href=\"http://creativecommons.org/licenses/by-sa/2.0/\">CC-BY-SA</a>'\n" + "});\n"; private double east = 180; private String filterTerm; private String geojsonField; private String geojsonIndex; private String hmRadius; private String hmTransparency; private String mapIndex; private String mapLocationField; private double north = 90; private double south = -90; private transient ImmutableOpenMap<String, MappingMetaData> typeMappings; private double west = -180; private String wmsAttribution; private String wmsFormat; private String wmsLayers; private String wmsName; private String wmsTileUrl; private int zoomLevel = 3; private final String tileServerConnection; private final Map<String, String> defaultParameterValues = new HashMap<>(); /** * Instantiates a new map widget page. * * @param parameters */ public MapWidgetBMPage(final PageParameters parameters) { // ///////////////////////////// // Set User Supplied Values tileServerConnection = parameters.get("tileUrl").toString(defaultTileServerConnection); // ///////////////////////////// // Elasticsearch getEsClient(); // ///////////////////////////// // // Initialize data from URL add(new AbstractDefaultAjaxBehavior() { private static final long serialVersionUID = 1L; @Override public void renderHead(Component component, IHeaderResponse response) { response.render(JavaScriptHeaderItem.forScript("function initializeMapData() { " + wrapAjaxJs("initializeMapData()", addDataToMap(parameters)) + "}", null)); } @Override protected void respond(AjaxRequestTarget target) { throw new UnsupportedOperationException("Not supported yet."); // To change body of generated methods, // choose Tools | Templates. } }); defaultParameterValues.put(NAME, ""); defaultParameterValues.put(ATTIBUTION, ""); defaultParameterValues.put(FORMAT, ""); defaultParameterValues.put(LAYERS, ""); defaultParameterValues.put(TILE_URL, ""); defaultParameterValues.put(GEOJSON_FIELD, ""); defaultParameterValues.put(WEST, "-180.0"); defaultParameterValues.put(EAST, "180.0"); defaultParameterValues.put(SOUTH, "-90.0"); defaultParameterValues.put(NORTH, "90.0"); defaultParameterValues.put(FILTER, ""); defaultParameterValues.put(ZOOM_LEVEL, "3"); defaultParameterValues.put(HM_RADIUS, "20"); defaultParameterValues.put(HM_TRANSPARENCY, "0.3"); defaultParameterValues.put(LOCATION, ""); defaultParameterValues.put(INDEX, ""); defaultParameterValues.put(TYPE, ""); defaultParameterValues.put(ZOOM_VIEW, "3"); defaultParameterValues.put(CENTER, "[0.0,0.0]"); } @Override public void renderHead(final IHeaderResponse response) { super.renderHead(response); response.render(JavaScriptHeaderItem.forScript(tileServerConnection, null)); } private String addDataToHeatmap(final String indexName, final String location) { final StringBuffer heatmapJs = new StringBuffer(); final StringBuffer markerJs = new StringBuffer(); SearchResponse results; if ((indexName != null) && !indexName.isEmpty()) { try { final SearchRequestBuilder aggSearch = getEsClient().prepareSearch(indexName); aggSearch.addAggregation( geohashGrid("geohashgrid").field(location).precision(zoomLevel).size(MAX_GEOHASH_GRID)); aggSearch.setSearchType(SearchType.COUNT); QueryBuilder query = QueryBuilders.matchAllQuery(); if ((filterTerm != null) && !filterTerm.isEmpty()) { query = QueryBuilders.matchQuery("_all", filterTerm); } if ((south > -90) && (north < 90) && (east < 180) && (west > -180)) { final QueryBuilder geoQuery = QueryBuilders.geoBoundingBoxQuery(location).topLeft(north, west) .bottomRight(south, east); aggSearch.setQuery(QueryBuilders.boolQuery().must(query).must(geoQuery)); } else { aggSearch.setQuery(query); } results = aggSearch.execute().actionGet(); generateHeatmapJs(results, heatmapJs); if (results.getHits().getTotalHits() < MAX_HITS_TO_SHOW) { final SearchRequestBuilder search = getEsClient().prepareSearch(indexName); search.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); search.setSize(MAX_HITS_TO_SHOW); if ((south > -90) && (north < 90) && (east < 180) && (west > -180)) { final QueryBuilder geoQuery = QueryBuilders.geoBoundingBoxQuery(location) .topLeft(north, west).bottomRight(south, east); aggSearch.setQuery(QueryBuilders.boolQuery().must(query).must(geoQuery)); } else { aggSearch.setQuery(query); } generateMarkerJs(search.execute().actionGet(), markerJs, location); } } catch (final ElasticsearchException e) { LOGGER.log(Level.SEVERE, "Invalid query - return empty results.", e); } } return heatmapJs.toString() + markerJs.toString(); } private String getWithDefault(INamedParameters.NamedPair parameter) { String returnValue = parameter.getValue(); if (returnValue == null) { returnValue = defaultParameterValues.get(parameter.getKey()); if (returnValue == null) { returnValue = "0"; } } return returnValue; } private String addDataToMap(final PageParameters parameters) { Map<String, String> configuration = new HashMap<>(); parameters.getAllNamed().stream().forEach((parameter) -> { configuration.put(parameter.getKey(), getWithDefault(parameter)); }); return addDataToMap(configuration); } private String addDataToMap(final Map<String, String> configuration) { StringBuilder js = new StringBuilder(); js.append(setMapViewableArea(configuration)); final String dataType = configuration.get(TYPE); if (dataType != null) { switch (dataType) { case "Query Results": mapIndex = configuration.get(INDEX); mapLocationField = configuration.get(LOCATION); hmTransparency = configuration.get(HM_TRANSPARENCY); hmRadius = configuration.get(HM_RADIUS); zoomLevel = Integer.parseInt(configuration.get(ZOOM_LEVEL)); filterTerm = configuration.get(FILTER); north = Double.parseDouble(configuration.get(NORTH)); south = Double.parseDouble(configuration.get(SOUTH)); east = Double.parseDouble(configuration.get(EAST)); west = Double.parseDouble(configuration.get(WEST)); typeMappings = esGetMappings(mapIndex); js.append(addHeatmapToMap()); js.append(addDataToHeatmap(mapIndex, mapLocationField)); break; case "GeoJSON": geojsonIndex = configuration.get(INDEX); geojsonField = configuration.get(GEOJSON_FIELD); js.append(addGeojsonToMap(geojsonIndex, geojsonField)); break; case "WMS": wmsTileUrl = configuration.get(TILE_URL); wmsLayers = configuration.get(LAYERS); wmsFormat = configuration.get(FORMAT); wmsAttribution = configuration.get(ATTIBUTION); wmsName = configuration.get(NAME); js.append(addWmsToMap(wmsTileUrl, wmsLayers, wmsFormat, wmsAttribution, wmsName)); break; default: LOGGER.log(Level.SEVERE, "Unknown dataType {0}", dataType); break; } } return js.toString(); } private static final String NAME = "Name"; private static final String ATTIBUTION = "Attibution"; private static final String FORMAT = "Format"; private static final String LAYERS = "Layers"; private static final String TILE_URL = "TileURL"; private static final String GEOJSON_FIELD = "GeojsonField"; private static final String WEST = "west"; private static final String EAST = "east"; private static final String SOUTH = "south"; private static final String NORTH = "north"; private static final String FILTER = "filter"; private static final String ZOOM_LEVEL = "zoomLevel"; private static final String HM_RADIUS = "HmRadius"; private static final String HM_TRANSPARENCY = "HmTransparency"; private static final String LOCATION = "Location"; private static final String INDEX = "Index"; private static final String TYPE = "Type"; private static final String ZOOM_VIEW = "zoomView"; private static final String CENTER = "center"; private String setMapViewableArea(final Map<String, String> configuration) { final StringBuilder js = new StringBuilder(); js.append("map.setView(L.latLng(").append(configuration.get(CENTER)).append("), ") .append(configuration.get(ZOOM_VIEW)).append(");"); return js.toString(); } private String addHeatmapToMap() { final StringBuffer js = new StringBuffer(); js.append("if(!(typeof heatmapLayer === 'undefined') && (heatmapLayer != null)) {"); js.append(" map.removeLayer(heatmapLayer);"); js.append(" controls.removeLayer(heatmapLayer);"); js.append(" markerLayer.clearLayers(); "); js.append("}"); js.append("heatmapLayer = new HeatmapOverlay({"); js.append(" radius: ").append(hmRadius).append(","); js.append(" opacity: ").append(hmTransparency).append(","); js.append(" scaleRadius: false,"); js.append(" useLocalExtrema: false,"); js.append(" latField: 'lat',"); js.append(" lngField: 'lng',"); js.append(" valueField: 'count'"); js.append(" });"); js.append("map.addLayer(heatmapLayer);"); js.append("controls.addOverlay(heatmapLayer,'Heat Map');"); return js.toString(); } @SuppressWarnings("unchecked") private void generateMarkerJs(final SearchResponse results, final StringBuffer markerJs, final String locationField) { for (final SearchHit hit : results.getHits().getHits()) { final GeoPoint location = new GeoPoint(); location.resetFromString(hit.getSource().get(locationField).toString()); final BigDecimal lat = new BigDecimal(location.lat()).setScale(11, BigDecimal.ROUND_HALF_DOWN); final BigDecimal lon = new BigDecimal(location.lon()).setScale(11, BigDecimal.ROUND_HALF_DOWN); markerJs.append("markerLayer.addLayer(L.marker(["); markerJs.append(Double.toString(lat.doubleValue())); markerJs.append(","); markerJs.append(Double.toString(lon.doubleValue())); final StringBuffer markerText = new StringBuffer(); hit.getSource().entrySet().stream().forEach((fields) -> { String markerValue = (String) fields.getValue(); try { markerText.append("<b>"); markerText.append(fields.getKey()); markerText.append(" :</b> "); if (isLink(fields.getKey(), ((Map<String, Object>) typeMappings.get(hit.getType()) .getSourceAsMap().get("_meta")))) { markerText.append("<a href=\""); markerText.append(markerValue); markerText.append("\">"); markerText.append(markerValue); markerText.append("</a>"); } else { if (markerValue.length() > MAX_POPUP_ROW_LENGTH) { markerValue = markerValue.substring(0, MAX_POPUP_ROW_LENGTH) + " ..."; } markerText.append(StringEscapeUtils.escapeHtml(markerValue).replace('\n', ' ')); } markerText.append("<br>"); } catch (final IOException e) { LOGGER.log(Level.SEVERE, "Cannot access type mappings", e); } catch (final Exception e2) { LOGGER.log(Level.SEVERE, e2.getMessage(), e2); } }); markerJs.append("]).bindPopup('"); markerJs.append(markerText.toString().replace("'", " ")); markerJs.append("', {maxWidth: 500}));\n"); } } /* * (non-Javadoc) * * @see org.apache.wicket.markup.html.WebPage#onAfterRender() */ @Override protected void onAfterRender() { super.onAfterRender(); // Increase the HTTP session timeout to 1 day ((HttpServletRequest) RequestCycle.get().getRequest().getContainerRequest()).getSession() .setMaxInactiveInterval(ONE_DAY_IN_SECONDS); } protected @Override void setHeaders(final org.apache.wicket.request.http.WebResponse response) { super.setHeaders(response); response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate"); // HTTP 1.1. response.setHeader("Pragma", "no-cache"); // HTTP 1.0. response.setDateHeader("Expires", Time.START_OF_UNIX_TIME); // Proxies. } }