Java tutorial
/** * Copyright (c) 2002-2011 "Neo Technology," * Network Engine for Objects in Lund AB [http://neotechnology.com] * * This file is part of Neo4j. * * Neo4j is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package gima.neo4j.testsuite.osmcheck; import static java.util.Arrays.asList; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; import java.nio.charset.Charset; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import org.apache.commons.collections.MapUtils; import org.geotools.referencing.datum.DefaultEllipsoid; import org.neo4j.collections.rtree.Envelope; import org.neo4j.collections.rtree.Listener; import org.neo4j.collections.rtree.NullListener; import org.neo4j.gis.spatial.Constants; import org.neo4j.gis.spatial.SpatialDatabaseService; import org.neo4j.gis.spatial.osm.OSMDataset; import org.neo4j.gis.spatial.osm.OSMGeometryEncoder; import org.neo4j.gis.spatial.osm.OSMLayer; import org.neo4j.gis.spatial.osm.OSMRelation; import org.neo4j.gis.spatial.osm.RoadDirection; import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.PropertyContainer; import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.ReturnableEvaluator; import org.neo4j.graphdb.StopEvaluator; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.Traverser; import org.neo4j.graphdb.Traverser.Order; import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.graphdb.index.Index; import org.neo4j.graphdb.index.IndexHits; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider; import org.neo4j.kernel.EmbeddedGraphDatabase; import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import org.neo4j.kernel.impl.batchinsert.SimpleRelationship; public class OSMImporter implements Constants { public static DefaultEllipsoid WGS84 = DefaultEllipsoid.WGS84; public static String INDEX_NAME_CHANGESET = "changeset"; public static String INDEX_NAME_USER = "user"; public static String INDEX_NAME_NODE = "node"; public static String INDEX_NAME_WAY = "node"; protected boolean nodesProcessingFinished = false; private String layerName; private StatsManager stats = new StatsManager(); private long osm_dataset = -1; private Listener monitor; private Charset charset = Charset.defaultCharset(); private static class TagStats { private String name; private int count = 0; private HashMap<String, Integer> stats = new HashMap<String, Integer>(); TagStats(String name) { this.name = name; } int add(String key) { count++; if (stats.containsKey(key)) { int num = stats.get(key); stats.put(key, ++num); return num; } else { stats.put(key, 1); return 1; } } /** * Return only reasonably commonly used tags. * * @return */ public String[] getTags() { if (stats.size() > 0) { int threshold = count / (stats.size() * 20); ArrayList<String> tags = new ArrayList<String>(); for (String key : stats.keySet()) { if (key.equals("waterway")) { System.out.println("debug[" + key + "]: " + stats.get(key)); } if (stats.get(key) > threshold) { tags.add(key); } } Collections.sort(tags); return tags.toArray(new String[tags.size()]); } else { return new String[0]; } } public String toString() { return "TagStats[" + name + "]: " + asList(getTags()); } } private static class StatsManager { private HashMap<String, TagStats> tagStats = new HashMap<String, TagStats>(); private HashMap<Integer, Integer> geomStats = new HashMap<Integer, Integer>(); ; protected TagStats getTagStats(String type) { if (!tagStats.containsKey(type)) { tagStats.put(type, new TagStats(type)); } return tagStats.get(type); } protected int addToTagStats(String type, String key) { getTagStats("all").add(key); return getTagStats(type).add(key); } protected int addToTagStats(String type, Collection<String> keys) { int count = 0; for (String key : keys) { count += addToTagStats(type, key); } return count; } protected void printTagStats() { System.out.println("Tag statistics for " + tagStats.size() + " types:"); for (String key : tagStats.keySet()) { TagStats stats = tagStats.get(key); System.out.println("\t" + key + ": " + stats); } } protected void addGeomStats(Node geomNode) { if (geomNode != null) { addGeomStats((Integer) geomNode.getProperty(PROP_TYPE, null)); } } protected void addGeomStats(Integer geom) { Integer count = geomStats.get(geom); geomStats.put(geom, count == null ? 1 : count + 1); } protected void dumpGeomStats() { System.out.println("Geometry statistics for " + geomStats.size() + " geometry types:"); for (Object key : geomStats.keySet()) { Integer count = geomStats.get(key); System.out.println( "\t" + SpatialDatabaseService.convertGeometryTypeToName((Integer) key) + ": " + count); } geomStats.clear(); } } public OSMImporter(String layerName) { this(layerName, null); } public OSMImporter(String layerName, Listener monitor) { this.layerName = layerName; if (monitor == null) { monitor = new NullListener(); } this.monitor = monitor; } public void reIndex(GraphDatabaseService database) { reIndex(database, 10000, true, false); } public void reIndex(GraphDatabaseService database, int commitInterval) { reIndex(database, commitInterval, true, false); } public void reIndex(GraphDatabaseService database, int commitInterval, boolean includePoints, boolean includeRelations) { if (commitInterval < 1) { throw new IllegalArgumentException("commitInterval must be >= 1"); } System.out.println( "Re-indexing with GraphDatabaseService: " + database + " (class: " + database.getClass() + ")"); setLogContext("Index"); SpatialDatabaseService spatialDatabase = new SpatialDatabaseService(database); OSMLayer layer = (OSMLayer) spatialDatabase.getOrCreateLayer(layerName, OSMGeometryEncoder.class, OSMLayer.class); // TODO: The next line creates the relationship between the dataset and // layer, but this seems more like a side-effect and should be done // explicitly OSMDataset dataset = layer.getDataset(osm_dataset); //layer.clear(); // clear the index without destroying underlying data long startTime = System.currentTimeMillis(); Traverser traverser = database.getNodeById(osm_dataset).traverse(Order.DEPTH_FIRST, StopEvaluator.END_OF_GRAPH, ReturnableEvaluator.ALL_BUT_START_NODE, OSMRelation.WAYS, Direction.OUTGOING, OSMRelation.NEXT, Direction.OUTGOING); Transaction tx = database.beginTx(); boolean useWays = false; int count = 0; try { layer.setExtraPropertyNames(stats.getTagStats("all").getTags()); if (useWays) { beginProgressMonitor(dataset.getWayCount()); for (Node way : traverser) { updateProgressMonitor(count); incrLogContext(); stats.addGeomStats(layer.addWay(way, true)); if (includePoints) { Node first = way.getSingleRelationship(OSMRelation.FIRST_NODE, Direction.OUTGOING) .getEndNode(); for (Node proxy : first.traverse(Order.DEPTH_FIRST, StopEvaluator.END_OF_GRAPH, ReturnableEvaluator.ALL, OSMRelation.NEXT, Direction.OUTGOING)) { Node node = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING) .getEndNode(); stats.addGeomStats(layer.addWay(node, true)); } } if (++count % commitInterval == 0) { tx.success(); tx.finish(); tx = database.beginTx(); } } // TODO ask charset to user? } else { beginProgressMonitor(dataset.getChangesetCount()); for (Node changeset : dataset.getAllChangesetNodes()) { updateProgressMonitor(count); incrLogContext(); for (Relationship rel : changeset.getRelationships(OSMRelation.CHANGESET, Direction.INCOMING)) { stats.addGeomStats(layer.addWay(rel.getStartNode(), true)); } if (++count % commitInterval == 0) { tx.success(); tx.finish(); tx = database.beginTx(); } } // TODO ask charset to user? } tx.success(); } finally { endProgressMonitor(); tx.finish(); } long stopTime = System.currentTimeMillis(); log("info | Re-indexing elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000.0)); stats.dumpGeomStats(); } private static class GeometryMetaData { private Envelope bbox = new Envelope(); private int vertices = 0; private int geometry = -1; public GeometryMetaData(int type) { this.geometry = type; } public int getGeometryType() { return geometry; } public void expandToIncludePoint(double[] location) { bbox.expandToInclude(location[0], location[1]); vertices++; geometry = -1; } public void expandToIncludeBBox(Map<String, Object> nodeProps) { double[] sbb = (double[]) nodeProps.get("bbox"); bbox.expandToInclude(sbb[0], sbb[2]); bbox.expandToInclude(sbb[1], sbb[3]); vertices += (Integer) nodeProps.get("vertices"); } public void checkSupportedGeometry(Integer memGType) { if ((memGType == null || memGType != GTYPE_LINESTRING) && geometry != GTYPE_POLYGON) { geometry = -1; } } public void setPolygon() { geometry = GTYPE_POLYGON; } public boolean isValid() { return geometry > 0; } public int getVertices() { return vertices; } public Envelope getBBox() { return bbox; } } private static abstract class OSMWriter<T> { protected StatsManager statsManager; protected OSMImporter osmImporter; protected T osm_dataset; private OSMWriter(StatsManager statsManager, OSMImporter osmImporter) { this.statsManager = statsManager; this.osmImporter = osmImporter; } public static OSMWriter<Long> fromBatchInserter(BatchInserter batchInserter, StatsManager stats, OSMImporter osmImporter) { return new OSMBatchWriter(batchInserter, stats, osmImporter); } public static OSMWriter<Node> fromGraphDatabase(GraphDatabaseService graphDb, StatsManager stats, OSMImporter osmImporter, int txInterval) { return new OSMGraphWriter(graphDb, stats, osmImporter, txInterval); } protected abstract T getOrCreateNode(String name, String type, T parent, RelationshipType relType); protected abstract T getOrCreateOSMDataset(String name); protected abstract void setDatasetProperties(Map<String, Object> extractProperties); protected abstract void addNodeTags(T node, LinkedHashMap<String, Object> tags, String type); protected abstract void addNodeGeometry(T node, int gtype, Envelope bbox, int vertices); protected abstract T addNode(String name, Map<String, Object> properties, String indexKey); protected abstract void createRelationship(T from, T to, RelationshipType relType, LinkedHashMap<String, Object> relProps); protected void createRelationship(T from, T to, RelationshipType relType) { createRelationship(from, to, relType, null); } protected HashMap<String, Integer> stats = new HashMap<String, Integer>(); protected HashMap<String, LogCounter> nodeFindStats = new HashMap<String, LogCounter>(); protected long logTime = 0; protected long findTime = 0; protected long firstFindTime = 0; protected long lastFindTime = 0; protected long firstLogTime = 0; protected static int foundNodes = 0; protected static int createdNodes = 0; protected int foundOSMNodes = 0; protected int missingUserCount = 0; protected void logMissingUser(Map<String, Object> nodeProps) { if (missingUserCount++ < 10) { System.err.println("Missing user or uid: " + nodeProps.toString()); } } private class LogCounter { private long count = 0; private long totalTime = 0; } protected void logNodeFoundFrom(String key) { LogCounter counter = nodeFindStats.get(key); if (counter == null) { counter = new LogCounter(); nodeFindStats.put(key, counter); } counter.count++; foundOSMNodes++; long currentTime = System.currentTimeMillis(); if (lastFindTime > 0) { counter.totalTime += currentTime - lastFindTime; } lastFindTime = currentTime; logNodesFound(currentTime); } protected void logNodesFound(long currentTime) { if (firstFindTime == 0) { firstFindTime = currentTime; findTime = currentTime; } if (currentTime == 0 || currentTime - findTime > 1432) { int duration = 0; if (currentTime > 0) { duration = (int) ((currentTime - firstFindTime) / 1000); } System.out.println(new Date(currentTime) + ": Found " + foundOSMNodes + " nodes during " + duration + "s way creation: "); for (String type : nodeFindStats.keySet()) { LogCounter found = nodeFindStats.get(type); double rate = 0.0f; if (found.totalTime > 0) { rate = (1000.0 * (float) found.count / (float) found.totalTime); } System.out.println("\t" + type + ": \t" + found.count + "/" + (found.totalTime / 1000) + "s" + " \t(" + rate + " nodes/second)"); } findTime = currentTime; } } protected void logNodeAddition(LinkedHashMap<String, Object> tags, String type) { Integer count = stats.get(type); if (count == null) { count = 1; } else { count++; } stats.put(type, count); long currentTime = System.currentTimeMillis(); if (firstLogTime == 0) { firstLogTime = currentTime; logTime = currentTime; } if (currentTime - logTime > 1432) { System.out.println(new Date(currentTime) + ": Saving " + type + " " + count + " \t(" + (1000.0 * (float) count / (float) (currentTime - firstLogTime)) + " " + type + "/second)"); logTime = currentTime; } } void describeLoaded() { logNodesFound(0); for (String type : new String[] { "node", "way", "relation" }) { Integer count = stats.get(type); if (count != null) { System.out.println("Loaded " + count + " " + type + "s"); } } } protected abstract long getDatasetId(); private int missingNodeCount = 0; private void missingNode(long ndRef) { if (missingNodeCount++ < 10) { osmImporter.error("Cannot find node for osm-id " + ndRef); } } private void describeMissing() { if (missingNodeCount > 0) { osmImporter.error("When processing the ways, there were " + missingNodeCount + " missing nodes"); } if (missingMemberCount > 0) { osmImporter.error( "When processing the relations, there were " + missingMemberCount + " missing members"); } } private int missingMemberCount = 0; private void missingMember(String description) { if (missingMemberCount++ < 10) { osmImporter.error("Cannot find member: " + description); } } protected T currentNode = null; protected T prev_way = null; protected T prev_relation = null; protected int nodeCount = 0; protected int poiCount = 0; protected int wayCount = 0; protected int relationCount = 0; protected int userCount = 0; protected int changesetCount = 0; /** * Add the BBox metadata to the dataset * @param bboxProperties */ protected void addOSMBBox(Map<String, Object> bboxProperties) { T bbox = addNode("bbox", bboxProperties, null); createRelationship(osm_dataset, bbox, OSMRelation.BBOX); } /** * Create a new OSM node from the specified attributes (including * location, user, changeset). The node is stored in the currentNode * field, so that it can be used in the subsequent call to * addOSMNodeTags after we close the XML tag for OSM nodes. * * @param nodeProps * HashMap of attributes for the OSM-node */ protected void createOSMNode(Map<String, Object> nodeProps) { T changesetNode = getChangesetNode(nodeProps); currentNode = addNode("node", nodeProps, "node_osm_id"); createRelationship(currentNode, changesetNode, OSMRelation.CHANGESET); nodeCount++; } ArrayList<String> pointTags = StyleReader.readNodes(); ArrayList<String> lineTags = StyleReader.readWays(); ArrayList<String> polyTags = StyleReader.readPolyCandidates(); private void addOSMNodeTags(boolean allPoints, LinkedHashMap<String, Object> currentNodeTags) { currentNodeTags.remove("created_by"); // redundant information // Nodes with tags get added to the index as point geometries if (inList(pointTags, currentNodeTags)) { if (allPoints || currentNodeTags.size() > 0) { Map<String, Object> nodeProps = getNodeProperties(currentNode); Envelope bbox = new Envelope(); double[] location = new double[] { (Double) nodeProps.get("lon"), (Double) nodeProps.get("lat") }; bbox.expandToInclude(location[0], location[1]); addNodeGeometry(currentNode, GTYPE_POINT, bbox, 1); poiCount++; } } addNodeTags(currentNode, currentNodeTags, "node"); } private boolean inList(ArrayList<String> list, LinkedHashMap<String, Object> currentNodeTags) { Iterator it = list.iterator(); while (it.hasNext()) { String tag = (String) it.next(); if (currentNodeTags.containsKey(tag)) { return true; } } return false; } protected void debugNodeWithId(T node, String idName, long[] idValues) { Map<String, Object> nodeProperties = getNodeProperties(node); String node_osm_id = nodeProperties.get(idName).toString(); for (long idValue : idValues) { if (node_osm_id.equals(Long.toString(idValue))) { System.out.println("Debug node: " + node_osm_id); } } } protected void createOSMWay(Map<String, Object> wayProperties, ArrayList<Long> wayNodes, LinkedHashMap<String, Object> wayTags) { RoadDirection direction = isOneway(wayTags); String name = (String) wayTags.get("name"); int geometry = GTYPE_LINESTRING; boolean isRoad = wayTags.containsKey("highway"); boolean nomineeLine = inList(lineTags, wayTags); boolean nomineePoly = inList(polyTags, wayTags); boolean hasAreaTag = false; if (wayTags.containsKey("area")) { hasAreaTag = wayTags.get("area").equals("yes"); } if (!nomineeLine) { return; } if (isRoad) { wayProperties.put("oneway", direction.toString()); wayProperties.put("highway", wayTags.get("highway")); } if (name != null) { // Copy name tag to way because this seems like a valuable location for // such a property wayProperties.put("name", name); } T changesetNode = getChangesetNode(wayProperties); T way = addNode(INDEX_NAME_WAY, wayProperties, "way_osm_id"); createRelationship(way, changesetNode, OSMRelation.CHANGESET); if (prev_way == null) { createRelationship(osm_dataset, way, OSMRelation.WAYS); } else { createRelationship(prev_way, way, OSMRelation.NEXT); } prev_way = way; addNodeTags(way, wayTags, "way"); Envelope bbox = new Envelope(); T firstNode = null; T prevNode = null; T prevProxy = null; Map<String, Object> prevProps = null; LinkedHashMap<String, Object> relProps = new LinkedHashMap<String, Object>(); HashMap<String, Object> directionProps = new HashMap<String, Object>(); directionProps.put("oneway", true); for (long nd_ref : wayNodes) { //long pointNode = batchIndexService.getSingleNode("node_osm_id", nd_ref); T pointNode = getOSMNode(nd_ref, changesetNode); if (pointNode == null) { /* * This can happen if we import not whole planet, so some referenced * nodes will be unavailable */ missingNode(nd_ref); continue; } T proxyNode = createProxyNode(); if (firstNode == null) { firstNode = pointNode; } if (prevNode == pointNode) { continue; } createRelationship(proxyNode, pointNode, OSMRelation.NODE, null); Map<String, Object> nodeProps = getNodeProperties(pointNode); double[] location = new double[] { (Double) nodeProps.get("lon"), (Double) nodeProps.get("lat") }; bbox.expandToInclude(location[0], location[1]); if (prevProxy == null) { createRelationship(way, proxyNode, OSMRelation.FIRST_NODE); } else { relProps.clear(); double[] prevLoc = new double[] { (Double) prevProps.get("lon"), (Double) prevProps.get("lat") }; double length = distance(prevLoc[0], prevLoc[1], location[0], location[1]); relProps.put("length", length); // We default to non-directional createRelationship(prevProxy, proxyNode, OSMRelation.NEXT, relProps); } prevNode = pointNode; prevProxy = proxyNode; prevProps = nodeProps; } if (prevNode.equals(firstNode) && wayNodes.size() >= 4) {//(prevNode.equals(firstNode)) { firstNode != null & if (nomineePoly || hasAreaTag) { geometry = GTYPE_POLYGON; } } if (prevNode.equals(firstNode) & wayNodes.size() == 2) {//(prevNode.equals(firstNode)) { firstNode != null & System.out.println("Invalid line: prevNode = firstNode & wayNodes.size() == 2"); return; } if (wayNodes.size() < 2) { System.out.println("Invalid line: wayNodes.size() < 2"); //geometry = GTYPE_POINT; return; } addNodeGeometry(way, geometry, bbox, wayNodes.size()); this.wayCount++; } private void createOSMRelation(Map<String, Object> relationProperties, ArrayList<Map<String, Object>> relationMembers, LinkedHashMap<String, Object> relationTags) { String name = (String) relationTags.get("name"); String type = (String) relationTags.get("type"); if (name != null) { // Copy name tag to way because this seems like a valuable location for // such a property relationProperties.put("name", name); } T relation = addNode("relation", relationProperties, "relation_osm_id"); if (prev_relation == null) { createRelationship(osm_dataset, relation, OSMRelation.RELATIONS); } else { createRelationship(prev_relation, relation, OSMRelation.NEXT); } prev_relation = relation; addNodeTags(relation, relationTags, "relation"); // We will test for cases that invalidate multilinestring further down GeometryMetaData metaGeom = new GeometryMetaData(GTYPE_MULTILINESTRING); if (type != null && type.length() > 0) { //relProps.put("type", type); if (type.equals("multipolygon")) { metaGeom = new GeometryMetaData(GTYPE_MULTIPOLYGON); } } T prevMember = null; LinkedHashMap<String, Object> relProps = new LinkedHashMap<String, Object>(); for (Map<String, Object> memberProps : relationMembers) { String memberType = (String) memberProps.get("type"); long member_ref = Long.parseLong(memberProps.get("ref").toString()); if (memberType != null) { T member = getSingleNode(memberType, memberType + "_osm_id", member_ref); if (null == member || prevMember == member) { /* * This can happen if we import not whole planet, so some * referenced nodes will be unavailable */ missingMember(memberProps.toString()); continue; } if (member == relation) { osmImporter.error("Cannot add relation to same member: relation[" + relationTags + "] - member[" + memberProps + "]"); continue; } Map<String, Object> nodeProps = getNodeProperties(member); if (memberType.equals("node")) { double[] location = new double[] { (Double) nodeProps.get("lon"), (Double) nodeProps.get("lat") }; metaGeom.expandToIncludePoint(location); } else if (memberType.equals("way")) { System.out.println(memberType); } else if (memberType.equals("nodes")) { System.err.println("Unexpected 'nodes' member type"); } else { updateGeometryMetaDataFromMember(member, metaGeom, nodeProps); } relProps.clear(); String role = (String) memberProps.get("role"); if (role != null && role.length() > 0) { relProps.put("role", role); if (role.equals("outer")) { metaGeom.setPolygon(); } } createRelationship(relation, member, OSMRelation.MEMBER, relProps); // members can belong to multiple relations, in multiple orders, so NEXT will clash (also with NEXT between ways in original way load) // if (prevMember < 0) { // batchGraphDb.createRelationship(relation, member, OSMRelation.MEMBERS, null); // } else { // batchGraphDb.createRelationship(prevMember, member, OSMRelation.NEXT, null); // } prevMember = member; } else { System.err.println("Cannot process invalid relation member: " + memberProps.toString()); } } if (metaGeom.isValid()) { addNodeGeometry(relation, metaGeom.getGeometryType(), metaGeom.getBBox(), metaGeom.getVertices()); } this.relationCount++; } /** * This method should be overridden by implementation that are able to perform database or index optimizations when requested, like the batch inserter. */ protected void optimize() { } protected abstract T getSingleNode(String name, String string, Object value); protected abstract Map<String, Object> getNodeProperties(T member); protected abstract T getOSMNode(long osmId, T changesetNode); protected abstract void updateGeometryMetaDataFromMember(T member, GeometryMetaData metaGeom, Map<String, Object> nodeProps); protected abstract void finish(); protected abstract T createProxyNode(); protected abstract T getChangesetNode(Map<String, Object> nodeProps); protected abstract T getUserNode(Map<String, Object> nodeProps); } private static class OSMGraphWriter extends OSMWriter<Node> { private GraphDatabaseService graphDb; private Node osm_root; private long currentChangesetId = -1; private Node currentChangesetNode; private long currentUserId = -1; private Node currentUserNode; private Node usersNode; private HashMap<Long, Node> changesetNodes = new HashMap<Long, Node>(); private Transaction tx; private int checkCount = 0; private int txInterval; private OSMGraphWriter(GraphDatabaseService graphDb, StatsManager statsManager, OSMImporter osmImporter, int txInterval) { super(statsManager, osmImporter); this.graphDb = graphDb; this.txInterval = txInterval; if (this.txInterval < 100) { System.err.println("Warning: Unusually short txInterval, expect bad insert performance"); } checkTx(); // Opens transaction for future writes } private void successTx() { if (tx != null) { tx.success(); tx.finish(); tx = null; checkCount = 0; } } private void checkTx() { if (checkCount++ > txInterval || tx == null) { successTx(); tx = graphDb.beginTx(); } } private Index<Node> indexFor(String indexName) { //return graphDb.index().forNodes( indexName, MapUtil.stringMap("type", "exact") ); return graphDb.index().forNodes(indexName); } private Node findNode(String name, Node parent, RelationshipType relType) { for (Relationship relationship : parent.getRelationships(relType, Direction.OUTGOING)) { Node node = relationship.getEndNode(); if (name.equals(node.getProperty("name"))) { return node; } } return null; } @Override protected Node getOrCreateNode(String name, String type, Node parent, RelationshipType relType) { Node node = findNode(name, parent, relType); if (node == null) { node = graphDb.createNode(); node.setProperty("name", name); node.setProperty("type", type); parent.createRelationshipTo(node, relType); checkTx(); } return node; } @Override protected Node getOrCreateOSMDataset(String name) { if (osm_dataset == null) { osm_root = getOrCreateNode("osm_root", "osm", graphDb.getReferenceNode(), OSMRelation.OSM); osm_dataset = getOrCreateNode(name, "osm", osm_root, OSMRelation.OSM); } return osm_dataset; } @Override protected void setDatasetProperties(Map<String, Object> extractProperties) { for (String key : extractProperties.keySet()) { osm_dataset.setProperty(key, extractProperties.get(key)); } } private void addProperties(PropertyContainer node, Map<String, Object> properties) { for (String property : properties.keySet()) { node.setProperty(property, properties.get(property)); } } @Override protected void addNodeTags(Node node, LinkedHashMap<String, Object> tags, String type) { logNodeAddition(tags, type); if (node != null && tags.size() > 0) { statsManager.addToTagStats(type, tags.keySet()); Node tagsNode = graphDb.createNode(); addProperties(tagsNode, tags); node.createRelationshipTo(tagsNode, OSMRelation.TAGS); tags.clear(); } } @Override protected void addNodeGeometry(Node node, int gtype, Envelope bbox, int vertices) { if (node != null && bbox.isValid() && vertices > 0) { if (gtype == GTYPE_GEOMETRY) { gtype = vertices > 1 ? GTYPE_MULTIPOINT : GTYPE_POINT; } Node geomNode = graphDb.createNode(); geomNode.setProperty("gtype", gtype); geomNode.setProperty("vertices", vertices); geomNode.setProperty("bbox", new double[] { bbox.getMinX(), bbox.getMaxX(), bbox.getMinY(), bbox.getMaxY() }); node.createRelationshipTo(geomNode, OSMRelation.GEOM); statsManager.addGeomStats(gtype); } } @Override protected Node addNode(String name, Map<String, Object> properties, String indexKey) { Node node = graphDb.createNode(); if (indexKey != null && properties.containsKey(indexKey)) { indexFor(name).add(node, indexKey, properties.get(indexKey)); properties.put(indexKey, Long.parseLong(properties.get(indexKey).toString())); } addProperties(node, properties); checkTx(); return node; } protected Node addNodeWithCheck(String name, Map<String, Object> properties, String indexKey) { Node node = null; Object indexValue = (indexKey == null) ? null : properties.get(indexKey); if (indexValue != null && (createdNodes + foundNodes < 100 || foundNodes > 10)) { node = indexFor(name).get(indexKey, properties.get(indexKey)).getSingle(); } if (node == null) { node = graphDb.createNode(); addProperties(node, properties); if (indexValue != null) { indexFor(name).add(node, indexKey, properties.get(indexKey)); } createdNodes++; checkTx(); } else { foundNodes++; } return node; } @Override protected void createRelationship(Node from, Node to, RelationshipType relType, LinkedHashMap<String, Object> relProps) { Relationship rel = from.createRelationshipTo(to, relType); if (relProps != null && relProps.size() > 0) { addProperties(rel, relProps); } } @Override protected long getDatasetId() { return osm_dataset.getId(); } @Override protected Node getSingleNode(String name, String string, Object value) { return indexFor(name).get(string, value).getSingle(); } @Override protected Map<String, Object> getNodeProperties(Node node) { LinkedHashMap<String, Object> properties = new LinkedHashMap<String, Object>(); for (String property : node.getPropertyKeys()) { properties.put(property, node.getProperty(property)); } return properties; } @Override protected Node getOSMNode(long osmId, Node changesetNode) { if (currentChangesetNode != changesetNode || changesetNodes.isEmpty()) { currentChangesetNode = changesetNode; changesetNodes.clear(); for (Relationship rel : changesetNode.getRelationships(OSMRelation.CHANGESET, Direction.INCOMING)) { Node node = rel.getStartNode(); Long nodeOsmId = (Long) node.getProperty("node_osm_id", null); if (nodeOsmId != null) { changesetNodes.put(nodeOsmId, node); } } } Node node = changesetNodes.get(osmId); if (node == null) { logNodeFoundFrom("node-index"); return indexFor("node").get("node_osm_id", osmId).getSingle(); } else { logNodeFoundFrom("changeset"); return node; } } @Override protected void updateGeometryMetaDataFromMember(Node member, GeometryMetaData metaGeom, Map<String, Object> nodeProps) { for (Relationship rel : member.getRelationships(OSMRelation.GEOM)) { nodeProps = getNodeProperties(rel.getEndNode()); metaGeom.checkSupportedGeometry((Integer) nodeProps.get("gtype")); metaGeom.expandToIncludeBBox(nodeProps); } } @Override protected void finish() { osm_dataset.setProperty("relationCount", (Integer) osm_dataset.getProperty("relationCount", 0) + relationCount); osm_dataset.setProperty("wayCount", (Integer) osm_dataset.getProperty("wayCount", 0) + wayCount); osm_dataset.setProperty("nodeCount", (Integer) osm_dataset.getProperty("nodeCount", 0) + nodeCount); osm_dataset.setProperty("poiCount", (Integer) osm_dataset.getProperty("poiCount", 0) + poiCount); osm_dataset.setProperty("changesetCount", (Integer) osm_dataset.getProperty("changesetCount", 0) + changesetCount); osm_dataset.setProperty("userCount", (Integer) osm_dataset.getProperty("userCount", 0) + userCount); successTx(); } @Override protected Node createProxyNode() { return graphDb.createNode(); } @Override protected Node getChangesetNode(Map<String, Object> nodeProps) { long changeset = Long.parseLong(nodeProps.remove(INDEX_NAME_CHANGESET).toString()); getUserNode(nodeProps); if (changeset != currentChangesetId) { currentChangesetId = changeset; IndexHits<Node> result = indexFor(INDEX_NAME_CHANGESET).get(INDEX_NAME_CHANGESET, currentChangesetId); if (result.size() > 0) { currentChangesetNode = result.getSingle(); } else { LinkedHashMap<String, Object> changesetProps = new LinkedHashMap<String, Object>(); changesetProps.put(INDEX_NAME_CHANGESET, currentChangesetId); changesetProps.put("timestamp", nodeProps.get("timestamp")); currentChangesetNode = (Node) addNode(INDEX_NAME_CHANGESET, changesetProps, INDEX_NAME_CHANGESET); changesetCount++; if (currentUserNode != null) { createRelationship(currentChangesetNode, currentUserNode, OSMRelation.USER); } } result.close(); } return currentChangesetNode; } @Override protected Node getUserNode(Map<String, Object> nodeProps) { try { long uid = Long.parseLong(nodeProps.remove("uid").toString()); String name = nodeProps.remove(INDEX_NAME_USER).toString(); if (uid != currentUserId) { currentUserId = uid; IndexHits<Node> result = indexFor(INDEX_NAME_USER).get("uid", currentUserId); if (result.size() > 0) { currentUserNode = indexFor(INDEX_NAME_USER).get("uid", currentUserId).getSingle(); } else { LinkedHashMap<String, Object> userProps = new LinkedHashMap<String, Object>(); userProps.put("uid", currentUserId); userProps.put("name", name); userProps.put("timestamp", nodeProps.get("timestamp")); currentUserNode = (Node) addNode(INDEX_NAME_USER, userProps, "uid"); userCount++; // if (currentChangesetNode != null) { // currentChangesetNode.createRelationshipTo(currentUserNode, OSMRelation.USER); // } if (usersNode == null) { usersNode = graphDb.createNode(); osm_dataset.createRelationshipTo(usersNode, OSMRelation.USERS); } usersNode.createRelationshipTo(currentUserNode, OSMRelation.OSM_USER); } result.close(); } } catch (Exception e) { currentUserId = -1; currentUserNode = null; logMissingUser(nodeProps); } return currentUserNode; } public String toString() { return "OSMGraphWriter: DatabaseService[" + graphDb + "]:txInterval[" + this.txInterval + "]"; } } private static class OSMBatchWriter extends OSMWriter<Long> { private BatchInserter batchInserter; private BatchInserterIndexProvider batchIndexService; private HashMap<String, BatchInserterIndex> batchIndices = new HashMap<String, BatchInserterIndex>(); private long osm_root; private long currentChangesetId = -1; private long currentChangesetNode = -1; private long currentUserId = -1; private long currentUserNode = -1; private long usersNode = -1; private HashMap<Long, Long> changesetNodes = new HashMap<Long, Long>(); private OSMBatchWriter(BatchInserter batchGraphDb, StatsManager statsManager, OSMImporter osmImporter) { super(statsManager, osmImporter); this.batchInserter = batchGraphDb; this.batchIndexService = new LuceneBatchInserterIndexProvider(batchGraphDb); } private BatchInserterIndex indexFor(String indexName) { BatchInserterIndex index = batchIndices.get(indexName); if (index == null) { index = batchIndexService.nodeIndex(indexName, MapUtil.stringMap("type", "exact")); batchIndices.put(indexName, index); } return index; } @Override public Long getOrCreateOSMDataset(String name) { if (osm_dataset == null || osm_dataset <= 0) { osm_root = getOrCreateNode("osm_root", "osm", batchInserter.getReferenceNode(), OSMRelation.OSM); osm_dataset = getOrCreateNode(name, "osm", osm_root, OSMRelation.OSM); } return osm_dataset; } private long findNode(BatchInserter batchInserter, String name, long parent, RelationshipType relType) { for (SimpleRelationship relationship : batchInserter.getRelationships(parent)) { if (relationship.getType().name().equals(relType.name())) { long node = relationship.getEndNode(); Object nodeName = batchInserter.getNodeProperties(node).get("name"); if (nodeName != null && name.equals(nodeName.toString())) { return node; } } } return -1; } @Override protected Long getOrCreateNode(String name, String type, Long parent, RelationshipType relType) { long node = findNode(batchInserter, name, parent, relType); if (node < 0) { HashMap<String, Object> properties = new HashMap<String, Object>(); properties.put("name", name); properties.put("type", type); node = batchInserter.createNode(properties); batchInserter.createRelationship(parent, node, relType, null); } return node; } public String toString() { return "OSMBatchWriter: BatchInserter[" + batchInserter.toString() + "]:IndexService[" + batchIndexService.toString() + "]"; } @Override protected void setDatasetProperties(Map<String, Object> extraProperties) { LinkedHashMap<String, Object> properties = new LinkedHashMap<String, Object>(); properties.putAll(batchInserter.getNodeProperties(osm_dataset)); properties.putAll(extraProperties); batchInserter.setNodeProperties(osm_dataset, properties); } @Override protected void addNodeTags(Long node, LinkedHashMap<String, Object> tags, String type) { logNodeAddition(tags, type); if (node > 0 && tags.size() > 0) { statsManager.addToTagStats(type, tags.keySet()); long id = batchInserter.createNode(tags); batchInserter.createRelationship(node, id, OSMRelation.TAGS, null); tags.clear(); } } @Override protected void addNodeGeometry(Long node, int gtype, Envelope bbox, int vertices) { if (node > 0 && bbox.isValid() && vertices > 0) { LinkedHashMap<String, Object> properties = new LinkedHashMap<String, Object>(); if (gtype == GTYPE_GEOMETRY) { gtype = vertices > 1 ? GTYPE_MULTIPOINT : GTYPE_POINT; } properties.put("gtype", gtype); properties.put("vertices", vertices); properties.put("bbox", new double[] { bbox.getMinX(), bbox.getMaxX(), bbox.getMinY(), bbox.getMaxY() }); long id = batchInserter.createNode(properties); batchInserter.createRelationship(node, id, OSMRelation.GEOM, null); properties.clear(); statsManager.addGeomStats(gtype); } } @Override protected Long addNode(String name, Map<String, Object> properties, String indexKey) { long id = -1; if (indexKey != null && properties.containsKey(indexKey)) { Map<String, Object> props = new HashMap<String, Object>(); props.put(indexKey, properties.get(indexKey).toString()); properties.put(indexKey, Long.parseLong(properties.get(indexKey).toString())); id = batchInserter.createNode(properties); indexFor(name).add(id, props); } else { id = batchInserter.createNode(properties); } return id; } protected Long addNodeWithCheck(String name, Map<String, Object> properties, String indexKey) { // TODO: This code allows for importing into existing data, but // slows the import down by almost three times. The problem is that // the batchIndexService cannot switch efficiently between read and // write mode. Rather we should use pure GraphDatabaseService API // for update mode. long id = -1; Object indexValue = (indexKey == null) ? null : properties.get(indexKey); if (indexValue != null && (createdNodes + foundNodes < 100 || foundNodes > 10)) { id = indexFor(name).get(indexKey, properties.get(indexKey)).getSingle(); } if (id < 0) { id = batchInserter.createNode(properties); if (indexValue != null) { Map<String, Object> props = new HashMap<String, Object>(); props.put(indexKey, properties.get(indexKey)); indexFor(name).add(id, props); } createdNodes++; } else { foundNodes++; } return id; } @Override protected void createRelationship(Long from, Long to, RelationshipType relType, LinkedHashMap<String, Object> relProps) { batchInserter.createRelationship(from, to, relType, relProps); } protected void optimize() { //TODO: optimize //batchIndexService.optimize(); for (String index : new String[] { "node", "way", "changeset", "user" }) { indexFor(index).flush(); } } @Override protected long getDatasetId() { return osm_dataset; } @Override protected Long getSingleNode(String name, String string, Object value) { //BatchInserterIndex indexFor = indexFor(name); //IndexHits<Long> indexHits = indexFor.get(string, value); //Long single = indexHits.getSingle(); return indexFor(name).get(string, value).getSingle(); } @Override protected Map<String, Object> getNodeProperties(Long member) { return batchInserter.getNodeProperties(member); } @Override protected Long getOSMNode(long osmId, Long changesetNode) { if (currentChangesetNode != changesetNode || changesetNodes.isEmpty()) { currentChangesetNode = changesetNode; changesetNodes.clear(); for (SimpleRelationship rel : batchInserter.getRelationships(changesetNode)) { if (rel.getType().name().equals(OSMRelation.CHANGESET.name())) { Long node = rel.getStartNode(); Map<String, Object> props = batchInserter.getNodeProperties(node); Long nodeOsmId = (Long) props.get("node_osm_id"); if (nodeOsmId != null) { changesetNodes.put(nodeOsmId, node); } } } } Long node = changesetNodes.get(osmId); if (node == null) { logNodeFoundFrom("node-index"); return indexFor(INDEX_NAME_NODE).get("node_osm_id", osmId).getSingle(); } else { logNodeFoundFrom("changeset"); return node; } } @Override protected void updateGeometryMetaDataFromMember(Long member, GeometryMetaData metaGeom, Map<String, Object> nodeProps) { for (SimpleRelationship rel : batchInserter.getRelationships(member)) { if (rel.getType().equals(OSMRelation.GEOM)) { nodeProps = getNodeProperties(rel.getEndNode()); metaGeom.checkSupportedGeometry((Integer) nodeProps.get("gtype")); metaGeom.expandToIncludeBBox(nodeProps); } } } @Override protected void finish() { HashMap<String, Object> dsProps = new HashMap<String, Object>( batchInserter.getNodeProperties(osm_dataset)); updateDSCounts(dsProps, "relationCount", relationCount); updateDSCounts(dsProps, "wayCount", wayCount); updateDSCounts(dsProps, "nodeCount", nodeCount); updateDSCounts(dsProps, "poiCount", poiCount); updateDSCounts(dsProps, "changesetCount", changesetCount); updateDSCounts(dsProps, "userCount", userCount); setDatasetProperties(dsProps); batchIndexService.shutdown(); batchIndexService = null; } private void updateDSCounts(HashMap<String, Object> dsProps, String name, int count) { Integer current = (Integer) dsProps.get(name); dsProps.put(name, (current == null ? 0 : current) + count); } @Override protected Long createProxyNode() { return batchInserter.createNode(null); } @Override protected Long getChangesetNode(Map<String, Object> nodeProps) { long changeset = Long.parseLong(nodeProps.remove("changeset").toString()); getUserNode(nodeProps); if (changeset != currentChangesetId) { currentChangesetId = changeset; changesetNodes.clear(); IndexHits<Long> results = indexFor("changeset").get("changeset", currentChangesetId); if (results.size() > 0) { currentChangesetNode = results.getSingle(); } else { LinkedHashMap<String, Object> changesetProps = new LinkedHashMap<String, Object>(); changesetProps.put("changeset", currentChangesetId); changesetProps.put("timestamp", nodeProps.get("timestamp")); currentChangesetNode = (Long) addNode("changeset", changesetProps, "changeset"); indexFor("changeset").flush(); if (currentUserNode > 0) { createRelationship(currentChangesetNode, currentUserNode, OSMRelation.USER); } } results.close(); } return currentChangesetNode; } @Override protected Long getUserNode(Map<String, Object> nodeProps) { try { long uid = Long.parseLong(nodeProps.remove("uid").toString()); String name = nodeProps.remove("user").toString(); if (uid != currentUserId) { currentUserId = uid; IndexHits<Long> results = indexFor(INDEX_NAME_USER).get("uid", currentUserId); if (results.size() > 0) { currentUserNode = results.getSingle(); } else { LinkedHashMap<String, Object> userProps = new LinkedHashMap<String, Object>(); userProps.put("uid", currentUserId); userProps.put("name", name); userProps.put("timestamp", nodeProps.get("timestamp")); currentUserNode = (Long) addNode("user", userProps, "uid"); indexFor(INDEX_NAME_USER).flush(); if (usersNode < 0) { usersNode = batchInserter.createNode(MapUtils.EMPTY_MAP); createRelationship(osm_dataset, usersNode, OSMRelation.USERS); } createRelationship(usersNode, currentUserNode, OSMRelation.OSM_USER); } results.close(); } } catch (Exception e) { currentUserId = -1; currentUserNode = -1; logMissingUser(nodeProps); } return currentUserNode; } } public void importFile(GraphDatabaseService database, String dataset) throws IOException, XMLStreamException { importFile(database, dataset, false, 5000); } public void importFile(GraphDatabaseService database, String dataset, int txInterval) throws IOException, XMLStreamException { importFile(database, dataset, false, txInterval); } public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws IOException, XMLStreamException { importFile(OSMWriter.fromGraphDatabase(database, stats, this, txInterval), dataset, allPoints, charset); } public void importFile(BatchInserter batchInserter, String dataset) throws IOException, XMLStreamException { importFile(batchInserter, dataset, false); } public void importFile(BatchInserter batchInserter, String dataset, boolean allPoints) throws IOException, XMLStreamException { importFile(OSMWriter.fromBatchInserter(batchInserter, stats, this), dataset, allPoints, charset); } public static class CountedFileReader extends InputStreamReader { private long length = 0; private long charsRead = 0; public CountedFileReader(String path, Charset charset) throws FileNotFoundException { super(new FileInputStream(path), charset); this.length = (new File(path)).length(); } public CountedFileReader(File file, Charset charset) throws FileNotFoundException { super(new FileInputStream(file), charset); this.length = file.length(); } public long getCharsRead() { return charsRead; } public long getlength() { return length; } public double getProgress() { return length > 0 ? (double) charsRead / (double) length : 0; } public int getPercentRead() { return (int) (100.0 * getProgress()); } public int read(char[] cbuf, int offset, int length) throws IOException { int read = super.read(cbuf, offset, length); if (read > 0) { charsRead += read; } return read; } } private int progress = 0; private long progressTime = 0; private void beginProgressMonitor(int length) { monitor.begin(length); progress = 0; progressTime = System.currentTimeMillis(); } private void updateProgressMonitor(int currentProgress) { if (currentProgress > this.progress) { long time = System.currentTimeMillis(); if (time - progressTime > 1000) { monitor.worked(currentProgress - progress); progress = currentProgress; progressTime = time; } } } private void endProgressMonitor() { monitor.done(); progress = 0; progressTime = 0; } public void setCharset(Charset charset) { this.charset = charset; } public void importFile(OSMWriter<?> osmWriter, String dataset, boolean allPoints, Charset charset) throws IOException, XMLStreamException { System.out.println("Importing with osm-writer: " + osmWriter); osmWriter.getOrCreateOSMDataset(layerName); osm_dataset = osmWriter.getDatasetId(); long startTime = System.currentTimeMillis(); long[] times = new long[] { 0L, 0L, 0L, 0L }; javax.xml.stream.XMLInputFactory factory = javax.xml.stream.XMLInputFactory.newInstance(); CountedFileReader reader = new CountedFileReader(dataset, charset); javax.xml.stream.XMLStreamReader parser = factory.createXMLStreamReader(reader); int countXMLTags = 0; beginProgressMonitor(100); setLogContext(dataset); boolean startedWays = false; boolean startedRelations = false; try { ArrayList<String> currentXMLTags = new ArrayList<String>(); int depth = 0; Map<String, Object> wayProperties = null; ArrayList<Long> wayNodes = new ArrayList<Long>(); Map<String, Object> relationProperties = null; ArrayList<Map<String, Object>> relationMembers = new ArrayList<Map<String, Object>>(); LinkedHashMap<String, Object> currentNodeTags = new LinkedHashMap<String, Object>(); while (true) { updateProgressMonitor(reader.getPercentRead()); incrLogContext(); int event = parser.next(); if (event == javax.xml.stream.XMLStreamConstants.END_DOCUMENT) { break; } switch (event) { case javax.xml.stream.XMLStreamConstants.START_ELEMENT: currentXMLTags.add(depth, parser.getLocalName()); String tagPath = currentXMLTags.toString(); if (tagPath.equals("[osm]")) { osmWriter.setDatasetProperties(extractProperties(parser)); } else if (tagPath.equals("[osm, bounds]")) { osmWriter.addOSMBBox(extractProperties("bbox", parser)); } else if (tagPath.equals("[osm, node]")) { // <node id="269682538" lat="56.0420950" lon="12.9693483" user="sanna" uid="31450" visible="true" version="1" changeset="133823" timestamp="2008-06-11T12:36:28Z"/> osmWriter.createOSMNode(extractProperties("node", parser)); } else if (tagPath.equals("[osm, way]")) { // <way id="27359054" user="spull" uid="61533" visible="true" version="8" changeset="4707351" timestamp="2010-05-15T15:39:57Z"> if (!startedWays) { startedWays = true; times[0] = System.currentTimeMillis(); osmWriter.optimize(); times[1] = System.currentTimeMillis(); } wayProperties = extractProperties("way", parser); wayNodes.clear(); } else if (tagPath.equals("[osm, way, nd]")) { Map<String, Object> properties = extractProperties(parser); wayNodes.add(Long.parseLong(properties.get("ref").toString())); } else if (tagPath.endsWith("tag]")) { Map<String, Object> properties = extractProperties(parser); currentNodeTags.put(properties.get("k").toString(), properties.get("v").toString()); } else if (tagPath.equals("[osm, relation]")) { // <relation id="77965" user="Grillo" uid="13957" visible="true" version="24" changeset="5465617" timestamp="2010-08-11T19:25:46Z"> if (!startedRelations) { startedRelations = true; times[2] = System.currentTimeMillis(); osmWriter.optimize(); times[3] = System.currentTimeMillis(); } relationProperties = extractProperties("relation", parser); relationMembers.clear(); } else if (tagPath.equals("[osm, relation, member]")) { relationMembers.add(extractProperties(parser)); } if (startedRelations) { if (countXMLTags < 10) { log("Starting tag at depth " + depth + ": " + currentXMLTags.get(depth) + " - " + currentXMLTags.toString()); for (int i = 0; i < parser.getAttributeCount(); i++) { log("\t" + currentXMLTags.toString() + ": " + parser.getAttributeLocalName(i) + "[" + parser.getAttributeNamespace(i) + "," + parser.getAttributePrefix(i) + "," + parser.getAttributeType(i) + "," + "] = " + parser.getAttributeValue(i)); } } countXMLTags++; } depth++; break; case javax.xml.stream.XMLStreamConstants.END_ELEMENT: if (currentXMLTags.toString().equals("[osm, node]")) { osmWriter.addOSMNodeTags(allPoints, currentNodeTags); } else if (currentXMLTags.toString().equals("[osm, way]")) { osmWriter.createOSMWay(wayProperties, wayNodes, currentNodeTags); } else if (currentXMLTags.toString().equals("[osm, relation]")) { osmWriter.createOSMRelation(relationProperties, relationMembers, currentNodeTags); } depth--; currentXMLTags.remove(depth); // log("Ending tag at depth "+depth+": "+currentTags.get(depth)); break; default: break; } } } finally { endProgressMonitor(); parser.close(); osmWriter.finish(); this.osm_dataset = osmWriter.getDatasetId(); } describeTimes(startTime, times); osmWriter.describeMissing(); osmWriter.describeLoaded(); long stopTime = System.currentTimeMillis(); log("info | Elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000.0)); stats.dumpGeomStats(); stats.printTagStats(); } private void describeTimes(long startTime, long[] times) { long endTime = System.currentTimeMillis(); log("Completed load in " + (1.0 * (endTime - startTime) / 1000.0) + "s"); log("\tImported nodes: " + (1.0 * (times[0] - startTime) / 1000.0) + "s"); log("\tOptimized index: " + (1.0 * (times[1] - times[0]) / 1000.0) + "s"); log("\tImported ways: " + (1.0 * (times[2] - times[1]) / 1000.0) + "s"); log("\tOptimized index: " + (1.0 * (times[3] - times[2]) / 1000.0) + "s"); log("\tImported rels: " + (1.0 * (endTime - times[3]) / 1000.0) + "s"); } private Map<String, Object> extractProperties(XMLStreamReader parser) { return extractProperties(null, parser); } private Map<String, Object> extractProperties(String name, XMLStreamReader parser) { // <node id="269682538" lat="56.0420950" lon="12.9693483" user="sanna" uid="31450" visible="true" version="1" changeset="133823" timestamp="2008-06-11T12:36:28Z"/> // <way id="27359054" user="spull" uid="61533" visible="true" version="8" changeset="4707351" timestamp="2010-05-15T15:39:57Z"> // <relation id="77965" user="Grillo" uid="13957" visible="true" version="24" changeset="5465617" timestamp="2010-08-11T19:25:46Z"> LinkedHashMap<String, Object> properties = new LinkedHashMap<String, Object>(); for (int i = 0; i < parser.getAttributeCount(); i++) { String prop = parser.getAttributeLocalName(i); String value = parser.getAttributeValue(i); if (name != null && prop.equals("id")) { prop = name + "_osm_id"; name = null; } if (prop.equals("lat") || prop.equals("lon")) { properties.put(prop, Double.parseDouble(value)); } else if (name != null && prop.equals("version")) { properties.put(prop, Integer.parseInt(value)); } else if (prop.equals("visible")) { if (!value.equals("true") && !value.equals("1")) { properties.put(prop, false); } } else if (prop.equals("timestamp")) { try { Date timestamp = timestampFormat.parse(value); properties.put(prop, timestamp.getTime()); } catch (ParseException e) { error("Error parsing timestamp", e); } } else { properties.put(prop, value); } } if (name != null) { properties.put("name", name); } return properties; } /** * Detects if road has the only direction * * @param wayProperties * @return RoadDirection */ public static RoadDirection isOneway(Map<String, Object> wayProperties) { String oneway = (String) wayProperties.get("oneway"); if (null != oneway) { if ("-1".equals(oneway)) { return RoadDirection.BACKWARD; } if ("1".equals(oneway) || "yes".equalsIgnoreCase(oneway) || "true".equalsIgnoreCase(oneway)) { return RoadDirection.FORWARD; } } return RoadDirection.BOTH; } /** * Calculate correct distance between 2 points on Earth. * * @param latA * @param lonA * @param latB * @param lonB * @return distance in meters */ public static double distance(double lonA, double latA, double lonB, double latB) { return WGS84.orthodromicDistance(lonA, latA, lonB, latB); } private void log(PrintStream out, String message, Exception e) { if (logContext != null) { message = logContext + "[" + contextLine + "]: " + message; } out.println(message); if (e != null) { e.printStackTrace(out); } } private void log(String message) { log(System.out, message, null); } private void error(String message) { log(System.err, message, null); } private void error(String message, Exception e) { log(System.err, message, e); } private String logContext = null; private int contextLine = 0; //"2008-06-11T12:36:28Z" private DateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); private void setLogContext(String context) { logContext = context; contextLine = 0; } private void incrLogContext() { contextLine++; } /** * This method allows for a console, command-line application for loading * one or more *.osm files into a new database. * * @param args * , the database directory followed by one or more osm files */ public static void main(String[] args) { if (args.length < 2) { System.out.println("Usage: osmimporter databasedir osmfile <..osmfiles..>"); } else { OSMImportManager importer = new OSMImportManager(args[0]); for (int i = 1; i < args.length; i++) { try { importer.loadTestOsmData(args[i], 5000); } catch (Exception e) { System.err.println("Error importing OSM file '" + args[i] + "': " + e); e.printStackTrace(); } finally { importer.shutdown(); } } } } private static class OSMImportManager { private GraphDatabaseService graphDb; private BatchInserter batchInserter; private File dbPath; private boolean useBatchInserter = false; public OSMImportManager(String path) { setDbPath(path); } public void setDbPath(String path) { dbPath = new File(path); if (dbPath.exists()) { if (!dbPath.isDirectory()) { throw new RuntimeException("Database path is an existing file: " + dbPath.getAbsolutePath()); } } else { dbPath.mkdirs(); } } private void loadTestOsmData(String layerName, int commitInterval) throws Exception { String osmPath = layerName; System.out.println("\n=== Loading layer " + layerName + " from " + osmPath + " ==="); long start = System.currentTimeMillis(); if (useBatchInserter) { switchToBatchInserter(); OSMImporter importer = new OSMImporter(layerName); importer.importFile(batchInserter, osmPath); switchToEmbeddedGraphDatabase(); importer.reIndex(graphDb, commitInterval); } else { switchToEmbeddedGraphDatabase(); OSMImporter importer = new OSMImporter(layerName); importer.importFile(graphDb, osmPath, false, commitInterval); importer.reIndex(graphDb, commitInterval); } shutdown(); System.out.println("=== Completed loading " + layerName + " in " + (System.currentTimeMillis() - start) / 1000.0 + " seconds ==="); } private void switchToEmbeddedGraphDatabase() { shutdown(); graphDb = new EmbeddedGraphDatabase(dbPath.getAbsolutePath()); } private void switchToBatchInserter() { shutdown(); batchInserter = new BatchInserterImpl(dbPath.getAbsolutePath()); graphDb = batchInserter.getGraphDbService(); } protected void shutdown() { if (graphDb != null) { graphDb.shutdown(); //batch graphDb = null; batchInserter = null; } } } }