Java tutorial
/******************************************************************************* * Copyright 2016, The IKANOW Open Source Project. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. *******************************************************************************/ package com.ikanow.aleph2.graph.titan.utils; import static org.junit.Assert.*; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Vertex; import com.thinkaurelius.titan.core.Cardinality; import org.apache.tinkerpop.gremlin.structure.io.IoCore; import com.fasterxml.jackson.databind.JsonNode; import com.ikanow.aleph2.data_model.utils.Lambdas; import com.ikanow.aleph2.data_model.utils.Optionals; import com.ikanow.aleph2.data_model.utils.UuidUtils; import com.thinkaurelius.titan.core.PropertyKey; import com.thinkaurelius.titan.core.SchemaViolationException; import com.thinkaurelius.titan.core.TitanEdge; import com.thinkaurelius.titan.core.TitanException; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.core.TitanTransaction; import com.thinkaurelius.titan.core.TitanVertex; import com.thinkaurelius.titan.core.attribute.Cmp; import com.thinkaurelius.titan.core.attribute.Contain; import com.thinkaurelius.titan.core.attribute.Text; import com.thinkaurelius.titan.core.schema.Mapping; import com.thinkaurelius.titan.core.schema.SchemaAction; import com.thinkaurelius.titan.core.schema.SchemaStatus; import com.thinkaurelius.titan.core.schema.TitanManagement; import fj.Unit; /** Some utilities to check out how Titan works * All @org.junit.Tests removed since not actually test code, that was just for convenience * @author Alex * */ public class TestMiscTitanProperties { protected static String showGraph(final TitanGraph titan) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); titan.io(IoCore.graphson()).writer().create().writeGraph(baos, titan); return baos.toString(); } protected static String showElement(final TitanGraph titan, final Element element) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); if (Vertex.class.isAssignableFrom(element.getClass())) { titan.io(IoCore.graphson()).writer().create().writeVertex(baos, (Vertex) element); } else { titan.io(IoCore.graphson()).writer().create().writeEdge(baos, (Edge) element); } return baos.toString(); } @SuppressWarnings("unchecked") //@org.junit.Test public void test_elementProperties_edge() throws IOException, InterruptedException { final String tmpdir = System.getProperty("java.io.tmpdir") + "/titan-test-" + UuidUtils.get().getRandomUuid(); try { FileUtils.deleteDirectory(new File(tmpdir)); } catch (Exception e) { } TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory") .set("index.search.backend", "elasticsearch").set("index.search.elasticsearch.local-mode", true) .set("index.search.directory", tmpdir) .set("index.search.cluster-name", UuidUtils.get().getRandomUuid()) .set("index.search.ignore-cluster-name", false).set("index.search.elasticsearch.client-only", false) //.set("query.force-index", true) //(disabled for testing) .open(); { TitanManagement mgmt = titan.openManagement(); // Without ES: //mgmt.makePropertyKey("paths").dataType(String.class).cardinality(Cardinality.SET).make(); //with ES as a search back-end, can do SET/LIST mgmt.buildIndex("pathQuery", Edge.class) .addKey(mgmt.makePropertyKey("paths").dataType(String.class).make(), Mapping.STRING.asParameter()) .buildMixedIndex("search"); //mgmt.buildIndex("pathQuery_nonAnalyzed", Vertex.class).addKey(mgmt.makePropertyKey("paths").dataType(String.class).cardinality(Cardinality.SET).make()).buildMixedIndex("search"); //.addKey("_b", Mapping.STRING.asParameter()).buildMixedIndex("search") mgmt.commit(); } Thread.sleep(1500L); // (mapping not generated in time?) buildSmallGraph(titan); Thread.sleep(3000L); // (mapping not generated in time?) final TitanTransaction tx = titan.buildTransaction().start(); Optionals.<TitanEdge>streamOf(tx.query().has("edge_prop", "edge_prop_val").edges(), false).findFirst() .map(e -> { System.out.println("FOUND EDGE"); return Unit.unit(); }).orElseGet(() -> { System.out.println("NOT FOUND"); return Unit.unit(); }); } @SuppressWarnings("unchecked") //@org.junit.Test public void test_elementProperties_vertex() throws IOException, InterruptedException { final String tmpdir = System.getProperty("java.io.tmpdir") + "/titan-test-" + UuidUtils.get().getRandomUuid(); try { FileUtils.deleteDirectory(new File(tmpdir)); } catch (Exception e) { } TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory") .set("index.search.backend", "elasticsearch").set("index.search.elasticsearch.local-mode", true) .set("index.search.directory", tmpdir) .set("index.search.cluster-name", UuidUtils.get().getRandomUuid()) .set("index.search.ignore-cluster-name", false).set("index.search.elasticsearch.client-only", false) //.set("query.force-index", true) //(disabled for testing) .open(); { TitanManagement mgmt = titan.openManagement(); // Without ES: //mgmt.makePropertyKey("paths").dataType(String.class).cardinality(Cardinality.SET).make(); //with ES as a search back-end, can do SET/LIST mgmt.buildIndex("pathQuery", Vertex.class).addKey( mgmt.makePropertyKey("paths").dataType(String.class).cardinality(Cardinality.SET).make(), Mapping.STRING.asParameter()).buildMixedIndex("search"); //mgmt.buildIndex("pathQuery_nonAnalyzed", Vertex.class).addKey(mgmt.makePropertyKey("paths").dataType(String.class).cardinality(Cardinality.SET).make()).buildMixedIndex("search"); //.addKey("_b", Mapping.STRING.asParameter()).buildMixedIndex("search") mgmt.commit(); } // Just check I can do this multiple times: try { TitanManagement mgmt = titan.openManagement(); mgmt.makePropertyKey("paths").dataType(String.class).cardinality(Cardinality.SET).make(); mgmt.commit(); } catch (SchemaViolationException e) { } // (can but throws this exception, which is fine) Thread.sleep(1500L); // (mapping not generated in time?) buildSmallGraph(titan); final TitanTransaction tx = titan.buildTransaction().start(); final TitanVertex v = Optionals.<TitanVertex>streamOf(tx.query().has("type", "rabbit").vertices(), false) .findFirst().get(); // These will fail because the property has not been declared // v.property(org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality.set, "animal", "mouse"); // v.property(org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality.set, "animal", "cat"); // v.property(org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality.set, "animal", "mouse"); // This all works as expected v.property(org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality.set, "paths", "mouse"); v.property(org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality.set, "paths", "cat"); v.property(org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality.set, "paths", "mouse"); // These will overwrite // v.property("_b", "mouse"); // v.property("_b", "cat"); // v.property("_b", "mouse"); // This does not work, so properties will be protected with a single bucket access v.property("type", "rabbit", "paths", Arrays.asList("a", "b")); //"[a, b]" (ie a string of a list) //v.property("type", "rabbit", "_b", Stream.of("a", "b").toArray()); // [L;Object] System.out.println(showElement(titan, v)); tx.commit(); // OK let's double check how we retrieve a list of properties final TitanVertex v2 = Optionals .<TitanVertex>streamOf(titan.query().has("type", "rabbit").vertices(), false).findFirst().get(); // Use "properties" with a single key to get the list of buckets System.out.println("paths = " + Optionals.streamOf(v2.properties("paths"), false) .map(vp -> vp.value().toString()).collect(Collectors.joining(";"))); // Double check a query on _b works assertEquals(0L, Optionals.streamOf(titan.query().has("paths", "cat").vertices(), false).count());//all these queries System.out.println("ES queries in 3s...:"); Thread.sleep(3000L); // (wait for ES to complete) assertEquals(1L, Optionals.streamOf(titan.indexQuery("pathQuery", "v.paths:cat").vertices(), false).count()); assertEquals(1L, Optionals.streamOf(titan.indexQuery("pathQuery", "v.paths:(rabbit cat)").vertices(), false) .count()); assertEquals(1L, Optionals.streamOf(titan.query().has("paths", Text.CONTAINS, "cat").vertices(), false).count()); final TitanTransaction tx2 = titan.buildTransaction().start(); //these all intermittently fail?! (well consistently either work or fail for some period, v unclear why) // but starting from a transaction instead of titan as below seems to fix this, so something in titan must not be getting updated assertEquals(1L, Optionals.streamOf(tx2.query().has("paths").vertices(), false).count()); // (Ah interestingly this doesn't seem to accept a scan?!) assertEquals(1L, Optionals.streamOf(tx2.query().has("paths", "cat").vertices(), false).count()); assertEquals(1L, Optionals.streamOf(tx2.query().has("paths", Cmp.EQUAL, "cat").vertices(), false).count()); // note this doesn't use the ES query annoyingly assertEquals(1L, Optionals .streamOf(tx2.query().has("paths", Contain.IN, Arrays.asList("cat")).vertices(), false).count()); assertEquals(1L, Optionals .streamOf(tx2.query().has("paths", Contain.IN, Arrays.asList("rabbit", "cat")).vertices(), false) .count()); tx2.commit(); // need to figure out how to handle analyzed vs non-analyzed, I think it's TEXT vs STRING? // https://groups.google.com/forum/#!topic/aureliusgraphs/VGv-RJwt8zI // actual docs: http://s3.thinkaurelius.com/docs/titan/1.0.0/index-parameters.html // graph.makeKey("name").dataType(String.class).indexed("search", Element.class, new Parameter[]{Parameter.of(Mapping.MAPPING_PREFIX,Mapping.STRING)}) .single().make(); // Can you do a simple query on a list type (ie single cardinality)? Nope only over the entire list { assertEquals(0L, Optionals.streamOf(titan.query().has("set", "val1").vertices(), false).count()); assertEquals(1L, Optionals .streamOf(titan.query().has("set", Arrays.asList("val1", "val2").toArray()).vertices(), false) .count()); } } @SuppressWarnings("unchecked") //@org.junit.Test public void test_someBasicGraphBehavior() throws IOException { // Test some basic properties and transferring to/from GraphSON TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory") //.set("query.force-index", true) //(disabled for testing) .open(); buildSmallGraph(titan); final TitanTransaction tx = titan.buildTransaction().start(); tx.query().vertices().forEach(v -> System.out.println(v.keys() + " ... " + v.label() + " .. " + v.id())); { System.out.println("---- entire graph ------"); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); titan.io(IoCore.graphson()).writer().create().writeGraph(baos, titan); System.out.println(baos.toString()); } System.out.println("---- per vertex ------"); tx.query().vertices().forEach(Lambdas.wrap_consumer_i(v -> { { System.out.println("type = " + v.getClass().toString()); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); titan.io(IoCore.graphson()).writer().create().writeVertex(baos, v); System.out.println(baos.toString()); } Optionals.streamOf(v.edges(Direction.BOTH), false).forEach(Lambdas.wrap_consumer_i(e -> { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); titan.io(IoCore.graphson()).writer().create().writeEdge(baos, e); System.out.println(baos.toString()); })); })); // OK here's a nice example of how to convert a vertex to JsonNode Optionals.streamOf(tx.query().vertices(), false).findFirst().map( v -> titan.io(IoCore.graphson()).mapper().create().createMapper().convertValue(v, JsonNode.class)) .ifPresent(j -> System.out.println("?? " + j.toString())); // Looking at how to import vertices .. final String s1 = "{\"id\":4200,\"label\":\"test2\",\"type\":\"vertex\",\"properties\":{\"set\":[{\"id\":\"1l9-38o-5j9\",\"value\":[\"val1\",\"val2\"]}],\"type\":[{\"id\":\"171-38o-4qt\",\"value\":\"rabbit\"}]}}"; final org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper m = titan.io(IoCore.graphson()).mapper() .create().createMapper(); // confirmation that can't go JsonNode -> vertex directly //final CacheVertex v1 = m.convertValue(s1, CacheVertex.class); //final StandardVertex v1 = m.convertValue(m.readTree(s1), StandardVertex.class); //m.convertValue(m.readTree(s1), org.apache.tinkerpop.gremlin.structure.util.star.StarGraph.StarVertex.class); // But string -> vertex does work final Vertex read_vertex1 = titan.io(IoCore.graphson()).reader().create() .readVertex(new ByteArrayInputStream(s1.getBytes(StandardCharsets.UTF_8)), v -> v.get()); System.out.println("read: " + read_vertex1.getClass().toString() + ": " + m.convertValue(read_vertex1, JsonNode.class).toString()); // (note the properties _have_ to be that complicated) System.out.println("---- property query ------"); Optionals.streamOf(tx.query().has("type", "rabbit").vertices(), false).findFirst().map( v -> titan.io(IoCore.graphson()).mapper().create().createMapper().convertValue(v, JsonNode.class)) .ifPresent(j -> System.out.println("?? " + j.toString())); System.out.println("---- label query, returns nothing (can query edge labels, not vertex labels) ------"); Optionals.streamOf(tx.query().has("label", "test2").vertices(), false).findFirst().map( v -> titan.io(IoCore.graphson()).mapper().create().createMapper().convertValue(v, JsonNode.class)) .ifPresent(j -> System.out.println("?? " + j.toString())); } //@org.junit.Test public void test_someGraphErrors() throws IOException { // Test some graph errors TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory").set("query.force-index", true) //(disabled for testing) .open(); buildSmallGraph(titan); // 1) Check what happens if you try to do a non-indexed query: try { Optionals.streamOf(titan.query().vertices(), false).findFirst(); } catch (TitanException e) { System.out.println( "Threw expected titan exception: " + e.getClass().toString() + " / cause = " + e.getCause()); } // 2) Play around with indexes // I THINK THIS DOESN'T WORK BECAUSE INDEXES AREN'T SUPPORTED IN THIS VERSION OF TITAN... // This fails because type isn't unique: try { titan.openManagement().makePropertyKey("type").dataType(String.class).make(); } catch (com.thinkaurelius.titan.core.SchemaViolationException e) { System.out.println( "Threw expected titan exception: " + e.getClass().toString() + " / cause = " + e.getCause()); } try { final TitanManagement mgmt = titan.openManagement(); final PropertyKey type_key = mgmt.getPropertyKey("type"); mgmt.buildIndex("byType", Vertex.class).addKey(type_key).buildCompositeIndex(); //(this fails, as you'd expect) //mgmt.updateIndex(mgmt.getGraphIndex("byType"), SchemaAction.REINDEX).get(); mgmt.commit(); } catch (Exception e) { throw new RuntimeException(e); } // (Reindex existing data) try { for (int i = 0; i < 10; ++i) { com.thinkaurelius.titan.graphdb.database.management.ManagementSystem.awaitGraphIndexStatus(titan, "byType"); final TitanManagement mgmt = titan.openManagement(); final PropertyKey type_key = mgmt.getPropertyKey("type"); final SchemaStatus status = mgmt.getGraphIndex("byType").getIndexStatus(type_key); if (status != SchemaStatus.INSTALLED) break; System.out.println(status.toString()); Thread.sleep(250L); } { final TitanManagement mgmt = titan.openManagement(); mgmt.updateIndex(mgmt.getGraphIndex("byType"), SchemaAction.REINDEX).get(); mgmt.commit(); } } catch (Exception e) { System.out.println( "Threw expected titan exception: " + e.getClass().toString() + " / cause = " + e.getCause()); } } @SuppressWarnings("unchecked") //@org.junit.Test public void test_concurrentChanges_conflicting() throws IOException { // Test some graph errors TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory").set("query.force-index", false) //(not supported) .open(); buildSmallGraph(titan); final Supplier<TitanTransaction> build_trans = () -> { final TitanTransaction tx = titan.buildTransaction().start(); tx.addVertex("test_label"); Optionals.<TitanVertex>streamOf(tx.query().has("type", "rabbit").vertices(), false) .forEach(v -> v.property("change", "something")); return tx; }; final TitanTransaction tx1 = build_trans.get(); final TitanTransaction tx2 = titan.buildTransaction().start(); Optionals.<TitanVertex>streamOf(tx2.query().has("type", "rabbit").vertices(), false) .forEach(v -> v.property("change", "something_else")); tx2.addVertex("test_label"); tx2.commit(); try { tx1.commit(); fail("Should have thrown"); } catch (TitanException e) { System.out.println("Threw expected titan exception: " + e.getClass().toString() + " / cause = " + e.getCause() + " .. " + e.getCause().getCause()); assertEquals(com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException.class, e.getCause().getCause().getClass()); } // Check can retry: build_trans.get().commit(); } @SuppressWarnings("unchecked") @org.junit.Test public void test_concurrentChanges_conflicting_threadedTrans() throws IOException { // Test some graph errors TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory").set("query.force-index", false) //(not supported) .open(); buildSmallGraph(titan); final Supplier<TitanTransaction> build_trans = () -> { final TitanTransaction tx = titan.newTransaction(); tx.addVertex("test_label"); Optionals.<TitanVertex>streamOf(tx.query().has("type", "rabbit").vertices(), false) .forEach(v -> v.property("change", "something")); return tx; }; final TitanTransaction tx1 = build_trans.get(); final TitanTransaction tx2 = titan.newTransaction(); Optionals.<TitanVertex>streamOf(tx2.query().has("type", "rabbit").vertices(), false) .forEach(v -> v.property("change", "something_else")); tx2.addVertex("test_label"); tx2.commit(); try { tx1.commit(); fail("Should have thrown"); } catch (TitanException e) { System.out.println("Threw expected titan exception: " + e.getClass().toString() + " / cause = " + e.getCause() + " .. " + e.getCause().getCause()); assertEquals(com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException.class, e.getCause().getCause().getClass()); } // Check can retry: build_trans.get().commit(); } @SuppressWarnings("unchecked") //@org.junit.Test public void test_concurrentChanges_nonConflicting() throws IOException { // Test some graph errors TitanGraph titan = TitanFactory.build().set("storage.backend", "inmemory").set("query.force-index", false) //(not supported) .open(); buildSmallGraph(titan); final Supplier<TitanTransaction> build_trans = () -> { final TitanTransaction tx = titan.buildTransaction().start(); Optionals.<TitanVertex>streamOf(tx.query().has("type", "rabbit").vertices(), false) .forEach(v -> v.property("change", "something")); return tx; }; final TitanTransaction tx1 = build_trans.get(); final TitanTransaction tx2 = titan.buildTransaction().start(); Optionals.<TitanVertex>streamOf(tx2.query().hasNot("type", "rabbit").vertices(), false) .forEach(v -> v.property("change", "something_else")); { System.out.println("---- entire graph ... tx1 ------"); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); titan.io(IoCore.graphson()).writer().create().writeGraph(baos, tx1); System.out.println(baos.toString()); } { System.out.println("---- entire graph ... tx2 ------"); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); titan.io(IoCore.graphson()).writer().create().writeGraph(baos, tx2); System.out.println(baos.toString()); } tx2.commit(); // I was expecting this to work, but it still fails ... might be an issue with the inmemory storage again? try { tx1.commit(); fail("Should have thrown"); } catch (TitanException e) { System.out.println("Threw expected titan exception: " + e.getClass().toString() + " / cause = " + e.getCause() + " .. " + e.getCause().getCause()); assertEquals(com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException.class, e.getCause().getCause().getClass()); } } ////////////////////////////////////// public void buildSmallGraph(TitanGraph test) { final TitanTransaction tx = test.buildTransaction().start(); final TitanVertex v1 = tx.addVertex("test1"); v1.property("unprotected", "hai"); v1.property("protected", "by_me", "test_meta", "test_meta_value"); final TitanVertex v2 = tx.addVertex("test2"); // Check ids are assigned immediately: System.out.println("Assigned vertices with ids " + Arrays.asList(v1.id(), v2.id())); assertTrue(Long.class.isAssignableFrom(v1.id().getClass())); assertTrue(Long.class.isAssignableFrom(v2.id().getClass())); // how to multiple? v2.property("type", "rabbit"); final Object[] arr = Arrays.asList("val1", "val2").toArray(); // (note this has the underlying type of [String, which is important - [Object won't work) v2.property("set", arr); final TitanEdge e1 = v1.addEdge("test_v1_v2", v2); e1.property("edge_prop", "edge_prop_val"); tx.commit(); } }