List of usage examples for org.apache.hadoop.conf Configuration write
@Override public void write(DataOutput out) throws IOException
From source file:gaffer.accumulostore.operation.spark.handler.GetJavaRDDOfAllElementsHandlerTest.java
License:Apache License
@Test public void checkGetAllElementsInJavaRDD() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); final Set<Element> expectedElements = new HashSet<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(TestGroups.ENTITY); entity.setVertex("" + i); final Edge edge1 = new Edge(TestGroups.EDGE); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*from ww w. j a va 2s .co m*/ edge1.putProperty(TestPropertyNames.COUNT, 2); final Edge edge2 = new Edge(TestGroups.EDGE); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty(TestPropertyNames.COUNT, 4); elements.add(edge1); elements.add(edge2); elements.add(entity); expectedElements.add(edge1); expectedElements.add(edge2); expectedElements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEntitySeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "gaffer.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for "1" final GetJavaRDDOfAllElements rddQuery = new GetJavaRDDOfAllElements.Builder() .javaSparkContext(sparkContext).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); final JavaRDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(rdd.collect()); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:gaffer.accumulostore.operation.spark.handler.GetJavaRDDOfElementsHandlerTest.java
License:Apache License
@Test public void checkGetCorrectElementsInJavaRDDForEntitySeed() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(ENTITY_GROUP); entity.setVertex("" + i); final Edge edge1 = new Edge(EDGE_GROUP); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*from w ww .j av a 2s .co m*/ edge1.putProperty("count", 2); final Edge edge2 = new Edge(EDGE_GROUP); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty("count", 4); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEntitySeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "gaffer.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for "1" GetJavaRDDOfElements<EntitySeed> rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>() .javaSparkContext(sparkContext).seeds(Collections.singleton(new EntitySeed("1"))).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); JavaRDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(rdd.collect()); final Set<Element> expectedElements = new HashSet<>(); final Entity entity1 = new Entity(ENTITY_GROUP); entity1.setVertex("1"); final Edge edge1B = new Edge(EDGE_GROUP); edge1B.setSource("1"); edge1B.setDestination("B"); edge1B.setDirected(false); edge1B.putProperty("count", 2); final Edge edge1C = new Edge(EDGE_GROUP); edge1C.setSource("1"); edge1C.setDestination("C"); edge1C.setDirected(false); edge1C.putProperty("count", 4); expectedElements.add(entity1); expectedElements.add(edge1B); expectedElements.add(edge1C); assertEquals(expectedElements, results); // Check get correct edges for "1" when specify entities only rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EntitySeed("1"))) .view(new View.Builder().entity(ENTITY_GROUP).build()).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); expectedElements.add(entity1); assertEquals(expectedElements, results); // Check get correct edges for "1" when specify edges only rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EntitySeed("1"))).view(new View.Builder().edge(EDGE_GROUP).build()) .build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); expectedElements.add(edge1B); expectedElements.add(edge1C); assertEquals(expectedElements, results); // Check get correct edges for "1" and "5" Set<EntitySeed> seeds = new HashSet<>(); seeds.add(new EntitySeed("1")); seeds.add(new EntitySeed("5")); rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>().javaSparkContext(sparkContext).seeds(seeds) .build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); final Entity entity5 = new Entity(ENTITY_GROUP); entity5.setVertex("5"); final Edge edge5B = new Edge(EDGE_GROUP); edge5B.setSource("5"); edge5B.setDestination("B"); edge5B.setDirected(false); edge5B.putProperty("count", 2); final Edge edge5C = new Edge(EDGE_GROUP); edge5C.setSource("5"); edge5C.setDestination("C"); edge5C.setDirected(false); edge5C.putProperty("count", 4); expectedElements.clear(); expectedElements.add(entity1); expectedElements.add(edge1B); expectedElements.add(edge1C); expectedElements.add(entity5); expectedElements.add(edge5B); expectedElements.add(edge5C); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:gaffer.accumulostore.operation.spark.handler.GetJavaRDDOfElementsHandlerTest.java
License:Apache License
@Test public void checkGetCorrectElementsInRDDForEdgeSeed() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(ENTITY_GROUP); entity.setVertex("" + i); final Edge edge1 = new Edge(EDGE_GROUP); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/* w w w . ja v a 2s . c om*/ edge1.putProperty("count", 2); final Edge edge2 = new Edge(EDGE_GROUP); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty("count", 4); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEdgeSeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "gaffer.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for EdgeSeed 1 -> B GetJavaRDDOfElements<EdgeSeed> rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>() .javaSparkContext(sparkContext).seeds(Collections.singleton(new EdgeSeed("1", "B", false))) .setIncludeEdges(GetOperation.IncludeEdgeType.ALL).setIncludeEntities(false).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); JavaRDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(); results.addAll(rdd.collect()); final Set<Element> expectedElements = new HashSet<>(); final Edge edge1B = new Edge(EDGE_GROUP); edge1B.setSource("1"); edge1B.setDestination("B"); edge1B.setDirected(false); edge1B.putProperty("count", 2); expectedElements.add(edge1B); assertEquals(expectedElements, results); // Check get entity for 1 when query for 1 -> B and specify entities only rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))).setIncludeEntities(true) .setIncludeEdges(GetOperation.IncludeEdgeType.NONE).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); final Entity entity1 = new Entity(ENTITY_GROUP); entity1.setVertex("1"); expectedElements.add(entity1); assertEquals(expectedElements, results); // Check get correct edges for 1 -> B when specify edges only rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))) .view(new View.Builder().edge(EDGE_GROUP).build()).setIncludeEntities(false) .setIncludeEdges(GetOperation.IncludeEdgeType.ALL).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); expectedElements.add(edge1B); assertEquals(expectedElements, results); // Check get correct edges for 1 -> B and 5 -> C Set<EdgeSeed> seeds = new HashSet<>(); seeds.add(new EdgeSeed("1", "B", false)); seeds.add(new EdgeSeed("5", "C", false)); rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext) .setIncludeEntities(false).seeds(seeds).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); final Edge edge5C = new Edge(EDGE_GROUP); edge5C.setSource("5"); edge5C.setDestination("C"); edge5C.setDirected(false); edge5C.putProperty("count", 4); expectedElements.clear(); expectedElements.add(edge1B); expectedElements.add(edge5C); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:gaffer.accumulostore.operation.spark.handler.GetRDDOfElementsHandlerTest.java
License:Apache License
@Test public void checkGetCorrectElementsInRDDForEntitySeed() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(ENTITY_GROUP); entity.setVertex("" + i); final Edge edge1 = new Edge(EDGE_GROUP); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*from ww w. java2 s . c o m*/ edge1.putProperty("count", 2); final Edge edge2 = new Edge(EDGE_GROUP); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty("count", 4); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInRDDForEntitySeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "gaffer.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final SparkContext sparkContext = new SparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for "1" GetRDDOfElements<EntitySeed> rddQuery = new GetRDDOfElements.Builder<EntitySeed>() .sparkContext(sparkContext).seeds(Collections.singleton(new EntitySeed("1"))).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); RDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } Set<Element> results = new HashSet<>(); // NB: IDE suggests the cast in the following line is unnecessary but compilation fails without it Element[] returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } final Set<Element> expectedElements = new HashSet<>(); final Entity entity1 = new Entity(ENTITY_GROUP); entity1.setVertex("1"); final Edge edge1B = new Edge(EDGE_GROUP); edge1B.setSource("1"); edge1B.setDestination("B"); edge1B.setDirected(false); edge1B.putProperty("count", 2); final Edge edge1C = new Edge(EDGE_GROUP); edge1C.setSource("1"); edge1C.setDestination("C"); edge1C.setDirected(false); edge1C.putProperty("count", 4); expectedElements.add(entity1); expectedElements.add(edge1B); expectedElements.add(edge1C); assertEquals(expectedElements, results); // Check get correct edges for "1" when specify entities only rddQuery = new GetRDDOfElements.Builder<EntitySeed>().sparkContext(sparkContext) .seeds(Collections.singleton(new EntitySeed("1"))) .view(new View.Builder().entity(ENTITY_GROUP).build()).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } expectedElements.clear(); expectedElements.add(entity1); assertEquals(expectedElements, results); // Check get correct edges for "1" when specify edges only rddQuery = new GetRDDOfElements.Builder<EntitySeed>().sparkContext(sparkContext) .seeds(Collections.singleton(new EntitySeed("1"))).view(new View.Builder().edge(EDGE_GROUP).build()) .build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } expectedElements.clear(); expectedElements.add(edge1B); expectedElements.add(edge1C); assertEquals(expectedElements, results); // Check get correct edges for "1" and "5" Set<EntitySeed> seeds = new HashSet<>(); seeds.add(new EntitySeed("1")); seeds.add(new EntitySeed("5")); rddQuery = new GetRDDOfElements.Builder<EntitySeed>().sparkContext(sparkContext).seeds(seeds).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } final Entity entity5 = new Entity(ENTITY_GROUP); entity5.setVertex("5"); final Edge edge5B = new Edge(EDGE_GROUP); edge5B.setSource("5"); edge5B.setDestination("B"); edge5B.setDirected(false); edge5B.putProperty("count", 2); final Edge edge5C = new Edge(EDGE_GROUP); edge5C.setSource("5"); edge5C.setDestination("C"); edge5C.setDirected(false); edge5C.putProperty("count", 4); expectedElements.clear(); expectedElements.add(entity1); expectedElements.add(edge1B); expectedElements.add(edge1C); expectedElements.add(entity5); expectedElements.add(edge5B); expectedElements.add(edge5C); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:gaffer.accumulostore.operation.spark.handler.GetRDDOfElementsHandlerTest.java
License:Apache License
@Test public void checkGetCorrectElementsInRDDForEdgeSeed() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(ENTITY_GROUP); entity.setVertex("" + i); final Edge edge1 = new Edge(EDGE_GROUP); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);// w w w. ja v a 2s . c o m edge1.putProperty("count", 2); final Edge edge2 = new Edge(EDGE_GROUP); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty("count", 4); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInRDDForEdgeSeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "gaffer.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final SparkContext sparkContext = new SparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for EdgeSeed 1 -> B GetRDDOfElements<EdgeSeed> rddQuery = new GetRDDOfElements.Builder<EdgeSeed>().sparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))) .setIncludeEdges(GetOperation.IncludeEdgeType.ALL).setIncludeEntities(false).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); RDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } Set<Element> results = new HashSet<>(); // NB: IDE suggests the cast in the following line is unnecessary but compilation fails without it Element[] returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } final Set<Element> expectedElements = new HashSet<>(); final Edge edge1B = new Edge(EDGE_GROUP); edge1B.setSource("1"); edge1B.setDestination("B"); edge1B.setDirected(false); edge1B.putProperty("count", 2); expectedElements.add(edge1B); assertEquals(expectedElements, results); // Check get entity for 1 when query for 1 -> B and specify entities only rddQuery = new GetRDDOfElements.Builder<EdgeSeed>().sparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))).setIncludeEntities(true) .setIncludeEdges(GetOperation.IncludeEdgeType.NONE).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } expectedElements.clear(); final Entity entity1 = new Entity(ENTITY_GROUP); entity1.setVertex("1"); expectedElements.add(entity1); assertEquals(expectedElements, results); // Check get correct edges for 1 -> B when specify edges only rddQuery = new GetRDDOfElements.Builder<EdgeSeed>().sparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))) .view(new View.Builder().edge(EDGE_GROUP).build()).setIncludeEntities(false) .setIncludeEdges(GetOperation.IncludeEdgeType.ALL).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } expectedElements.clear(); expectedElements.add(edge1B); assertEquals(expectedElements, results); // Check get correct edges for 1 -> B and 5 -> C Set<EdgeSeed> seeds = new HashSet<>(); seeds.add(new EdgeSeed("1", "B", false)); seeds.add(new EdgeSeed("5", "C", false)); rddQuery = new GetRDDOfElements.Builder<EdgeSeed>().sparkContext(sparkContext).setIncludeEntities(false) .seeds(seeds).build(); rddQuery.addOption(AbstractGetRDDOperationHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); returnedElements = (Element[]) rdd.collect(); for (int i = 0; i < returnedElements.length; i++) { results.add(returnedElements[i]); } final Edge edge5C = new Edge(EDGE_GROUP); edge5C.setSource("5"); edge5C.setDestination("C"); edge5C.setDirected(false); edge5C.putProperty("count", 4); expectedElements.clear(); expectedElements.add(edge1B); expectedElements.add(edge5C); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:org.apache.hama.bsp.ResourceManager.java
License:Apache License
private byte[] getConfigurationOverride(Long groomRPCPort, Long groomPeerPort, Integer maxTasks, Long slotJVMHeap) {//from w ww . ja v a 2 s . c o m // Create a configuration from the current configuration and // override properties as appropriate for the Groom server. Configuration overrides = new Configuration(conf); overrides.set("bsp.groom.rpc.port", groomRPCPort.toString()); overrides.set("bsp.peer.port", groomPeerPort.toString()); overrides.set("bsp.tasks.maximum", maxTasks.toString()); overrides.set("bsp.child.java.opts", conf.get("bsp.child.java.opts") + slotJVMHeap + "m"); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { overrides.write(new DataOutputStream(baos)); baos.flush(); } catch (IOException e) { log.warn("Failed to serialize configuration.", e); System.exit(1); } return baos.toByteArray(); }
From source file:uk.gov.gchq.gaffer.sparkaccumulo.operation.handler.javardd.GetJavaRDDOfAllElementsHandlerTest.java
License:Apache License
@Test public void checkGetAllElementsInJavaRDD() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); final Set<Element> expectedElements = new HashSet<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(TestGroups.ENTITY); entity.setVertex("" + i); final Edge edge1 = new Edge(TestGroups.EDGE); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*from w ww . java2 s . c o m*/ edge1.putProperty(TestPropertyNames.COUNT, 2); final Edge edge2 = new Edge(TestGroups.EDGE); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty(TestPropertyNames.COUNT, 4); elements.add(edge1); elements.add(edge2); elements.add(entity); expectedElements.add(edge1); expectedElements.add(edge2); expectedElements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEntitySeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "uk.gov.gchq.gaffer.spark.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for "1" final GetJavaRDDOfAllElements rddQuery = new GetJavaRDDOfAllElements.Builder() .javaSparkContext(sparkContext).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); final JavaRDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(rdd.collect()); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:uk.gov.gchq.gaffer.sparkaccumulo.operation.handler.javardd.GetJavaRDDOfAllElementsHandlerTest.java
License:Apache License
@Test public void checkGetAllElementsInJavaRDDWithVisibility() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchemaWithVisibility.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 1; i++) { final Entity entity = new Entity(TestGroups.ENTITY); entity.setVertex("" + i); entity.putProperty("visibility", "public"); final Edge edge1 = new Edge(TestGroups.EDGE); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*from ww w. ja v a 2 s . c o m*/ edge1.putProperty(TestPropertyNames.COUNT, 2); edge1.putProperty("visibility", "private"); final Edge edge2 = new Edge(TestGroups.EDGE); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty(TestPropertyNames.COUNT, 4); edge2.putProperty("visibility", "public"); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User("user", Collections.singleton("public")); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEntitySeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "uk.gov.gchq.gaffer.spark.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Create user with just public auth, and user with both private and public final Set<String> publicNotPrivate = new HashSet<>(); publicNotPrivate.add("public"); final User userWithPublicNotPrivate = new User("user1", publicNotPrivate); final Set<String> privateAuth = new HashSet<>(); privateAuth.add("public"); privateAuth.add("private"); final User userWithPrivate = new User("user2", privateAuth); // Calculate correct results for 2 users final Set<Element> expectedElementsPublicNotPrivate = new HashSet<>(); final Set<Element> expectedElementsPrivate = new HashSet<>(); for (final Element element : elements) { expectedElementsPrivate.add(element); if (element.getProperty("visibility").equals("public")) { expectedElementsPublicNotPrivate.add(element); } } // Check get correct edges for user with just public GetJavaRDDOfAllElements rddQuery = new GetJavaRDDOfAllElements.Builder().javaSparkContext(sparkContext) .build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); JavaRDD<Element> rdd = graph1.execute(rddQuery, userWithPublicNotPrivate); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(rdd.collect()); assertEquals(expectedElementsPublicNotPrivate, results); // Check get correct edges for user with both private and public rddQuery = new GetJavaRDDOfAllElements.Builder().javaSparkContext(sparkContext).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, userWithPrivate); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); assertEquals(expectedElementsPrivate, results); sparkContext.stop(); }
From source file:uk.gov.gchq.gaffer.sparkaccumulo.operation.handler.javardd.GetJavaRDDOfElementsHandlerTest.java
License:Apache License
@Test public void checkGetCorrectElementsInJavaRDDForEntitySeed() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(ENTITY_GROUP); entity.setVertex("" + i); final Edge edge1 = new Edge(EDGE_GROUP); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*ww w. j a v a 2 s.c o m*/ edge1.putProperty("count", 2); final Edge edge2 = new Edge(EDGE_GROUP); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty("count", 4); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEntitySeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "uk.gov.gchq.gaffer.spark.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for "1" GetJavaRDDOfElements<EntitySeed> rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>() .javaSparkContext(sparkContext).seeds(Collections.singleton(new EntitySeed("1"))).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); JavaRDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(rdd.collect()); final Set<Element> expectedElements = new HashSet<>(); final Entity entity1 = new Entity(ENTITY_GROUP); entity1.setVertex("1"); final Edge edge1B = new Edge(EDGE_GROUP); edge1B.setSource("1"); edge1B.setDestination("B"); edge1B.setDirected(false); edge1B.putProperty("count", 2); final Edge edge1C = new Edge(EDGE_GROUP); edge1C.setSource("1"); edge1C.setDestination("C"); edge1C.setDirected(false); edge1C.putProperty("count", 4); expectedElements.add(entity1); expectedElements.add(edge1B); expectedElements.add(edge1C); assertEquals(expectedElements, results); // Check get correct edges for "1" when specify entities only rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EntitySeed("1"))) .view(new View.Builder().entity(ENTITY_GROUP).build()).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); expectedElements.add(entity1); assertEquals(expectedElements, results); // Check get correct edges for "1" when specify edges only rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EntitySeed("1"))).view(new View.Builder().edge(EDGE_GROUP).build()) .build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); expectedElements.add(edge1B); expectedElements.add(edge1C); assertEquals(expectedElements, results); // Check get correct edges for "1" and "5" Set<EntitySeed> seeds = new HashSet<>(); seeds.add(new EntitySeed("1")); seeds.add(new EntitySeed("5")); rddQuery = new GetJavaRDDOfElements.Builder<EntitySeed>().javaSparkContext(sparkContext).seeds(seeds) .build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); final Entity entity5 = new Entity(ENTITY_GROUP); entity5.setVertex("5"); final Edge edge5B = new Edge(EDGE_GROUP); edge5B.setSource("5"); edge5B.setDestination("B"); edge5B.setDirected(false); edge5B.putProperty("count", 2); final Edge edge5C = new Edge(EDGE_GROUP); edge5C.setSource("5"); edge5C.setDestination("C"); edge5C.setDirected(false); edge5C.putProperty("count", 4); expectedElements.clear(); expectedElements.add(entity1); expectedElements.add(edge1B); expectedElements.add(edge1C); expectedElements.add(entity5); expectedElements.add(edge5B); expectedElements.add(edge5C); assertEquals(expectedElements, results); sparkContext.stop(); }
From source file:uk.gov.gchq.gaffer.sparkaccumulo.operation.handler.javardd.GetJavaRDDOfElementsHandlerTest.java
License:Apache License
@Test public void checkGetCorrectElementsInRDDForEdgeSeed() throws OperationException, IOException { final Graph graph1 = new Graph.Builder() .addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")) .addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")) .addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")) .storeProperties(getClass().getResourceAsStream("/store.properties")).build(); final List<Element> elements = new ArrayList<>(); for (int i = 0; i < 10; i++) { final Entity entity = new Entity(ENTITY_GROUP); entity.setVertex("" + i); final Edge edge1 = new Edge(EDGE_GROUP); edge1.setSource("" + i); edge1.setDestination("B"); edge1.setDirected(false);/*from w w w . java 2 s . c o m*/ edge1.putProperty("count", 2); final Edge edge2 = new Edge(EDGE_GROUP); edge2.setSource("" + i); edge2.setDestination("C"); edge2.setDirected(false); edge2.putProperty("count", 4); elements.add(edge1); elements.add(edge2); elements.add(entity); } final User user = new User(); graph1.execute(new AddElements(elements), user); final SparkConf sparkConf = new SparkConf().setMaster("local") .setAppName("testCheckGetCorrectElementsInJavaRDDForEdgeSeed") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "uk.gov.gchq.gaffer.spark.serialisation.kryo.Registrator") .set("spark.driver.allowMultipleContexts", "true"); final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); // Create Hadoop configuration and serialise to a string final Configuration configuration = new Configuration(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); configuration.write(new DataOutputStream(baos)); final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8); // Check get correct edges for EdgeSeed 1 -> B GetJavaRDDOfElements<EdgeSeed> rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>() .javaSparkContext(sparkContext).seeds(Collections.singleton(new EdgeSeed("1", "B", false))) .setIncludeEdges(GetOperation.IncludeEdgeType.ALL).setIncludeEntities(false).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); JavaRDD<Element> rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } final Set<Element> results = new HashSet<>(); results.addAll(rdd.collect()); final Set<Element> expectedElements = new HashSet<>(); final Edge edge1B = new Edge(EDGE_GROUP); edge1B.setSource("1"); edge1B.setDestination("B"); edge1B.setDirected(false); edge1B.putProperty("count", 2); expectedElements.add(edge1B); assertEquals(expectedElements, results); // Check get entity for 1 when query for 1 -> B and specify entities only rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))).setIncludeEntities(true) .setIncludeEdges(GetOperation.IncludeEdgeType.NONE).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); final Entity entity1 = new Entity(ENTITY_GROUP); entity1.setVertex("1"); expectedElements.add(entity1); assertEquals(expectedElements, results); // Check get correct edges for 1 -> B when specify edges only rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext) .seeds(Collections.singleton(new EdgeSeed("1", "B", false))) .view(new View.Builder().edge(EDGE_GROUP).build()).setIncludeEntities(false) .setIncludeEdges(GetOperation.IncludeEdgeType.ALL).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); expectedElements.clear(); expectedElements.add(edge1B); assertEquals(expectedElements, results); // Check get correct edges for 1 -> B and 5 -> C Set<EdgeSeed> seeds = new HashSet<>(); seeds.add(new EdgeSeed("1", "B", false)); seeds.add(new EdgeSeed("5", "C", false)); rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext) .setIncludeEntities(false).seeds(seeds).build(); rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString); rdd = graph1.execute(rddQuery, user); if (rdd == null) { fail("No RDD returned"); } results.clear(); results.addAll(rdd.collect()); final Edge edge5C = new Edge(EDGE_GROUP); edge5C.setSource("5"); edge5C.setDestination("C"); edge5C.setDirected(false); edge5C.putProperty("count", 4); expectedElements.clear(); expectedElements.add(edge1B); expectedElements.add(edge5C); assertEquals(expectedElements, results); sparkContext.stop(); }