Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.atlas; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Provider; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypesChangeListener; import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.graph.GraphBackedMetadataRepository; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.GremlinVersion; import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.services.DefaultMetadataService; import org.apache.atlas.services.MetadataService; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.typesystem.IInstance; import org.apache.atlas.typesystem.ITypedReferenceableInstance; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.types.*; import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; import org.apache.atlas.typesystem.types.cache.TypeCache; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.util.AtlasRepositoryConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; import org.codehaus.jettison.json.JSONArray; import org.testng.Assert; import org.testng.SkipException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import static org.apache.atlas.typesystem.types.utils.TypesUtil.*; import static org.testng.Assert.assertEquals; /** * Test utility class. */ public final class TestUtils { public static final long TEST_DATE_IN_LONG = 1418265358440L; public static final String EMPLOYEES_ATTR = "employees"; public static final String DEPARTMENT_ATTR = "department"; public static final String ASSETS_ATTR = "assets"; public static final String POSITIONS_ATTR = "positions"; public static final String ASSET_TYPE = "TestAsset"; public static final String DATABASE_TYPE = "hive_database"; public static final String DATABASE_NAME = "foo"; public static final String TABLE_TYPE = "hive_table"; public static final String PROCESS_TYPE = "hive_process"; public static final String COLUMN_TYPE = "column_type"; public static final String TABLE_NAME = "bar"; public static final String CLASSIFICATION = "classification"; public static final String PII = "PII"; public static final String SUPER_TYPE_NAME = "Base"; public static final String STORAGE_DESC_TYPE = "hive_storagedesc"; public static final String PARTITION_STRUCT_TYPE = "partition_struct_type"; public static final String PARTITION_CLASS_TYPE = "partition_class_type"; public static final String SERDE_TYPE = "serdeType"; public static final String COLUMNS_MAP = "columnsMap"; public static final String COLUMNS_ATTR_NAME = "columns"; public static final String NAME = "name"; private TestUtils() { } /** * Dumps the graph in GSON format in the path returned. * * @param graph handle to graph * @return path to the dump file * @throws Exception */ public static String dumpGraph(AtlasGraph<?, ?> graph) throws Exception { File tempFile = File.createTempFile("graph", ".gson"); System.out.println("tempFile.getPath() = " + tempFile.getPath()); GraphHelper.dumpToLog(graph); FileOutputStream os = null; try { os = new FileOutputStream(tempFile); graph.exportToGson(os); } finally { if (os != null) { try { os.close(); } catch (IOException e) { e.printStackTrace(); } } } return tempFile.getPath(); } /** * Class Hierarchy is: * Department(name : String, employees : Array[Person]) * Person(name : String, department : Department, manager : Manager) * Manager(subordinates : Array[Person]) extends Person * <p/> * Persons can have SecurityClearance(level : Int) clearance. */ public static void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException { String _description = "_description"; EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", "OrgLevel" + _description, new EnumValue("L1", 1), new EnumValue("L2", 2)); StructTypeDefinition addressDetails = createStructTypeDef("Address", "Address" + _description, createRequiredAttrDef("street", DataTypes.STRING_TYPE), createRequiredAttrDef("city", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department" + _description, ImmutableSet.<String>of(), createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), new AttributeDefinition(EMPLOYEES_ATTR, String.format("array<%s>", "Person"), Multiplicity.OPTIONAL, true, DEPARTMENT_ATTR), new AttributeDefinition(POSITIONS_ATTR, String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL, false, null)); HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person" + _description, ImmutableSet.<String>of(), createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createOptionalAttrDef("orgLevel", "OrgLevel"), createOptionalAttrDef("address", "Address"), new AttributeDefinition(DEPARTMENT_ATTR, "Department", Multiplicity.REQUIRED, false, EMPLOYEES_ATTR), new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"), new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null), new AttributeDefinition(ASSETS_ATTR, String.format("array<%s>", ASSET_TYPE), Multiplicity.OPTIONAL, false, null), createOptionalAttrDef("birthday", DataTypes.DATE_TYPE), createOptionalAttrDef("hasPets", DataTypes.BOOLEAN_TYPE), createOptionalAttrDef("numberOfCars", DataTypes.BYTE_TYPE), createOptionalAttrDef("houseNumber", DataTypes.SHORT_TYPE), createOptionalAttrDef("carMileage", DataTypes.INT_TYPE), createOptionalAttrDef("shares", DataTypes.LONG_TYPE), createOptionalAttrDef("salary", DataTypes.DOUBLE_TYPE), createOptionalAttrDef("age", DataTypes.FLOAT_TYPE), createOptionalAttrDef("numberOfStarsEstimate", DataTypes.BIGINTEGER_TYPE), createOptionalAttrDef("approximationOfPi", DataTypes.BIGDECIMAL_TYPE), createOptionalAttrDef("isOrganDonor", DataTypes.BOOLEAN_TYPE)); HierarchicalTypeDefinition<ClassType> assetTypeDef = createClassTypeDef(ASSET_TYPE, "Asset" + _description, ImmutableSet.<String>of(), createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), new AttributeDefinition("childAssets", String.format("array<%s>", ASSET_TYPE), Multiplicity.OPTIONAL, false, null)); HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager" + _description, ImmutableSet.of("Person"), new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false, "manager")); HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef("SecurityClearance", "SecurityClearance" + _description, ImmutableSet.<String>of(), createRequiredAttrDef("level", DataTypes.INT_TYPE)); ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails), ImmutableList.of(securityClearanceTypeDef), ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef, assetTypeDef)); } public static final String DEPARTMENT_TYPE = "Department"; public static final String PERSON_TYPE = "Person"; public static ITypedReferenceableInstance createDeptEg1(TypeSystem ts) throws AtlasException { Referenceable hrDept = new Referenceable(DEPARTMENT_TYPE); Referenceable john = new Referenceable(PERSON_TYPE); Referenceable jane = new Referenceable("Manager", "SecurityClearance"); Referenceable johnAddr = new Referenceable("Address"); Referenceable janeAddr = new Referenceable("Address"); Referenceable julius = new Referenceable("Manager"); Referenceable juliusAddr = new Referenceable("Address"); Referenceable max = new Referenceable("Person"); Referenceable maxAddr = new Referenceable("Address"); hrDept.set(NAME, "hr"); john.set(NAME, "John"); john.set(DEPARTMENT_ATTR, hrDept); johnAddr.set("street", "Stewart Drive"); johnAddr.set("city", "Sunnyvale"); john.set("address", johnAddr); john.set("birthday", new Date(1950, 5, 15)); john.set("isOrganDonor", true); john.set("hasPets", true); john.set("numberOfCars", 1); john.set("houseNumber", 153); john.set("carMileage", 13364); john.set("shares", 15000); john.set("salary", 123345.678); john.set("age", 50); john.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000")); john.set("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286")); jane.set(NAME, "Jane"); jane.set(DEPARTMENT_ATTR, hrDept); janeAddr.set("street", "Great America Parkway"); janeAddr.set("city", "Santa Clara"); jane.set("address", janeAddr); janeAddr.set("street", "Great America Parkway"); julius.set(NAME, "Julius"); julius.set(DEPARTMENT_ATTR, hrDept); juliusAddr.set("street", "Madison Ave"); juliusAddr.set("city", "Newtonville"); julius.set("address", juliusAddr); julius.set("subordinates", ImmutableList.<Referenceable>of()); max.set(NAME, "Max"); max.set(DEPARTMENT_ATTR, hrDept); maxAddr.set("street", "Ripley St"); maxAddr.set("city", "Newton"); max.set("address", maxAddr); max.set("manager", jane); max.set("mentor", julius); max.set("birthday", new Date(1979, 3, 15)); max.set("isOrganDonor", true); max.set("hasPets", true); max.set("age", 36); max.set("numberOfCars", 2); max.set("houseNumber", 17); max.set("carMileage", 13); max.set("shares", Long.MAX_VALUE); max.set("salary", Double.MAX_VALUE); max.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000")); max.set("approximationOfPi", new BigDecimal("3.1415926535897932")); john.set("manager", jane); john.set("mentor", max); hrDept.set(EMPLOYEES_ATTR, ImmutableList.of(john, jane, julius, max)); jane.set("subordinates", ImmutableList.of(john, max)); jane.getTrait("SecurityClearance").set("level", 1); ClassType deptType = ts.getDataType(ClassType.class, "Department"); ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED); Assert.assertNotNull(hrDept2); return hrDept2; } public static TypesDef simpleType() { HierarchicalTypeDefinition<ClassType> superTypeDefinition = createClassTypeDef("h_type", ImmutableSet.<String>of(), createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", new AttributeDefinition[] { createRequiredAttrDef(NAME, DataTypes.STRING_TYPE) }); HierarchicalTypeDefinition<TraitType> traitTypeDefinition = createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); EnumValue values[] = { new EnumValue("ONE", 1), }; EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values); return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition)); } public static TypesDef simpleTypeUpdated() { HierarchicalTypeDefinition<ClassType> superTypeDefinition = createClassTypeDef("h_type", ImmutableSet.<String>of(), createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition = createClassTypeDef("new_h_type", ImmutableSet.<String>of(), createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType", new AttributeDefinition[] { createRequiredAttrDef(NAME, DataTypes.STRING_TYPE) }); HierarchicalTypeDefinition<TraitType> traitTypeDefinition = createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of()); EnumValue values[] = { new EnumValue("ONE", 1), }; EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values); return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition)); } public static TypesDef simpleTypeUpdatedDiff() { HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition = createClassTypeDef("new_h_type", ImmutableSet.<String>of(), createOptionalAttrDef("attr", DataTypes.STRING_TYPE)); return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition)); } public static TypesDef defineHiveTypes() { String _description = "_description"; HierarchicalTypeDefinition<ClassType> superTypeDefinition = createClassTypeDef(SUPER_TYPE_NAME, ImmutableSet.<String>of(), createOptionalAttrDef("namespace", DataTypes.STRING_TYPE), createOptionalAttrDef("cluster", DataTypes.STRING_TYPE), createOptionalAttrDef("colo", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createOptionalAttrDef("created", DataTypes.DATE_TYPE), createOptionalAttrDef("isReplicated", DataTypes.BOOLEAN_TYPE), new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null), createRequiredAttrDef("description", DataTypes.STRING_TYPE)); StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description, new AttributeDefinition[] { createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createRequiredAttrDef("serde", DataTypes.STRING_TYPE), createOptionalAttrDef("description", DataTypes.STRING_TYPE) }); EnumValue values[] = { new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2), }; EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values); HierarchicalTypeDefinition<ClassType> columnsDefinition = createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(), createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE)); StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description, new AttributeDefinition[] { createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), }); AttributeDefinition[] attributeDefinitions = new AttributeDefinition[] { new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false, null), new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null), }; HierarchicalTypeDefinition<ClassType> storageDescClsDef = new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), attributeDefinitions); AttributeDefinition[] partClsAttributes = new AttributeDefinition[] { new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("table", TABLE_TYPE, Multiplicity.REQUIRED, false, null), new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false, null), new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, null), new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null), }; HierarchicalTypeDefinition<ClassType> partClsDef = new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description, ImmutableSet.of(SUPER_TYPE_NAME), partClsAttributes); HierarchicalTypeDefinition<ClassType> processClsType = new HierarchicalTypeDefinition<>(ClassType.class, PROCESS_TYPE, PROCESS_TYPE + _description, ImmutableSet.<String>of(), new AttributeDefinition[] { new AttributeDefinition("outputs", "array<" + TABLE_TYPE + ">", Multiplicity.OPTIONAL, false, null) }); HierarchicalTypeDefinition<ClassType> tableTypeDefinition = createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME), TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), createRequiredAttrDef("description", DataTypes.STRING_TYPE), createRequiredAttrDef("type", DataTypes.STRING_TYPE), createOptionalAttrDef("created", DataTypes.DATE_TYPE), // enum new AttributeDefinition("tableType", "tableType", Multiplicity.REQUIRED, false, null), // array of strings new AttributeDefinition("columnNames", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null), // array of classes new AttributeDefinition("columns", String.format("array<%s>", COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), // array of structs new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"), Multiplicity.OPTIONAL, true, null), // map of primitives new AttributeDefinition("parametersMap", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, true, null), //map of classes - new AttributeDefinition(COLUMNS_MAP, DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), COLUMN_TYPE), Multiplicity.OPTIONAL, true, null), //map of structs new AttributeDefinition("partitionsMap", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), "partition_struct_type"), Multiplicity.OPTIONAL, true, null), // struct reference new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null), new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null), // class reference new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null), //class reference as composite new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null)); HierarchicalTypeDefinition<TraitType> piiTypeDefinition = createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of()); HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(), createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition = createTraitTypeDef( "fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION), createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition, partitionDefinition), ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition), ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition, storageDescClsDef, partClsDef, processClsType)); } public static Collection<IDataType> createHiveTypes(TypeSystem typeSystem) throws Exception { if (!typeSystem.isRegistered(TABLE_TYPE)) { TypesDef typesDef = defineHiveTypes(); return typeSystem.defineTypes(typesDef).values(); } return new ArrayList<>(); } public static final String randomString() { return RandomStringUtils.randomAlphanumeric(10); } public static Referenceable createDBEntity() { Referenceable entity = new Referenceable(DATABASE_TYPE); String dbName = RandomStringUtils.randomAlphanumeric(10); entity.set(NAME, dbName); entity.set("description", "us db"); return entity; } public static Referenceable createTableEntity(String dbId) { Referenceable entity = new Referenceable(TABLE_TYPE); String tableName = RandomStringUtils.randomAlphanumeric(10); entity.set(NAME, tableName); entity.set("description", "random table"); entity.set("type", "type"); entity.set("tableType", "MANAGED"); entity.set("database", new Id(dbId, 0, DATABASE_TYPE)); entity.set("created", new Date()); return entity; } public static Referenceable createColumnEntity() { Referenceable entity = new Referenceable(COLUMN_TYPE); entity.set(NAME, RandomStringUtils.randomAlphanumeric(10)); entity.set("type", "VARCHAR(32)"); return entity; } /** * Creates an entity in the graph and does basic validation * of the GuidMapping that was created in the process. * */ public static String createInstance(MetadataService metadataService, Referenceable entity) throws Exception { RequestContext.createContext(); String entityjson = InstanceSerialization.toJson(entity, true); JSONArray entitiesJson = new JSONArray(); entitiesJson.put(entityjson); CreateUpdateEntitiesResult creationResult = metadataService.createEntities(entitiesJson.toString()); Map<String, String> guidMap = creationResult.getGuidMapping().getGuidAssignments(); Map<Id, Referenceable> referencedObjects = findReferencedObjects(entity); for (Map.Entry<Id, Referenceable> entry : referencedObjects.entrySet()) { Id foundId = entry.getKey(); if (foundId.isUnassigned()) { String guid = guidMap.get(entry.getKey()._getId()); Referenceable obj = entry.getValue(); loadAndDoSimpleValidation(guid, obj, metadataService); } } List<String> guids = creationResult.getCreatedEntities(); if (guids != null && guids.size() > 0) { return guids.get(guids.size() - 1); } return null; } private static Map<Id, Referenceable> findReferencedObjects(Referenceable ref) { Map<Id, Referenceable> result = new HashMap<>(); findReferencedObjects(ref, result); return result; } private static void findReferencedObjects(Referenceable ref, Map<Id, Referenceable> seen) { Id guid = ref.getId(); if (seen.containsKey(guid)) { return; } seen.put(guid, ref); for (Map.Entry<String, Object> attr : ref.getValuesMap().entrySet()) { Object value = attr.getValue(); if (value instanceof Referenceable) { findReferencedObjects((Referenceable) value, seen); } else if (value instanceof List) { for (Object o : (List) value) { if (o instanceof Referenceable) { findReferencedObjects((Referenceable) o, seen); } } } else if (value instanceof Map) { for (Object o : ((Map) value).values()) { if (o instanceof Referenceable) { findReferencedObjects((Referenceable) o, seen); } } } } } /** * Clears the state in the request context. * */ public static void resetRequestContext() { //reset the context while preserving the user String user = RequestContext.get().getUser(); RequestContext.createContext(); RequestContext.get().setUser(user); } /** * Triggers the Atlas initialization process using the specified MetadataRepository. * This causes the built-in types and their indices to be created. */ public static void setupGraphProvider(MetadataRepository repo) throws AtlasException { TypeCache typeCache = null; try { typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance(); } catch (Throwable t) { typeCache = new DefaultTypeCache(); } final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry()); Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() { @Override public TypesChangeListener get() { return indexer; } }; Configuration config = ApplicationProperties.get(); ITypeStore typeStore = new GraphBackedTypeStore(); DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo, typeStore, Collections.singletonList(indexerProvider), new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache); //commit the created types getGraph().commit(); } public static AtlasGraph getGraph() { return AtlasGraphProvider.getGraphInstance(); } /** * Adds a proxy wrapper around the specified MetadataService that automatically * resets the request context before every call. * * @param delegate * @return */ public static MetadataService addSessionCleanupWrapper(final MetadataService delegate) { return (MetadataService) Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(), new Class[] { MetadataService.class }, new InvocationHandler() { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { try { resetRequestContext(); Object result = method.invoke(delegate, args); return result; } catch (InvocationTargetException e) { e.getCause().printStackTrace(); throw e.getCause(); } catch (Throwable t) { t.printStackTrace(); throw t; } } }); } /** * Adds a proxy wrapper around the specified MetadataRepository that automatically * resets the request context before every call and either commits or rolls * back the graph transaction after every call. * * @param delegate * @return */ public static MetadataRepository addTransactionWrapper(final MetadataRepository delegate) { return (MetadataRepository) Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(), new Class[] { MetadataRepository.class }, new InvocationHandler() { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { boolean useTransaction = GraphBackedMetadataRepository.class .getMethod(method.getName(), method.getParameterTypes()) .isAnnotationPresent(GraphTransaction.class); try { resetRequestContext(); Object result = method.invoke(delegate, args); if (useTransaction) { System.out.println("Committing changes"); getGraph().commit(); System.out.println("Commit succeeded."); } return result; } catch (InvocationTargetException e) { e.getCause().printStackTrace(); if (useTransaction) { System.out.println("Rolling back changes due to exception."); getGraph().rollback(); } throw e.getCause(); } catch (Throwable t) { t.printStackTrace(); if (useTransaction) { System.out.println("Rolling back changes due to exception."); getGraph().rollback(); } throw t; } } }); } /** * Loads the entity and does sanity testing of the GuidMapping that was * created during the operation. * */ public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataRepository repositoryService) throws AtlasException { ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid); doSimpleValidation(original, loaded); return loaded; } /** * Loads the entity and does sanity testing of the GuidMapping that was * created during the operation. * */ public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataService repositoryService) throws AtlasException { ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid); doSimpleValidation(original, loaded); return loaded; } private static void doSimpleValidation(Referenceable original, IInstance loaded) throws AtlasException { assertEquals(loaded.getTypeName(), original.getTypeName()); ClassType ct = TypeSystem.getInstance().getDataType(ClassType.class, loaded.getTypeName()); //compare primitive fields for (AttributeInfo field : ct.fieldMapping.fields.values()) { if (field.dataType().getTypeCategory() == TypeCategory.PRIMITIVE) { if (original.get(field.name) != null) { Object rawLoadedValue = loaded.get(field.name); Object rawProvidedValue = original.get(field.name); Object convertedLoadedValue = field.dataType().convert(rawLoadedValue, Multiplicity.REQUIRED); Object convertedProvidedValue = field.dataType().convert(rawProvidedValue, Multiplicity.REQUIRED); assertEquals(convertedLoadedValue, convertedProvidedValue); } } } } /** * Validates that the two String Collections contain the same items, without * regard to order. * */ public static void assertContentsSame(Collection<String> actual, Collection<String> expected) { assertEquals(actual.size(), expected.size()); Set<String> checker = new HashSet<>(); checker.addAll(expected); checker.removeAll(actual); assertEquals(checker.size(), 0); } public static void skipForGremlin3EnabledGraphDb() throws SkipException { //ATLAS-1579 Currently, some tests are skipped for titan1 backened. As these tests are hard coded to use Gremlin2. See ATLAS-1579, ATLAS-1591 once it is fixed, please remove it. if (TestUtils.getGraph().getSupportedGremlinVersion() == GremlinVersion.THREE) { throw new SkipException("This test requires Gremlin2. Skipping test "); } } }