From 9b00a9ddce40150fd66d87ddaae65250247bf196 Mon Sep 17 00:00:00 2001 From: Shwetha GS <sshivalingamurthy@hortonworks.com> Date: Wed, 22 Jun 2016 10:56:54 +0530 Subject: [PATCH] ATLAS-856 Lazy-load type cache provider (dkantor via shwethags) --- common/src/main/java/org/apache/atlas/ApplicationProperties.java | 5 ++--- release-log.txt | 1 + repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java | 49 +++++++++++++++++++++++++++++++++++++++---------- repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java | 14 ++++++++++++++ repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java | 9 +++++++++ repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java | 20 +++++++++++++++++--- repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java | 44 +++++++++++++++++++++++++++++--------------- repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java | 43 +++++++++++++++++++++++++++++++++++++++++++ repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java | 212 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java | 42 ++++++++++++++++++++++++++++++++++++++++++ repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java | 16 +++++++--------- typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java | 42 +++++++----------------------------------- typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java | 229 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java | 137 ----------------------------------------------------------------------------------------------------------------------------------------- typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java | 137 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java | 11 ++++++++--- typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProviderTest.java | 383 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheTest.java | 383 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java | 4 +++- 22 files changed, 1513 insertions(+), 828 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java create mode 100644 repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java create mode 100644 repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java create mode 100644 repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java create mode 100644 repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java create mode 100644 typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java delete mode 100644 typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java delete mode 100644 typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java create mode 100644 typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java delete mode 100644 typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProviderTest.java create mode 100644 typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheTest.java diff --git a/common/src/main/java/org/apache/atlas/ApplicationProperties.java b/common/src/main/java/org/apache/atlas/ApplicationProperties.java index 741e1a6..9b1e9cd 100644 --- a/common/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/common/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -108,10 +108,9 @@ public final class ApplicationProperties extends PropertiesConfiguration { return inConf.subset(prefix); } - public static Class getClass(String propertyName, String defaultValue, Class assignableClass) - throws AtlasException { + public static Class getClass(Configuration configuration, String propertyName, String defaultValue, + Class assignableClass) throws AtlasException { try { - Configuration configuration = get(); String propertyValue = configuration.getString(propertyName, defaultValue); Class<?> clazz = Class.forName(propertyValue); if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) { diff --git a/release-log.txt b/release-log.txt index 661bf21..235cb97 100644 --- a/release-log.txt +++ b/release-log.txt @@ -40,6 +40,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-856 Lazy-load type cache provider (dkantor via shwethags) ATLAS-931 Delete entities fails when hard delete is configured (dkantor via sumasai) ATLAS-932 UI: 'create tag' button does not work (mneethiraj via sumasai) ATLAS-928 UI is not showing the name column for hive tables in the schema tab (yhemanth via sumasai) diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index c4c7678..3486436 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers; import com.google.inject.multibindings.Multibinder; import com.google.inject.throwingproviders.ThrowingProviderBinder; import com.thinkaurelius.titan.core.TitanGraph; + import org.aopalliance.intercept.MethodInterceptor; import org.apache.atlas.discovery.DiscoveryService; import org.apache.atlas.discovery.DataSetLineageService; @@ -50,6 +51,9 @@ import org.apache.atlas.services.MetadataService; import org.apache.atlas.services.ReservedTypesRegistrar; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.TypeSystemProvider; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.apache.commons.configuration.Configuration; /** * Guice module for Repository module. @@ -85,9 +89,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton(); - bindAuditRepository(binder()); + Configuration configuration = getConfiguration(); + bindAuditRepository(binder(), configuration); + + bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton(); - bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton(); + bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton(); //Add EntityAuditListener as EntityChangeListener Multibinder<EntityChangeListener> entityChangeListenerBinder = @@ -99,9 +106,17 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor); } - protected void bindAuditRepository(Binder binder) { + protected Configuration getConfiguration() { + try { + return ApplicationProperties.get(); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } + + protected void bindAuditRepository(Binder binder, Configuration configuration) { - Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(); + Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration()); //Map EntityAuditRepository interface to configured implementation binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton(); @@ -117,10 +132,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl"; - private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() { + private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) { try { - return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, - HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); + return ApplicationProperties.getClass(configuration, + AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); } catch (AtlasException e) { throw new RuntimeException(e); } @@ -128,12 +143,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl"; - private Class<? extends DeleteHandler> getDeleteHandlerImpl() { + private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) { try { - return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY, - SoftDeleteHandler.class.getName(), DeleteHandler.class); + return ApplicationProperties.getClass(configuration, + DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class); } catch (AtlasException e) { throw new RuntimeException(e); } } + + public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl"; + + protected Class<? extends TypeCache> getTypeCache(Configuration configuration) { + + // Get the type cache implementation class from Atlas configuration. + try { + return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY, + DefaultTypeCache.class.getName(), TypeCache.class); + } catch (AtlasException e) { + throw new RuntimeException("Error getting TypeCache implementation class", e); + } + } + } diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java index 5ed9e02..4503899 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java @@ -214,6 +214,20 @@ public class GraphBackedTypeStore implements ITypeStore { Iterator vertices = titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator(); + return getTypesFromVertices(vertices); + } + + @Override + @GraphTransaction + public TypesDef restoreType(String typeName) throws AtlasException { + // Get vertex for the specified type name. + Iterator vertices = + titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator(); + + return getTypesFromVertices(vertices); + } + + private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException { ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder(); ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder(); ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder(); diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java index 790c4b3..25f5f8b 100755 --- a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java @@ -40,4 +40,13 @@ public interface ITypeStore { * @throws AtlasException */ TypesDef restore() throws AtlasException; + + /** + * Restore the specified type definition + * + * @param typeName name of requested type + * @return persisted type definition + * @throws AtlasException + */ + TypesDef restoreType(String typeName) throws AtlasException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java new file mode 100644 index 0000000..d0c6f6a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java @@ -0,0 +1,250 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeSystem.TransientTypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.utils.TypesUtil; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Singleton; + + +/** + * An extension of {@link DefaultTypeCache} which loads + * the requested type from the type store if it is not found in the cache, + * and adds it to the cache if it's found in the store. + * Any attribute and super types that are required by the requested type + * are also loaded from the store if they are not already in the cache. + */ +@Singleton +public class StoreBackedTypeCache extends DefaultTypeCache { + + private ITypeStore typeStore; + + private ImmutableList<String> coreTypes; + private TypeSystem typeSystem; + + @Inject + public StoreBackedTypeCache(final ITypeStore typeStore) { + this.typeStore = typeStore; + typeSystem = TypeSystem.getInstance(); + coreTypes = typeSystem.getCoreTypes(); + } + + private static class Context { + ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder(); + ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder(); + ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder(); + ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder(); + Set<String> loadedFromStore = new HashSet<>(); + + public void addTypesDefToLists(TypesDef typesDef) { + + List<EnumTypeDefinition> enumTypesAsJavaList = typesDef.enumTypesAsJavaList(); + enums.addAll(enumTypesAsJavaList); + for (EnumTypeDefinition etd : enumTypesAsJavaList) { + loadedFromStore.add(etd.name); + } + List<StructTypeDefinition> structTypesAsJavaList = typesDef.structTypesAsJavaList(); + structs.addAll(structTypesAsJavaList); + for (StructTypeDefinition std : structTypesAsJavaList) { + loadedFromStore.add(std.typeName); + } + List<HierarchicalTypeDefinition<ClassType>> classTypesAsJavaList = typesDef.classTypesAsJavaList(); + classTypes.addAll(classTypesAsJavaList); + for (HierarchicalTypeDefinition<ClassType> classTypeDef : classTypesAsJavaList) { + loadedFromStore.add(classTypeDef.typeName); + } + List<HierarchicalTypeDefinition<TraitType>> traitTypesAsJavaList = typesDef.traitTypesAsJavaList(); + traits.addAll(traitTypesAsJavaList); + for (HierarchicalTypeDefinition<TraitType> traitTypeDef : traitTypesAsJavaList) { + loadedFromStore.add(traitTypeDef.typeName); + } + } + + public boolean isLoadedFromStore(String typeName) { + return loadedFromStore.contains(typeName); + } + + public TypesDef getTypesDef() { + return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build()); + } + } + + @Override + public boolean has(String typeName) throws AtlasException { + + return (get(typeName) != null); + } + + /** + * Checks whether the specified type is cached in memory and does *not* + * access the type store. Used for testing. + * + * @param typeName + * @return + */ + public boolean isCachedInMemory(String typeName) throws AtlasException { + return super.has(typeName); + } + + /** + * Gets the requested type from the cache. + * This implementation will check the type store if the type is + * not already cached. If found in the type store, the type and + * any required super and attribute types are loaded from the type store, and + * added to the cache. + * + * @see org.apache.atlas.typesystem.types.cache.DefaultTypeCache#get(java.lang.String) + */ + @Override + public IDataType get(String typeName) throws AtlasException { + + IDataType type = super.get(typeName); + if (type != null) { + return type; + } + + // Type is not cached - check the type store. + // Any super and attribute types needed by the requested type + // which are not cached will also be loaded from the store. + Context context = new Context(); + TypesDef typesDef = getTypeFromStore(typeName, context); + if (typesDef.isEmpty()) { + // Type not found in the type store. + return null; + } + + // Add all types that were loaded from the store to the cache. + TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(context.getTypesDef(), false); + Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded(); + putAll(typesAdded.values()); + return typesAdded.get(typeName); + } + + private void getTypeFromCacheOrStore(String typeName, Context context) + throws AtlasException { + + if (coreTypes.contains(typeName) || super.has(typeName)) { + return; + } + + if (context.isLoadedFromStore(typeName)) { + return; + } + + // Type not cached and hasn't been loaded during this operation, so check the store. + TypesDef typesDef = getTypeFromStore(typeName, context); + if (typesDef.isEmpty()) { + // Attribute type not found in cache or store. + throw new AtlasException(typeName + " not found in type store"); + } + } + + private TypesDef getTypeFromStore(String typeName, Context context) + throws AtlasException { + + TypesDef typesDef = typeStore.restoreType(typeName); + if (!typesDef.isEmpty()) { + // Type found in store, add it to lists. + context.addTypesDefToLists(typesDef); + + // Check the attribute and super types that are + // used by the requested type, and restore them + // as needed. + checkAttributeAndSuperTypes(typesDef, context); + } + return typesDef; + } + + private void checkAttributeAndSuperTypes(TypesDef typesDef, Context context) + throws AtlasException { + + // Check the cache and store for attribute types and super types. + for (HierarchicalTypeDefinition<ClassType> classTypeDef : typesDef.classTypesAsJavaList()) { + checkAttributeTypes(classTypeDef.attributeDefinitions, context); + for (String superTypeName : classTypeDef.superTypes) { + getTypeFromCacheOrStore(superTypeName, context); + } + } + for (HierarchicalTypeDefinition<TraitType> traitTypeDef : typesDef.traitTypesAsJavaList()) { + checkAttributeTypes(traitTypeDef.attributeDefinitions, context); + for (String superTypeName : traitTypeDef.superTypes) { + getTypeFromCacheOrStore(superTypeName, context); + } + } + for (StructTypeDefinition structTypeDef : typesDef.structTypesAsJavaList()) { + checkAttributeTypes(structTypeDef.attributeDefinitions, context); + } + } + + private void checkAttributeTypes(AttributeDefinition[] attributeDefinitions, + Context context) throws AtlasException { + + for (AttributeDefinition attrDef : attributeDefinitions) { + checkAttributeType(attrDef, context); + } + } + + private void checkAttributeType(AttributeDefinition attrDef, Context context) throws AtlasException { + + List<String> typeNamesToLookup = new ArrayList<>(2); + + // Get the attribute type(s). + String elementTypeName = TypeUtils.parseAsArrayType(attrDef.dataTypeName); + if (elementTypeName != null) { + // Array attribute, lookup the element type. + typeNamesToLookup.add(elementTypeName); + } + else { + String[] mapTypeNames = TypeUtils.parseAsMapType(attrDef.dataTypeName); + if (mapTypeNames != null) { + // Map attribute, lookup the key and value types. + typeNamesToLookup.addAll(Arrays.asList(mapTypeNames)); + } + else { + // Not an array or map, lookup the attribute type. + typeNamesToLookup.add(attrDef.dataTypeName); + } + } + + for (String typeName : typeNamesToLookup) { + getTypeFromCacheOrStore(typeName, context); + } + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 37e7b66..99f0036 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Provider; + import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; @@ -60,6 +61,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.typesystem.types.ValueConversionException; +import org.apache.atlas.typesystem.types.cache.TypeCache; import org.apache.atlas.typesystem.types.utils.TypesUtil; import org.apache.atlas.utils.ParamChecker; import org.apache.commons.configuration.Configuration; @@ -71,6 +73,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; + import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -109,10 +112,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, final IBootstrapTypesRegistrar typesRegistrar, final Collection<Provider<TypesChangeListener>> typeListenerProviders, - final Collection<Provider<EntityChangeListener>> entityListenerProviders) + final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache) throws AtlasException { this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders, - TypeSystem.getInstance(), ApplicationProperties.get()); + TypeSystem.getInstance(), ApplicationProperties.get(), typeCache); } DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, @@ -120,10 +123,21 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang final Collection<Provider<TypesChangeListener>> typeListenerProviders, final Collection<Provider<EntityChangeListener>> entityListenerProviders, final TypeSystem typeSystem, - final Configuration configuration) throws AtlasException { + final Configuration configuration, TypeCache typeCache) throws AtlasException { this.typeStore = typeStore; this.typesRegistrar = typesRegistrar; this.typeSystem = typeSystem; + /** + * Ideally a TypeCache implementation should have been injected in the TypeSystemProvider, + * but a singleton of TypeSystem is constructed privately within the class so that + * clients of TypeSystem would never instantiate a TypeSystem object directly in + * their code. As soon as a client makes a call to TypeSystem.getInstance(), they + * should have the singleton ready for consumption. Manually inject TypeSystem with + * the Guice-instantiated type cache here, before types are restored. + * This allows cache implementations to participate in Guice dependency injection. + */ + this.typeSystem.setTypeCache(typeCache); + this.repository = repository; for (Provider<TypesChangeListener> provider : typeListenerProviders) { diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java index dd530a7..da47dc1 100755 --- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java @@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup; import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; + import org.apache.atlas.AtlasException; import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.TestUtils; @@ -53,6 +54,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.inject.Inject; + import java.util.Iterator; import java.util.List; import java.util.Map; @@ -64,6 +66,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructType @Guice(modules = RepositoryMetadataModule.class) public class GraphBackedTypeStoreTest { + private static final String DESCRIPTION = "_description"; + @Inject private GraphProvider<TitanGraph> graphProvider; @@ -97,6 +101,12 @@ public class GraphBackedTypeStoreTest { dumpGraph(); } + @Test(dependsOnMethods = "testStore") + public void testRestoreType() throws Exception { + TypesDef typesDef = ((GraphBackedTypeStore)typeStore).restoreType("Manager"); + verifyRestoredClassType(typesDef, "Manager"); + } + private void dumpGraph() { TitanGraph graph = graphProvider.get(); for (Vertex v : graph.getVertices()) { @@ -109,7 +119,6 @@ public class GraphBackedTypeStoreTest { @Test(dependsOnMethods = "testStore") public void testRestore() throws Exception { - String description = "_description"; TypesDef types = typeStore.restore(); //validate enum @@ -117,7 +126,7 @@ public class GraphBackedTypeStoreTest { Assert.assertEquals(1, enumTypes.size()); EnumTypeDefinition orgLevel = enumTypes.get(0); Assert.assertEquals(orgLevel.name, "OrgLevel"); - Assert.assertEquals(orgLevel.description, "OrgLevel"+description); + Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION); Assert.assertEquals(orgLevel.enumValues.length, 2); EnumValue enumValue = orgLevel.enumValues[0]; Assert.assertEquals(enumValue.value, "L1"); @@ -127,25 +136,14 @@ public class GraphBackedTypeStoreTest { List<StructTypeDefinition> structTypes = types.structTypesAsJavaList(); Assert.assertEquals(1, structTypes.size()); - boolean clsTypeFound = false; - List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList(); - for (HierarchicalTypeDefinition<ClassType> classType : classTypes) { - if (classType.typeName.equals("Manager")) { - ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName); - Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length); - Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size()); - Assert.assertEquals(classType.typeDescription, classType.typeName+description); - clsTypeFound = true; - } - } - Assert.assertTrue(clsTypeFound, "Manager type not restored"); + verifyRestoredClassType(types, "Manager"); //validate trait List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList(); Assert.assertEquals(1, traitTypes.size()); HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0); Assert.assertEquals("SecurityClearance", trait.typeName); - Assert.assertEquals(trait.typeName+description, trait.typeDescription); + Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription); Assert.assertEquals(1, trait.attributeDefinitions.length); AttributeDefinition attribute = trait.attributeDefinitions[0]; Assert.assertEquals("level", attribute.name); @@ -229,4 +227,20 @@ public class GraphBackedTypeStoreTest { } return edgeCount; } + + private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException { + boolean clsTypeFound = false; + List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList(); + for (HierarchicalTypeDefinition<ClassType> classType : classTypes) { + if (classType.typeName.equals(typeName)) { + ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName); + Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length); + Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size()); + Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION); + clsTypeFound = true; + } + } + Assert.assertTrue(clsTypeFound, typeName + " type not restored"); + } + } diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java new file mode 100644 index 0000000..e62af85 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.testng.Assert; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.google.inject.Inject; + +/** + * Verify Guice can successfully instantiate and inject StoreBackTypeCache. + * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration + * which has type cache implementation class set to {@link StoreBackedTypeCache}. + */ +@Guice(modules = StoreBackedTypeCacheTestModule.class) +public class StoreBackedTypeCacheConfigurationTest { + + @Inject + private TypeCache typeCache; + + @Test + public void testConfigureAsTypeCache() throws Exception { + // Verify Guice successfully instantiated and injected StoreBackTypeCache + Assert.assertTrue(typeCache instanceof StoreBackedTypeCache); + } +} \ No newline at end of file diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java new file mode 100644 index 0000000..ee4cc28 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import java.util.HashMap; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.TestUtils; +import org.apache.atlas.repository.graph.GraphProvider; +import org.apache.atlas.typesystem.types.AttributeInfo; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.atlas.typesystem.types.HierarchicalType; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; + + +/** + * Unit test for {@link StoreBackedTypeCache} + */ +@Guice(modules = RepositoryMetadataModule.class) +public class StoreBackedTypeCacheTest { + + @Inject + private GraphProvider<TitanGraph> graphProvider; + + @Inject + private ITypeStore typeStore; + + @Inject + private StoreBackedTypeCache typeCache; + + private TypeSystem ts; + + private Map<String, ClassType> classTypesToTest = new HashMap<>(); + + @BeforeClass + public void setUp() throws Exception { + ts = TypeSystem.getInstance(); + ts.reset(); + + // Populate the type store for testing. + TestUtils.defineDeptEmployeeTypes(ts); + TestUtils.createHiveTypes(ts); + ImmutableList<String> typeNames = ts.getTypeNames(); + typeStore.store(ts, typeNames); + + ClassType type = ts.getDataType(ClassType.class, "Manager"); + classTypesToTest.put("Manager", type); + type = ts.getDataType(ClassType.class, TestUtils.TABLE_TYPE); + classTypesToTest.put(TestUtils.TABLE_TYPE, type); + } + + @AfterClass + public void tearDown() throws Exception { + ts.reset(); + try { + graphProvider.get().shutdown(); + } + catch(Exception e) { + e.printStackTrace(); + } + + try { + TitanCleanup.clear(graphProvider.get()); + } + catch(Exception e) { + e.printStackTrace(); + } + } + + @BeforeMethod + public void setupTestMethod() throws Exception { + typeCache.clear(); + } + + @Test + public void testGetClassType() throws Exception { + for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey())); + + IDataType dataType = typeCache.get(typeEntry.getKey()); + // Verify the type is now cached. + Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey())); + + Assert.assertTrue(dataType instanceof ClassType); + ClassType cachedType = (ClassType)dataType; + // Verify that get() also loaded and cached any dependencies of this type from the type store. + verifyHierarchicalType(cachedType, typeEntry.getValue()); + } + } + + @Test + public void testHasClassType() throws Exception { + for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey())); + + // Calling has() should result in type and its dependencies + // loaded from the type store and added to the cache. + Assert.assertTrue(typeCache.has(typeEntry.getKey())); + + // Verify the type is now cached in memory. + Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey())); + } + } + + @Test + public void testGetTraitType() throws Exception { + ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT); + for (String traitTypeName : traitNames) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName)); + + IDataType dataType = typeCache.get(traitTypeName); + // Verify the type is now cached. + Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName)); + + Assert.assertTrue(dataType instanceof TraitType); + TraitType cachedType = (TraitType)dataType; + // Verify that get() also loaded and cached any dependencies of this type from the type store. + verifyHierarchicalType(cachedType, ts.getDataType(TraitType.class, traitTypeName)); + } + } + + @Test + public void testHasTraitType() throws Exception { + ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT); + for (String traitTypeName : traitNames) { + // Not cached yet + Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName)); + + // Calling has() should result in type and its dependencies + // loaded from the type store and added to the cache. + Assert.assertTrue(typeCache.has(traitTypeName)); + + // Verify the type is now cached. + Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName)); + } + } + + private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException { + Assert.assertEquals(dataType.numFields, expectedDataType.numFields); + Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size()); + Assert.assertEquals(dataType.fieldMapping().fields.size(), expectedDataType.fieldMapping().fields.size()); + ImmutableSet<String> superTypes = dataType.superTypes; + Assert.assertEquals(superTypes.size(), expectedDataType.superTypes.size()); + + // Verify that any attribute and super types were also cached. + for (String superTypeName : superTypes) { + Assert.assertTrue(typeCache.has(superTypeName)); + } + for (AttributeInfo attrInfo : dataType.fieldMapping().fields.values()) { + switch (attrInfo.dataType().getTypeCategory()) { + case CLASS: + case STRUCT: + case ENUM: + Assert.assertTrue(typeCache.has(attrInfo.dataType().getName()), attrInfo.dataType().getName() + " should be cached"); + break; + case ARRAY: + String elementTypeName = TypeUtils.parseAsArrayType(attrInfo.dataType().getName()); + if (!ts.getCoreTypes().contains(elementTypeName)) { + Assert.assertTrue(typeCache.has(elementTypeName), elementTypeName + " should be cached"); + } + break; + case MAP: + String[] mapTypeNames = TypeUtils.parseAsMapType(attrInfo.dataType().getName()); + for (String typeName : mapTypeNames) { + if (!ts.getCoreTypes().contains(typeName)) { + Assert.assertTrue(typeCache.has(typeName), typeName + " should be cached"); + } + } + break; + default: + break; + } + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java new file mode 100644 index 0000000..058ed4d --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.typestore; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RepositoryMetadataModule; +import org.apache.commons.configuration.Configuration; + + +/** + * Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}. + * + */ +public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule { + @Override + protected Configuration getConfiguration() { + try { + Configuration configuration = ApplicationProperties.get(); + configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY, + StoreBackedTypeCache.class.getName()); + return configuration; + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java new file mode 100644 index 0000000..4cb5c67 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.service; + +import org.apache.atlas.TestUtils; +import org.apache.atlas.repository.typestore.ITypeStore; +import org.apache.atlas.repository.typestore.StoreBackedTypeCache; +import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule; +import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.cache.TypeCache; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; + + +/** + * Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store. + * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration + * which has type cache implementation class set to {@link StoreBackedTypeCache}. + */ +@Guice(modules = StoreBackedTypeCacheTestModule.class) +public class StoreBackedTypeCacheMetadataServiceTest +{ + @Inject + private MetadataService metadataService; + + @Inject + private ITypeStore typeStore; + + @Inject + TypeCache typeCache; + + private TypeSystem ts; + + @BeforeClass + public void setUp() throws Exception { + ts = TypeSystem.getInstance(); + ts.reset(); + + // Populate the type store for testing. + TestUtils.defineDeptEmployeeTypes(ts); + TestUtils.createHiveTypes(ts); + ImmutableList<String> typeNames = ts.getTypeNames(); + typeStore.store(ts, typeNames); + ts.reset(); + } + + @Test + public void testIt() throws Exception { + Assert.assertTrue(typeCache instanceof StoreBackedTypeCache); + StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache; + + // Cache should be empty + Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager")); + + // Type lookup on MetadataService should cause Manager type to be loaded from the type store + // and cached. + Assert.assertNotNull(metadataService.getTypeDefinition("Manager")); + Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager")); + } +} diff --git a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java index effee2a..393b539 100644 --- a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java +++ b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java @@ -19,6 +19,7 @@ package org.apache.atlas.services; import com.google.inject.Provider; + import org.apache.atlas.AtlasException; import org.apache.atlas.listener.EntityChangeListener; import org.apache.atlas.listener.TypesChangeListener; @@ -26,12 +27,8 @@ import org.apache.atlas.repository.MetadataRepository; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.atlas.ha.HAConfiguration; -import org.apache.atlas.listener.TypesChangeListener; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.TypeSystem; import org.apache.commons.configuration.Configuration; import org.mockito.Matchers; import org.mockito.Mock; @@ -44,6 +41,7 @@ import java.util.HashMap; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; @@ -78,7 +76,7 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class), mock(ITypeStore.class), typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, defaultMetadataService); @@ -91,10 +89,10 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository, typeStore, typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); verifyZeroInteractions(typeStore); - verifyZeroInteractions(typeSystem); + verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any()); verifyZeroInteractions(typesRegistrar); } @@ -109,7 +107,7 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository, typeStore, typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); defaultMetadataService.instanceIsActive(); verify(typeStore).restore(); @@ -134,7 +132,7 @@ public class DefaultMetadataServiceMockTest { DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository, typeStore, typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), - new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration); + new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null); defaultMetadataService.instanceIsActive(); defaultMetadataService.instanceIsPassive(); diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java index 14f1968..90b5815 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java @@ -30,14 +30,13 @@ import java.util.concurrent.ConcurrentHashMap; import javax.inject.Singleton; -import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.exception.TypeExistsException; import org.apache.atlas.typesystem.exception.TypeNotFoundException; -import org.apache.atlas.typesystem.types.cache.DefaultTypeCacheProvider; -import org.apache.atlas.typesystem.types.cache.ITypeCacheProvider; +import org.apache.atlas.typesystem.types.cache.DefaultTypeCache; +import org.apache.atlas.typesystem.types.cache.TypeCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +47,6 @@ import com.google.common.collect.ImmutableSet; @InterfaceAudience.Private public class TypeSystem { private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class); - private static final String CACHE_PROVIDER_CLASS_PROPERTY = "atlas.typesystem.cache.provider"; private static final TypeSystem INSTANCE = new TypeSystem(); private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() { @@ -60,7 +58,7 @@ public class TypeSystem { } }; - private ITypeCacheProvider typeCache; + private TypeCache typeCache = new DefaultTypeCache(); private IdType idType; private Map<String, IDataType> coreTypes; @@ -84,44 +82,18 @@ public class TypeSystem { return this; } + public void setTypeCache(TypeCache typeCache) { + this.typeCache = typeCache; + } + private void initialize() { - initCacheProvider(); coreTypes = new ConcurrentHashMap<>(); registerPrimitiveTypes(); registerCoreTypes(); } - /** - * Ideally a cache provider should have been injected in the TypeSystemProvider, - * but a singleton of TypeSystem is constructed privately within the class so that - * clients of TypeSystem would never instantiate a TypeSystem object directly in - * their code. As soon as a client makes a call to TypeSystem.getInstance(), they - * should have the singleton ready for consumption. To enable such an access pattern, - * it kind of becomes imperative to initialize the cache provider within the - * TypeSystem constructor (bypassing the GUICE way of injecting a cache provider) - */ - private void initCacheProvider() { - - // read the pluggable cache provider from Atlas configuration - final String defaultCacheProvider = DefaultTypeCacheProvider.class.getName(); - Class cacheProviderClass; - try { - cacheProviderClass = ApplicationProperties.getClass(CACHE_PROVIDER_CLASS_PROPERTY, - defaultCacheProvider, ITypeCacheProvider.class); - } catch (AtlasException e) { - throw new RuntimeException("Error getting type cache provider implementation class", e); - } - - try { - typeCache = (ITypeCacheProvider)cacheProviderClass.newInstance(); - } - catch (Exception e) { - throw new RuntimeException("Error creating instance of type cache provider implementation class " + cacheProviderClass.getName(), e); - } - } - public ImmutableList<String> getCoreTypes() { return ImmutableList.copyOf(coreTypes.keySet()); } diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java new file mode 100644 index 0000000..f683ed0 --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.typesystem.types.cache; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.atlas.typesystem.types.EnumType; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.StructType; +import org.apache.atlas.typesystem.types.TraitType; + +import com.google.inject.Singleton; + +/** + * Caches the types in-memory within the same process space. + */ +@Singleton +@SuppressWarnings("rawtypes") +public class DefaultTypeCache implements TypeCache { + + private Map<String, IDataType> types_ = new ConcurrentHashMap<>(); + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#has(java.lang + * .String) + */ + @Override + public boolean has(String typeName) throws AtlasException { + + return types_.containsKey(typeName); + } + + /* (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#has(org. + * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) + */ + @Override + public boolean has(TypeCategory typeCategory, String typeName) + throws AtlasException { + + assertValidTypeCategory(typeCategory); + return has(typeName); + } + + private void assertValidTypeCategory(TypeCategory typeCategory) throws + AtlasException { + + // there might no need of 'typeCategory' in this implementation for + // certain API, but for a distributed cache, it might help for the + // implementers to partition the types per their category + // while persisting so that look can be efficient + + if (typeCategory == null) { + throw new AtlasException("Category of the types to be filtered is null."); + } + + boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) || + typeCategory.equals(TypeCategory.TRAIT) || + typeCategory.equals(TypeCategory.ENUM) || + typeCategory.equals(TypeCategory.STRUCT); + + if (!validTypeCategory) { + throw new AtlasException("Category of the types should be one of CLASS " + + "| TRAIT | ENUM | STRUCT."); + } + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#get(java.lang + * .String) + */ + @Override + public IDataType get(String typeName) throws AtlasException { + + return types_.get(typeName); + } + + /* (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#get(org.apache. + * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) + */ + @Override + public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException { + + assertValidTypeCategory(typeCategory); + return get(typeName); + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#getNames(org + * .apache.atlas.typesystem.types.DataTypes.TypeCategory) + */ + @Override + public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException { + + assertValidTypeCategory(typeCategory); + + List<String> typeNames = new ArrayList<>(); + for (Entry<String, IDataType> typeEntry : types_.entrySet()) { + String name = typeEntry.getKey(); + IDataType type = typeEntry.getValue(); + + if (type.getTypeCategory().equals(typeCategory)) { + typeNames.add(name); + } + } + return typeNames; + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#getAllNames() + */ + @Override + public Collection<String> getAllTypeNames() throws AtlasException { + + return types_.keySet(); + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#put(org.apache + * .atlas.typesystem.types.IDataType) + */ + @Override + public void put(IDataType type) throws AtlasException { + + assertValidType(type); + types_.put(type.getName(), type); + } + + private void assertValidType(IDataType type) throws + AtlasException { + + if (type == null) { + throw new AtlasException("type is null."); + } + + boolean validTypeCategory = (type instanceof ClassType) || + (type instanceof TraitType) || + (type instanceof EnumType) || + (type instanceof StructType); + + if (!validTypeCategory) { + throw new AtlasException("Category of the types should be one of ClassType | " + + "TraitType | EnumType | StructType."); + } + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#putAll(java + * .util.Collection) + */ + @Override + public void putAll(Collection<IDataType> types) throws AtlasException { + + for (IDataType type : types) { + assertValidType(type); + types_.put(type.getName(), type); + } + } + + /* + * (non-Javadoc) + * @see + * org.apache.atlas.typesystem.types.cache.TypeCache#remove(java + * .lang.String) + */ + @Override + public void remove(String typeName) throws AtlasException { + + types_.remove(typeName); + } + + /* (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#remove(org. + * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) + */ + @Override + public void remove(TypeCategory typeCategory, String typeName) + throws AtlasException { + + assertValidTypeCategory(typeCategory); + remove(typeName); + } + + /* + * (non-Javadoc) + * @see org.apache.atlas.typesystem.types.cache.TypeCache#clear() + */ + @Override + public void clear() { + + types_.clear(); + } +} diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java deleted file mode 100644 index 700bda6..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.typesystem.types.cache; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TraitType; - -import com.google.inject.Singleton; - -/** - * Caches the types in-memory within the same process space. - */ -@Singleton -@SuppressWarnings("rawtypes") -public class DefaultTypeCacheProvider implements ITypeCacheProvider { - - private Map<String, IDataType> types_ = new ConcurrentHashMap<>(); - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang - * .String) - */ - @Override - public boolean has(String typeName) throws AtlasException { - - return types_.containsKey(typeName); - } - - /* (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org. - * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) - */ - @Override - public boolean has(TypeCategory typeCategory, String typeName) - throws AtlasException { - - assertValidTypeCategory(typeCategory); - return has(typeName); - } - - private void assertValidTypeCategory(TypeCategory typeCategory) throws - AtlasException { - - // there might no need of 'typeCategory' in this implementation for - // certain API, but for a distributed cache, it might help for the - // implementers to partition the types per their category - // while persisting so that look can be efficient - - if (typeCategory == null) { - throw new AtlasException("Category of the types to be filtered is null."); - } - - boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) || - typeCategory.equals(TypeCategory.TRAIT) || - typeCategory.equals(TypeCategory.ENUM) || - typeCategory.equals(TypeCategory.STRUCT); - - if (!validTypeCategory) { - throw new AtlasException("Category of the types should be one of CLASS " - + "| TRAIT | ENUM | STRUCT."); - } - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang - * .String) - */ - @Override - public IDataType get(String typeName) throws AtlasException { - - return types_.get(typeName); - } - - /* (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache. - * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) - */ - @Override - public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException { - - assertValidTypeCategory(typeCategory); - return get(typeName); - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org - * .apache.atlas.typesystem.types.DataTypes.TypeCategory) - */ - @Override - public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException { - - assertValidTypeCategory(typeCategory); - - List<String> typeNames = new ArrayList<>(); - for (Entry<String, IDataType> typeEntry : types_.entrySet()) { - String name = typeEntry.getKey(); - IDataType type = typeEntry.getValue(); - - if (type.getTypeCategory().equals(typeCategory)) { - typeNames.add(name); - } - } - return typeNames; - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames() - */ - @Override - public Collection<String> getAllTypeNames() throws AtlasException { - - return types_.keySet(); - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache - * .atlas.typesystem.types.IDataType) - */ - @Override - public void put(IDataType type) throws AtlasException { - - assertValidType(type); - types_.put(type.getName(), type); - } - - private void assertValidType(IDataType type) throws - AtlasException { - - if (type == null) { - throw new AtlasException("type is null."); - } - - boolean validTypeCategory = (type instanceof ClassType) || - (type instanceof TraitType) || - (type instanceof EnumType) || - (type instanceof StructType); - - if (!validTypeCategory) { - throw new AtlasException("Category of the types should be one of ClassType | " - + "TraitType | EnumType | StructType."); - } - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java - * .util.Collection) - */ - @Override - public void putAll(Collection<IDataType> types) throws AtlasException { - - for (IDataType type : types) { - assertValidType(type); - types_.put(type.getName(), type); - } - } - - /* - * (non-Javadoc) - * @see - * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java - * .lang.String) - */ - @Override - public void remove(String typeName) throws AtlasException { - - types_.remove(typeName); - } - - /* (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org. - * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String) - */ - @Override - public void remove(TypeCategory typeCategory, String typeName) - throws AtlasException { - - assertValidTypeCategory(typeCategory); - remove(typeName); - } - - /* - * (non-Javadoc) - * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear() - */ - @Override - public void clear() { - - types_.clear(); - } -} diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java deleted file mode 100644 index 55e9ddc..0000000 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.typesystem.types.cache; - -import java.util.Collection; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.IDataType; - -/** - * The types are cached to allow faster lookup when type info is needed during - * creation/updation of entities, DSL query translation/execution. - * Implementations of this can chose to plugin a distributed cache provider - * or an in-memory cache synched across nodes in an Altas cluster. <br> - * <br> - * Type entries in the cache can be one of ... <br> - * {@link org.apache.atlas.typesystem.types.ClassType} <br> - * {@link org.apache.atlas.typesystem.types.TraitType} <br> - * {@link org.apache.atlas.typesystem.types.StructType} <br> - * {@link org.apache.atlas.typesystem.types.EnumType} - */ -@SuppressWarnings("rawtypes") -public interface ITypeCacheProvider { - - /** - * @param typeName - * @return true if the type exists in cache, false otherwise. - * @throws AtlasException - */ - boolean has(String typeName) throws AtlasException; - - /** - * @param typeCategory Non-null category of type. The category can be one of - * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. - * @param typeName - * @return true if the type of given category exists in cache, false otherwise. - * @throws AtlasException - */ - boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; - - /** - * @param name The name of the type. - * @return returns non-null type if cached, otherwise null - * @throws AtlasException - */ - public IDataType get(String typeName) throws AtlasException; - - /** - * @param typeCategory Non-null category of type. The category can be one of - * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. - * @param typeName - * @return returns non-null type (of the specified category) if cached, otherwise null - * @throws AtlasException - */ - public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; - - /** - * @param typeCategory The category of types to filter the returned types. Cannot be null. - * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT | - * TypeCategory.STRUCT | TypeCategory.ENUM. - * @return - * @throws AtlasException - */ - Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException; - - /** - * This is a convenience API to get the names of all types. - * - * @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory) - * @return - * @throws AtlasException - */ - Collection<String> getAllTypeNames() throws AtlasException; - - /** - * @param type The type to be added to the cache. The type should not be - * null, otherwise throws NullPointerException. <br> - * Type entries in the cache can be one of ... <br> - * {@link org.apache.atlas.typesystem.types.ClassType} <br> - * {@link org.apache.atlas.typesystem.types.TraitType} <br> - * {@link org.apache.atlas.typesystem.types.StructType} <br> - * {@link org.apache.atlas.typesystem.types.EnumType} - * @throws AtlasException - */ - void put(IDataType type) throws AtlasException; - - /** - * @param types The types to be added to the cache. The type should not be - * null, otherwise throws NullPointerException. <br> - * Type entries in the cache can be one of ... <br> - * {@link org.apache.atlas.typesystem.types.ClassType} <br> - * {@link org.apache.atlas.typesystem.types.TraitType} <br> - * {@link org.apache.atlas.typesystem.types.StructType} <br> - * {@link org.apache.atlas.typesystem.types.EnumType} - * @throws AtlasException - */ - void putAll(Collection<IDataType> types) throws AtlasException; - - /** - * @param typeName Name of the type to be removed from the cache. If type - * exists, it will be removed, otherwise does nothing. - * @throws AtlasException - */ - void remove(String typeName) throws AtlasException; - - /** - * @param typeCategory Non-null category of type. The category can be one of - * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. - * @param typeName Name of the type to be removed from the cache. If type - * exists, it will be removed, otherwise does nothing. - * @throws AtlasException - */ - void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; - - /** - * Clear the type cache - * - */ - void clear(); -} diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java new file mode 100644 index 0000000..27622c2 --- /dev/null +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.typesystem.types.cache; + +import java.util.Collection; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.IDataType; + +/** + * The types are cached to allow faster lookup when type info is needed during + * creation/updation of entities, DSL query translation/execution. + * Implementations of this can chose to plugin a distributed cache + * or an in-memory cache synched across nodes in an Altas cluster. <br> + * <br> + * Type entries in the cache can be one of ... <br> + * {@link org.apache.atlas.typesystem.types.ClassType} <br> + * {@link org.apache.atlas.typesystem.types.TraitType} <br> + * {@link org.apache.atlas.typesystem.types.StructType} <br> + * {@link org.apache.atlas.typesystem.types.EnumType} + */ +@SuppressWarnings("rawtypes") +public interface TypeCache { + + /** + * @param typeName + * @return true if the type exists in cache, false otherwise. + * @throws AtlasException + */ + boolean has(String typeName) throws AtlasException; + + /** + * @param typeCategory Non-null category of type. The category can be one of + * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. + * @param typeName + * @return true if the type of given category exists in cache, false otherwise. + * @throws AtlasException + */ + boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; + + /** + * @param name The name of the type. + * @return returns non-null type if cached, otherwise null + * @throws AtlasException + */ + public IDataType get(String typeName) throws AtlasException; + + /** + * @param typeCategory Non-null category of type. The category can be one of + * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. + * @param typeName + * @return returns non-null type (of the specified category) if cached, otherwise null + * @throws AtlasException + */ + public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; + + /** + * @param typeCategory The category of types to filter the returned types. Cannot be null. + * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT | + * TypeCategory.STRUCT | TypeCategory.ENUM. + * @return + * @throws AtlasException + */ + Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException; + + /** + * This is a convenience API to get the names of all types. + * + * @see TypeCache#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory) + * @return + * @throws AtlasException + */ + Collection<String> getAllTypeNames() throws AtlasException; + + /** + * @param type The type to be added to the cache. The type should not be + * null, otherwise throws NullPointerException. <br> + * Type entries in the cache can be one of ... <br> + * {@link org.apache.atlas.typesystem.types.ClassType} <br> + * {@link org.apache.atlas.typesystem.types.TraitType} <br> + * {@link org.apache.atlas.typesystem.types.StructType} <br> + * {@link org.apache.atlas.typesystem.types.EnumType} + * @throws AtlasException + */ + void put(IDataType type) throws AtlasException; + + /** + * @param types The types to be added to the cache. The type should not be + * null, otherwise throws NullPointerException. <br> + * Type entries in the cache can be one of ... <br> + * {@link org.apache.atlas.typesystem.types.ClassType} <br> + * {@link org.apache.atlas.typesystem.types.TraitType} <br> + * {@link org.apache.atlas.typesystem.types.StructType} <br> + * {@link org.apache.atlas.typesystem.types.EnumType} + * @throws AtlasException + */ + void putAll(Collection<IDataType> types) throws AtlasException; + + /** + * @param typeName Name of the type to be removed from the cache. If type + * exists, it will be removed, otherwise does nothing. + * @throws AtlasException + */ + void remove(String typeName) throws AtlasException; + + /** + * @param typeCategory Non-null category of type. The category can be one of + * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM. + * @param typeName Name of the type to be removed from the cache. If type + * exists, it will be removed, otherwise does nothing. + * @throws AtlasException + */ + void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException; + + /** + * Clear the type cache + * + */ + void clear(); +} diff --git a/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java b/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java index 0679642..6d43359 100644 --- a/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java +++ b/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java @@ -58,17 +58,22 @@ public class ApplicationPropertiesTest { @Test public void testGetClass() throws Exception { + Configuration configuration = ApplicationProperties.get(); + //read from atlas-application.properties - Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class); + Class cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl", + ApplicationProperties.class.getName(), TypeSystem.class); assertEquals(cls.getName(), TypeSystem.class.getName()); //default value - cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class); + cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem2.impl", + TypeSystem.class.getName(), TypeSystem.class); assertEquals(cls.getName(), TypeSystem.class.getName()); //incompatible assignTo class, should throw AtlasException try { - cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class); + cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl", + ApplicationProperties.class.getName(), ApplicationProperties.class); Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown."); } catch (AtlasException e) { diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProviderTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProviderTest.java deleted file mode 100644 index c426d50..0000000 --- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProviderTest.java +++ /dev/null @@ -1,383 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.typesystem.types.cache; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.atlas.AtlasException; -import org.apache.atlas.AtlasException; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.EnumValue; -import org.apache.atlas.typesystem.types.IDataType; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import com.google.common.collect.ImmutableSet; - -/** - * Tests functional behavior of {@link DefaultTypeCacheProvider} - */ -@SuppressWarnings("rawtypes") -public class DefaultTypeCacheProviderTest { - - private String CLASSTYPE_CUSTOMER = "Customer"; - private String STRUCTTYPE_ADDRESS = "Address"; - private String TRAITTYPE_PRIVILEGED = "Privileged"; - private String ENUMTYPE_SHIPPING = "Shipping"; - - private String UNKNOWN_TYPE = "UndefinedType"; - - private ClassType customerType; - private StructType addressType; - private TraitType privilegedTrait; - private EnumType shippingEnum; - - private DefaultTypeCacheProvider cacheProvider; - - @BeforeClass - public void onetimeSetup() throws Exception { - - // init TypeSystem - TypeSystem ts = TypeSystem.getInstance().reset(); - - // Customer ClassType - customerType = ts.defineClassType(TypesUtil - .createClassTypeDef(CLASSTYPE_CUSTOMER, ImmutableSet.<String>of(), - TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("id", DataTypes.LONG_TYPE))); - - // Address StructType - addressType = ts.defineStructType(STRUCTTYPE_ADDRESS, true, - TypesUtil.createRequiredAttrDef("first line", DataTypes.STRING_TYPE), - TypesUtil.createOptionalAttrDef("second line", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("city", DataTypes.STRING_TYPE), - TypesUtil.createRequiredAttrDef("pincode", DataTypes.INT_TYPE)); - - // Privileged TraitType - privilegedTrait = ts.defineTraitType(TypesUtil - .createTraitTypeDef(TRAITTYPE_PRIVILEGED, ImmutableSet.<String>of(), - TypesUtil.createRequiredAttrDef("category", DataTypes.INT_TYPE))); - - // Shipping EnumType - shippingEnum = ts.defineEnumType(TypesUtil.createEnumTypeDef(ENUMTYPE_SHIPPING, - new EnumValue("Domestic", 1), new EnumValue("International", 2))); - } - - @BeforeMethod - public void eachTestSetup() throws Exception { - - cacheProvider = new DefaultTypeCacheProvider(); - - cacheProvider.put(customerType); - cacheProvider.put(addressType); - cacheProvider.put(privilegedTrait); - cacheProvider.put(shippingEnum); - } - - @Test - public void testCacheGetType() throws Exception { - - IDataType custType = cacheProvider.get(CLASSTYPE_CUSTOMER); - verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class); - - IDataType addrType = cacheProvider.get(STRUCTTYPE_ADDRESS); - verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class); - - IDataType privTrait = cacheProvider.get(TRAITTYPE_PRIVILEGED); - verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class); - - IDataType shippingEnum = cacheProvider.get(ENUMTYPE_SHIPPING); - verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class); - - assertNull(cacheProvider.get(UNKNOWN_TYPE)); - } - - @Test - public void testCacheGetTypeByCategory() throws Exception { - - IDataType custType = cacheProvider.get(TypeCategory.CLASS, CLASSTYPE_CUSTOMER); - verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class); - - IDataType addrType = cacheProvider.get(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS); - verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class); - - IDataType privTrait = cacheProvider.get(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED); - verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class); - - IDataType shippingEnum = cacheProvider.get(TypeCategory.ENUM, ENUMTYPE_SHIPPING); - verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class); - - assertNull(cacheProvider.get(UNKNOWN_TYPE)); - } - - private void verifyType(IDataType actualType, String expectedName, Class<? extends IDataType> typeClass) { - - assertNotNull(actualType, "The " + expectedName + " type not in cache"); - assertTrue(typeClass.isInstance(actualType)); - assertEquals(actualType.getName(), expectedName, "The type name does not match"); - } - - @Test - public void testCacheHasType() throws Exception { - - assertTrue(cacheProvider.has(CLASSTYPE_CUSTOMER)); - assertTrue(cacheProvider.has(STRUCTTYPE_ADDRESS)); - assertTrue(cacheProvider.has(TRAITTYPE_PRIVILEGED)); - assertTrue(cacheProvider.has(ENUMTYPE_SHIPPING)); - - assertFalse(cacheProvider.has(UNKNOWN_TYPE)); - } - - @Test - public void testCacheHasTypeByCategory() throws Exception { - - assertTrue(cacheProvider.has(TypeCategory.CLASS, CLASSTYPE_CUSTOMER)); - assertTrue(cacheProvider.has(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS)); - assertTrue(cacheProvider.has(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED)); - assertTrue(cacheProvider.has(TypeCategory.ENUM, ENUMTYPE_SHIPPING)); - - assertFalse(cacheProvider.has(UNKNOWN_TYPE)); - } - - @Test - public void testCacheGetAllTypeNames() throws Exception { - - List<String> allTypeNames = new ArrayList<String>(cacheProvider.getAllTypeNames()); - Collections.sort(allTypeNames); - - final int EXPECTED_TYPE_COUNT = 4; - assertEquals(allTypeNames.size(), EXPECTED_TYPE_COUNT, "Total number of types does not match."); - - assertEquals(STRUCTTYPE_ADDRESS, allTypeNames.get(0)); - assertEquals(CLASSTYPE_CUSTOMER, allTypeNames.get(1)); - assertEquals(TRAITTYPE_PRIVILEGED, allTypeNames.get(2)); - assertEquals(ENUMTYPE_SHIPPING, allTypeNames.get(3)); - } - - @Test - public void testCacheGetTypeNamesByCategory() throws Exception { - - List<String> classTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.CLASS)); - final int EXPECTED_CLASSTYPE_COUNT = 1; - assertEquals(classTypes.size(), EXPECTED_CLASSTYPE_COUNT); - assertEquals(CLASSTYPE_CUSTOMER, classTypes.get(0)); - - List<String> structTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.STRUCT)); - final int EXPECTED_STRUCTTYPE_COUNT = 1; - assertEquals(structTypes.size(), EXPECTED_STRUCTTYPE_COUNT); - assertEquals(STRUCTTYPE_ADDRESS, structTypes.get(0)); - - List<String> traitTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.TRAIT)); - final int EXPECTED_TRAITTYPE_COUNT = 1; - assertEquals(traitTypes.size(), EXPECTED_TRAITTYPE_COUNT); - assertEquals(TRAITTYPE_PRIVILEGED, traitTypes.get(0)); - - List<String> enumTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.ENUM)); - final int EXPECTED_ENUMTYPE_COUNT = 1; - assertEquals(enumTypes.size(), EXPECTED_ENUMTYPE_COUNT); - assertEquals(ENUMTYPE_SHIPPING, enumTypes.get(0)); - } - - @Test - public void testCacheBulkInsert() throws Exception { - - List<IDataType> allTypes = new ArrayList<>(); - allTypes.add(customerType); - allTypes.add(addressType); - allTypes.add(privilegedTrait); - allTypes.add(shippingEnum); - - // create a new cache provider instead of using the one setup for every method call - cacheProvider = new DefaultTypeCacheProvider(); - cacheProvider.putAll(allTypes); - - IDataType custType = cacheProvider.get(CLASSTYPE_CUSTOMER); - verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class); - - IDataType addrType = cacheProvider.get(STRUCTTYPE_ADDRESS); - verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class); - - IDataType privTrait = cacheProvider.get(TRAITTYPE_PRIVILEGED); - verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class); - - IDataType shippingEnum = cacheProvider.get(ENUMTYPE_SHIPPING); - verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class); - } - - @Test - public void testCacheRemove() throws Exception { - - cacheProvider.remove(CLASSTYPE_CUSTOMER); - assertNull(cacheProvider.get(CLASSTYPE_CUSTOMER)); - assertFalse(cacheProvider.has(CLASSTYPE_CUSTOMER)); - assertTrue(cacheProvider.getTypeNames(TypeCategory.CLASS).isEmpty()); - - final int EXPECTED_TYPE_COUNT = 3; - assertEquals(cacheProvider.getAllTypeNames().size(), EXPECTED_TYPE_COUNT); - } - - @Test - public void testCacheRemoveByCategory() throws Exception { - - cacheProvider.remove(TypeCategory.CLASS, CLASSTYPE_CUSTOMER); - assertNull(cacheProvider.get(CLASSTYPE_CUSTOMER)); - assertFalse(cacheProvider.has(CLASSTYPE_CUSTOMER)); - assertTrue(cacheProvider.getTypeNames(TypeCategory.CLASS).isEmpty()); - - final int EXPECTED_TYPE_COUNT = 3; - assertEquals(cacheProvider.getAllTypeNames().size(), EXPECTED_TYPE_COUNT); - } - - @Test - public void testCacheClear() throws Exception { - - cacheProvider.clear(); - - assertNull(cacheProvider.get(CLASSTYPE_CUSTOMER)); - assertFalse(cacheProvider.has(CLASSTYPE_CUSTOMER)); - - assertNull(cacheProvider.get(STRUCTTYPE_ADDRESS)); - assertFalse(cacheProvider.has(STRUCTTYPE_ADDRESS)); - - assertNull(cacheProvider.get(TRAITTYPE_PRIVILEGED)); - assertFalse(cacheProvider.has(TRAITTYPE_PRIVILEGED)); - - assertNull(cacheProvider.get(ENUMTYPE_SHIPPING)); - assertFalse(cacheProvider.has(ENUMTYPE_SHIPPING)); - - assertTrue(cacheProvider.getTypeNames(TypeCategory.CLASS).isEmpty()); - assertTrue(cacheProvider.getTypeNames(TypeCategory.STRUCT).isEmpty()); - assertTrue(cacheProvider.getTypeNames(TypeCategory.TRAIT).isEmpty()); - assertTrue(cacheProvider.getTypeNames(TypeCategory.ENUM).isEmpty()); - - assertTrue(cacheProvider.getAllTypeNames().isEmpty()); - } - - @Test(expectedExceptions = AtlasException.class) - public void testPutTypeWithNullType() throws Exception { - - cacheProvider.put(null); - fail("Null type should be not allowed in 'put'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testPutTypeWithInvalidType() throws Exception { - - cacheProvider.put(DataTypes.BOOLEAN_TYPE); - fail("type should only be an instance of ClassType | EnumType | StructType | TraitType in 'put'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testGetTypeWithNullCategory() throws Exception { - - cacheProvider.get(null, CLASSTYPE_CUSTOMER); - fail("Null TypeCategory should be not allowed in 'get'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testGetTypeWithInvalidCategory() throws Exception { - - cacheProvider.get(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName()); - fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'get'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheHasTypeWithNullCategory() throws Exception { - - cacheProvider.has(null, CLASSTYPE_CUSTOMER); - fail("Null TypeCategory should be not allowed in 'has'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheHasTypeWithInvalidCategory() throws Exception { - - cacheProvider.has(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName()); - fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'has'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheGetTypeNamesByNullCategory() throws Exception { - - cacheProvider.getTypeNames(null); - fail("Null TypeCategory should be not allowed in 'getNames'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheGetTypeNamesByInvalidCategory() throws Exception { - - cacheProvider.getTypeNames(TypeCategory.PRIMITIVE); - fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'getNames'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheBulkInsertWithNullType() throws Exception { - - List<IDataType> allTypes = new ArrayList<>(); - allTypes.add(null); - - // create a new cache provider instead of using the one setup for every method call - cacheProvider = new DefaultTypeCacheProvider(); - cacheProvider.putAll(allTypes); - - fail("Null type should be not allowed in 'putAll'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheBulkInsertWithInvalidType() throws Exception { - - List<IDataType> allTypes = new ArrayList<>(); - allTypes.add(DataTypes.BOOLEAN_TYPE); - - // create a new cache provider instead of using the one setup for every method call - cacheProvider = new DefaultTypeCacheProvider(); - cacheProvider.putAll(allTypes); - - fail("type should only one of ClassType | EnumType | StructType | TraitType in 'putAll'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheRemoveByNullCategory() throws Exception { - - cacheProvider.remove(null, CLASSTYPE_CUSTOMER); - fail("Null type should be not allowed in 'remove'"); - } - - @Test(expectedExceptions = AtlasException.class) - public void testCacheRemoveByInvalidCategory() throws Exception { - - cacheProvider.remove(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName()); - fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'remove'"); - } -} diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheTest.java new file mode 100644 index 0000000..5ed0d5e --- /dev/null +++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheTest.java @@ -0,0 +1,383 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.typesystem.types.cache; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.DataTypes.TypeCategory; +import org.apache.atlas.typesystem.types.EnumType; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.StructType; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableSet; + +/** + * Tests functional behavior of {@link DefaultTypeCache} + */ +@SuppressWarnings("rawtypes") +public class DefaultTypeCacheTest { + + private String CLASSTYPE_CUSTOMER = "Customer"; + private String STRUCTTYPE_ADDRESS = "Address"; + private String TRAITTYPE_PRIVILEGED = "Privileged"; + private String ENUMTYPE_SHIPPING = "Shipping"; + + private String UNKNOWN_TYPE = "UndefinedType"; + + private ClassType customerType; + private StructType addressType; + private TraitType privilegedTrait; + private EnumType shippingEnum; + + private DefaultTypeCache cache; + + @BeforeClass + public void onetimeSetup() throws Exception { + + // init TypeSystem + TypeSystem ts = TypeSystem.getInstance().reset(); + + // Customer ClassType + customerType = ts.defineClassType(TypesUtil + .createClassTypeDef(CLASSTYPE_CUSTOMER, ImmutableSet.<String>of(), + TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("id", DataTypes.LONG_TYPE))); + + // Address StructType + addressType = ts.defineStructType(STRUCTTYPE_ADDRESS, true, + TypesUtil.createRequiredAttrDef("first line", DataTypes.STRING_TYPE), + TypesUtil.createOptionalAttrDef("second line", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("city", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("pincode", DataTypes.INT_TYPE)); + + // Privileged TraitType + privilegedTrait = ts.defineTraitType(TypesUtil + .createTraitTypeDef(TRAITTYPE_PRIVILEGED, ImmutableSet.<String>of(), + TypesUtil.createRequiredAttrDef("category", DataTypes.INT_TYPE))); + + // Shipping EnumType + shippingEnum = ts.defineEnumType(TypesUtil.createEnumTypeDef(ENUMTYPE_SHIPPING, + new EnumValue("Domestic", 1), new EnumValue("International", 2))); + } + + @BeforeMethod + public void eachTestSetup() throws Exception { + + cache = new DefaultTypeCache(); + + cache.put(customerType); + cache.put(addressType); + cache.put(privilegedTrait); + cache.put(shippingEnum); + } + + @Test + public void testCacheGetType() throws Exception { + + IDataType custType = cache.get(CLASSTYPE_CUSTOMER); + verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class); + + IDataType addrType = cache.get(STRUCTTYPE_ADDRESS); + verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class); + + IDataType privTrait = cache.get(TRAITTYPE_PRIVILEGED); + verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class); + + IDataType shippingEnum = cache.get(ENUMTYPE_SHIPPING); + verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class); + + assertNull(cache.get(UNKNOWN_TYPE)); + } + + @Test + public void testCacheGetTypeByCategory() throws Exception { + + IDataType custType = cache.get(TypeCategory.CLASS, CLASSTYPE_CUSTOMER); + verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class); + + IDataType addrType = cache.get(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS); + verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class); + + IDataType privTrait = cache.get(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED); + verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class); + + IDataType shippingEnum = cache.get(TypeCategory.ENUM, ENUMTYPE_SHIPPING); + verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class); + + assertNull(cache.get(UNKNOWN_TYPE)); + } + + private void verifyType(IDataType actualType, String expectedName, Class<? extends IDataType> typeClass) { + + assertNotNull(actualType, "The " + expectedName + " type not in cache"); + assertTrue(typeClass.isInstance(actualType)); + assertEquals(actualType.getName(), expectedName, "The type name does not match"); + } + + @Test + public void testCacheHasType() throws Exception { + + assertTrue(cache.has(CLASSTYPE_CUSTOMER)); + assertTrue(cache.has(STRUCTTYPE_ADDRESS)); + assertTrue(cache.has(TRAITTYPE_PRIVILEGED)); + assertTrue(cache.has(ENUMTYPE_SHIPPING)); + + assertFalse(cache.has(UNKNOWN_TYPE)); + } + + @Test + public void testCacheHasTypeByCategory() throws Exception { + + assertTrue(cache.has(TypeCategory.CLASS, CLASSTYPE_CUSTOMER)); + assertTrue(cache.has(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS)); + assertTrue(cache.has(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED)); + assertTrue(cache.has(TypeCategory.ENUM, ENUMTYPE_SHIPPING)); + + assertFalse(cache.has(UNKNOWN_TYPE)); + } + + @Test + public void testCacheGetAllTypeNames() throws Exception { + + List<String> allTypeNames = new ArrayList<String>(cache.getAllTypeNames()); + Collections.sort(allTypeNames); + + final int EXPECTED_TYPE_COUNT = 4; + assertEquals(allTypeNames.size(), EXPECTED_TYPE_COUNT, "Total number of types does not match."); + + assertEquals(STRUCTTYPE_ADDRESS, allTypeNames.get(0)); + assertEquals(CLASSTYPE_CUSTOMER, allTypeNames.get(1)); + assertEquals(TRAITTYPE_PRIVILEGED, allTypeNames.get(2)); + assertEquals(ENUMTYPE_SHIPPING, allTypeNames.get(3)); + } + + @Test + public void testCacheGetTypeNamesByCategory() throws Exception { + + List<String> classTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.CLASS)); + final int EXPECTED_CLASSTYPE_COUNT = 1; + assertEquals(classTypes.size(), EXPECTED_CLASSTYPE_COUNT); + assertEquals(CLASSTYPE_CUSTOMER, classTypes.get(0)); + + List<String> structTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.STRUCT)); + final int EXPECTED_STRUCTTYPE_COUNT = 1; + assertEquals(structTypes.size(), EXPECTED_STRUCTTYPE_COUNT); + assertEquals(STRUCTTYPE_ADDRESS, structTypes.get(0)); + + List<String> traitTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.TRAIT)); + final int EXPECTED_TRAITTYPE_COUNT = 1; + assertEquals(traitTypes.size(), EXPECTED_TRAITTYPE_COUNT); + assertEquals(TRAITTYPE_PRIVILEGED, traitTypes.get(0)); + + List<String> enumTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.ENUM)); + final int EXPECTED_ENUMTYPE_COUNT = 1; + assertEquals(enumTypes.size(), EXPECTED_ENUMTYPE_COUNT); + assertEquals(ENUMTYPE_SHIPPING, enumTypes.get(0)); + } + + @Test + public void testCacheBulkInsert() throws Exception { + + List<IDataType> allTypes = new ArrayList<>(); + allTypes.add(customerType); + allTypes.add(addressType); + allTypes.add(privilegedTrait); + allTypes.add(shippingEnum); + + // create a new cache instead of using the one setup for every method call + cache = new DefaultTypeCache(); + cache.putAll(allTypes); + + IDataType custType = cache.get(CLASSTYPE_CUSTOMER); + verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class); + + IDataType addrType = cache.get(STRUCTTYPE_ADDRESS); + verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class); + + IDataType privTrait = cache.get(TRAITTYPE_PRIVILEGED); + verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class); + + IDataType shippingEnum = cache.get(ENUMTYPE_SHIPPING); + verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class); + } + + @Test + public void testCacheRemove() throws Exception { + + cache.remove(CLASSTYPE_CUSTOMER); + assertNull(cache.get(CLASSTYPE_CUSTOMER)); + assertFalse(cache.has(CLASSTYPE_CUSTOMER)); + assertTrue(cache.getTypeNames(TypeCategory.CLASS).isEmpty()); + + final int EXPECTED_TYPE_COUNT = 3; + assertEquals(cache.getAllTypeNames().size(), EXPECTED_TYPE_COUNT); + } + + @Test + public void testCacheRemoveByCategory() throws Exception { + + cache.remove(TypeCategory.CLASS, CLASSTYPE_CUSTOMER); + assertNull(cache.get(CLASSTYPE_CUSTOMER)); + assertFalse(cache.has(CLASSTYPE_CUSTOMER)); + assertTrue(cache.getTypeNames(TypeCategory.CLASS).isEmpty()); + + final int EXPECTED_TYPE_COUNT = 3; + assertEquals(cache.getAllTypeNames().size(), EXPECTED_TYPE_COUNT); + } + + @Test + public void testCacheClear() throws Exception { + + cache.clear(); + + assertNull(cache.get(CLASSTYPE_CUSTOMER)); + assertFalse(cache.has(CLASSTYPE_CUSTOMER)); + + assertNull(cache.get(STRUCTTYPE_ADDRESS)); + assertFalse(cache.has(STRUCTTYPE_ADDRESS)); + + assertNull(cache.get(TRAITTYPE_PRIVILEGED)); + assertFalse(cache.has(TRAITTYPE_PRIVILEGED)); + + assertNull(cache.get(ENUMTYPE_SHIPPING)); + assertFalse(cache.has(ENUMTYPE_SHIPPING)); + + assertTrue(cache.getTypeNames(TypeCategory.CLASS).isEmpty()); + assertTrue(cache.getTypeNames(TypeCategory.STRUCT).isEmpty()); + assertTrue(cache.getTypeNames(TypeCategory.TRAIT).isEmpty()); + assertTrue(cache.getTypeNames(TypeCategory.ENUM).isEmpty()); + + assertTrue(cache.getAllTypeNames().isEmpty()); + } + + @Test(expectedExceptions = AtlasException.class) + public void testPutTypeWithNullType() throws Exception { + + cache.put(null); + fail("Null type should be not allowed in 'put'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testPutTypeWithInvalidType() throws Exception { + + cache.put(DataTypes.BOOLEAN_TYPE); + fail("type should only be an instance of ClassType | EnumType | StructType | TraitType in 'put'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testGetTypeWithNullCategory() throws Exception { + + cache.get(null, CLASSTYPE_CUSTOMER); + fail("Null TypeCategory should be not allowed in 'get'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testGetTypeWithInvalidCategory() throws Exception { + + cache.get(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName()); + fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'get'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheHasTypeWithNullCategory() throws Exception { + + cache.has(null, CLASSTYPE_CUSTOMER); + fail("Null TypeCategory should be not allowed in 'has'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheHasTypeWithInvalidCategory() throws Exception { + + cache.has(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName()); + fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'has'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheGetTypeNamesByNullCategory() throws Exception { + + cache.getTypeNames(null); + fail("Null TypeCategory should be not allowed in 'getNames'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheGetTypeNamesByInvalidCategory() throws Exception { + + cache.getTypeNames(TypeCategory.PRIMITIVE); + fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'getNames'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheBulkInsertWithNullType() throws Exception { + + List<IDataType> allTypes = new ArrayList<>(); + allTypes.add(null); + + // create a new cache instead of using the one setup for every method call + cache = new DefaultTypeCache(); + cache.putAll(allTypes); + + fail("Null type should be not allowed in 'putAll'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheBulkInsertWithInvalidType() throws Exception { + + List<IDataType> allTypes = new ArrayList<>(); + allTypes.add(DataTypes.BOOLEAN_TYPE); + + // create a new cache instead of using the one setup for every method call + cache = new DefaultTypeCache(); + cache.putAll(allTypes); + + fail("type should only one of ClassType | EnumType | StructType | TraitType in 'putAll'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheRemoveByNullCategory() throws Exception { + + cache.remove(null, CLASSTYPE_CUSTOMER); + fail("Null type should be not allowed in 'remove'"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testCacheRemoveByInvalidCategory() throws Exception { + + cache.remove(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName()); + fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'remove'"); + } +} diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java b/webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java index 9cb76d3..b00080f 100644 --- a/webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java +++ b/webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java @@ -19,13 +19,15 @@ package org.apache.atlas.web.listeners; import com.google.inject.Binder; + import org.apache.atlas.RepositoryMetadataModule; import org.apache.atlas.repository.audit.EntityAuditRepository; import org.apache.atlas.repository.audit.InMemoryEntityAuditRepository; +import org.apache.commons.configuration.Configuration; public class TestModule extends RepositoryMetadataModule { @Override - protected void bindAuditRepository(Binder binder) { + protected void bindAuditRepository(Binder binder, Configuration configuration) { //Map EntityAuditRepository interface to hbase based implementation binder.bind(EntityAuditRepository.class).to(InMemoryEntityAuditRepository.class).asEagerSingleton(); } -- libgit2 0.27.1