Skip to content

A
atlas
Commit 9b00a9dd by Shwetha GS
Options

ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)

parent a7869bee
Showing with 861 additions and 174 deletions
common/src/main/java/org/apache/atlas/ApplicationProperties.java
......@@ -108,10 +108,9 @@ public final class ApplicationProperties extends PropertiesConfiguration {
return inConf.subset(prefix);
}
public static Class getClass(String propertyName, String defaultValue, Class assignableClass)
throws AtlasException {
public static Class getClass(Configuration configuration, String propertyName, String defaultValue,
Class assignableClass) throws AtlasException {
try {
Configuration configuration = get();
String propertyValue = configuration.getString(propertyName, defaultValue);
Class<?> clazz = Class.forName(propertyValue);
if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) {
......
release-log.txt
......@@ -40,6 +40,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)
ATLAS-931 Delete entities fails when hard delete is configured (dkantor via sumasai)
ATLAS-932 UI: 'create tag' button does not work (mneethiraj via sumasai)
ATLAS-928 UI is not showing the name column for hive tables in the schema tab (yhemanth via sumasai)
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.DataSetLineageService;
......@@ -50,6 +51,9 @@ import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeSystemProvider;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.commons.configuration.Configuration;
/**
* Guice module for Repository module.
......@@ -85,9 +89,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton();
bindAuditRepository(binder());
Configuration configuration = getConfiguration();
bindAuditRepository(binder(), configuration);
bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton();
bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton();
bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton();
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
......@@ -99,9 +106,17 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
}
protected void bindAuditRepository(Binder binder) {
protected Configuration getConfiguration() {
try {
return ApplicationProperties.get();
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
protected void bindAuditRepository(Binder binder, Configuration configuration) {
Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl();
Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration());
//Map EntityAuditRepository interface to configured implementation
binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
......@@ -117,10 +132,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY,
HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
return ApplicationProperties.getClass(configuration,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
......@@ -128,12 +143,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
private Class<? extends DeleteHandler> getDeleteHandlerImpl() {
private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
SoftDeleteHandler.class.getName(), DeleteHandler.class);
return ApplicationProperties.getClass(configuration,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
protected Class<? extends TypeCache> getTypeCache(Configuration configuration) {
// Get the type cache implementation class from Atlas configuration.
try {
return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
DefaultTypeCache.class.getName(), TypeCache.class);
} catch (AtlasException e) {
throw new RuntimeException("Error getting TypeCache implementation class", e);
}
}
}
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -214,6 +214,20 @@ public class GraphBackedTypeStore implements ITypeStore {
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
return getTypesFromVertices(vertices);
}
@Override
@GraphTransaction
public TypesDef restoreType(String typeName) throws AtlasException {
// Get vertex for the specified type name.
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
return getTypesFromVertices(vertices);
}
private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
......
repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
......@@ -40,4 +40,13 @@ public interface ITypeStore {
* @throws AtlasException
*/
TypesDef restore() throws AtlasException;
/**
* Restore the specified type definition
*
* @param typeName name of requested type
* @return persisted type definition
* @throws AtlasException
*/
TypesDef restoreType(String typeName) throws AtlasException;
}
repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeSystem.TransientTypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.google.inject.Singleton;
/**
* An extension of {@link DefaultTypeCache} which loads
* the requested type from the type store if it is not found in the cache,
* and adds it to the cache if it's found in the store.
* Any attribute and super types that are required by the requested type
* are also loaded from the store if they are not already in the cache.
*/
@Singleton
public class StoreBackedTypeCache extends DefaultTypeCache {
private ITypeStore typeStore;
private ImmutableList<String> coreTypes;
private TypeSystem typeSystem;
@Inject
public StoreBackedTypeCache(final ITypeStore typeStore) {
this.typeStore = typeStore;
typeSystem = TypeSystem.getInstance();
coreTypes = typeSystem.getCoreTypes();
}
private static class Context {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
Set<String> loadedFromStore = new HashSet<>();
public void addTypesDefToLists(TypesDef typesDef) {
List<EnumTypeDefinition> enumTypesAsJavaList = typesDef.enumTypesAsJavaList();
enums.addAll(enumTypesAsJavaList);
for (EnumTypeDefinition etd : enumTypesAsJavaList) {
loadedFromStore.add(etd.name);
}
List<StructTypeDefinition> structTypesAsJavaList = typesDef.structTypesAsJavaList();
structs.addAll(structTypesAsJavaList);
for (StructTypeDefinition std : structTypesAsJavaList) {
loadedFromStore.add(std.typeName);
}
List<HierarchicalTypeDefinition<ClassType>> classTypesAsJavaList = typesDef.classTypesAsJavaList();
classTypes.addAll(classTypesAsJavaList);
for (HierarchicalTypeDefinition<ClassType> classTypeDef : classTypesAsJavaList) {
loadedFromStore.add(classTypeDef.typeName);
}
List<HierarchicalTypeDefinition<TraitType>> traitTypesAsJavaList = typesDef.traitTypesAsJavaList();
traits.addAll(traitTypesAsJavaList);
for (HierarchicalTypeDefinition<TraitType> traitTypeDef : traitTypesAsJavaList) {
loadedFromStore.add(traitTypeDef.typeName);
}
}
public boolean isLoadedFromStore(String typeName) {
return loadedFromStore.contains(typeName);
}
public TypesDef getTypesDef() {
return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
}
@Override
public boolean has(String typeName) throws AtlasException {
return (get(typeName) != null);
}
/**
* Checks whether the specified type is cached in memory and does *not*
* access the type store. Used for testing.
*
* @param typeName
* @return
*/
public boolean isCachedInMemory(String typeName) throws AtlasException {
return super.has(typeName);
}
/**
* Gets the requested type from the cache.
* This implementation will check the type store if the type is
* not already cached. If found in the type store, the type and
* any required super and attribute types are loaded from the type store, and
* added to the cache.
*
* @see org.apache.atlas.typesystem.types.cache.DefaultTypeCache#get(java.lang.String)
*/
@Override
public IDataType get(String typeName) throws AtlasException {
IDataType type = super.get(typeName);
if (type != null) {
return type;
}
// Type is not cached - check the type store.
// Any super and attribute types needed by the requested type
// which are not cached will also be loaded from the store.
Context context = new Context();
TypesDef typesDef = getTypeFromStore(typeName, context);
if (typesDef.isEmpty()) {
// Type not found in the type store.
return null;
}
// Add all types that were loaded from the store to the cache.
TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(context.getTypesDef(), false);
Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
putAll(typesAdded.values());
return typesAdded.get(typeName);
}
private void getTypeFromCacheOrStore(String typeName, Context context)
throws AtlasException {
if (coreTypes.contains(typeName) || super.has(typeName)) {
return;
}
if (context.isLoadedFromStore(typeName)) {
return;
}
// Type not cached and hasn't been loaded during this operation, so check the store.
TypesDef typesDef = getTypeFromStore(typeName, context);
if (typesDef.isEmpty()) {
// Attribute type not found in cache or store.
throw new AtlasException(typeName + " not found in type store");
}
}
private TypesDef getTypeFromStore(String typeName, Context context)
throws AtlasException {
TypesDef typesDef = typeStore.restoreType(typeName);
if (!typesDef.isEmpty()) {
// Type found in store, add it to lists.
context.addTypesDefToLists(typesDef);
// Check the attribute and super types that are
// used by the requested type, and restore them
// as needed.
checkAttributeAndSuperTypes(typesDef, context);
}
return typesDef;
}
private void checkAttributeAndSuperTypes(TypesDef typesDef, Context context)
throws AtlasException {
// Check the cache and store for attribute types and super types.
for (HierarchicalTypeDefinition<ClassType> classTypeDef : typesDef.classTypesAsJavaList()) {
checkAttributeTypes(classTypeDef.attributeDefinitions, context);
for (String superTypeName : classTypeDef.superTypes) {
getTypeFromCacheOrStore(superTypeName, context);
}
}
for (HierarchicalTypeDefinition<TraitType> traitTypeDef : typesDef.traitTypesAsJavaList()) {
checkAttributeTypes(traitTypeDef.attributeDefinitions, context);
for (String superTypeName : traitTypeDef.superTypes) {
getTypeFromCacheOrStore(superTypeName, context);
}
}
for (StructTypeDefinition structTypeDef : typesDef.structTypesAsJavaList()) {
checkAttributeTypes(structTypeDef.attributeDefinitions, context);
}
}
private void checkAttributeTypes(AttributeDefinition[] attributeDefinitions,
Context context) throws AtlasException {
for (AttributeDefinition attrDef : attributeDefinitions) {
checkAttributeType(attrDef, context);
}
}
private void checkAttributeType(AttributeDefinition attrDef, Context context) throws AtlasException {
List<String> typeNamesToLookup = new ArrayList<>(2);
// Get the attribute type(s).
String elementTypeName = TypeUtils.parseAsArrayType(attrDef.dataTypeName);
if (elementTypeName != null) {
// Array attribute, lookup the element type.
typeNamesToLookup.add(elementTypeName);
}
else {
String[] mapTypeNames = TypeUtils.parseAsMapType(attrDef.dataTypeName);
if (mapTypeNames != null) {
// Map attribute, lookup the key and value types.
typeNamesToLookup.addAll(Arrays.asList(mapTypeNames));
}
else {
// Not an array or map, lookup the attribute type.
typeNamesToLookup.add(attrDef.dataTypeName);
}
}
for (String typeName : typeNamesToLookup) {
getTypeFromCacheOrStore(typeName, context);
}
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -60,6 +61,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.configuration.Configuration;
......@@ -71,6 +73,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
......@@ -109,10 +112,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders)
final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
throws AtlasException {
this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
TypeSystem.getInstance(), ApplicationProperties.get());
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
......@@ -120,10 +123,21 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
final TypeSystem typeSystem,
final Configuration configuration) throws AtlasException {
final Configuration configuration, TypeCache typeCache) throws AtlasException {
this.typeStore = typeStore;
this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem;
/**
* Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
* but a singleton of TypeSystem is constructed privately within the class so that
* clients of TypeSystem would never instantiate a TypeSystem object directly in
* their code. As soon as a client makes a call to TypeSystem.getInstance(), they
* should have the singleton ready for consumption. Manually inject TypeSystem with
* the Guice-instantiated type cache here, before types are restored.
* This allows cache implementations to participate in Guice dependency injection.
*/
this.typeSystem.setTypeCache(typeCache);
this.repository = repository;
for (Provider<TypesChangeListener> provider : typeListenerProviders) {
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
......@@ -53,6 +54,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
......@@ -64,6 +66,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructType
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedTypeStoreTest {
private static final String DESCRIPTION = "_description";
@Inject
private GraphProvider<TitanGraph> graphProvider;
......@@ -97,6 +101,12 @@ public class GraphBackedTypeStoreTest {
dumpGraph();
}
@Test(dependsOnMethods = "testStore")
public void testRestoreType() throws Exception {
TypesDef typesDef = ((GraphBackedTypeStore)typeStore).restoreType("Manager");
verifyRestoredClassType(typesDef, "Manager");
}
private void dumpGraph() {
TitanGraph graph = graphProvider.get();
for (Vertex v : graph.getVertices()) {
......@@ -109,7 +119,6 @@ public class GraphBackedTypeStoreTest {
@Test(dependsOnMethods = "testStore")
public void testRestore() throws Exception {
String description = "_description";
TypesDef types = typeStore.restore();
//validate enum
......@@ -117,7 +126,7 @@ public class GraphBackedTypeStoreTest {
Assert.assertEquals(1, enumTypes.size());
EnumTypeDefinition orgLevel = enumTypes.get(0);
Assert.assertEquals(orgLevel.name, "OrgLevel");
Assert.assertEquals(orgLevel.description, "OrgLevel"+description);
Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION);
Assert.assertEquals(orgLevel.enumValues.length, 2);
EnumValue enumValue = orgLevel.enumValues[0];
Assert.assertEquals(enumValue.value, "L1");
......@@ -127,25 +136,14 @@ public class GraphBackedTypeStoreTest {
List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
Assert.assertEquals(1, structTypes.size());
boolean clsTypeFound = false;
List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
if (classType.typeName.equals("Manager")) {
ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
Assert.assertEquals(classType.typeDescription, classType.typeName+description);
clsTypeFound = true;
}
}
Assert.assertTrue(clsTypeFound, "Manager type not restored");
verifyRestoredClassType(types, "Manager");
//validate trait
List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
Assert.assertEquals(1, traitTypes.size());
HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
Assert.assertEquals("SecurityClearance", trait.typeName);
Assert.assertEquals(trait.typeName+description, trait.typeDescription);
Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription);
Assert.assertEquals(1, trait.attributeDefinitions.length);
AttributeDefinition attribute = trait.attributeDefinitions[0];
Assert.assertEquals("level", attribute.name);
......@@ -229,4 +227,20 @@ public class GraphBackedTypeStoreTest {
}
return edgeCount;
}
private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException {
boolean clsTypeFound = false;
List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
if (classType.typeName.equals(typeName)) {
ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION);
clsTypeFound = true;
}
}
Assert.assertTrue(clsTypeFound, typeName + " type not restored");
}
}
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.inject.Inject;
/**
* Verify Guice can successfully instantiate and inject StoreBackTypeCache.
* StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
* which has type cache implementation class set to {@link StoreBackedTypeCache}.
*/
@Guice(modules = StoreBackedTypeCacheTestModule.class)
public class StoreBackedTypeCacheConfigurationTest {
@Inject
private TypeCache typeCache;
@Test
public void testConfigureAsTypeCache() throws Exception {
// Verify Guice successfully instantiated and injected StoreBackTypeCache
Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
/**
* Unit test for {@link StoreBackedTypeCache}
*/
@Guice(modules = RepositoryMetadataModule.class)
public class StoreBackedTypeCacheTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private ITypeStore typeStore;
@Inject
private StoreBackedTypeCache typeCache;
private TypeSystem ts;
private Map<String, ClassType> classTypesToTest = new HashMap<>();
@BeforeClass
public void setUp() throws Exception {
ts = TypeSystem.getInstance();
ts.reset();
// Populate the type store for testing.
TestUtils.defineDeptEmployeeTypes(ts);
TestUtils.createHiveTypes(ts);
ImmutableList<String> typeNames = ts.getTypeNames();
typeStore.store(ts, typeNames);
ClassType type = ts.getDataType(ClassType.class, "Manager");
classTypesToTest.put("Manager", type);
type = ts.getDataType(ClassType.class, TestUtils.TABLE_TYPE);
classTypesToTest.put(TestUtils.TABLE_TYPE, type);
}
@AfterClass
public void tearDown() throws Exception {
ts.reset();
try {
graphProvider.get().shutdown();
}
catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
}
catch(Exception e) {
e.printStackTrace();
}
}
@BeforeMethod
public void setupTestMethod() throws Exception {
typeCache.clear();
}
@Test
public void testGetClassType() throws Exception {
for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
IDataType dataType = typeCache.get(typeEntry.getKey());
// Verify the type is now cached.
Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
Assert.assertTrue(dataType instanceof ClassType);
ClassType cachedType = (ClassType)dataType;
// Verify that get() also loaded and cached any dependencies of this type from the type store.
verifyHierarchicalType(cachedType, typeEntry.getValue());
}
}
@Test
public void testHasClassType() throws Exception {
for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
// Calling has() should result in type and its dependencies
// loaded from the type store and added to the cache.
Assert.assertTrue(typeCache.has(typeEntry.getKey()));
// Verify the type is now cached in memory.
Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
}
}
@Test
public void testGetTraitType() throws Exception {
ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
for (String traitTypeName : traitNames) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
IDataType dataType = typeCache.get(traitTypeName);
// Verify the type is now cached.
Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
Assert.assertTrue(dataType instanceof TraitType);
TraitType cachedType = (TraitType)dataType;
// Verify that get() also loaded and cached any dependencies of this type from the type store.
verifyHierarchicalType(cachedType, ts.getDataType(TraitType.class, traitTypeName));
}
}
@Test
public void testHasTraitType() throws Exception {
ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
for (String traitTypeName : traitNames) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
// Calling has() should result in type and its dependencies
// loaded from the type store and added to the cache.
Assert.assertTrue(typeCache.has(traitTypeName));
// Verify the type is now cached.
Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
}
}
private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException {
Assert.assertEquals(dataType.numFields, expectedDataType.numFields);
Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size());
Assert.assertEquals(dataType.fieldMapping().fields.size(), expectedDataType.fieldMapping().fields.size());
ImmutableSet<String> superTypes = dataType.superTypes;
Assert.assertEquals(superTypes.size(), expectedDataType.superTypes.size());
// Verify that any attribute and super types were also cached.
for (String superTypeName : superTypes) {
Assert.assertTrue(typeCache.has(superTypeName));
}
for (AttributeInfo attrInfo : dataType.fieldMapping().fields.values()) {
switch (attrInfo.dataType().getTypeCategory()) {
case CLASS:
case STRUCT:
case ENUM:
Assert.assertTrue(typeCache.has(attrInfo.dataType().getName()), attrInfo.dataType().getName() + " should be cached");
break;
case ARRAY:
String elementTypeName = TypeUtils.parseAsArrayType(attrInfo.dataType().getName());
if (!ts.getCoreTypes().contains(elementTypeName)) {
Assert.assertTrue(typeCache.has(elementTypeName), elementTypeName + " should be cached");
}
break;
case MAP:
String[] mapTypeNames = TypeUtils.parseAsMapType(attrInfo.dataType().getName());
for (String typeName : mapTypeNames) {
if (!ts.getCoreTypes().contains(typeName)) {
Assert.assertTrue(typeCache.has(typeName), typeName + " should be cached");
}
}
break;
default:
break;
}
}
}
}
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.commons.configuration.Configuration;
/**
* Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}.
*
*/
public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule {
@Override
protected Configuration getConfiguration() {
try {
Configuration configuration = ApplicationProperties.get();
configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY,
StoreBackedTypeCache.class.getName());
return configuration;
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
}
repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.service;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
/**
* Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store.
* StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
* which has type cache implementation class set to {@link StoreBackedTypeCache}.
*/
@Guice(modules = StoreBackedTypeCacheTestModule.class)
public class StoreBackedTypeCacheMetadataServiceTest
{
@Inject
private MetadataService metadataService;
@Inject
private ITypeStore typeStore;
@Inject
TypeCache typeCache;
private TypeSystem ts;
@BeforeClass
public void setUp() throws Exception {
ts = TypeSystem.getInstance();
ts.reset();
// Populate the type store for testing.
TestUtils.defineDeptEmployeeTypes(ts);
TestUtils.createHiveTypes(ts);
ImmutableList<String> typeNames = ts.getTypeNames();
typeStore.store(ts, typeNames);
ts.reset();
}
@Test
public void testIt() throws Exception {
Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache;
// Cache should be empty
Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager"));
// Type lookup on MetadataService should cause Manager type to be loaded from the type store
// and cached.
Assert.assertNotNull(metadataService.getTypeDefinition("Manager"));
Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager"));
}
}
repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.services;
import com.google.inject.Provider;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
......@@ -26,12 +27,8 @@ import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.configuration.Configuration;
import org.mockito.Matchers;
import org.mockito.Mock;
......@@ -44,6 +41,7 @@ import java.util.HashMap;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
......@@ -78,7 +76,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
mock(ITypeStore.class),
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
typeSystem, defaultMetadataService);
......@@ -91,10 +89,10 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verifyZeroInteractions(typeStore);
verifyZeroInteractions(typeSystem);
verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any());
verifyZeroInteractions(typesRegistrar);
}
......@@ -109,7 +107,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
verify(typeStore).restore();
......@@ -134,7 +132,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
defaultMetadataService.instanceIsPassive();
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -30,14 +30,13 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCacheProvider;
import org.apache.atlas.typesystem.types.cache.ITypeCacheProvider;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -48,7 +47,6 @@ import com.google.common.collect.ImmutableSet;
@InterfaceAudience.Private
public class TypeSystem {
private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class);
private static final String CACHE_PROVIDER_CLASS_PROPERTY = "atlas.typesystem.cache.provider";
private static final TypeSystem INSTANCE = new TypeSystem();
private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() {
......@@ -60,7 +58,7 @@ public class TypeSystem {
}
};
private ITypeCacheProvider typeCache;
private TypeCache typeCache = new DefaultTypeCache();
private IdType idType;
private Map<String, IDataType> coreTypes;
......@@ -84,44 +82,18 @@ public class TypeSystem {
return this;
}
public void setTypeCache(TypeCache typeCache) {
this.typeCache = typeCache;
}
private void initialize() {
initCacheProvider();
coreTypes = new ConcurrentHashMap<>();
registerPrimitiveTypes();
registerCoreTypes();
}
/**
* Ideally a cache provider should have been injected in the TypeSystemProvider,
* but a singleton of TypeSystem is constructed privately within the class so that
* clients of TypeSystem would never instantiate a TypeSystem object directly in
* their code. As soon as a client makes a call to TypeSystem.getInstance(), they
* should have the singleton ready for consumption. To enable such an access pattern,
* it kind of becomes imperative to initialize the cache provider within the
* TypeSystem constructor (bypassing the GUICE way of injecting a cache provider)
*/
private void initCacheProvider() {
// read the pluggable cache provider from Atlas configuration
final String defaultCacheProvider = DefaultTypeCacheProvider.class.getName();
Class cacheProviderClass;
try {
cacheProviderClass = ApplicationProperties.getClass(CACHE_PROVIDER_CLASS_PROPERTY,
defaultCacheProvider, ITypeCacheProvider.class);
} catch (AtlasException e) {
throw new RuntimeException("Error getting type cache provider implementation class", e);
}
try {
typeCache = (ITypeCacheProvider)cacheProviderClass.newInstance();
}
catch (Exception e) {
throw new RuntimeException("Error creating instance of type cache provider implementation class " + cacheProviderClass.getName(), e);
}
}
public ImmutableList<String> getCoreTypes() {
return ImmutableList.copyOf(coreTypes.keySet());
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java → typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
......@@ -39,14 +39,14 @@ import com.google.inject.Singleton;
*/
@Singleton
@SuppressWarnings("rawtypes")
public class DefaultTypeCacheProvider implements ITypeCacheProvider {
public class DefaultTypeCache implements TypeCache {
private Map<String, IDataType> types_ = new ConcurrentHashMap<>();
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang
* org.apache.atlas.typesystem.types.cache.TypeCache#has(java.lang
* .String)
*/
@Override
......@@ -56,7 +56,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
}
/* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org.
* @see org.apache.atlas.typesystem.types.cache.TypeCache#has(org.
* apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
*/
@Override
......@@ -93,7 +93,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang
* org.apache.atlas.typesystem.types.cache.TypeCache#get(java.lang
* .String)
*/
@Override
......@@ -103,7 +103,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
}
/* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache.
* @see org.apache.atlas.typesystem.types.cache.TypeCache#get(org.apache.
* atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
*/
@Override
......@@ -116,7 +116,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org
* org.apache.atlas.typesystem.types.cache.TypeCache#getNames(org
* .apache.atlas.typesystem.types.DataTypes.TypeCategory)
*/
@Override
......@@ -139,7 +139,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames()
* org.apache.atlas.typesystem.types.cache.TypeCache#getAllNames()
*/
@Override
public Collection<String> getAllTypeNames() throws AtlasException {
......@@ -150,7 +150,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache
* org.apache.atlas.typesystem.types.cache.TypeCache#put(org.apache
* .atlas.typesystem.types.IDataType)
*/
@Override
......@@ -181,7 +181,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java
* org.apache.atlas.typesystem.types.cache.TypeCache#putAll(java
* .util.Collection)
*/
@Override
......@@ -196,7 +196,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java
* org.apache.atlas.typesystem.types.cache.TypeCache#remove(java
* .lang.String)
*/
@Override
......@@ -206,7 +206,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
}
/* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org.
* @see org.apache.atlas.typesystem.types.cache.TypeCache#remove(org.
* apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
*/
@Override
......@@ -219,7 +219,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear()
* @see org.apache.atlas.typesystem.types.cache.TypeCache#clear()
*/
@Override
public void clear() {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java → typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
......@@ -27,7 +27,7 @@ import org.apache.atlas.typesystem.types.IDataType;
/**
* The types are cached to allow faster lookup when type info is needed during
* creation/updation of entities, DSL query translation/execution.
* Implementations of this can chose to plugin a distributed cache provider
* Implementations of this can chose to plugin a distributed cache
* or an in-memory cache synched across nodes in an Altas cluster. <br>
* <br>
* Type entries in the cache can be one of ... <br>
......@@ -37,7 +37,7 @@ import org.apache.atlas.typesystem.types.IDataType;
* {@link org.apache.atlas.typesystem.types.EnumType}
*/
@SuppressWarnings("rawtypes")
public interface ITypeCacheProvider {
public interface TypeCache {
/**
* @param typeName
......@@ -83,7 +83,7 @@ public interface ITypeCacheProvider {
/**
* This is a convenience API to get the names of all types.
*
* @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
* @see TypeCache#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
* @return
* @throws AtlasException
*/
......
typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
......@@ -58,17 +58,22 @@ public class ApplicationPropertiesTest {
@Test
public void testGetClass() throws Exception {
Configuration configuration = ApplicationProperties.get();
//read from atlas-application.properties
Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class);
Class cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
ApplicationProperties.class.getName(), TypeSystem.class);
assertEquals(cls.getName(), TypeSystem.class.getName());
//default value
cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class);
cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem2.impl",
TypeSystem.class.getName(), TypeSystem.class);
assertEquals(cls.getName(), TypeSystem.class.getName());
//incompatible assignTo class, should throw AtlasException
try {
cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class);
cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
ApplicationProperties.class.getName(), ApplicationProperties.class);
Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown.");
}
catch (AtlasException e) {
......
typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProviderTest.java → typesystem/src/test/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheTest.java
......@@ -47,10 +47,10 @@ import org.testng.annotations.Test;
import com.google.common.collect.ImmutableSet;
/**
* Tests functional behavior of {@link DefaultTypeCacheProvider}
* Tests functional behavior of {@link DefaultTypeCache}
*/
@SuppressWarnings("rawtypes")
public class DefaultTypeCacheProviderTest {
public class DefaultTypeCacheTest {
private String CLASSTYPE_CUSTOMER = "Customer";
private String STRUCTTYPE_ADDRESS = "Address";
......@@ -64,7 +64,7 @@ public class DefaultTypeCacheProviderTest {
private TraitType privilegedTrait;
private EnumType shippingEnum;
private DefaultTypeCacheProvider cacheProvider;
private DefaultTypeCache cache;
@BeforeClass
public void onetimeSetup() throws Exception {
......@@ -98,48 +98,48 @@ public class DefaultTypeCacheProviderTest {
@BeforeMethod
public void eachTestSetup() throws Exception {
cacheProvider = new DefaultTypeCacheProvider();
cache = new DefaultTypeCache();
cacheProvider.put(customerType);
cacheProvider.put(addressType);
cacheProvider.put(privilegedTrait);
cacheProvider.put(shippingEnum);
cache.put(customerType);
cache.put(addressType);
cache.put(privilegedTrait);
cache.put(shippingEnum);
}
@Test
public void testCacheGetType() throws Exception {
IDataType custType = cacheProvider.get(CLASSTYPE_CUSTOMER);
IDataType custType = cache.get(CLASSTYPE_CUSTOMER);
verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class);
IDataType addrType = cacheProvider.get(STRUCTTYPE_ADDRESS);
IDataType addrType = cache.get(STRUCTTYPE_ADDRESS);
verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class);
IDataType privTrait = cacheProvider.get(TRAITTYPE_PRIVILEGED);
IDataType privTrait = cache.get(TRAITTYPE_PRIVILEGED);
verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class);
IDataType shippingEnum = cacheProvider.get(ENUMTYPE_SHIPPING);
IDataType shippingEnum = cache.get(ENUMTYPE_SHIPPING);
verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class);
assertNull(cacheProvider.get(UNKNOWN_TYPE));
assertNull(cache.get(UNKNOWN_TYPE));
}
@Test
public void testCacheGetTypeByCategory() throws Exception {
IDataType custType = cacheProvider.get(TypeCategory.CLASS, CLASSTYPE_CUSTOMER);
IDataType custType = cache.get(TypeCategory.CLASS, CLASSTYPE_CUSTOMER);
verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class);
IDataType addrType = cacheProvider.get(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS);
IDataType addrType = cache.get(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS);
verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class);
IDataType privTrait = cacheProvider.get(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED);
IDataType privTrait = cache.get(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED);
verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class);
IDataType shippingEnum = cacheProvider.get(TypeCategory.ENUM, ENUMTYPE_SHIPPING);
IDataType shippingEnum = cache.get(TypeCategory.ENUM, ENUMTYPE_SHIPPING);
verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class);
assertNull(cacheProvider.get(UNKNOWN_TYPE));
assertNull(cache.get(UNKNOWN_TYPE));
}
private void verifyType(IDataType actualType, String expectedName, Class<? extends IDataType> typeClass) {
......@@ -152,29 +152,29 @@ public class DefaultTypeCacheProviderTest {
@Test
public void testCacheHasType() throws Exception {
assertTrue(cacheProvider.has(CLASSTYPE_CUSTOMER));
assertTrue(cacheProvider.has(STRUCTTYPE_ADDRESS));
assertTrue(cacheProvider.has(TRAITTYPE_PRIVILEGED));
assertTrue(cacheProvider.has(ENUMTYPE_SHIPPING));
assertTrue(cache.has(CLASSTYPE_CUSTOMER));
assertTrue(cache.has(STRUCTTYPE_ADDRESS));
assertTrue(cache.has(TRAITTYPE_PRIVILEGED));
assertTrue(cache.has(ENUMTYPE_SHIPPING));
assertFalse(cacheProvider.has(UNKNOWN_TYPE));
assertFalse(cache.has(UNKNOWN_TYPE));
}
@Test
public void testCacheHasTypeByCategory() throws Exception {
assertTrue(cacheProvider.has(TypeCategory.CLASS, CLASSTYPE_CUSTOMER));
assertTrue(cacheProvider.has(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS));
assertTrue(cacheProvider.has(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED));
assertTrue(cacheProvider.has(TypeCategory.ENUM, ENUMTYPE_SHIPPING));
assertTrue(cache.has(TypeCategory.CLASS, CLASSTYPE_CUSTOMER));
assertTrue(cache.has(TypeCategory.STRUCT, STRUCTTYPE_ADDRESS));
assertTrue(cache.has(TypeCategory.TRAIT, TRAITTYPE_PRIVILEGED));
assertTrue(cache.has(TypeCategory.ENUM, ENUMTYPE_SHIPPING));
assertFalse(cacheProvider.has(UNKNOWN_TYPE));
assertFalse(cache.has(UNKNOWN_TYPE));
}
@Test
public void testCacheGetAllTypeNames() throws Exception {
List<String> allTypeNames = new ArrayList<String>(cacheProvider.getAllTypeNames());
List<String> allTypeNames = new ArrayList<String>(cache.getAllTypeNames());
Collections.sort(allTypeNames);
final int EXPECTED_TYPE_COUNT = 4;
......@@ -189,22 +189,22 @@ public class DefaultTypeCacheProviderTest {
@Test
public void testCacheGetTypeNamesByCategory() throws Exception {
List<String> classTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.CLASS));
List<String> classTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.CLASS));
final int EXPECTED_CLASSTYPE_COUNT = 1;
assertEquals(classTypes.size(), EXPECTED_CLASSTYPE_COUNT);
assertEquals(CLASSTYPE_CUSTOMER, classTypes.get(0));
List<String> structTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.STRUCT));
List<String> structTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.STRUCT));
final int EXPECTED_STRUCTTYPE_COUNT = 1;
assertEquals(structTypes.size(), EXPECTED_STRUCTTYPE_COUNT);
assertEquals(STRUCTTYPE_ADDRESS, structTypes.get(0));
List<String> traitTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.TRAIT));
List<String> traitTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.TRAIT));
final int EXPECTED_TRAITTYPE_COUNT = 1;
assertEquals(traitTypes.size(), EXPECTED_TRAITTYPE_COUNT);
assertEquals(TRAITTYPE_PRIVILEGED, traitTypes.get(0));
List<String> enumTypes = new ArrayList<String>(cacheProvider.getTypeNames(TypeCategory.ENUM));
List<String> enumTypes = new ArrayList<String>(cache.getTypeNames(TypeCategory.ENUM));
final int EXPECTED_ENUMTYPE_COUNT = 1;
assertEquals(enumTypes.size(), EXPECTED_ENUMTYPE_COUNT);
assertEquals(ENUMTYPE_SHIPPING, enumTypes.get(0));
......@@ -219,125 +219,125 @@ public class DefaultTypeCacheProviderTest {
allTypes.add(privilegedTrait);
allTypes.add(shippingEnum);
// create a new cache provider instead of using the one setup for every method call
cacheProvider = new DefaultTypeCacheProvider();
cacheProvider.putAll(allTypes);
// create a new cache instead of using the one setup for every method call
cache = new DefaultTypeCache();
cache.putAll(allTypes);
IDataType custType = cacheProvider.get(CLASSTYPE_CUSTOMER);
IDataType custType = cache.get(CLASSTYPE_CUSTOMER);
verifyType(custType, CLASSTYPE_CUSTOMER, ClassType.class);
IDataType addrType = cacheProvider.get(STRUCTTYPE_ADDRESS);
IDataType addrType = cache.get(STRUCTTYPE_ADDRESS);
verifyType(addrType, STRUCTTYPE_ADDRESS, StructType.class);
IDataType privTrait = cacheProvider.get(TRAITTYPE_PRIVILEGED);
IDataType privTrait = cache.get(TRAITTYPE_PRIVILEGED);
verifyType(privTrait, TRAITTYPE_PRIVILEGED, TraitType.class);
IDataType shippingEnum = cacheProvider.get(ENUMTYPE_SHIPPING);
IDataType shippingEnum = cache.get(ENUMTYPE_SHIPPING);
verifyType(shippingEnum, ENUMTYPE_SHIPPING, EnumType.class);
}
@Test
public void testCacheRemove() throws Exception {
cacheProvider.remove(CLASSTYPE_CUSTOMER);
assertNull(cacheProvider.get(CLASSTYPE_CUSTOMER));
assertFalse(cacheProvider.has(CLASSTYPE_CUSTOMER));
assertTrue(cacheProvider.getTypeNames(TypeCategory.CLASS).isEmpty());
cache.remove(CLASSTYPE_CUSTOMER);
assertNull(cache.get(CLASSTYPE_CUSTOMER));
assertFalse(cache.has(CLASSTYPE_CUSTOMER));
assertTrue(cache.getTypeNames(TypeCategory.CLASS).isEmpty());
final int EXPECTED_TYPE_COUNT = 3;
assertEquals(cacheProvider.getAllTypeNames().size(), EXPECTED_TYPE_COUNT);
assertEquals(cache.getAllTypeNames().size(), EXPECTED_TYPE_COUNT);
}
@Test
public void testCacheRemoveByCategory() throws Exception {
cacheProvider.remove(TypeCategory.CLASS, CLASSTYPE_CUSTOMER);
assertNull(cacheProvider.get(CLASSTYPE_CUSTOMER));
assertFalse(cacheProvider.has(CLASSTYPE_CUSTOMER));
assertTrue(cacheProvider.getTypeNames(TypeCategory.CLASS).isEmpty());
cache.remove(TypeCategory.CLASS, CLASSTYPE_CUSTOMER);
assertNull(cache.get(CLASSTYPE_CUSTOMER));
assertFalse(cache.has(CLASSTYPE_CUSTOMER));
assertTrue(cache.getTypeNames(TypeCategory.CLASS).isEmpty());
final int EXPECTED_TYPE_COUNT = 3;
assertEquals(cacheProvider.getAllTypeNames().size(), EXPECTED_TYPE_COUNT);
assertEquals(cache.getAllTypeNames().size(), EXPECTED_TYPE_COUNT);
}
@Test
public void testCacheClear() throws Exception {
cacheProvider.clear();
cache.clear();
assertNull(cacheProvider.get(CLASSTYPE_CUSTOMER));
assertFalse(cacheProvider.has(CLASSTYPE_CUSTOMER));
assertNull(cache.get(CLASSTYPE_CUSTOMER));
assertFalse(cache.has(CLASSTYPE_CUSTOMER));
assertNull(cacheProvider.get(STRUCTTYPE_ADDRESS));
assertFalse(cacheProvider.has(STRUCTTYPE_ADDRESS));
assertNull(cache.get(STRUCTTYPE_ADDRESS));
assertFalse(cache.has(STRUCTTYPE_ADDRESS));
assertNull(cacheProvider.get(TRAITTYPE_PRIVILEGED));
assertFalse(cacheProvider.has(TRAITTYPE_PRIVILEGED));
assertNull(cache.get(TRAITTYPE_PRIVILEGED));
assertFalse(cache.has(TRAITTYPE_PRIVILEGED));
assertNull(cacheProvider.get(ENUMTYPE_SHIPPING));
assertFalse(cacheProvider.has(ENUMTYPE_SHIPPING));
assertNull(cache.get(ENUMTYPE_SHIPPING));
assertFalse(cache.has(ENUMTYPE_SHIPPING));
assertTrue(cacheProvider.getTypeNames(TypeCategory.CLASS).isEmpty());
assertTrue(cacheProvider.getTypeNames(TypeCategory.STRUCT).isEmpty());
assertTrue(cacheProvider.getTypeNames(TypeCategory.TRAIT).isEmpty());
assertTrue(cacheProvider.getTypeNames(TypeCategory.ENUM).isEmpty());
assertTrue(cache.getTypeNames(TypeCategory.CLASS).isEmpty());
assertTrue(cache.getTypeNames(TypeCategory.STRUCT).isEmpty());
assertTrue(cache.getTypeNames(TypeCategory.TRAIT).isEmpty());
assertTrue(cache.getTypeNames(TypeCategory.ENUM).isEmpty());
assertTrue(cacheProvider.getAllTypeNames().isEmpty());
assertTrue(cache.getAllTypeNames().isEmpty());
}
@Test(expectedExceptions = AtlasException.class)
public void testPutTypeWithNullType() throws Exception {
cacheProvider.put(null);
cache.put(null);
fail("Null type should be not allowed in 'put'");
}
@Test(expectedExceptions = AtlasException.class)
public void testPutTypeWithInvalidType() throws Exception {
cacheProvider.put(DataTypes.BOOLEAN_TYPE);
cache.put(DataTypes.BOOLEAN_TYPE);
fail("type should only be an instance of ClassType | EnumType | StructType | TraitType in 'put'");
}
@Test(expectedExceptions = AtlasException.class)
public void testGetTypeWithNullCategory() throws Exception {
cacheProvider.get(null, CLASSTYPE_CUSTOMER);
cache.get(null, CLASSTYPE_CUSTOMER);
fail("Null TypeCategory should be not allowed in 'get'");
}
@Test(expectedExceptions = AtlasException.class)
public void testGetTypeWithInvalidCategory() throws Exception {
cacheProvider.get(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName());
cache.get(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName());
fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'get'");
}
@Test(expectedExceptions = AtlasException.class)
public void testCacheHasTypeWithNullCategory() throws Exception {
cacheProvider.has(null, CLASSTYPE_CUSTOMER);
cache.has(null, CLASSTYPE_CUSTOMER);
fail("Null TypeCategory should be not allowed in 'has'");
}
@Test(expectedExceptions = AtlasException.class)
public void testCacheHasTypeWithInvalidCategory() throws Exception {
cacheProvider.has(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName());
cache.has(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName());
fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'has'");
}
@Test(expectedExceptions = AtlasException.class)
public void testCacheGetTypeNamesByNullCategory() throws Exception {
cacheProvider.getTypeNames(null);
cache.getTypeNames(null);
fail("Null TypeCategory should be not allowed in 'getNames'");
}
@Test(expectedExceptions = AtlasException.class)
public void testCacheGetTypeNamesByInvalidCategory() throws Exception {
cacheProvider.getTypeNames(TypeCategory.PRIMITIVE);
cache.getTypeNames(TypeCategory.PRIMITIVE);
fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'getNames'");
}
......@@ -347,9 +347,9 @@ public class DefaultTypeCacheProviderTest {
List<IDataType> allTypes = new ArrayList<>();
allTypes.add(null);
// create a new cache provider instead of using the one setup for every method call
cacheProvider = new DefaultTypeCacheProvider();
cacheProvider.putAll(allTypes);
// create a new cache instead of using the one setup for every method call
cache = new DefaultTypeCache();
cache.putAll(allTypes);
fail("Null type should be not allowed in 'putAll'");
}
......@@ -360,9 +360,9 @@ public class DefaultTypeCacheProviderTest {
List<IDataType> allTypes = new ArrayList<>();
allTypes.add(DataTypes.BOOLEAN_TYPE);
// create a new cache provider instead of using the one setup for every method call
cacheProvider = new DefaultTypeCacheProvider();
cacheProvider.putAll(allTypes);
// create a new cache instead of using the one setup for every method call
cache = new DefaultTypeCache();
cache.putAll(allTypes);
fail("type should only one of ClassType | EnumType | StructType | TraitType in 'putAll'");
}
......@@ -370,14 +370,14 @@ public class DefaultTypeCacheProviderTest {
@Test(expectedExceptions = AtlasException.class)
public void testCacheRemoveByNullCategory() throws Exception {
cacheProvider.remove(null, CLASSTYPE_CUSTOMER);
cache.remove(null, CLASSTYPE_CUSTOMER);
fail("Null type should be not allowed in 'remove'");
}
@Test(expectedExceptions = AtlasException.class)
public void testCacheRemoveByInvalidCategory() throws Exception {
cacheProvider.remove(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName());
cache.remove(TypeCategory.PRIMITIVE, DataTypes.BOOLEAN_TYPE.getName());
fail("TypeCategory should only be one of TypeCategory.CLASS | ENUM | STRUCT | TRAIT in 'remove'");
}
}
webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java
......@@ -19,13 +19,15 @@
package org.apache.atlas.web.listeners;
import com.google.inject.Binder;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.InMemoryEntityAuditRepository;
import org.apache.commons.configuration.Configuration;
public class TestModule extends RepositoryMetadataModule {
@Override
protected void bindAuditRepository(Binder binder) {
protected void bindAuditRepository(Binder binder, Configuration configuration) {
//Map EntityAuditRepository interface to hbase based implementation
binder.bind(EntityAuditRepository.class).to(InMemoryEntityAuditRepository.class).asEagerSingleton();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment