Skip to content

A
atlas
Commit 9b00a9dd by Shwetha GS
Options

ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)

parent a7869bee
Showing with 781 additions and 94 deletions
common/src/main/java/org/apache/atlas/ApplicationProperties.java
......@@ -108,10 +108,9 @@ public final class ApplicationProperties extends PropertiesConfiguration {
return inConf.subset(prefix);
}
public static Class getClass(String propertyName, String defaultValue, Class assignableClass)
throws AtlasException {
public static Class getClass(Configuration configuration, String propertyName, String defaultValue,
Class assignableClass) throws AtlasException {
try {
Configuration configuration = get();
String propertyValue = configuration.getString(propertyName, defaultValue);
Class<?> clazz = Class.forName(propertyValue);
if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) {
......
release-log.txt
......@@ -40,6 +40,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)
ATLAS-931 Delete entities fails when hard delete is configured (dkantor via sumasai)
ATLAS-932 UI: 'create tag' button does not work (mneethiraj via sumasai)
ATLAS-928 UI is not showing the name column for hive tables in the schema tab (yhemanth via sumasai)
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.DataSetLineageService;
......@@ -50,6 +51,9 @@ import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeSystemProvider;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.commons.configuration.Configuration;
/**
* Guice module for Repository module.
......@@ -85,9 +89,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton();
bindAuditRepository(binder());
Configuration configuration = getConfiguration();
bindAuditRepository(binder(), configuration);
bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton();
bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton();
bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton();
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
......@@ -99,9 +106,17 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
}
protected void bindAuditRepository(Binder binder) {
protected Configuration getConfiguration() {
try {
return ApplicationProperties.get();
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
protected void bindAuditRepository(Binder binder, Configuration configuration) {
Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl();
Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration());
//Map EntityAuditRepository interface to configured implementation
binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
......@@ -117,10 +132,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY,
HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
return ApplicationProperties.getClass(configuration,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
......@@ -128,12 +143,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
private Class<? extends DeleteHandler> getDeleteHandlerImpl() {
private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) {
try {
return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
SoftDeleteHandler.class.getName(), DeleteHandler.class);
return ApplicationProperties.getClass(configuration,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
protected Class<? extends TypeCache> getTypeCache(Configuration configuration) {
// Get the type cache implementation class from Atlas configuration.
try {
return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
DefaultTypeCache.class.getName(), TypeCache.class);
} catch (AtlasException e) {
throw new RuntimeException("Error getting TypeCache implementation class", e);
}
}
}
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -214,6 +214,20 @@ public class GraphBackedTypeStore implements ITypeStore {
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
return getTypesFromVertices(vertices);
}
@Override
@GraphTransaction
public TypesDef restoreType(String typeName) throws AtlasException {
// Get vertex for the specified type name.
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
return getTypesFromVertices(vertices);
}
private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
......
repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
......@@ -40,4 +40,13 @@ public interface ITypeStore {
* @throws AtlasException
*/
TypesDef restore() throws AtlasException;
/**
* Restore the specified type definition
*
* @param typeName name of requested type
* @return persisted type definition
* @throws AtlasException
*/
TypesDef restoreType(String typeName) throws AtlasException;
}
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
......@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
......@@ -60,6 +61,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.configuration.Configuration;
......@@ -71,6 +73,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
......@@ -109,10 +112,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders)
final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
throws AtlasException {
this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
TypeSystem.getInstance(), ApplicationProperties.get());
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
......@@ -120,10 +123,21 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
final TypeSystem typeSystem,
final Configuration configuration) throws AtlasException {
final Configuration configuration, TypeCache typeCache) throws AtlasException {
this.typeStore = typeStore;
this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem;
/**
* Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
* but a singleton of TypeSystem is constructed privately within the class so that
* clients of TypeSystem would never instantiate a TypeSystem object directly in
* their code. As soon as a client makes a call to TypeSystem.getInstance(), they
* should have the singleton ready for consumption. Manually inject TypeSystem with
* the Guice-instantiated type cache here, before types are restored.
* This allows cache implementations to participate in Guice dependency injection.
*/
this.typeSystem.setTypeCache(typeCache);
this.repository = repository;
for (Provider<TypesChangeListener> provider : typeListenerProviders) {
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
......@@ -53,6 +54,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
......@@ -64,6 +66,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructType
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedTypeStoreTest {
private static final String DESCRIPTION = "_description";
@Inject
private GraphProvider<TitanGraph> graphProvider;
......@@ -97,6 +101,12 @@ public class GraphBackedTypeStoreTest {
dumpGraph();
}
@Test(dependsOnMethods = "testStore")
public void testRestoreType() throws Exception {
TypesDef typesDef = ((GraphBackedTypeStore)typeStore).restoreType("Manager");
verifyRestoredClassType(typesDef, "Manager");
}
private void dumpGraph() {
TitanGraph graph = graphProvider.get();
for (Vertex v : graph.getVertices()) {
......@@ -109,7 +119,6 @@ public class GraphBackedTypeStoreTest {
@Test(dependsOnMethods = "testStore")
public void testRestore() throws Exception {
String description = "_description";
TypesDef types = typeStore.restore();
//validate enum
......@@ -117,7 +126,7 @@ public class GraphBackedTypeStoreTest {
Assert.assertEquals(1, enumTypes.size());
EnumTypeDefinition orgLevel = enumTypes.get(0);
Assert.assertEquals(orgLevel.name, "OrgLevel");
Assert.assertEquals(orgLevel.description, "OrgLevel"+description);
Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION);
Assert.assertEquals(orgLevel.enumValues.length, 2);
EnumValue enumValue = orgLevel.enumValues[0];
Assert.assertEquals(enumValue.value, "L1");
......@@ -127,25 +136,14 @@ public class GraphBackedTypeStoreTest {
List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
Assert.assertEquals(1, structTypes.size());
boolean clsTypeFound = false;
List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
if (classType.typeName.equals("Manager")) {
ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
Assert.assertEquals(classType.typeDescription, classType.typeName+description);
clsTypeFound = true;
}
}
Assert.assertTrue(clsTypeFound, "Manager type not restored");
verifyRestoredClassType(types, "Manager");
//validate trait
List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
Assert.assertEquals(1, traitTypes.size());
HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
Assert.assertEquals("SecurityClearance", trait.typeName);
Assert.assertEquals(trait.typeName+description, trait.typeDescription);
Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription);
Assert.assertEquals(1, trait.attributeDefinitions.length);
AttributeDefinition attribute = trait.attributeDefinitions[0];
Assert.assertEquals("level", attribute.name);
......@@ -229,4 +227,20 @@ public class GraphBackedTypeStoreTest {
}
return edgeCount;
}
private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException {
boolean clsTypeFound = false;
List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
if (classType.typeName.equals(typeName)) {
ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION);
clsTypeFound = true;
}
}
Assert.assertTrue(clsTypeFound, typeName + " type not restored");
}
}
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.testng.Assert;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.inject.Inject;
/**
* Verify Guice can successfully instantiate and inject StoreBackTypeCache.
* StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
* which has type cache implementation class set to {@link StoreBackedTypeCache}.
*/
@Guice(modules = StoreBackedTypeCacheTestModule.class)
public class StoreBackedTypeCacheConfigurationTest {
@Inject
private TypeCache typeCache;
@Test
public void testConfigureAsTypeCache() throws Exception {
// Verify Guice successfully instantiated and injected StoreBackTypeCache
Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
}
}
\ No newline at end of file
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
/**
* Unit test for {@link StoreBackedTypeCache}
*/
@Guice(modules = RepositoryMetadataModule.class)
public class StoreBackedTypeCacheTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
@Inject
private ITypeStore typeStore;
@Inject
private StoreBackedTypeCache typeCache;
private TypeSystem ts;
private Map<String, ClassType> classTypesToTest = new HashMap<>();
@BeforeClass
public void setUp() throws Exception {
ts = TypeSystem.getInstance();
ts.reset();
// Populate the type store for testing.
TestUtils.defineDeptEmployeeTypes(ts);
TestUtils.createHiveTypes(ts);
ImmutableList<String> typeNames = ts.getTypeNames();
typeStore.store(ts, typeNames);
ClassType type = ts.getDataType(ClassType.class, "Manager");
classTypesToTest.put("Manager", type);
type = ts.getDataType(ClassType.class, TestUtils.TABLE_TYPE);
classTypesToTest.put(TestUtils.TABLE_TYPE, type);
}
@AfterClass
public void tearDown() throws Exception {
ts.reset();
try {
graphProvider.get().shutdown();
}
catch(Exception e) {
e.printStackTrace();
}
try {
TitanCleanup.clear(graphProvider.get());
}
catch(Exception e) {
e.printStackTrace();
}
}
@BeforeMethod
public void setupTestMethod() throws Exception {
typeCache.clear();
}
@Test
public void testGetClassType() throws Exception {
for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
IDataType dataType = typeCache.get(typeEntry.getKey());
// Verify the type is now cached.
Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
Assert.assertTrue(dataType instanceof ClassType);
ClassType cachedType = (ClassType)dataType;
// Verify that get() also loaded and cached any dependencies of this type from the type store.
verifyHierarchicalType(cachedType, typeEntry.getValue());
}
}
@Test
public void testHasClassType() throws Exception {
for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
// Calling has() should result in type and its dependencies
// loaded from the type store and added to the cache.
Assert.assertTrue(typeCache.has(typeEntry.getKey()));
// Verify the type is now cached in memory.
Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
}
}
@Test
public void testGetTraitType() throws Exception {
ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
for (String traitTypeName : traitNames) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
IDataType dataType = typeCache.get(traitTypeName);
// Verify the type is now cached.
Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
Assert.assertTrue(dataType instanceof TraitType);
TraitType cachedType = (TraitType)dataType;
// Verify that get() also loaded and cached any dependencies of this type from the type store.
verifyHierarchicalType(cachedType, ts.getDataType(TraitType.class, traitTypeName));
}
}
@Test
public void testHasTraitType() throws Exception {
ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
for (String traitTypeName : traitNames) {
// Not cached yet
Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
// Calling has() should result in type and its dependencies
// loaded from the type store and added to the cache.
Assert.assertTrue(typeCache.has(traitTypeName));
// Verify the type is now cached.
Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
}
}
private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException {
Assert.assertEquals(dataType.numFields, expectedDataType.numFields);
Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size());
Assert.assertEquals(dataType.fieldMapping().fields.size(), expectedDataType.fieldMapping().fields.size());
ImmutableSet<String> superTypes = dataType.superTypes;
Assert.assertEquals(superTypes.size(), expectedDataType.superTypes.size());
// Verify that any attribute and super types were also cached.
for (String superTypeName : superTypes) {
Assert.assertTrue(typeCache.has(superTypeName));
}
for (AttributeInfo attrInfo : dataType.fieldMapping().fields.values()) {
switch (attrInfo.dataType().getTypeCategory()) {
case CLASS:
case STRUCT:
case ENUM:
Assert.assertTrue(typeCache.has(attrInfo.dataType().getName()), attrInfo.dataType().getName() + " should be cached");
break;
case ARRAY:
String elementTypeName = TypeUtils.parseAsArrayType(attrInfo.dataType().getName());
if (!ts.getCoreTypes().contains(elementTypeName)) {
Assert.assertTrue(typeCache.has(elementTypeName), elementTypeName + " should be cached");
}
break;
case MAP:
String[] mapTypeNames = TypeUtils.parseAsMapType(attrInfo.dataType().getName());
for (String typeName : mapTypeNames) {
if (!ts.getCoreTypes().contains(typeName)) {
Assert.assertTrue(typeCache.has(typeName), typeName + " should be cached");
}
}
break;
default:
break;
}
}
}
}
repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.typestore;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.commons.configuration.Configuration;
/**
* Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}.
*
*/
public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule {
@Override
protected Configuration getConfiguration() {
try {
Configuration configuration = ApplicationProperties.get();
configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY,
StoreBackedTypeCache.class.getName());
return configuration;
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
}
repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.service;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
/**
* Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store.
* StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
* which has type cache implementation class set to {@link StoreBackedTypeCache}.
*/
@Guice(modules = StoreBackedTypeCacheTestModule.class)
public class StoreBackedTypeCacheMetadataServiceTest
{
@Inject
private MetadataService metadataService;
@Inject
private ITypeStore typeStore;
@Inject
TypeCache typeCache;
private TypeSystem ts;
@BeforeClass
public void setUp() throws Exception {
ts = TypeSystem.getInstance();
ts.reset();
// Populate the type store for testing.
TestUtils.defineDeptEmployeeTypes(ts);
TestUtils.createHiveTypes(ts);
ImmutableList<String> typeNames = ts.getTypeNames();
typeStore.store(ts, typeNames);
ts.reset();
}
@Test
public void testIt() throws Exception {
Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache;
// Cache should be empty
Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager"));
// Type lookup on MetadataService should cause Manager type to be loaded from the type store
// and cached.
Assert.assertNotNull(metadataService.getTypeDefinition("Manager"));
Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager"));
}
}
repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.services;
import com.google.inject.Provider;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
......@@ -26,12 +27,8 @@ import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.configuration.Configuration;
import org.mockito.Matchers;
import org.mockito.Mock;
......@@ -44,6 +41,7 @@ import java.util.HashMap;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
......@@ -78,7 +76,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
mock(ITypeStore.class),
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
typeSystem, defaultMetadataService);
......@@ -91,10 +89,10 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verifyZeroInteractions(typeStore);
verifyZeroInteractions(typeSystem);
verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any());
verifyZeroInteractions(typesRegistrar);
}
......@@ -109,7 +107,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
verify(typeStore).restore();
......@@ -134,7 +132,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
defaultMetadataService.instanceIsPassive();
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
......@@ -30,14 +30,13 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.inject.Singleton;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCacheProvider;
import org.apache.atlas.typesystem.types.cache.ITypeCacheProvider;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -48,7 +47,6 @@ import com.google.common.collect.ImmutableSet;
@InterfaceAudience.Private
public class TypeSystem {
private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class);
private static final String CACHE_PROVIDER_CLASS_PROPERTY = "atlas.typesystem.cache.provider";
private static final TypeSystem INSTANCE = new TypeSystem();
private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() {
......@@ -60,7 +58,7 @@ public class TypeSystem {
}
};
private ITypeCacheProvider typeCache;
private TypeCache typeCache = new DefaultTypeCache();
private IdType idType;
private Map<String, IDataType> coreTypes;
......@@ -84,44 +82,18 @@ public class TypeSystem {
return this;
}
public void setTypeCache(TypeCache typeCache) {
this.typeCache = typeCache;
}
private void initialize() {
initCacheProvider();
coreTypes = new ConcurrentHashMap<>();
registerPrimitiveTypes();
registerCoreTypes();
}
/**
* Ideally a cache provider should have been injected in the TypeSystemProvider,
* but a singleton of TypeSystem is constructed privately within the class so that
* clients of TypeSystem would never instantiate a TypeSystem object directly in
* their code. As soon as a client makes a call to TypeSystem.getInstance(), they
* should have the singleton ready for consumption. To enable such an access pattern,
* it kind of becomes imperative to initialize the cache provider within the
* TypeSystem constructor (bypassing the GUICE way of injecting a cache provider)
*/
private void initCacheProvider() {
// read the pluggable cache provider from Atlas configuration
final String defaultCacheProvider = DefaultTypeCacheProvider.class.getName();
Class cacheProviderClass;
try {
cacheProviderClass = ApplicationProperties.getClass(CACHE_PROVIDER_CLASS_PROPERTY,
defaultCacheProvider, ITypeCacheProvider.class);
} catch (AtlasException e) {
throw new RuntimeException("Error getting type cache provider implementation class", e);
}
try {
typeCache = (ITypeCacheProvider)cacheProviderClass.newInstance();
}
catch (Exception e) {
throw new RuntimeException("Error creating instance of type cache provider implementation class " + cacheProviderClass.getName(), e);
}
}
public ImmutableList<String> getCoreTypes() {
return ImmutableList.copyOf(coreTypes.keySet());
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java → typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
......@@ -39,14 +39,14 @@ import com.google.inject.Singleton;
*/
@Singleton
@SuppressWarnings("rawtypes")
public class DefaultTypeCacheProvider implements ITypeCacheProvider {
public class DefaultTypeCache implements TypeCache {
private Map<String, IDataType> types_ = new ConcurrentHashMap<>();
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang
* org.apache.atlas.typesystem.types.cache.TypeCache#has(java.lang
* .String)
*/
@Override
......@@ -56,7 +56,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
}
/* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org.
* @see org.apache.atlas.typesystem.types.cache.TypeCache#has(org.
* apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
*/
@Override
......@@ -93,7 +93,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang
* org.apache.atlas.typesystem.types.cache.TypeCache#get(java.lang
* .String)
*/
@Override
......@@ -103,7 +103,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
}
/* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache.
* @see org.apache.atlas.typesystem.types.cache.TypeCache#get(org.apache.
* atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
*/
@Override
......@@ -116,7 +116,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org
* org.apache.atlas.typesystem.types.cache.TypeCache#getNames(org
* .apache.atlas.typesystem.types.DataTypes.TypeCategory)
*/
@Override
......@@ -139,7 +139,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames()
* org.apache.atlas.typesystem.types.cache.TypeCache#getAllNames()
*/
@Override
public Collection<String> getAllTypeNames() throws AtlasException {
......@@ -150,7 +150,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache
* org.apache.atlas.typesystem.types.cache.TypeCache#put(org.apache
* .atlas.typesystem.types.IDataType)
*/
@Override
......@@ -181,7 +181,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java
* org.apache.atlas.typesystem.types.cache.TypeCache#putAll(java
* .util.Collection)
*/
@Override
......@@ -196,7 +196,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see
* org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java
* org.apache.atlas.typesystem.types.cache.TypeCache#remove(java
* .lang.String)
*/
@Override
......@@ -206,7 +206,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
}
/* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org.
* @see org.apache.atlas.typesystem.types.cache.TypeCache#remove(org.
* apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
*/
@Override
......@@ -219,7 +219,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
/*
* (non-Javadoc)
* @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear()
* @see org.apache.atlas.typesystem.types.cache.TypeCache#clear()
*/
@Override
public void clear() {
......
typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java → typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
......@@ -27,7 +27,7 @@ import org.apache.atlas.typesystem.types.IDataType;
/**
* The types are cached to allow faster lookup when type info is needed during
* creation/updation of entities, DSL query translation/execution.
* Implementations of this can chose to plugin a distributed cache provider
* Implementations of this can chose to plugin a distributed cache
* or an in-memory cache synched across nodes in an Altas cluster. <br>
* <br>
* Type entries in the cache can be one of ... <br>
......@@ -37,7 +37,7 @@ import org.apache.atlas.typesystem.types.IDataType;
* {@link org.apache.atlas.typesystem.types.EnumType}
*/
@SuppressWarnings("rawtypes")
public interface ITypeCacheProvider {
public interface TypeCache {
/**
* @param typeName
......@@ -83,7 +83,7 @@ public interface ITypeCacheProvider {
/**
* This is a convenience API to get the names of all types.
*
* @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
* @see TypeCache#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
* @return
* @throws AtlasException
*/
......
typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
......@@ -58,17 +58,22 @@ public class ApplicationPropertiesTest {
@Test
public void testGetClass() throws Exception {
Configuration configuration = ApplicationProperties.get();
//read from atlas-application.properties
Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class);
Class cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
ApplicationProperties.class.getName(), TypeSystem.class);
assertEquals(cls.getName(), TypeSystem.class.getName());
//default value
cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class);
cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem2.impl",
TypeSystem.class.getName(), TypeSystem.class);
assertEquals(cls.getName(), TypeSystem.class.getName());
//incompatible assignTo class, should throw AtlasException
try {
cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class);
cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
ApplicationProperties.class.getName(), ApplicationProperties.class);
Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown.");
}
catch (AtlasException e) {
......
webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java
......@@ -19,13 +19,15 @@
package org.apache.atlas.web.listeners;
import com.google.inject.Binder;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.InMemoryEntityAuditRepository;
import org.apache.commons.configuration.Configuration;
public class TestModule extends RepositoryMetadataModule {
@Override
protected void bindAuditRepository(Binder binder) {
protected void bindAuditRepository(Binder binder, Configuration configuration) {
//Map EntityAuditRepository interface to hbase based implementation
binder.bind(EntityAuditRepository.class).to(InMemoryEntityAuditRepository.class).asEagerSingleton();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment