ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)

common/src/main/java/org/apache/atlas/ApplicationProperties.java

@@ -108,10 +108,9 @@ public final class ApplicationProperties extends PropertiesConfiguration {
         return inConf.subset(prefix);
     }
 
-    public static Class getClass(String propertyName, String defaultValue, Class assignableClass)
-        throws AtlasException {
+    public static Class getClass(Configuration configuration, String propertyName, String defaultValue,
+                                 Class assignableClass) throws AtlasException {
         try {
-            Configuration configuration = get();
             String propertyValue = configuration.getString(propertyName, defaultValue);
             Class<?> clazz = Class.forName(propertyValue);
             if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) {

release-log.txt

@@ -40,6 +40,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)
 ATLAS-931 Delete entities fails when hard delete is configured (dkantor via sumasai)
 ATLAS-932 UI: 'create tag' button does not work (mneethiraj via sumasai)
 ATLAS-928 UI is not showing the name column for hive tables in the schema tab (yhemanth via sumasai)

repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java

@@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers;
 import com.google.inject.multibindings.Multibinder;
 import com.google.inject.throwingproviders.ThrowingProviderBinder;
 import com.thinkaurelius.titan.core.TitanGraph;
+
 import org.aopalliance.intercept.MethodInterceptor;
 import org.apache.atlas.discovery.DiscoveryService;
 import org.apache.atlas.discovery.DataSetLineageService;
@@ -50,6 +51,9 @@ import org.apache.atlas.services.MetadataService;
 import org.apache.atlas.services.ReservedTypesRegistrar;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.TypeSystemProvider;
+import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.apache.commons.configuration.Configuration;
 
 /**
  * Guice module for Repository module.
@@ -85,9 +89,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 
         bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton();
 
-        bindAuditRepository(binder());
+        Configuration configuration = getConfiguration();
+        bindAuditRepository(binder(), configuration);
+
+        bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton();
 
-        bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton();
+        bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton();
 
         //Add EntityAuditListener as EntityChangeListener
         Multibinder<EntityChangeListener> entityChangeListenerBinder =
@@ -99,9 +106,17 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
         bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
     }
 
-    protected void bindAuditRepository(Binder binder) {
+    protected Configuration getConfiguration() {
+        try {
+            return ApplicationProperties.get();
+        } catch (AtlasException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected void bindAuditRepository(Binder binder, Configuration configuration) {
 
-        Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl();
+        Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration());
 
         //Map EntityAuditRepository interface to configured implementation
         binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
@@ -117,10 +132,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 
     private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
 
-    private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
+    private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) {
         try {
-            return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY,
-                    HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
+            return ApplicationProperties.getClass(configuration,
+                    AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
         } catch (AtlasException e) {
             throw new RuntimeException(e);
         }
@@ -128,12 +143,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
 
     private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
 
-    private Class<? extends DeleteHandler> getDeleteHandlerImpl() {
+    private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) {
         try {
-            return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
-                    SoftDeleteHandler.class.getName(), DeleteHandler.class);
+            return ApplicationProperties.getClass(configuration,
+                    DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
         } catch (AtlasException e) {
             throw new RuntimeException(e);
         }
     }
+
+    public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
+
+    protected Class<? extends TypeCache> getTypeCache(Configuration configuration) {
+
+        // Get the type cache implementation class from Atlas configuration.
+        try {
+            return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
+                DefaultTypeCache.class.getName(), TypeCache.class);
+        } catch (AtlasException e) {
+            throw new RuntimeException("Error getting TypeCache implementation class", e);
+        }
+    }
+
 }

repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java

@@ -214,6 +214,20 @@ public class GraphBackedTypeStore implements ITypeStore {
         Iterator vertices =
                 titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
 
+        return getTypesFromVertices(vertices);
+    }
+
+    @Override
+    @GraphTransaction
+    public TypesDef restoreType(String typeName) throws AtlasException {
+        // Get vertex for the specified type name.
+        Iterator vertices =
+            titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
+
+        return getTypesFromVertices(vertices);
+    }
+
+    private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException {
         ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
         ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
         ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();

repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java

@@ -40,4 +40,13 @@ public interface ITypeStore {
      * @throws AtlasException
      */
     TypesDef restore() throws AtlasException;
+
+    /**
+     * Restore the specified type definition
+     *
+     * @param typeName name of requested type
+     * @return persisted type definition
+     * @throws AtlasException
+     */
+    TypesDef restoreType(String typeName) throws AtlasException;
 }

repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java

@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Provider;
+
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasClient;
 import org.apache.atlas.AtlasException;
@@ -60,6 +61,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
 import org.apache.atlas.typesystem.types.TraitType;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.typesystem.types.ValueConversionException;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
 import org.apache.atlas.typesystem.types.utils.TypesUtil;
 import org.apache.atlas.utils.ParamChecker;
 import org.apache.commons.configuration.Configuration;
@@ -71,6 +73,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashSet;
@@ -109,10 +112,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
     DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
                            final IBootstrapTypesRegistrar typesRegistrar,
                            final Collection<Provider<TypesChangeListener>> typeListenerProviders,
-                           final Collection<Provider<EntityChangeListener>> entityListenerProviders)
+                           final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
             throws AtlasException {
         this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
-                TypeSystem.getInstance(), ApplicationProperties.get());
+                TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
     }
 
     DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
@@ -120,10 +123,21 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
                            final Collection<Provider<TypesChangeListener>> typeListenerProviders,
                            final Collection<Provider<EntityChangeListener>> entityListenerProviders,
                            final TypeSystem typeSystem,
-                           final Configuration configuration) throws AtlasException {
+                           final Configuration configuration, TypeCache typeCache) throws AtlasException {
         this.typeStore = typeStore;
         this.typesRegistrar = typesRegistrar;
         this.typeSystem = typeSystem;
+        /**
+         * Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
+         * but a singleton of TypeSystem is constructed privately within the class so that
+         * clients of TypeSystem would never instantiate a TypeSystem object directly in
+         * their code. As soon as a client makes a call to TypeSystem.getInstance(), they
+         * should have the singleton ready for consumption. Manually inject TypeSystem with
+         * the Guice-instantiated type cache here, before types are restored.
+         * This allows cache implementations to participate in Guice dependency injection.
+         */
+        this.typeSystem.setTypeCache(typeCache);
+
         this.repository = repository;
 
         for (Provider<TypesChangeListener> provider : typeListenerProviders) {

repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java

@@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
 import com.tinkerpop.blueprints.Direction;
 import com.tinkerpop.blueprints.Edge;
 import com.tinkerpop.blueprints.Vertex;
+
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.TestUtils;
@@ -53,6 +54,7 @@ import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
 import javax.inject.Inject;
+
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -64,6 +66,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructType
 
 @Guice(modules = RepositoryMetadataModule.class)
 public class GraphBackedTypeStoreTest {
+    private static final String DESCRIPTION = "_description";
+
     @Inject
     private GraphProvider<TitanGraph> graphProvider;
 
@@ -97,6 +101,12 @@ public class GraphBackedTypeStoreTest {
         dumpGraph();
     }
 
+    @Test(dependsOnMethods = "testStore")
+    public void testRestoreType() throws Exception {
+        TypesDef typesDef = ((GraphBackedTypeStore)typeStore).restoreType("Manager");
+        verifyRestoredClassType(typesDef, "Manager");
+    }
+
     private void dumpGraph() {
         TitanGraph graph = graphProvider.get();
         for (Vertex v : graph.getVertices()) {
@@ -109,7 +119,6 @@ public class GraphBackedTypeStoreTest {
 
     @Test(dependsOnMethods = "testStore")
     public void testRestore() throws Exception {
-        String description = "_description";
         TypesDef types = typeStore.restore();
 
         //validate enum
@@ -117,7 +126,7 @@ public class GraphBackedTypeStoreTest {
         Assert.assertEquals(1, enumTypes.size());
         EnumTypeDefinition orgLevel = enumTypes.get(0);
         Assert.assertEquals(orgLevel.name, "OrgLevel");
-        Assert.assertEquals(orgLevel.description, "OrgLevel"+description);
+        Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION);
         Assert.assertEquals(orgLevel.enumValues.length, 2);
         EnumValue enumValue = orgLevel.enumValues[0];
         Assert.assertEquals(enumValue.value, "L1");
@@ -127,25 +136,14 @@ public class GraphBackedTypeStoreTest {
         List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
         Assert.assertEquals(1, structTypes.size());
 
-        boolean clsTypeFound = false;
-        List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
-        for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
-            if (classType.typeName.equals("Manager")) {
-                ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
-                Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
-                Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
-                Assert.assertEquals(classType.typeDescription, classType.typeName+description);
-                clsTypeFound = true;
-            }
-        }
-        Assert.assertTrue(clsTypeFound, "Manager type not restored");
+        verifyRestoredClassType(types, "Manager");
 
         //validate trait
         List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
         Assert.assertEquals(1, traitTypes.size());
         HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
         Assert.assertEquals("SecurityClearance", trait.typeName);
-        Assert.assertEquals(trait.typeName+description, trait.typeDescription);
+        Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription);
         Assert.assertEquals(1, trait.attributeDefinitions.length);
         AttributeDefinition attribute = trait.attributeDefinitions[0];
         Assert.assertEquals("level", attribute.name);
@@ -229,4 +227,20 @@ public class GraphBackedTypeStoreTest {
         }
         return edgeCount;
     }
+
+    private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException {
+        boolean clsTypeFound = false;
+        List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
+        for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
+            if (classType.typeName.equals(typeName)) {
+                ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
+                Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
+                Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
+                Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION);
+                clsTypeFound = true;
+            }
+        }
+        Assert.assertTrue(clsTypeFound, typeName + " type not restored");
+    }
+
 }

repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.testng.Assert;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.google.inject.Inject;
+
+/**
+ *  Verify Guice can successfully instantiate and inject StoreBackTypeCache.
+ *  StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
+ *  which has type cache implementation class set to {@link StoreBackedTypeCache}.
+ */
+@Guice(modules = StoreBackedTypeCacheTestModule.class)
+public class StoreBackedTypeCacheConfigurationTest {
+
+    @Inject
+    private TypeCache typeCache;
+
+    @Test
+    public void testConfigureAsTypeCache() throws Exception {
+        // Verify Guice successfully instantiated and injected StoreBackTypeCache
+        Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
+    }
+}
\ No newline at end of file

repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+
+
+/**
+ * Unit test for {@link StoreBackedTypeCache}
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class StoreBackedTypeCacheTest {
+
+    @Inject
+    private GraphProvider<TitanGraph> graphProvider;
+
+    @Inject
+    private ITypeStore typeStore;
+
+    @Inject
+    private StoreBackedTypeCache typeCache;
+
+    private TypeSystem ts;
+
+    private Map<String, ClassType> classTypesToTest = new HashMap<>();
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        ts = TypeSystem.getInstance();
+        ts.reset();
+
+        // Populate the type store for testing.
+        TestUtils.defineDeptEmployeeTypes(ts);
+        TestUtils.createHiveTypes(ts);
+        ImmutableList<String> typeNames = ts.getTypeNames();
+        typeStore.store(ts, typeNames);
+
+        ClassType type = ts.getDataType(ClassType.class, "Manager");
+        classTypesToTest.put("Manager", type);
+        type = ts.getDataType(ClassType.class, TestUtils.TABLE_TYPE);
+        classTypesToTest.put(TestUtils.TABLE_TYPE, type);
+    }
+
+    @AfterClass
+    public void tearDown() throws Exception {
+        ts.reset();
+        try {
+            graphProvider.get().shutdown();
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+
+        try {
+            TitanCleanup.clear(graphProvider.get());
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @BeforeMethod
+    public void setupTestMethod() throws Exception {
+        typeCache.clear();
+    }
+
+    @Test
+    public void testGetClassType() throws Exception {
+        for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
+            // Not cached yet
+            Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
+
+            IDataType dataType = typeCache.get(typeEntry.getKey());
+            // Verify the type is now cached.
+            Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
+
+            Assert.assertTrue(dataType instanceof ClassType);
+            ClassType cachedType = (ClassType)dataType;
+            // Verify that get() also loaded and cached any dependencies of this type from the type store.
+            verifyHierarchicalType(cachedType, typeEntry.getValue());
+        }
+    }
+
+    @Test
+    public void testHasClassType() throws Exception {
+        for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
+            // Not cached yet
+            Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
+
+            // Calling has() should result in type and its dependencies
+            // loaded from the type store and added to the cache.
+            Assert.assertTrue(typeCache.has(typeEntry.getKey()));
+
+            // Verify the type is now cached in memory.
+            Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
+        }
+    }
+
+    @Test
+    public void testGetTraitType() throws Exception {
+        ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
+        for (String traitTypeName : traitNames) {
+            // Not cached yet
+            Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
+
+            IDataType dataType = typeCache.get(traitTypeName);
+            // Verify the type is now cached.
+            Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
+
+            Assert.assertTrue(dataType instanceof TraitType);
+            TraitType cachedType = (TraitType)dataType;
+            // Verify that get() also loaded and cached any dependencies of this type from the type store.
+            verifyHierarchicalType(cachedType, ts.getDataType(TraitType.class, traitTypeName));
+        }
+    }
+
+    @Test
+    public void testHasTraitType() throws Exception {
+        ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
+        for (String traitTypeName : traitNames) {
+            // Not cached yet
+            Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
+
+            // Calling has() should result in type and its dependencies
+            // loaded from the type store and added to the cache.
+            Assert.assertTrue(typeCache.has(traitTypeName));
+
+            // Verify the type is now cached.
+            Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
+        }
+    }
+
+    private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException {
+        Assert.assertEquals(dataType.numFields, expectedDataType.numFields);
+        Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size());
+        Assert.assertEquals(dataType.fieldMapping().fields.size(), expectedDataType.fieldMapping().fields.size());
+        ImmutableSet<String> superTypes = dataType.superTypes;
+        Assert.assertEquals(superTypes.size(), expectedDataType.superTypes.size());
+
+        // Verify that any attribute and super types were also cached.
+        for (String superTypeName : superTypes) {
+            Assert.assertTrue(typeCache.has(superTypeName));
+        }
+        for (AttributeInfo attrInfo : dataType.fieldMapping().fields.values()) {
+            switch (attrInfo.dataType().getTypeCategory()) {
+            case CLASS:
+            case STRUCT:
+            case ENUM:
+                Assert.assertTrue(typeCache.has(attrInfo.dataType().getName()), attrInfo.dataType().getName() + " should be cached");
+                break;
+            case ARRAY:
+                String elementTypeName = TypeUtils.parseAsArrayType(attrInfo.dataType().getName());
+                if (!ts.getCoreTypes().contains(elementTypeName)) {
+                    Assert.assertTrue(typeCache.has(elementTypeName), elementTypeName + " should be cached");
+                }
+                break;
+            case MAP:
+                String[] mapTypeNames = TypeUtils.parseAsMapType(attrInfo.dataType().getName());
+                for (String typeName : mapTypeNames) {
+                    if (!ts.getCoreTypes().contains(typeName)) {
+                        Assert.assertTrue(typeCache.has(typeName), typeName + " should be cached");
+                    }
+                }
+                break;
+            default:
+                break;
+            }
+        }
+    }
+}

repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.commons.configuration.Configuration;
+
+
+/**
+ * Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}.
+ *
+ */
+public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule {
+    @Override
+    protected Configuration getConfiguration() {
+        try {
+            Configuration configuration = ApplicationProperties.get();
+            configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY,
+                    StoreBackedTypeCache.class.getName());
+            return configuration;
+        } catch (AtlasException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.service;
+
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.typestore.ITypeStore;
+import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
+import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule;
+import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+
+
+/**
+ *  Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store.
+ *  StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
+ *  which has type cache implementation class set to {@link StoreBackedTypeCache}.
+ */
+@Guice(modules = StoreBackedTypeCacheTestModule.class)
+public class StoreBackedTypeCacheMetadataServiceTest
+{
+    @Inject
+    private MetadataService metadataService;
+
+    @Inject
+    private ITypeStore typeStore;
+
+    @Inject 
+    TypeCache typeCache;
+
+    private TypeSystem ts;
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        ts = TypeSystem.getInstance();
+        ts.reset();
+
+        // Populate the type store for testing.
+        TestUtils.defineDeptEmployeeTypes(ts);
+        TestUtils.createHiveTypes(ts);
+        ImmutableList<String> typeNames = ts.getTypeNames();
+        typeStore.store(ts, typeNames);
+        ts.reset();
+    }
+
+    @Test
+    public void testIt() throws Exception {
+        Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
+        StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache;
+
+        // Cache should be empty
+        Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager"));
+
+        // Type lookup on MetadataService should cause Manager type to be loaded from the type store
+        // and cached.
+        Assert.assertNotNull(metadataService.getTypeDefinition("Manager"));
+        Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager"));
+    }
+}

repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java

@@ -19,6 +19,7 @@
 package org.apache.atlas.services;
 
 import com.google.inject.Provider;
+
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.listener.EntityChangeListener;
 import org.apache.atlas.listener.TypesChangeListener;
@@ -26,12 +27,8 @@ import org.apache.atlas.repository.MetadataRepository;
 import org.apache.atlas.repository.typestore.ITypeStore;
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.atlas.ha.HAConfiguration;
-import org.apache.atlas.listener.TypesChangeListener;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.typestore.ITypeStore;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.commons.configuration.Configuration;
 import org.mockito.Matchers;
 import org.mockito.Mock;
@@ -44,6 +41,7 @@ import java.util.HashMap;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
@@ -78,7 +76,7 @@ public class DefaultMetadataServiceMockTest {
         DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
                 mock(ITypeStore.class),
                 typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
-                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
 
         verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
                 typeSystem, defaultMetadataService);
@@ -91,10 +89,10 @@ public class DefaultMetadataServiceMockTest {
         DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
                 typeStore,
                 typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
-                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
 
         verifyZeroInteractions(typeStore);
-        verifyZeroInteractions(typeSystem);
+        verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any());
         verifyZeroInteractions(typesRegistrar);
     }
 
@@ -109,7 +107,7 @@ public class DefaultMetadataServiceMockTest {
         DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
                 typeStore,
                 typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
-                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
         defaultMetadataService.instanceIsActive();
 
         verify(typeStore).restore();
@@ -134,7 +132,7 @@ public class DefaultMetadataServiceMockTest {
         DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
                 typeStore,
                 typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
-                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+                new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
 
         defaultMetadataService.instanceIsActive();
         defaultMetadataService.instanceIsPassive();

typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java

@@ -30,14 +30,13 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import javax.inject.Singleton;
 
-import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasException;
 import org.apache.atlas.classification.InterfaceAudience;
 import org.apache.atlas.typesystem.TypesDef;
 import org.apache.atlas.typesystem.exception.TypeExistsException;
 import org.apache.atlas.typesystem.exception.TypeNotFoundException;
-import org.apache.atlas.typesystem.types.cache.DefaultTypeCacheProvider;
-import org.apache.atlas.typesystem.types.cache.ITypeCacheProvider;
+import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -48,7 +47,6 @@ import com.google.common.collect.ImmutableSet;
 @InterfaceAudience.Private
 public class TypeSystem {
     private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class);
-    private static final String CACHE_PROVIDER_CLASS_PROPERTY = "atlas.typesystem.cache.provider";
 
     private static final TypeSystem INSTANCE = new TypeSystem();
     private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() {
@@ -60,7 +58,7 @@ public class TypeSystem {
         }
     };
 
-    private ITypeCacheProvider typeCache;
+    private TypeCache typeCache  = new DefaultTypeCache();
     private IdType idType;
     private Map<String, IDataType> coreTypes;
 
@@ -84,44 +82,18 @@ public class TypeSystem {
         return this;
     }
 
+    public void setTypeCache(TypeCache typeCache) {
+        this.typeCache = typeCache;
+    }
+
     private void initialize() {
 
-        initCacheProvider();
         coreTypes = new ConcurrentHashMap<>();
 
         registerPrimitiveTypes();
         registerCoreTypes();
     }
 
-    /**
-     * Ideally a cache provider should have been injected in the TypeSystemProvider,
-     * but a singleton of TypeSystem is constructed privately within the class so that
-     * clients of TypeSystem would never instantiate a TypeSystem object directly in
-     * their code. As soon as a client makes a call to TypeSystem.getInstance(), they
-     * should have the singleton ready for consumption. To enable such an access pattern,
-     * it kind of becomes imperative to initialize the cache provider within the
-     * TypeSystem constructor (bypassing the GUICE way of injecting a cache provider)
-     */
-    private void initCacheProvider() {
-
-        // read the pluggable cache provider from Atlas configuration
-        final String defaultCacheProvider = DefaultTypeCacheProvider.class.getName();
-        Class cacheProviderClass;
-        try {
-            cacheProviderClass = ApplicationProperties.getClass(CACHE_PROVIDER_CLASS_PROPERTY,
-                defaultCacheProvider, ITypeCacheProvider.class);
-        } catch (AtlasException e) {
-            throw new RuntimeException("Error getting type cache provider implementation class", e);
-        }
-
-        try {
-            typeCache = (ITypeCacheProvider)cacheProviderClass.newInstance();
-        }
-        catch (Exception e) {
-            throw new RuntimeException("Error creating instance of type cache provider implementation class " + cacheProviderClass.getName(), e);
-        }
-    }
-
     public ImmutableList<String> getCoreTypes() {
         return ImmutableList.copyOf(coreTypes.keySet());
     }

typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java → typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java

@@ -39,14 +39,14 @@ import com.google.inject.Singleton;
  */
 @Singleton
 @SuppressWarnings("rawtypes")
-public class DefaultTypeCacheProvider implements ITypeCacheProvider {
+public class DefaultTypeCache implements TypeCache {
 
     private Map<String, IDataType> types_ = new ConcurrentHashMap<>();
 
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang
+     * org.apache.atlas.typesystem.types.cache.TypeCache#has(java.lang
      * .String)
      */
     @Override
@@ -56,7 +56,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     }
 
     /* (non-Javadoc)
-     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org.
+     * @see org.apache.atlas.typesystem.types.cache.TypeCache#has(org.
      * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
      */
     @Override
@@ -93,7 +93,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang
+     * org.apache.atlas.typesystem.types.cache.TypeCache#get(java.lang
      * .String)
      */
     @Override
@@ -103,7 +103,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     }
 
     /* (non-Javadoc)
-     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache.
+     * @see org.apache.atlas.typesystem.types.cache.TypeCache#get(org.apache.
      * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
      */
     @Override
@@ -116,7 +116,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org
+     * org.apache.atlas.typesystem.types.cache.TypeCache#getNames(org
      * .apache.atlas.typesystem.types.DataTypes.TypeCategory)
      */
     @Override
@@ -139,7 +139,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames()
+     * org.apache.atlas.typesystem.types.cache.TypeCache#getAllNames()
      */
     @Override
     public Collection<String> getAllTypeNames() throws AtlasException {
@@ -150,7 +150,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache
+     * org.apache.atlas.typesystem.types.cache.TypeCache#put(org.apache
      * .atlas.typesystem.types.IDataType)
      */
     @Override
@@ -181,7 +181,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java
+     * org.apache.atlas.typesystem.types.cache.TypeCache#putAll(java
      * .util.Collection)
      */
     @Override
@@ -196,7 +196,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     /*
      * (non-Javadoc)
      * @see
-     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java
+     * org.apache.atlas.typesystem.types.cache.TypeCache#remove(java
      * .lang.String)
      */
     @Override
@@ -206,7 +206,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
     }
 
     /* (non-Javadoc)
-     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org.
+     * @see org.apache.atlas.typesystem.types.cache.TypeCache#remove(org.
      * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
      */
     @Override
@@ -219,7 +219,7 @@ public class DefaultTypeCacheProvider implements ITypeCacheProvider {
 
     /*
      * (non-Javadoc)
-     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear()
+     * @see org.apache.atlas.typesystem.types.cache.TypeCache#clear()
      */
     @Override
     public void clear() {

typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java → typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java

@@ -27,7 +27,7 @@ import org.apache.atlas.typesystem.types.IDataType;
 /**
  * The types are cached to allow faster lookup when type info is needed during
  * creation/updation of entities, DSL query translation/execution.
- * Implementations of this can chose to plugin a distributed cache provider
+ * Implementations of this can chose to plugin a distributed cache
  * or an in-memory cache synched across nodes in an Altas cluster. <br>
  * <br>
  * Type entries in the cache can be one of ... <br>
@@ -37,7 +37,7 @@ import org.apache.atlas.typesystem.types.IDataType;
  * {@link org.apache.atlas.typesystem.types.EnumType}
  */
 @SuppressWarnings("rawtypes")
-public interface ITypeCacheProvider {
+public interface TypeCache {
 
     /**
      * @param typeName
@@ -83,7 +83,7 @@ public interface ITypeCacheProvider {
     /**
      * This is a convenience API to get the names of all types.
      *
-     * @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
+     * @see TypeCache#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
      * @return
      * @throws AtlasException
      */

typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java

@@ -58,17 +58,22 @@ public class ApplicationPropertiesTest {
 
     @Test
     public void testGetClass() throws Exception {
+        Configuration configuration = ApplicationProperties.get();
+
         //read from atlas-application.properties
-        Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class);
+        Class cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
+            ApplicationProperties.class.getName(), TypeSystem.class);
         assertEquals(cls.getName(), TypeSystem.class.getName());
 
         //default value
-        cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class);
+        cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem2.impl",
+            TypeSystem.class.getName(), TypeSystem.class);
         assertEquals(cls.getName(), TypeSystem.class.getName());
 
         //incompatible assignTo class, should throw AtlasException
         try {
-            cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class);
+            cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
+                ApplicationProperties.class.getName(), ApplicationProperties.class);
             Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown.");
         }
         catch (AtlasException e) {

webapp/src/test/java/org/apache/atlas/web/listeners/TestModule.java

@@ -19,13 +19,15 @@
 package org.apache.atlas.web.listeners;
 
 import com.google.inject.Binder;
+
 import org.apache.atlas.RepositoryMetadataModule;
 import org.apache.atlas.repository.audit.EntityAuditRepository;
 import org.apache.atlas.repository.audit.InMemoryEntityAuditRepository;
+import org.apache.commons.configuration.Configuration;
 
 public class TestModule extends RepositoryMetadataModule {
     @Override
-    protected void bindAuditRepository(Binder binder) {
+    protected void bindAuditRepository(Binder binder, Configuration configuration) {
         //Map EntityAuditRepository interface to hbase based implementation
         binder.bind(EntityAuditRepository.class).to(InMemoryEntityAuditRepository.class).asEagerSingleton();
     }