ATLAS-683 Refactor local type-system cache with cache provider interface (vmadugun via shwethags)

.gitignore

@@ -60,7 +60,9 @@ distro/hbase/*.tar.gz
 #solr package downloaded
 distro/solr/*.tgz
 
+# Scala-IDE specific
 .cache-main
+.cache-tests
 
 # emacs files
 *#

common/src/main/java/org/apache/atlas/ApplicationProperties.java

@@ -92,13 +92,22 @@ public final class ApplicationProperties extends PropertiesConfiguration {
         return inConf.subset(prefix);
     }
 
-    public static Class getClass(String propertyName, String defaultValue) {
+    public static Class getClass(String propertyName, String defaultValue, Class assignableClass)
+        throws AtlasException {
         try {
             Configuration configuration = get();
             String propertyValue = configuration.getString(propertyName, defaultValue);
-            return Class.forName(propertyValue);
+            Class<?> clazz = Class.forName(propertyValue);
+            if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) {
+                return clazz;
+            } else {
+                String message = "Class " + clazz.getName() + " specified in property " + propertyName
+                        + " is not assignable to class " + assignableClass.getName();
+                LOG.error(message);
+                throw new AtlasException(message);
+            }
         } catch (Exception e) {
-            throw new RuntimeException(e);
+            throw new AtlasException(e);
         }
     }
 }

distro/src/conf/atlas-application.properties

@@ -122,3 +122,8 @@ atlas.login.credentials.file=${sys:atlas.home}/conf/users-credentials.properties
 #########POLICY FILE PATH #########
 atlas.auth.policy.file=${sys:atlas.home}/conf/policy-store.txt
 
+#########  Type Cache Provider Implementation ########
+# A type cache provider class which implements
+# org.apache.atlas.typesystem.types.cache.ITypeCacheProvider.
+# The default is DefaultTypeCacheProvider which is a local in-memory type cache.
+#atlas.typesystem.cache.provider=

release-log.txt

@@ -21,6 +21,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
 ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
 
 ALL CHANGES:
+ATLAS-683 Refactor local type-system cache with cache provider interface (vmadugun via shwethags)
 ATLAS-802 New look UI to show Business Catalog functionalities (kevalbhatt18 via yhemanth)
 ATLAS-658 Improve Lineage with Backbone porting (kevalbhatt18 via yhemanth)
 ATLAS-491 Business Catalog / Taxonomy (jspeidel via yhemanth)

repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java

@@ -111,7 +111,11 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
     private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
 
     private Class<? extends DeleteHandler> getDeleteHandler() {
-        return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
-                SoftDeleteHandler.class.getName());
+        try {
+            return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
+                    SoftDeleteHandler.class.getName(), DeleteHandler.class);
+        } catch (AtlasException e) {
+            throw new RuntimeException(e);
+        }
     }
 }

typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.typesystem.types.cache;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+
+import com.google.inject.Singleton;
+
+/**
+ * Caches the types in-memory within the same process space.
+ */
+@Singleton
+@SuppressWarnings("rawtypes")
+public class DefaultTypeCacheProvider implements ITypeCacheProvider {
+
+    private Map<String, IDataType> types_ = new ConcurrentHashMap<>();
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang
+     * .String)
+     */
+    @Override
+    public boolean has(String typeName) throws AtlasException {
+
+        return types_.containsKey(typeName);
+    }
+
+    /* (non-Javadoc)
+     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org.
+     * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
+     */
+    @Override
+    public boolean has(TypeCategory typeCategory, String typeName)
+        throws AtlasException {
+
+        assertValidTypeCategory(typeCategory);
+        return has(typeName);
+    }
+
+    private void assertValidTypeCategory(TypeCategory typeCategory) throws
+        AtlasException {
+
+        // there might no need of 'typeCategory' in this implementation for
+        // certain API, but for a distributed cache, it might help for the
+        // implementers to partition the types per their category
+        // while persisting so that look can be efficient
+
+        if (typeCategory == null) {
+            throw new AtlasException("Category of the types to be filtered is null.");
+        }
+
+        boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) ||
+            typeCategory.equals(TypeCategory.TRAIT) ||
+            typeCategory.equals(TypeCategory.ENUM) ||
+            typeCategory.equals(TypeCategory.STRUCT);
+
+        if (!validTypeCategory) {
+            throw new AtlasException("Category of the types should be one of CLASS "
+                + "| TRAIT | ENUM | STRUCT.");
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang
+     * .String)
+     */
+    @Override
+    public IDataType get(String typeName) throws AtlasException {
+
+        return types_.get(typeName);
+    }
+
+    /* (non-Javadoc)
+     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache.
+     * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
+     */
+    @Override
+    public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException {
+
+        assertValidTypeCategory(typeCategory);
+        return get(typeName);
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org
+     * .apache.atlas.typesystem.types.DataTypes.TypeCategory)
+     */
+    @Override
+    public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException {
+
+        assertValidTypeCategory(typeCategory);
+
+        List<String> typeNames = new ArrayList<>();
+        for (Entry<String, IDataType> typeEntry : types_.entrySet()) {
+            String name = typeEntry.getKey();
+            IDataType type = typeEntry.getValue();
+
+            if (type.getTypeCategory().equals(typeCategory)) {
+                typeNames.add(name);
+            }
+        }
+        return typeNames;
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames()
+     */
+    @Override
+    public Collection<String> getAllTypeNames() throws AtlasException {
+
+        return types_.keySet();
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache
+     * .atlas.typesystem.types.IDataType)
+     */
+    @Override
+    public void put(IDataType type) throws AtlasException {
+
+        assertValidType(type);
+        types_.put(type.getName(), type);
+    }
+
+    private void assertValidType(IDataType type) throws
+        AtlasException {
+
+        if (type == null) {
+            throw new AtlasException("type is null.");
+        }
+
+        boolean validTypeCategory = (type instanceof ClassType) ||
+            (type instanceof TraitType) ||
+            (type instanceof EnumType) ||
+            (type instanceof StructType);
+
+        if (!validTypeCategory) {
+            throw new AtlasException("Category of the types should be one of ClassType | "
+                + "TraitType | EnumType | StructType.");
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java
+     * .util.Collection)
+     */
+    @Override
+    public void putAll(Collection<IDataType> types) throws AtlasException {
+
+        for (IDataType type : types) {
+            assertValidType(type);
+            types_.put(type.getName(), type);
+        }
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see
+     * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java
+     * .lang.String)
+     */
+    @Override
+    public void remove(String typeName) throws AtlasException {
+
+        types_.remove(typeName);
+    }
+
+    /* (non-Javadoc)
+     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org.
+     * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
+     */
+    @Override
+    public void remove(TypeCategory typeCategory, String typeName)
+            throws AtlasException {
+
+        assertValidTypeCategory(typeCategory);
+        remove(typeName);
+    }
+
+    /*
+     * (non-Javadoc)
+     * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear()
+     */
+    @Override
+    public void clear() {
+
+        types_.clear();
+    }
+}

typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types.cache;
+
+import java.util.Collection;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+
+/**
+ * The types are cached to allow faster lookup when type info is needed during
+ * creation/updation of entities, DSL query translation/execution.
+ * Implementations of this can chose to plugin a distributed cache provider
+ * or an in-memory cache synched across nodes in an Altas cluster. <br>
+ * <br>
+ * Type entries in the cache can be one of ... <br>
+ * {@link org.apache.atlas.typesystem.types.ClassType} <br>
+ * {@link org.apache.atlas.typesystem.types.TraitType} <br>
+ * {@link org.apache.atlas.typesystem.types.StructType} <br>
+ * {@link org.apache.atlas.typesystem.types.EnumType}
+ */
+@SuppressWarnings("rawtypes")
+public interface ITypeCacheProvider {
+
+    /**
+     * @param typeName
+     * @return true if the type exists in cache, false otherwise.
+     * @throws AtlasException
+     */
+    boolean has(String typeName) throws AtlasException;
+
+    /**
+     * @param typeCategory Non-null category of type. The category can be one of
+     * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
+     * @param typeName
+     * @return true if the type of given category exists in cache, false otherwise.
+     * @throws AtlasException
+     */
+    boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
+
+    /**
+     * @param name The name of the type.
+     * @return returns non-null type if cached, otherwise null
+     * @throws AtlasException
+     */
+    public IDataType get(String typeName) throws AtlasException;
+
+    /**
+     * @param typeCategory Non-null category of type. The category can be one of
+     * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
+     * @param typeName
+     * @return returns non-null type (of the specified category) if cached, otherwise null
+     * @throws AtlasException
+     */
+    public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
+
+    /**
+     * @param typeCategory The category of types to filter the returned types. Cannot be null.
+     * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT |
+     * TypeCategory.STRUCT | TypeCategory.ENUM.
+     * @return
+     * @throws AtlasException
+     */
+    Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException;
+
+    /**
+     * This is a convenience API to get the names of all types.
+     *
+     * @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
+     * @return
+     * @throws AtlasException
+     */
+    Collection<String> getAllTypeNames() throws AtlasException;
+
+    /**
+     * @param type The type to be added to the cache. The type should not be
+     * null, otherwise throws NullPointerException. <br>
+     * Type entries in the cache can be one of ... <br>
+     * {@link org.apache.atlas.typesystem.types.ClassType} <br>
+     * {@link org.apache.atlas.typesystem.types.TraitType} <br>
+     * {@link org.apache.atlas.typesystem.types.StructType} <br>
+     * {@link org.apache.atlas.typesystem.types.EnumType}
+     * @throws AtlasException
+     */
+    void put(IDataType type) throws AtlasException;
+
+    /**
+     * @param types The types to be added to the cache. The type should not be
+     * null, otherwise throws NullPointerException. <br>
+     * Type entries in the cache can be one of ... <br>
+     * {@link org.apache.atlas.typesystem.types.ClassType} <br>
+     * {@link org.apache.atlas.typesystem.types.TraitType} <br>
+     * {@link org.apache.atlas.typesystem.types.StructType} <br>
+     * {@link org.apache.atlas.typesystem.types.EnumType}
+     * @throws AtlasException
+     */
+    void putAll(Collection<IDataType> types) throws AtlasException;
+
+    /**
+     * @param typeName Name of the type to be removed from the cache. If type
+     * exists, it will be removed, otherwise does nothing.
+     * @throws AtlasException
+     */
+    void remove(String typeName) throws AtlasException;
+
+    /**
+     * @param typeCategory Non-null category of type. The category can be one of
+     * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
+     * @param typeName Name of the type to be removed from the cache. If type
+     * exists, it will be removed, otherwise does nothing.
+     * @throws AtlasException
+     */
+    void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
+
+    /**
+     * Clear the type cache
+     *
+     */
+    void clear();
+}

typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java

@@ -19,6 +19,7 @@ package org.apache.atlas;
 
 import org.apache.atlas.typesystem.types.TypeSystem;
 import org.apache.commons.configuration.Configuration;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import static org.testng.Assert.assertEquals;
@@ -58,11 +59,20 @@ public class ApplicationPropertiesTest {
     @Test
     public void testGetClass() throws Exception {
         //read from atlas-application.properties
-        Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName());
+        Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class);
         assertEquals(cls.getName(), TypeSystem.class.getName());
 
         //default value
-        cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName());
+        cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class);
         assertEquals(cls.getName(), TypeSystem.class.getName());
+
+        //incompatible assignTo class, should throw AtlasException
+        try {
+            cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class);
+            Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown.");
+        }
+        catch (AtlasException e) {
+            // good
+        }
     }
 }

typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java

@@ -209,7 +209,7 @@ public class TypeSystemTest extends BaseTest {
     }
 
     @Test
-    public void testTypeNamesAreNotDuplicated() {
+    public void testTypeNamesAreNotDuplicated() throws Exception {
         TypeSystem typeSystem = getTypeSystem();
         ImmutableList<String> traitNames = typeSystem.getTypeNamesByCategory(DataTypes.TypeCategory.TRAIT);
         int numTraits = traitNames.size();