ATLAS-806 Create default taxonomy at server startup (jspeidel via yhemanth)

catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java

@@ -18,9 +18,14 @@
 
 package org.apache.atlas.catalog;
 
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
 import org.apache.atlas.catalog.definition.TaxonomyResourceDefinition;
 import org.apache.atlas.catalog.exception.*;
 import org.apache.atlas.catalog.query.AtlasQuery;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.*;
 
@@ -28,8 +33,18 @@ import java.util.*;
  * Provider for taxonomy resources.
  */
 public class TaxonomyResourceProvider extends BaseResourceProvider implements ResourceProvider {
+    private static final Logger LOG = LoggerFactory.getLogger(TaxonomyResourceProvider.class);
+    public static final String DEFAULT_TAXONOMY_NAME = "Catalog";
+    public static final String DEFAULT_TAXONOMY_DESCRIPTION = "Business Catalog";
     private final TermResourceProvider termResourceProvider;
 
+    // This is a cached value to prevent checking for taxonomy objects in every API call.
+    // It is updated once per lifetime of the application.
+    // TODO: If a taxonomy is deleted outside of this application, this value is not updated
+    // TODO: and there is no way in which a taxonomy will be auto-created.
+    // TODO: Assumption is that if a taxonomy is deleted externally, it will be created externally as well.
+    private static boolean taxonomyAutoInitializationChecked = false;
+
     public TaxonomyResourceProvider(AtlasTypeSystem typeSystem) {
         super(typeSystem, new TaxonomyResourceDefinition());
         termResourceProvider = new TermResourceProvider(typeSystem);
@@ -37,38 +52,36 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re
 
     @Override
     public Result getResourceById(Request request) throws ResourceNotFoundException {
-        AtlasQuery atlasQuery;
-        try {
-            atlasQuery = queryFactory.createTaxonomyQuery(request);
-        } catch (InvalidQueryException e) {
-            throw new CatalogRuntimeException("Unable to compile internal Taxonomy query: " + e, e);
+        synchronized (TaxonomyResourceProvider.class) {
+            createDefaultTaxonomyIfNeeded();
         }
-        Collection<Map<String, Object>> results = atlasQuery.execute();
-        if (results.isEmpty()) {
-            throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.",
-                    request.getProperty(resourceDefinition.getIdPropertyName())));
-        }
-        return new Result(results);
+        return doGetResourceById(request);
     }
 
     @Override
     public Result getResources(Request request) throws InvalidQueryException, ResourceNotFoundException {
-        AtlasQuery atlasQuery = queryFactory.createTaxonomyQuery(request);
-        return new Result(atlasQuery.execute());
+        synchronized (TaxonomyResourceProvider.class) {
+            createDefaultTaxonomyIfNeeded();
+        }
+        return doGetResources(request);
     }
 
     @Override
-    public synchronized void createResource(Request request)
+    public void createResource(Request request)
             throws InvalidPayloadException, ResourceAlreadyExistsException {
 
+        // not checking for default taxonomy in create per requirements
         resourceDefinition.validateCreatePayload(request);
+        synchronized (TaxonomyResourceProvider.class) {
             ensureTaxonomyDoesntExist(request);
-        typeSystem.createEntity(resourceDefinition, request);
+            doCreateResource(request);
+        }
     }
 
     @Override
     public Collection<String> createResources(Request request) throws InvalidQueryException, ResourceNotFoundException {
-        throw new UnsupportedOperationException("Creating multiple Taxonomies in a request is not currently supported");
+        throw new UnsupportedOperationException(
+                "Creating multiple Taxonomies in a request is not currently supported");
     }
 
     @Override
@@ -81,21 +94,36 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re
     @Override
     public void updateResourceById(Request request) throws ResourceNotFoundException, InvalidPayloadException {
         resourceDefinition.validateUpdatePayload(request);
+
         AtlasQuery atlasQuery;
         try {
             atlasQuery = queryFactory.createTaxonomyQuery(request);
         } catch (InvalidQueryException e) {
             throw new CatalogRuntimeException("Unable to compile internal Term query: " + e, e);
         }
+
+        synchronized (TaxonomyResourceProvider.class) {
+            createDefaultTaxonomyIfNeeded();
             if (atlasQuery.execute(request.getUpdateProperties()).isEmpty()) {
                 throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.",
                         request.getQueryProperties().get("name")));
             }
         }
+    }
+
+    private String getResourceId(Request request) throws ResourceNotFoundException {
+        request.addAdditionalSelectProperties(Collections.singleton("id"));
+        // will result in expected ResourceNotFoundException if taxonomy doesn't exist
+        Result result = getResourceById(request);
+        return String.valueOf(result.getPropertyMaps().iterator().next().get("id"));
+    }
 
+    //todo: this is currently required because the expected exception isn't thrown by the Atlas repository
+    //todo: when an attempt is made to create an entity that already exists
+    // must be called from within class monitor
     private void ensureTaxonomyDoesntExist(Request request) throws ResourceAlreadyExistsException {
         try {
-            getResourceById(request);
+            doGetResourceById(request);
             throw new ResourceAlreadyExistsException(String.format("Taxonomy '%s' already exists.",
                     request.getProperty("name")));
         } catch (ResourceNotFoundException e) {
@@ -103,11 +131,73 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re
         }
     }
 
-    private String getResourceId(Request request) throws ResourceNotFoundException {
-        request.addAdditionalSelectProperties(Collections.singleton("id"));
-        // will result in expected ResourceNotFoundException if taxonomy doesn't exist
-        Result result = getResourceById(request);
-        return String.valueOf(result.getPropertyMaps().iterator().next().get("id"));
+    // must be called from within class monitor
+    private Result doGetResourceById(Request request) throws ResourceNotFoundException {
+        AtlasQuery atlasQuery;
+        try {
+            atlasQuery = queryFactory.createTaxonomyQuery(request);
+        } catch (InvalidQueryException e) {
+            throw new CatalogRuntimeException("Unable to compile internal Taxonomy query: " + e, e);
+        }
+
+        Collection<Map<String, Object>> resultSet = atlasQuery.execute();
+        if (resultSet.isEmpty()) {
+            throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.",
+                    request.getProperty(resourceDefinition.getIdPropertyName())));
+        }
+        return new Result(resultSet);
+    }
+
+    // must be called from within class monitor
+    private Result doGetResources(Request request) throws InvalidQueryException, ResourceNotFoundException {
+        AtlasQuery atlasQuery = queryFactory.createTaxonomyQuery(request);
+        return new Result(atlasQuery.execute());
+    }
+
+    // must be called from within class monitor
+    private void doCreateResource(Request request) throws ResourceAlreadyExistsException {
+        typeSystem.createEntity(resourceDefinition, request);
+        taxonomyAutoInitializationChecked = true;
+    }
+
+    // must be called from within class monitor
+    private void createDefaultTaxonomyIfNeeded() {
+        if (! autoInitializationChecked()) {
+            try {
+                LOG.info("Checking if default taxonomy needs to be created.");
+                // if any business taxonomy has been created, don't create one more - hence searching to
+                // see if any taxonomy exists.
+                if (doGetResources(new CollectionRequest(null, null)).getPropertyMaps().isEmpty()) {
+                    LOG.info("No taxonomies found - going to create default taxonomy.");
+                    Map<String, Object> requestProperties = new HashMap<>();
+                    String defaultTaxonomyName = DEFAULT_TAXONOMY_NAME;
+                    try {
+                        Configuration configuration = ApplicationProperties.get();
+                        defaultTaxonomyName = configuration.getString("atlas.taxonomy.default.name",
+                                defaultTaxonomyName);
+                    } catch (AtlasException e) {
+                        LOG.warn("Unable to read Atlas configuration, will use {} as default taxonomy name",
+                                defaultTaxonomyName, e);
+                    }
+                    requestProperties.put("name", defaultTaxonomyName);
+                    requestProperties.put("description", DEFAULT_TAXONOMY_DESCRIPTION);
+
+                    doCreateResource(new InstanceRequest(requestProperties));
+                    LOG.info("Successfully created default taxonomy {}.", defaultTaxonomyName);
+                } else {
+                    taxonomyAutoInitializationChecked = true;
+                    LOG.info("Some taxonomy exists, not creating default taxonomy");
+                }
+            } catch (InvalidQueryException | ResourceNotFoundException e) {
+                LOG.error("Unable to query for existing taxonomies due to internal error.", e);
+            } catch (ResourceAlreadyExistsException e) {
+                LOG.info("Attempted to create default taxonomy and it already exists.");
+            }
+        }
+    }
+
+    protected boolean autoInitializationChecked() {
+        return taxonomyAutoInitializationChecked;
     }
 
     protected TermResourceProvider getTermResourceProvider() {

catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java

@@ -58,7 +58,110 @@ public class TaxonomyResourceProviderTest {
         expect(query.execute()).andReturn(queryResult);
         replay(typeSystem, queryFactory, query);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
+        provider.setQueryFactory(queryFactory);
+
+        Map<String, Object> requestProperties = new HashMap<>();
+        requestProperties.put("name", "taxonomyName");
+        Request userRequest = new InstanceRequest(requestProperties);
+
+        Result result = provider.getResourceById(userRequest);
+
+        assertEquals(1, result.getPropertyMaps().size());
+        assertEquals(queryResultRow, result.getPropertyMaps().iterator().next());
+
+        Request request = requestCapture.getValue();
+        assertNull(request.getQueryString());
+        assertEquals(0, request.getAdditionalSelectProperties().size());
+        assertEquals(requestProperties, request.getQueryProperties());
+
+        verify(typeSystem, queryFactory, query);
+    }
+
+    @Test
+    public void testGetResourceById_notInitialized_createDefaultTaxonomy() throws Exception {
+        AtlasTypeSystem typeSystem = createStrictMock(AtlasTypeSystem.class);
+        QueryFactory queryFactory = createStrictMock(QueryFactory.class);
+        AtlasQuery query = createStrictMock(AtlasQuery.class);
+        Capture<Request> checkForAnyTaxonomiesCapture = newCapture();
+        Capture<Request> createDefaultTaxonomyRequestCapture = newCapture();
+        Capture<Request> requestCapture = newCapture();
+        Capture<ResourceDefinition> resourceDefinitionCapture = newCapture();
+
+        Collection<Map<String, Object>> queryResult = new ArrayList<>();
+        Map<String, Object> queryResultRow = new HashMap<>();
+        queryResult.add(queryResultRow);
+        queryResultRow.put("name", "taxonomyName");
+        queryResultRow.put("description", "test taxonomy description");
+        queryResultRow.put("creation_time", "04/20/2016");
+
+        // mock expectations
+        expect(queryFactory.createTaxonomyQuery(capture(checkForAnyTaxonomiesCapture))).andReturn(query);
+        expect(query.execute()).andReturn(Collections.<Map<String, Object>>emptySet());
+        typeSystem.createEntity(capture(resourceDefinitionCapture), capture(createDefaultTaxonomyRequestCapture));
+        expect(queryFactory.createTaxonomyQuery(capture(requestCapture))).andReturn(query);
+        expect(query.execute()).andReturn(queryResult);
+        replay(typeSystem, queryFactory, query);
+
+        TestTaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
+        provider.setInitialized(false);
+        provider.setQueryFactory(queryFactory);
+
+        Map<String, Object> requestProperties = new HashMap<>();
+        requestProperties.put("name", "taxonomyName");
+        Request userRequest = new InstanceRequest(requestProperties);
+
+        Result result = provider.getResourceById(userRequest);
+
+        assertEquals(1, result.getPropertyMaps().size());
+        assertEquals(queryResultRow, result.getPropertyMaps().iterator().next());
+
+        Request request = requestCapture.getValue();
+        assertNull(request.getQueryString());
+        assertEquals(0, request.getAdditionalSelectProperties().size());
+        assertEquals(requestProperties, request.getQueryProperties());
+
+        Request checkForAnyTaxonomiesRequest = checkForAnyTaxonomiesCapture.getValue();
+        assertNull(checkForAnyTaxonomiesRequest.getQueryString());
+        assertEquals(checkForAnyTaxonomiesRequest.getAdditionalSelectProperties().size(), 0);
+        assertEquals(checkForAnyTaxonomiesRequest.getQueryProperties().size(), 0);
+
+        Request createDefaultTaxonomyRequest = createDefaultTaxonomyRequestCapture.getValue();
+        assertNull(createDefaultTaxonomyRequest.getQueryString());
+        assertEquals(createDefaultTaxonomyRequest.getAdditionalSelectProperties().size(), 0);
+        assertEquals(createDefaultTaxonomyRequest.getQueryProperties().size(), 2);
+        assertEquals(createDefaultTaxonomyRequest.getQueryProperties().get("name"),
+                TaxonomyResourceProvider.DEFAULT_TAXONOMY_NAME);
+        assertEquals(createDefaultTaxonomyRequest.getQueryProperties().get("description"),
+                TaxonomyResourceProvider.DEFAULT_TAXONOMY_DESCRIPTION);
+
+        verify(typeSystem, queryFactory, query);
+    }
+
+    @Test
+    public void testGetResourceById_notInitialized_taxonomyAlreadyExists() throws Exception {
+        AtlasTypeSystem typeSystem = createStrictMock(AtlasTypeSystem.class);
+        QueryFactory queryFactory = createStrictMock(QueryFactory.class);
+        AtlasQuery query = createStrictMock(AtlasQuery.class);
+        Capture<Request> checkForAnyTaxonomiesCapture = newCapture();
+        Capture<Request> requestCapture = newCapture();
+
+        Collection<Map<String, Object>> queryResult = new ArrayList<>();
+        Map<String, Object> queryResultRow = new HashMap<>();
+        queryResult.add(queryResultRow);
+        queryResultRow.put("name", "taxonomyName");
+        queryResultRow.put("description", "test taxonomy description");
+        queryResultRow.put("creation_time", "04/20/2016");
+
+        // mock expectations
+        expect(queryFactory.createTaxonomyQuery(capture(checkForAnyTaxonomiesCapture))).andReturn(query);
+        expect(query.execute()).andReturn(queryResult);
+        expect(queryFactory.createTaxonomyQuery(capture(requestCapture))).andReturn(query);
+        expect(query.execute()).andReturn(queryResult);
+        replay(typeSystem, queryFactory, query);
+
+        TestTaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
+        provider.setInitialized(false);
         provider.setQueryFactory(queryFactory);
 
         Map<String, Object> requestProperties = new HashMap<>();
@@ -93,7 +196,7 @@ public class TaxonomyResourceProviderTest {
         expect(query.execute()).andReturn(emptyResponse);
         replay(typeSystem, queryFactory, query);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         Map<String, Object> requestProperties = new HashMap<>();
@@ -130,7 +233,7 @@ public class TaxonomyResourceProviderTest {
         expect(query.execute()).andReturn(queryResult);
         replay(typeSystem, queryFactory, query);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         Request userRequest = new CollectionRequest(Collections.<String, Object>emptyMap(), "name:taxonomy*");
@@ -163,7 +266,7 @@ public class TaxonomyResourceProviderTest {
         expect(query.execute()).andReturn(queryResult);
         replay(typeSystem, queryFactory, query);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         Request userRequest = new CollectionRequest(Collections.<String, Object>emptyMap(), "name:taxonomy*");
@@ -193,7 +296,7 @@ public class TaxonomyResourceProviderTest {
         requestProperties.put("description", "test");
         Request userRequest = new InstanceRequest(requestProperties);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         provider.createResource(userRequest);
@@ -225,7 +328,7 @@ public class TaxonomyResourceProviderTest {
         requestProperties.put("name", "taxonomyName");
         Request userRequest = new InstanceRequest(requestProperties);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         provider.createResource(userRequest);
@@ -252,7 +355,7 @@ public class TaxonomyResourceProviderTest {
         requestProperties.put("name", "taxonomyName");
         Request userRequest = new InstanceRequest(requestProperties);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         provider.createResource(userRequest);
@@ -279,7 +382,7 @@ public class TaxonomyResourceProviderTest {
         requestProperties.put("name", "taxonomyName");
         Request userRequest = new InstanceRequest(requestProperties);
 
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
 
         provider.createResources(userRequest);
@@ -385,7 +488,7 @@ public class TaxonomyResourceProviderTest {
         replay(typeSystem, queryFactory, query);
 
         // instantiate resource provider and invoke method being tested
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
         provider.updateResourceById(userRequest);
 
@@ -424,7 +527,7 @@ public class TaxonomyResourceProviderTest {
         replay(typeSystem, queryFactory, query);
 
         // instantiate resource provider and invoke method being tested
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
         provider.updateResourceById(userRequest);
 
@@ -450,7 +553,7 @@ public class TaxonomyResourceProviderTest {
         replay(typeSystem, queryFactory, query);
 
         // instantiate resource provider and invoke method being tested
-        TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
+        TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
         provider.setQueryFactory(queryFactory);
         provider.updateResourceById(userRequest);
 
@@ -459,14 +562,30 @@ public class TaxonomyResourceProviderTest {
 
     private static class TestTaxonomyResourceProvider extends TaxonomyResourceProvider {
         private final TermResourceProvider termResourceProvider;
+        private boolean isInitialized = true;
+
+        public TestTaxonomyResourceProvider(AtlasTypeSystem typeSystem) {
+            super(typeSystem);
+            this.termResourceProvider = null;
+        }
+
         public TestTaxonomyResourceProvider(AtlasTypeSystem typeSystem, TermResourceProvider termResourceProvider) {
             super(typeSystem);
             this.termResourceProvider = termResourceProvider;
         }
 
+        public void setInitialized(boolean isInitialized) {
+            this.isInitialized = isInitialized;
+        }
+
         @Override
         protected synchronized TermResourceProvider getTermResourceProvider() {
             return termResourceProvider;
         }
+
+        @Override
+        protected boolean autoInitializationChecked() {
+            return isInitialized;
+        }
     }
 }

distro/src/conf/atlas-application.properties

@@ -176,3 +176,6 @@ atlas.authorizer.impl=SIMPLE
 #########  Performance Configs  #########
 #atlas.graph.storage.lock.retries=10
 #atlas.graph.storage.cache.db-cache-time=120000
+
+#########  Business Catalog  #########
+atlas.taxonomy.default.name=Catalog
\ No newline at end of file

release-log.txt

@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
 
 
 ALL CHANGES:
+ATLAS-806 Create default taxonomy at server startup (jspeidel via yhemanth)
 ATLAS-942 Jenkins build failure - GraphRepoMapperScaleTest (shwethags)
 ATLAS-920 Lineage graph is broken when there are multiple paths from same source table (kevalbhatt18 via sumasai)
 ATLAS-940 Type cache implementation property name in atlas-application.properties is incorrect ( dkantor via sumasai)