Skip to content

A
atlas
Commit 7993de0e by Hemanth Yamijala
Options

ATLAS-806 Create default taxonomy at server startup (jspeidel via yhemanth)

parent 78d787fe
Showing with 247 additions and 33 deletions
catalog/src/main/java/org/apache/atlas/catalog/TaxonomyResourceProvider.java
......@@ -18,9 +18,14 @@
package org.apache.atlas.catalog;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.catalog.definition.TaxonomyResourceDefinition;
import org.apache.atlas.catalog.exception.*;
import org.apache.atlas.catalog.query.AtlasQuery;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
......@@ -28,8 +33,18 @@ import java.util.*;
* Provider for taxonomy resources.
*/
public class TaxonomyResourceProvider extends BaseResourceProvider implements ResourceProvider {
private static final Logger LOG = LoggerFactory.getLogger(TaxonomyResourceProvider.class);
public static final String DEFAULT_TAXONOMY_NAME = "Catalog";
public static final String DEFAULT_TAXONOMY_DESCRIPTION = "Business Catalog";
private final TermResourceProvider termResourceProvider;
// This is a cached value to prevent checking for taxonomy objects in every API call.
// It is updated once per lifetime of the application.
// TODO: If a taxonomy is deleted outside of this application, this value is not updated
// TODO: and there is no way in which a taxonomy will be auto-created.
// TODO: Assumption is that if a taxonomy is deleted externally, it will be created externally as well.
private static boolean taxonomyAutoInitializationChecked = false;
public TaxonomyResourceProvider(AtlasTypeSystem typeSystem) {
super(typeSystem, new TaxonomyResourceDefinition());
termResourceProvider = new TermResourceProvider(typeSystem);
......@@ -37,38 +52,36 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re
@Override
public Result getResourceById(Request request) throws ResourceNotFoundException {
AtlasQuery atlasQuery;
try {
atlasQuery = queryFactory.createTaxonomyQuery(request);
} catch (InvalidQueryException e) {
throw new CatalogRuntimeException("Unable to compile internal Taxonomy query: " + e, e);
synchronized (TaxonomyResourceProvider.class) {
createDefaultTaxonomyIfNeeded();
}
Collection<Map<String, Object>> results = atlasQuery.execute();
if (results.isEmpty()) {
throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.",
request.getProperty(resourceDefinition.getIdPropertyName())));
}
return new Result(results);
return doGetResourceById(request);
}
@Override
public Result getResources(Request request) throws InvalidQueryException, ResourceNotFoundException {
AtlasQuery atlasQuery = queryFactory.createTaxonomyQuery(request);
return new Result(atlasQuery.execute());
synchronized (TaxonomyResourceProvider.class) {
createDefaultTaxonomyIfNeeded();
}
return doGetResources(request);
}
@Override
public synchronized void createResource(Request request)
public void createResource(Request request)
throws InvalidPayloadException, ResourceAlreadyExistsException {
// not checking for default taxonomy in create per requirements
resourceDefinition.validateCreatePayload(request);
synchronized (TaxonomyResourceProvider.class) {
ensureTaxonomyDoesntExist(request);
typeSystem.createEntity(resourceDefinition, request);
doCreateResource(request);
}
}
@Override
public Collection<String> createResources(Request request) throws InvalidQueryException, ResourceNotFoundException {
throw new UnsupportedOperationException("Creating multiple Taxonomies in a request is not currently supported");
throw new UnsupportedOperationException(
"Creating multiple Taxonomies in a request is not currently supported");
}
@Override
......@@ -81,21 +94,36 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re
@Override
public void updateResourceById(Request request) throws ResourceNotFoundException, InvalidPayloadException {
resourceDefinition.validateUpdatePayload(request);
AtlasQuery atlasQuery;
try {
atlasQuery = queryFactory.createTaxonomyQuery(request);
} catch (InvalidQueryException e) {
throw new CatalogRuntimeException("Unable to compile internal Term query: " + e, e);
}
synchronized (TaxonomyResourceProvider.class) {
createDefaultTaxonomyIfNeeded();
if (atlasQuery.execute(request.getUpdateProperties()).isEmpty()) {
throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.",
request.getQueryProperties().get("name")));
}
}
}
private String getResourceId(Request request) throws ResourceNotFoundException {
request.addAdditionalSelectProperties(Collections.singleton("id"));
// will result in expected ResourceNotFoundException if taxonomy doesn't exist
Result result = getResourceById(request);
return String.valueOf(result.getPropertyMaps().iterator().next().get("id"));
}
//todo: this is currently required because the expected exception isn't thrown by the Atlas repository
//todo: when an attempt is made to create an entity that already exists
// must be called from within class monitor
private void ensureTaxonomyDoesntExist(Request request) throws ResourceAlreadyExistsException {
try {
getResourceById(request);
doGetResourceById(request);
throw new ResourceAlreadyExistsException(String.format("Taxonomy '%s' already exists.",
request.getProperty("name")));
} catch (ResourceNotFoundException e) {
......@@ -103,11 +131,73 @@ public class TaxonomyResourceProvider extends BaseResourceProvider implements Re
}
}
private String getResourceId(Request request) throws ResourceNotFoundException {
request.addAdditionalSelectProperties(Collections.singleton("id"));
// will result in expected ResourceNotFoundException if taxonomy doesn't exist
Result result = getResourceById(request);
return String.valueOf(result.getPropertyMaps().iterator().next().get("id"));
// must be called from within class monitor
private Result doGetResourceById(Request request) throws ResourceNotFoundException {
AtlasQuery atlasQuery;
try {
atlasQuery = queryFactory.createTaxonomyQuery(request);
} catch (InvalidQueryException e) {
throw new CatalogRuntimeException("Unable to compile internal Taxonomy query: " + e, e);
}
Collection<Map<String, Object>> resultSet = atlasQuery.execute();
if (resultSet.isEmpty()) {
throw new ResourceNotFoundException(String.format("Taxonomy '%s' not found.",
request.getProperty(resourceDefinition.getIdPropertyName())));
}
return new Result(resultSet);
}
// must be called from within class monitor
private Result doGetResources(Request request) throws InvalidQueryException, ResourceNotFoundException {
AtlasQuery atlasQuery = queryFactory.createTaxonomyQuery(request);
return new Result(atlasQuery.execute());
}
// must be called from within class monitor
private void doCreateResource(Request request) throws ResourceAlreadyExistsException {
typeSystem.createEntity(resourceDefinition, request);
taxonomyAutoInitializationChecked = true;
}
// must be called from within class monitor
private void createDefaultTaxonomyIfNeeded() {
if (! autoInitializationChecked()) {
try {
LOG.info("Checking if default taxonomy needs to be created.");
// if any business taxonomy has been created, don't create one more - hence searching to
// see if any taxonomy exists.
if (doGetResources(new CollectionRequest(null, null)).getPropertyMaps().isEmpty()) {
LOG.info("No taxonomies found - going to create default taxonomy.");
Map<String, Object> requestProperties = new HashMap<>();
String defaultTaxonomyName = DEFAULT_TAXONOMY_NAME;
try {
Configuration configuration = ApplicationProperties.get();
defaultTaxonomyName = configuration.getString("atlas.taxonomy.default.name",
defaultTaxonomyName);
} catch (AtlasException e) {
LOG.warn("Unable to read Atlas configuration, will use {} as default taxonomy name",
defaultTaxonomyName, e);
}
requestProperties.put("name", defaultTaxonomyName);
requestProperties.put("description", DEFAULT_TAXONOMY_DESCRIPTION);
doCreateResource(new InstanceRequest(requestProperties));
LOG.info("Successfully created default taxonomy {}.", defaultTaxonomyName);
} else {
taxonomyAutoInitializationChecked = true;
LOG.info("Some taxonomy exists, not creating default taxonomy");
}
} catch (InvalidQueryException | ResourceNotFoundException e) {
LOG.error("Unable to query for existing taxonomies due to internal error.", e);
} catch (ResourceAlreadyExistsException e) {
LOG.info("Attempted to create default taxonomy and it already exists.");
}
}
}
protected boolean autoInitializationChecked() {
return taxonomyAutoInitializationChecked;
}
protected TermResourceProvider getTermResourceProvider() {
......
catalog/src/test/java/org/apache/atlas/catalog/TaxonomyResourceProviderTest.java
......@@ -58,7 +58,110 @@ public class TaxonomyResourceProviderTest {
expect(query.execute()).andReturn(queryResult);
replay(typeSystem, queryFactory, query);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
Map<String, Object> requestProperties = new HashMap<>();
requestProperties.put("name", "taxonomyName");
Request userRequest = new InstanceRequest(requestProperties);
Result result = provider.getResourceById(userRequest);
assertEquals(1, result.getPropertyMaps().size());
assertEquals(queryResultRow, result.getPropertyMaps().iterator().next());
Request request = requestCapture.getValue();
assertNull(request.getQueryString());
assertEquals(0, request.getAdditionalSelectProperties().size());
assertEquals(requestProperties, request.getQueryProperties());
verify(typeSystem, queryFactory, query);
}
@Test
public void testGetResourceById_notInitialized_createDefaultTaxonomy() throws Exception {
AtlasTypeSystem typeSystem = createStrictMock(AtlasTypeSystem.class);
QueryFactory queryFactory = createStrictMock(QueryFactory.class);
AtlasQuery query = createStrictMock(AtlasQuery.class);
Capture<Request> checkForAnyTaxonomiesCapture = newCapture();
Capture<Request> createDefaultTaxonomyRequestCapture = newCapture();
Capture<Request> requestCapture = newCapture();
Capture<ResourceDefinition> resourceDefinitionCapture = newCapture();
Collection<Map<String, Object>> queryResult = new ArrayList<>();
Map<String, Object> queryResultRow = new HashMap<>();
queryResult.add(queryResultRow);
queryResultRow.put("name", "taxonomyName");
queryResultRow.put("description", "test taxonomy description");
queryResultRow.put("creation_time", "04/20/2016");
// mock expectations
expect(queryFactory.createTaxonomyQuery(capture(checkForAnyTaxonomiesCapture))).andReturn(query);
expect(query.execute()).andReturn(Collections.<Map<String, Object>>emptySet());
typeSystem.createEntity(capture(resourceDefinitionCapture), capture(createDefaultTaxonomyRequestCapture));
expect(queryFactory.createTaxonomyQuery(capture(requestCapture))).andReturn(query);
expect(query.execute()).andReturn(queryResult);
replay(typeSystem, queryFactory, query);
TestTaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setInitialized(false);
provider.setQueryFactory(queryFactory);
Map<String, Object> requestProperties = new HashMap<>();
requestProperties.put("name", "taxonomyName");
Request userRequest = new InstanceRequest(requestProperties);
Result result = provider.getResourceById(userRequest);
assertEquals(1, result.getPropertyMaps().size());
assertEquals(queryResultRow, result.getPropertyMaps().iterator().next());
Request request = requestCapture.getValue();
assertNull(request.getQueryString());
assertEquals(0, request.getAdditionalSelectProperties().size());
assertEquals(requestProperties, request.getQueryProperties());
Request checkForAnyTaxonomiesRequest = checkForAnyTaxonomiesCapture.getValue();
assertNull(checkForAnyTaxonomiesRequest.getQueryString());
assertEquals(checkForAnyTaxonomiesRequest.getAdditionalSelectProperties().size(), 0);
assertEquals(checkForAnyTaxonomiesRequest.getQueryProperties().size(), 0);
Request createDefaultTaxonomyRequest = createDefaultTaxonomyRequestCapture.getValue();
assertNull(createDefaultTaxonomyRequest.getQueryString());
assertEquals(createDefaultTaxonomyRequest.getAdditionalSelectProperties().size(), 0);
assertEquals(createDefaultTaxonomyRequest.getQueryProperties().size(), 2);
assertEquals(createDefaultTaxonomyRequest.getQueryProperties().get("name"),
TaxonomyResourceProvider.DEFAULT_TAXONOMY_NAME);
assertEquals(createDefaultTaxonomyRequest.getQueryProperties().get("description"),
TaxonomyResourceProvider.DEFAULT_TAXONOMY_DESCRIPTION);
verify(typeSystem, queryFactory, query);
}
@Test
public void testGetResourceById_notInitialized_taxonomyAlreadyExists() throws Exception {
AtlasTypeSystem typeSystem = createStrictMock(AtlasTypeSystem.class);
QueryFactory queryFactory = createStrictMock(QueryFactory.class);
AtlasQuery query = createStrictMock(AtlasQuery.class);
Capture<Request> checkForAnyTaxonomiesCapture = newCapture();
Capture<Request> requestCapture = newCapture();
Collection<Map<String, Object>> queryResult = new ArrayList<>();
Map<String, Object> queryResultRow = new HashMap<>();
queryResult.add(queryResultRow);
queryResultRow.put("name", "taxonomyName");
queryResultRow.put("description", "test taxonomy description");
queryResultRow.put("creation_time", "04/20/2016");
// mock expectations
expect(queryFactory.createTaxonomyQuery(capture(checkForAnyTaxonomiesCapture))).andReturn(query);
expect(query.execute()).andReturn(queryResult);
expect(queryFactory.createTaxonomyQuery(capture(requestCapture))).andReturn(query);
expect(query.execute()).andReturn(queryResult);
replay(typeSystem, queryFactory, query);
TestTaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setInitialized(false);
provider.setQueryFactory(queryFactory);
Map<String, Object> requestProperties = new HashMap<>();
......@@ -93,7 +196,7 @@ public class TaxonomyResourceProviderTest {
expect(query.execute()).andReturn(emptyResponse);
replay(typeSystem, queryFactory, query);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
Map<String, Object> requestProperties = new HashMap<>();
......@@ -130,7 +233,7 @@ public class TaxonomyResourceProviderTest {
expect(query.execute()).andReturn(queryResult);
replay(typeSystem, queryFactory, query);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
Request userRequest = new CollectionRequest(Collections.<String, Object>emptyMap(), "name:taxonomy*");
......@@ -163,7 +266,7 @@ public class TaxonomyResourceProviderTest {
expect(query.execute()).andReturn(queryResult);
replay(typeSystem, queryFactory, query);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
Request userRequest = new CollectionRequest(Collections.<String, Object>emptyMap(), "name:taxonomy*");
......@@ -193,7 +296,7 @@ public class TaxonomyResourceProviderTest {
requestProperties.put("description", "test");
Request userRequest = new InstanceRequest(requestProperties);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.createResource(userRequest);
......@@ -225,7 +328,7 @@ public class TaxonomyResourceProviderTest {
requestProperties.put("name", "taxonomyName");
Request userRequest = new InstanceRequest(requestProperties);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.createResource(userRequest);
......@@ -252,7 +355,7 @@ public class TaxonomyResourceProviderTest {
requestProperties.put("name", "taxonomyName");
Request userRequest = new InstanceRequest(requestProperties);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.createResource(userRequest);
......@@ -279,7 +382,7 @@ public class TaxonomyResourceProviderTest {
requestProperties.put("name", "taxonomyName");
Request userRequest = new InstanceRequest(requestProperties);
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.createResources(userRequest);
......@@ -385,7 +488,7 @@ public class TaxonomyResourceProviderTest {
replay(typeSystem, queryFactory, query);
// instantiate resource provider and invoke method being tested
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.updateResourceById(userRequest);
......@@ -424,7 +527,7 @@ public class TaxonomyResourceProviderTest {
replay(typeSystem, queryFactory, query);
// instantiate resource provider and invoke method being tested
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.updateResourceById(userRequest);
......@@ -450,7 +553,7 @@ public class TaxonomyResourceProviderTest {
replay(typeSystem, queryFactory, query);
// instantiate resource provider and invoke method being tested
TaxonomyResourceProvider provider = new TaxonomyResourceProvider(typeSystem);
TaxonomyResourceProvider provider = new TestTaxonomyResourceProvider(typeSystem);
provider.setQueryFactory(queryFactory);
provider.updateResourceById(userRequest);
......@@ -459,14 +562,30 @@ public class TaxonomyResourceProviderTest {
private static class TestTaxonomyResourceProvider extends TaxonomyResourceProvider {
private final TermResourceProvider termResourceProvider;
private boolean isInitialized = true;
public TestTaxonomyResourceProvider(AtlasTypeSystem typeSystem) {
super(typeSystem);
this.termResourceProvider = null;
}
public TestTaxonomyResourceProvider(AtlasTypeSystem typeSystem, TermResourceProvider termResourceProvider) {
super(typeSystem);
this.termResourceProvider = termResourceProvider;
}
public void setInitialized(boolean isInitialized) {
this.isInitialized = isInitialized;
}
@Override
protected synchronized TermResourceProvider getTermResourceProvider() {
return termResourceProvider;
}
@Override
protected boolean autoInitializationChecked() {
return isInitialized;
}
}
}
distro/src/conf/atlas-application.properties
......@@ -176,3 +176,6 @@ atlas.authorizer.impl=SIMPLE
######### Performance Configs #########
#atlas.graph.storage.lock.retries=10
#atlas.graph.storage.cache.db-cache-time=120000
######### Business Catalog #########
atlas.taxonomy.default.name=Catalog
\ No newline at end of file
release-log.txt
......@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
ATLAS-806 Create default taxonomy at server startup (jspeidel via yhemanth)
ATLAS-942 Jenkins build failure - GraphRepoMapperScaleTest (shwethags)
ATLAS-920 Lineage graph is broken when there are multiple paths from same source table (kevalbhatt18 via sumasai)
ATLAS-940 Type cache implementation property name in atlas-application.properties is incorrect ( dkantor via sumasai)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment