Skip to content

A
atlas
Commit 40ee9492 by Shwetha GS
Options

ATLAS-622 Introduce soft delete (shwethags)

parent daf812aa
Showing with 656 additions and 232 deletions
addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
......@@ -459,7 +459,7 @@ public class HiveMetaStoreBridge {
final String[] parts = tableQualifiedName.split("@");
final String tableName = parts[0];
final String clusterName = parts[1];
return String.format("%s.%s@%s", tableName, colName, clusterName);
return String.format("%s.%s@%s", tableName, colName.toLowerCase(), clusterName);
}
public List<Referenceable> getColumns(List<FieldSchema> schemaList, String tableQualifiedName) throws Exception {
......
addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
......@@ -481,7 +481,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return entitiesCreatedOrUpdated;
}
private String normalize(String str) {
public static String normalize(String str) {
if (StringUtils.isEmpty(str)) {
return null;
}
......
client/src/main/java/org/apache/atlas/AtlasClient.java
......@@ -57,6 +57,7 @@ import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
*/
public class AtlasClient {
private static final Logger LOG = LoggerFactory.getLogger(AtlasClient.class);
public static final String NAME = "name";
public static final String GUID = "GUID";
public static final String TYPE = "type";
......@@ -403,6 +404,7 @@ public class AtlasClient {
* @throws AtlasServiceException
*/
public List<String> createType(String typeAsJson) throws AtlasServiceException {
LOG.debug("Creating type definition: {}", typeAsJson);
JSONObject response = callAPI(API.CREATE_TYPE, typeAsJson);
return extractResults(response, AtlasClient.TYPES, new ExtractOperation<String, JSONObject>() {
@Override
......@@ -429,6 +431,7 @@ public class AtlasClient {
* @throws AtlasServiceException
*/
public List<String> updateType(String typeAsJson) throws AtlasServiceException {
LOG.debug("Updating tyep definition: {}", typeAsJson);
JSONObject response = callAPI(API.UPDATE_TYPE, typeAsJson);
return extractResults(response, AtlasClient.TYPES, new ExtractOperation<String, JSONObject>() {
@Override
......@@ -474,6 +477,7 @@ public class AtlasClient {
* @throws AtlasServiceException
*/
public JSONArray createEntity(JSONArray entities) throws AtlasServiceException {
LOG.debug("Creating entities: {}", entities);
JSONObject response = callAPI(API.CREATE_ENTITY, entities.toString());
try {
return response.getJSONArray(GUID);
......@@ -522,6 +526,7 @@ public class AtlasClient {
public JSONArray updateEntities(Collection<Referenceable> entities) throws AtlasServiceException {
JSONArray entitiesArray = getEntitiesArray(entities);
LOG.debug("Updating entities: {}", entitiesArray);
JSONObject response = callAPI(API.UPDATE_ENTITY, entitiesArray.toString());
try {
return response.getJSONArray(GUID);
......@@ -538,6 +543,7 @@ public class AtlasClient {
* @param value property value
*/
public void updateEntityAttribute(final String guid, final String attribute, String value) throws AtlasServiceException {
LOG.debug("Updating entity id: {}, attribute name: {}, attribute value: {}", guid, attribute, value);
callAPIWithRetries(API.UPDATE_ENTITY_PARTIAL, value, new ResourceCreator() {
@Override
public WebResource createResource() {
......@@ -555,7 +561,7 @@ public class AtlasClient {
for (int i = 0; i < getNumberOfRetries(); i++) {
WebResource resource = resourceCreator.createResource();
try {
LOG.info("using resource {} for {} times", resource.getURI(), i);
LOG.debug("Using resource {} for {} times", resource.getURI(), i);
JSONObject result = callAPIWithResource(api, resource, requestObject);
return result;
} catch (ClientHandlerException che) {
......@@ -578,6 +584,7 @@ public class AtlasClient {
*/
public void updateEntity(String guid, Referenceable entity) throws AtlasServiceException {
String entityJson = InstanceSerialization.toJson(entity, true);
LOG.debug("Updating entity id {} with {}", guid, entityJson);
callAPI(API.UPDATE_ENTITY_PARTIAL, entityJson, guid);
}
......@@ -904,6 +911,7 @@ public class AtlasClient {
clientResponse = resource.accept(JSON_MEDIA_TYPE).type(JSON_MEDIA_TYPE)
.method(api.getMethod(), ClientResponse.class, requestObject);
LOG.debug("API {} returned status {}", resource.getURI(), clientResponse.getStatus());
if (clientResponse.getStatus() == api.getExpectedStatus().getStatusCode()) {
String responseAsString = clientResponse.getEntity(String.class);
try {
......
common/src/main/java/org/apache/atlas/ApplicationProperties.java
......@@ -91,4 +91,14 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static Configuration getSubsetConfiguration(Configuration inConf, String prefix) {
return inConf.subset(prefix);
}
public static Class getClass(String propertyName, String defaultValue) {
try {
Configuration configuration = get();
String propertyValue = configuration.getString(propertyName, defaultValue);
return Class.forName(propertyValue);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
release-log.txt
......@@ -3,6 +3,7 @@ Apache Atlas Release Notes
--trunk - unreleased
INCOMPATIBLE CHANGES:
ATLAS-622 Introduce soft delete (shwethags)
ATLAS-494 UI Authentication (nixonrodrigues via shwethags)
ATLAS-621 Introduce entity state in Id object (shwethags)
ATLAS-474 Server does not start if the type is updated with same super type class information (dkantor via shwethags)
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -35,9 +35,11 @@ import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graph.TitanGraphProvider;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
......@@ -85,6 +87,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bindAuditRepository(binder());
bind(DeleteHandler.class).to(getDeleteHandler()).asEagerSingleton();
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
Multibinder.newSetBinder(binder(), EntityChangeListener.class);
......@@ -103,4 +107,11 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder, Service.class);
serviceBinder.addBinding().to(HBaseBasedAuditRepository.class);
}
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
private Class<? extends DeleteHandler> getDeleteHandler() {
return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
SoftDeleteHandler.class.getName());
}
}
repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java
......@@ -45,7 +45,7 @@ public class EntityAuditListener implements EntityChangeListener {
@Override
public void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
List<EntityAuditEvent> events = new ArrayList<>();
long currentTime = System.currentTimeMillis();
long currentTime = RequestContext.get().getRequestTime();
for (ITypedReferenceableInstance entity : entities) {
EntityAuditEvent event = createEvent(entity, currentTime, EntityAuditEvent.EntityAuditAction.ENTITY_CREATE,
"Created: " + InstanceSerialization.toJson(entity, true));
......@@ -62,7 +62,7 @@ public class EntityAuditListener implements EntityChangeListener {
@Override
public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
List<EntityAuditEvent> events = new ArrayList<>();
long currentTime = System.currentTimeMillis();
long currentTime = RequestContext.get().getRequestTime();
for (ITypedReferenceableInstance entity : entities) {
EntityAuditEvent event = createEvent(entity, currentTime, EntityAuditEvent.EntityAuditAction.ENTITY_UPDATE,
"Updated: " + InstanceSerialization.toJson(entity, true));
......@@ -73,7 +73,7 @@ public class EntityAuditListener implements EntityChangeListener {
@Override
public void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait) throws AtlasException {
EntityAuditEvent event = createEvent(entity, System.currentTimeMillis(),
EntityAuditEvent event = createEvent(entity, RequestContext.get().getRequestTime(),
EntityAuditEvent.EntityAuditAction.TAG_ADD,
"Added trait: " + InstanceSerialization.toJson(trait, true));
auditRepository.putEvents(event);
......@@ -81,7 +81,7 @@ public class EntityAuditListener implements EntityChangeListener {
@Override
public void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException {
EntityAuditEvent event = createEvent(entity, System.currentTimeMillis(),
EntityAuditEvent event = createEvent(entity, RequestContext.get().getRequestTime(),
EntityAuditEvent.EntityAuditAction.TAG_DELETE, "Deleted trait: " + traitName);
auditRepository.putEvents(event);
}
......@@ -89,7 +89,7 @@ public class EntityAuditListener implements EntityChangeListener {
@Override
public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
List<EntityAuditEvent> events = new ArrayList<>();
long currentTime = System.currentTimeMillis();
long currentTime = RequestContext.get().getRequestTime();
for (ITypedReferenceableInstance entity : entities) {
EntityAuditEvent event = createEvent(entity, currentTime,
EntityAuditEvent.EntityAuditAction.ENTITY_DELETE, "Deleted entity");
......
repository/src/main/java/org/apache/atlas/repository/graph/AtlasEdgeLabel.java
......@@ -31,9 +31,6 @@ public class AtlasEdgeLabel {
private final String qualifiedAttributeName_;
public AtlasEdgeLabel(String edgeLabel) {
if (!edgeLabel.startsWith(GraphHelper.EDGE_LABEL_PREFIX)) {
throw new IllegalArgumentException("Invalid edge label " + edgeLabel + ": missing required prefix " + GraphHelper.EDGE_LABEL_PREFIX);
}
String labelWithoutPrefix = edgeLabel.substring(GraphHelper.EDGE_LABEL_PREFIX.length());
String[] fields = labelWithoutPrefix.split("\\.", 3);
if (fields.length < 2 || fields.length > 3) {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
......@@ -100,7 +100,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX);
// create a composite index for guid as its unique
createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true);
createCompositeAndMixedIndex(management, Constants.GUID_PROPERTY_KEY, String.class, true,
Cardinality.SINGLE, true);
// create a composite index for entity state
createCompositeAndMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true);
// create a composite and mixed index for type since it can be combined with other keys
createCompositeAndMixedIndex(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE,
......@@ -223,13 +227,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
switch (field.dataType().getTypeCategory()) {
case PRIMITIVE:
Cardinality cardinality = getCardinality(field.multiplicity);
createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique,
createCompositeAndMixedIndex(management, propertyName, getPrimitiveClass(field.dataType()), false,
cardinality, false);
break;
case ENUM:
cardinality = getCardinality(field.multiplicity);
createCompositeAndMixedIndex(management, propertyName, String.class, field.isUnique, cardinality, false);
createCompositeAndMixedIndex(management, propertyName, String.class, false, cardinality, false);
break;
case ARRAY:
......
repository/src/main/java/org/apache/atlas/repository/graph/HardDeleteHandler.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.types.TypeSystem;
public class HardDeleteHandler extends DeleteHandler {
private static final GraphHelper graphHelper = GraphHelper.getInstance();
@Inject
public HardDeleteHandler(TypeSystem typeSystem) {
super(typeSystem, true);
}
@Override
protected void _deleteVertex(Vertex instanceVertex) {
graphHelper.removeVertex(instanceVertex);
}
@Override
protected void deleteEdge(Edge edge) throws AtlasException {
graphHelper.removeEdge(edge);
}
}
repository/src/main/java/org/apache/atlas/repository/graph/SoftDeleteHandler.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.google.inject.Inject;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.TypeSystem;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
public class SoftDeleteHandler extends DeleteHandler {
@Inject
public SoftDeleteHandler(TypeSystem typeSystem) {
super(typeSystem, false);
}
@Override
protected void _deleteVertex(Vertex instanceVertex) {
Id.EntityState state = Id.EntityState.valueOf((String) instanceVertex.getProperty(STATE_PROPERTY_KEY));
if (state != Id.EntityState.DELETED) {
GraphHelper.setProperty(instanceVertex, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper.setProperty(instanceVertex, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
}
}
@Override
protected void deleteEdge(Edge edge) throws AtlasException {
Id.EntityState state = Id.EntityState.valueOf((String) edge.getProperty(STATE_PROPERTY_KEY));
if (state != Id.EntityState.DELETED) {
GraphHelper.setProperty(edge, STATE_PROPERTY_KEY, Id.EntityState.DELETED.name());
GraphHelper.setProperty(edge, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
}
}
}
repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
......@@ -206,9 +206,10 @@ public class GraphBackedTypeStore implements ITypeStore {
}
private void addEdge(Vertex fromVertex, Vertex toVertex, String label) {
Iterable<Edge> edges = GraphHelper.getOutGoingEdgesByLabel(fromVertex, label);
Iterator<Edge> edges = GraphHelper.getOutGoingEdgesByLabel(fromVertex, label);
// ATLAS-474: Check if this type system edge already exists, to avoid duplicates.
for (Edge edge : edges) {
while (edges.hasNext()) {
Edge edge = edges.next();
if (edge.getVertex(Direction.IN).equals(toVertex)) {
LOG.debug("Edge from {} to {} with label {} already exists",
toString(fromVertex), toString(toVertex), label);
......
repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
......@@ -69,6 +69,7 @@ public class BaseHiveRepositoryTest {
protected void setUp() throws Exception {
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
RequestContext.createContext();
setupInstances();
TestUtils.dumpGraph(graphProvider.get());
}
......
repository/src/test/java/org/apache/atlas/TestUtils.java
......@@ -21,14 +21,12 @@ package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.thinkaurelius.titan.core.TitanGraph;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
......@@ -46,12 +44,14 @@ import org.testng.Assert;
import java.io.File;
import java.util.Collection;
import java.util.Date;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
/**
* Test utility class.
......@@ -75,16 +75,7 @@ public final class TestUtils {
System.out.println("tempFile.getPath() = " + tempFile.getPath());
GraphSONWriter.outputGraph(titanGraph, tempFile.getPath());
System.out.println("Vertices:");
for (Vertex vertex : titanGraph.getVertices()) {
System.out.println(GraphHelper.vertexString(vertex));
}
System.out.println("Edges:");
for (Edge edge : titanGraph.getEdges()) {
System.out.println(GraphHelper.edgeString(edge));
}
GraphHelper.dumpToLog(titanGraph);
return tempFile.getPath();
}
......@@ -106,9 +97,9 @@ public final class TestUtils {
createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createRequiredAttrDef("city", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description, ImmutableSet.<String>of(),
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, "department"));
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(),
......@@ -132,9 +123,13 @@ public final class TestUtils {
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
}
public static Referenceable createDeptEg1(TypeSystem ts) throws AtlasException {
Referenceable hrDept = new Referenceable(ENTITY_TYPE);
Referenceable john = new Referenceable("Person");
public static final String DEPARTMENT_TYPE = "Department";
public static final String PERSON_TYPE = "Person";
public static ITypedReferenceableInstance createDeptEg1(TypeSystem ts) throws AtlasException {
Referenceable hrDept = new Referenceable(DEPARTMENT_TYPE);
Referenceable john = new Referenceable(PERSON_TYPE);
Referenceable jane = new Referenceable("Manager", "SecurityClearance");
Referenceable johnAddr = new Referenceable("Address");
Referenceable janeAddr = new Referenceable("Address");
......@@ -183,13 +178,13 @@ public final class TestUtils {
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
Assert.assertNotNull(hrDept2);
return hrDept;
return hrDept2;
}
public static final String ENTITY_TYPE = "Department";
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
public static final String PROCESS_TYPE = "hive_process";
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
......@@ -200,6 +195,9 @@ public final class TestUtils {
public static final String PARTITION_CLASS_TYPE = "partition_class_type";
public static final String SERDE_TYPE = "serdeType";
public static final String COLUMNS_MAP = "columnsMap";
public static final String COLUMNS_ATTR_NAME = "columns";
public static final String NAME = "name";
public static TypesDef defineHiveTypes() {
String _description = "_description";
......@@ -211,7 +209,7 @@ public final class TestUtils {
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createOptionalAttrDef("created", DataTypes.DATE_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
......@@ -227,7 +225,7 @@ public final class TestUtils {
HierarchicalTypeDefinition<ClassType> columnsDefinition =
createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description,
......@@ -268,6 +266,12 @@ public final class TestUtils {
new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description,
ImmutableSet.of(SUPER_TYPE_NAME), partClsAttributes);
HierarchicalTypeDefinition<ClassType> processClsType =
new HierarchicalTypeDefinition<>(ClassType.class, PROCESS_TYPE, PROCESS_TYPE + _description,
ImmutableSet.<String>of(), new AttributeDefinition[]{
new AttributeDefinition("outputs", "array<" + TABLE_TYPE + ">", Multiplicity.OPTIONAL, false, null)
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
......@@ -322,7 +326,8 @@ public final class TestUtils {
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition),
ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition, storageDescClsDef, partClsDef));
ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition,
storageDescClsDef, partClsDef, processClsType));
}
public static Collection<IDataType> createHiveTypes(TypeSystem typeSystem) throws Exception {
......@@ -336,4 +341,31 @@ public final class TestUtils {
public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10);
}
public static Referenceable createDBEntity() {
Referenceable entity = new Referenceable(DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10);
entity.set(NAME, dbName);
entity.set("description", "us db");
return entity;
}
public static Referenceable createTableEntity(String dbId) {
Referenceable entity = new Referenceable(TABLE_TYPE);
String tableName = RandomStringUtils.randomAlphanumeric(10);
entity.set(NAME, tableName);
entity.set("description", "random table");
entity.set("type", "type");
entity.set("tableType", "MANAGED");
entity.set("database", new Id(dbId, 0, DATABASE_TYPE));
entity.set("created", new Date());
return entity;
}
public static Referenceable createColumnEntity() {
Referenceable entity = new Referenceable(COLUMN_TYPE);
entity.set(NAME, RandomStringUtils.randomAlphanumeric(10));
entity.set("type", "VARCHAR(32)");
return entity;
}
}
repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.discovery;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
......@@ -38,6 +39,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -67,11 +69,8 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
TypeSystem typeSystem = TypeSystem.getInstance();
TestUtils.defineDeptEmployeeTypes(typeSystem);
Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
repositoryService.createEntities(hrDept2);
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
repositoryService.createEntities(hrDept);
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
Id janeGuid = jane.getId();
......@@ -81,6 +80,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
repositoryService.updateEntities(instance);
}
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
}
@AfterClass
public void tearDown() throws Exception {
super.tearDown();
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
......@@ -28,6 +28,7 @@ import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
......@@ -53,6 +54,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -99,6 +101,10 @@ public class GraphBackedMetadataRepositoryTest {
TestUtils.createHiveTypes(typeSystem);
}
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
}
@AfterClass
public void tearDown() throws Exception {
......@@ -116,14 +122,11 @@ public class GraphBackedMetadataRepositoryTest {
}
}
@Test
public void testSubmitEntity() throws Exception {
Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
List<String> guids = repositoryService.createEntities(hrDept2);
List<String> guids = repositoryService.createEntities(hrDept);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
guid = guids.get(4);
......@@ -147,7 +150,7 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityList() throws Exception {
List<String> entityList = repositoryService.getEntityList(TestUtils.ENTITY_TYPE);
List<String> entityList = repositoryService.getEntityList(TestUtils.DEPARTMENT_TYPE);
System.out.println("entityList = " + entityList);
Assert.assertNotNull(entityList);
Assert.assertTrue(entityList.contains(guid));
......@@ -247,7 +250,7 @@ public class GraphBackedMetadataRepositoryTest {
final String aGUID = getGUID();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNull(modificationTimestampPreUpdate);
Assert.assertNotNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
System.out.println("traitNames = " + traitNames);
......@@ -499,85 +502,6 @@ public class GraphBackedMetadataRepositoryTest {
row = (JSONObject) results.get(0);
Assert.assertEquals(row.get("typeName"), "Person");
}
@Test(dependsOnMethods = "testSubmitEntity")
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
Id johnGuid = john.getId();
ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
String maxGuid = max.getId()._getId();
Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(creationTimestamp);
Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNull(modificationTimestampPreUpdate);
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
Id janeGuid = jane.getId();
// Update max's mentor reference to john.
ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
ITypedReferenceableInstance instance = personType.createInstance(max.getId());
instance.set("mentor", johnGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - john should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
Object object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPostUpdate);
Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
// Update max's mentor reference to jane.
instance.set("mentor", janeGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - jane should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
// Verify modification timestamp was updated.
vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
Assert.assertNotNull(modificationTimestampPost2ndUpdate);
Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
ITypedReferenceableInstance julius = repositoryService.getEntityDefinition("Person", "name", "Julius");
Id juliusGuid = julius.getId();
instance = personType.createInstance(max.getId());
instance.set("manager", juliusGuid);
repositoryService.updatePartial(instance);
// Verify the update was applied correctly - julius should now be max's manager.
max = repositoryService.getEntityDefinition(maxGuid);
object = max.get("manager");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), juliusGuid._getId());
// Verify that max is no longer a subordinate of jane.
jane = repositoryService.getEntityDefinition(janeGuid._getId());
Object refValue = jane.get("subordinates");
Assert.assertTrue(refValue instanceof List);
List<Object> subordinates = (List<Object>)refValue;
Assert.assertEquals(subordinates.size(), 1);
Object listValue = subordinates.get(0);
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance subordinate = (ITypedReferenceableInstance) listValue;
Assert.assertNotEquals(subordinate.getId()._getId(), maxGuid);
}
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
Referenceable tableInstance = new Referenceable(TestUtils.TABLE_TYPE, TestUtils.CLASSIFICATION);
......
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import java.util.List;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.fail;
import static org.testng.AssertJUnit.assertNotNull;
public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepositoryDeleteTestBase {
@Override
DeleteHandler getDeleteHandler(TypeSystem typeSystem) {
return new HardDeleteHandler(typeSystem);
}
@Override
protected void assertTestDeleteReference(ITypedReferenceableInstance processInstance) throws Exception {
//assert that outputs is empty
ITypedReferenceableInstance newProcess =
repositoryService.getEntityDefinition(processInstance.getId()._getId());
assertNull(newProcess.get(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS));
}
@Override
protected void assertEntityDeleted(String id) throws Exception {
try {
repositoryService.getEntityDefinition(id);
fail("Expected EntityNotFoundException");
} catch(EntityNotFoundException e) {
//expected
}
}
@Override
protected void assertTestDeleteEntities(ITypedReferenceableInstance tableInstance) {
int vertexCount = getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, TestUtils.TABLE_TYPE).size();
assertEquals(vertexCount, 0);
vertexCount = getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, TestUtils.COLUMN_TYPE).size();
assertEquals(vertexCount, 0);
}
@Override
protected void assertVerticesDeleted(List<Vertex> vertices) {
assertEquals(vertices.size(), 0);
}
@Override
protected void assertTestUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
// Verify that max is no longer a subordinate of jane.
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane");
List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates");
Assert.assertEquals(subordinates.size(), 1);
}
@Override
protected void assertTestDisconnectBidirectionalReferences() throws Exception {
// Verify that the Manager.subordinates reference to the deleted employee
// Max was disconnected.
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane");
List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates");
assertEquals(subordinates.size(), 1);
}
@Override
protected void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(String structContainerGuid)
throws Exception {
// Verify that the unidirectional references from the struct and trait instances
// to the deleted entities were disconnected.
ITypedReferenceableInstance structContainerConvertedEntity =
repositoryService.getEntityDefinition(structContainerGuid);
ITypedStruct struct = (ITypedStruct) structContainerConvertedEntity.get("struct");
assertNull(struct.get("target"));
IStruct trait = structContainerConvertedEntity.getTrait("TestTrait");
assertNotNull(trait);
assertNull(trait.get("target"));
}
@Override
protected void assertTestDisconnectMapReferenceFromClassType(String mapOwnerGuid) throws Exception {
// Verify map references from mapOwner were disconnected.
ITypedReferenceableInstance mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid);
assertNull(mapOwnerInstance.get("map"));
assertNull(mapOwnerInstance.get("biMap"));
Vertex mapOwnerVertex = GraphHelper.getInstance().getVertexForGUID(mapOwnerGuid);
Object object = mapOwnerVertex.getProperty("MapOwner.map.value1");
assertNull(object);
object = mapOwnerVertex.getProperty("MapOwner.biMap.value1");
assertNull(object);
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositorySoftDeleteTest.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import java.util.List;
import java.util.Map;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
public class GraphBackedRepositorySoftDeleteTest extends GraphBackedMetadataRepositoryDeleteTestBase {
@Override
DeleteHandler getDeleteHandler(TypeSystem typeSystem) {
return new SoftDeleteHandler(typeSystem);
}
@Override
protected void assertTestDeleteReference(ITypedReferenceableInstance expected) throws Exception {
ITypedReferenceableInstance process = repositoryService.getEntityDefinition(expected.getId()._getId());
List<ITypedReferenceableInstance> outputs =
(List<ITypedReferenceableInstance>) process.get(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS);
List<ITypedReferenceableInstance> expectedOutputs =
(List<ITypedReferenceableInstance>) process.get(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS);
assertEquals(outputs.size(), expectedOutputs.size());
}
@Override
protected void assertEntityDeleted(String id) throws Exception {
ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(id);
assertEquals(entity.getId().getState(), Id.EntityState.DELETED);
}
@Override
protected void assertTestDeleteEntities(ITypedReferenceableInstance expected) throws Exception {
//Assert that the deleted table can be fully constructed back
ITypedReferenceableInstance table = repositoryService.getEntityDefinition(expected.getId()._getId());
List<ITypedReferenceableInstance> columns =
(List<ITypedReferenceableInstance>) table.get(TestUtils.COLUMNS_ATTR_NAME);
List<ITypedReferenceableInstance> expectedColumns =
(List<ITypedReferenceableInstance>) table.get(TestUtils.COLUMNS_ATTR_NAME);
assertEquals(columns.size(), expectedColumns.size());
}
@Override
protected void assertVerticesDeleted(List<Vertex> vertices) {
for (Vertex vertex : vertices) {
assertEquals(vertex.getProperty(Constants.STATE_PROPERTY_KEY), Id.EntityState.DELETED.name());
}
}
@Override
protected void assertTestUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
// Verify that max is no longer a subordinate of jane.
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane");
List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates");
Assert.assertEquals(subordinates.size(), 2);
}
@Override
protected void assertTestDisconnectBidirectionalReferences() throws Exception {
// Verify that the Manager.subordinates still references deleted employee
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Manager", "name", "Jane");
List<ITypedReferenceableInstance> subordinates = (List<ITypedReferenceableInstance>) jane.get("subordinates");
assertEquals(subordinates.size(), 2);
}
@Override
protected void assertTestDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes(String structContainerGuid)
throws Exception {
// Verify that the unidirectional references from the struct and trait instances
// to the deleted entities were disconnected.
ITypedReferenceableInstance structContainerConvertedEntity =
repositoryService.getEntityDefinition(structContainerGuid);
ITypedStruct struct = (ITypedStruct) structContainerConvertedEntity.get("struct");
assertNotNull(struct.get("target"));
IStruct trait = structContainerConvertedEntity.getTrait("TestTrait");
assertNotNull(trait);
assertNotNull(trait.get("target"));
}
@Override
protected void assertTestDisconnectMapReferenceFromClassType(String mapOwnerGuid) throws Exception {
ITypedReferenceableInstance mapOwnerInstance = repositoryService.getEntityDefinition(mapOwnerGuid);
Map<String, ITypedReferenceableInstance> map =
(Map<String, ITypedReferenceableInstance>) mapOwnerInstance.get("map");
assertNotNull(map);
assertEquals(map.size(), 1);
Map<String, ITypedReferenceableInstance> biMap =
(Map<String, ITypedReferenceableInstance>) mapOwnerInstance.get("biMap");
assertNotNull(biMap);
assertEquals(biMap.size(), 1);
}
}
repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
......@@ -27,6 +27,7 @@ import com.tinkerpop.blueprints.Predicate;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
......@@ -39,6 +40,7 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
......@@ -75,6 +77,11 @@ public class GraphRepoMapperScaleTest {
searchIndexer.onAdd(typesAdded);
}
@BeforeMethod
public void setupContext() {
RequestContext.createContext();
}
@AfterClass
public void tearDown() throws Exception {
TypeSystem.getInstance().reset();
......
repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
......@@ -178,7 +178,7 @@ public class GraphBackedTypeStoreTest {
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description,
ImmutableSet.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, "department"));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
......@@ -227,7 +227,7 @@ public class GraphBackedTypeStoreTest {
createOptionalAttrDef("name", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department",
ImmutableSet.of("Division", superTypeDef2.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, "department"));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
......@@ -269,9 +269,9 @@ public class GraphBackedTypeStoreTest {
private int countOutgoingEdges(Vertex typeVertex, String edgeLabel) {
Iterable<Edge> outGoingEdgesByLabel = GraphHelper.getOutGoingEdgesByLabel(typeVertex, edgeLabel);
Iterator<Edge> outGoingEdgesByLabel = GraphHelper.getOutGoingEdgesByLabel(typeVertex, edgeLabel);
int edgeCount = 0;
for (Iterator<Edge> iterator = outGoingEdgesByLabel.iterator(); iterator.hasNext();) {
for (Iterator<Edge> iterator = outGoingEdgesByLabel; iterator.hasNext();) {
iterator.next();
edgeCount++;
}
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
......@@ -37,7 +37,7 @@ class GremlinTest extends BaseGremlinTest {
TypeSystem.getInstance().reset()
QueryTestsUtils.setupTypes
gProvider = new TitanGraphProvider()
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider))
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider, null))
g = QueryTestsUtils.setupTestGraph(gProvider)
}
......
repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala
......@@ -37,7 +37,7 @@ class GremlinTest2 extends BaseGremlinTest {
TypeSystem.getInstance().reset()
QueryTestsUtils.setupTypes
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider))
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider, null))
g = QueryTestsUtils.setupTestGraph(gProvider)
}
......
repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
......@@ -37,7 +37,7 @@ class LineageQueryTest extends BaseGremlinTest {
TypeSystem.getInstance().reset()
QueryTestsUtils.setupTypes
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider))
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider, null))
g = QueryTestsUtils.setupTestGraph(gProvider)
}
......
repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
......@@ -164,7 +164,7 @@ object QueryTestsUtils extends GraphUtils {
val bindings: Bindings = engine.createBindings
bindings.put("g", g)
val hiveGraphFile = FileUtils.getTempDirectory().getPath.toString + File.separator + System.nanoTime() + ".gson"
val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
HiveTitanSample.writeGson(hiveGraphFile)
bindings.put("hiveGraphFile", hiveGraphFile)
......
server-api/src/main/java/org/apache/atlas/RequestContext.java
......@@ -18,15 +18,33 @@
package org.apache.atlas;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public class RequestContext {
private static final Logger LOG = LoggerFactory.getLogger(RequestContext.class);
private static final ThreadLocal<RequestContext> CURRENT_CONTEXT = new ThreadLocal<>();
private Set<String> createdEntityIds = new LinkedHashSet<>();
private Set<String> updatedEntityIds = new LinkedHashSet<>();
private Set<String> deletedEntityIds = new LinkedHashSet<>();
private List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>();
private String user;
private long requestTime;
TypeSystem typeSystem = TypeSystem.getInstance();
private RequestContext() {
}
......@@ -37,6 +55,7 @@ public class RequestContext {
public static RequestContext createContext() {
RequestContext context = new RequestContext();
context.requestTime = System.currentTimeMillis();
CURRENT_CONTEXT.set(context);
return context;
}
......@@ -52,4 +71,40 @@ public class RequestContext {
public void setUser(String user) {
this.user = user;
}
public void recordCreatedEntities(Collection<String> createdEntityIds) {
this.createdEntityIds.addAll(createdEntityIds);
}
public void recordUpdatedEntities(Collection<String> updatedEntityIds) {
this.updatedEntityIds.addAll(updatedEntityIds);
}
public void recordDeletedEntity(String entityId, String typeName) throws AtlasException {
ClassType type = typeSystem.getDataType(ClassType.class, typeName);
ITypedReferenceableInstance entity = type.createInstance(new Id(entityId, 0, typeName));
if (deletedEntityIds.add(entityId)) {
deletedEntities.add(entity);
}
}
public List<String> getCreatedEntityIds() {
return new ArrayList<>(createdEntityIds);
}
public List<String> getUpdatedEntityIds() {
return new ArrayList<>(updatedEntityIds);
}
public List<String> getDeletedEntityIds() {
return new ArrayList<>(deletedEntityIds);
}
public List<ITypedReferenceableInstance> getDeletedEntities() {
return deletedEntities;
}
public long getRequestTime() {
return requestTime;
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
......@@ -37,4 +37,5 @@ public interface IInstance {
Map<String, Object> getValuesMap() throws AtlasException;
String toShortString();
}
typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
......@@ -166,6 +166,11 @@ public class Referenceable extends Struct implements IReferenceableInstance {
'}';
}
@Override
public String toShortString() {
return String.format("entity[type=%s guid=%s]", typeName, id._getId());
}
public void replaceWithNewId(Id id) {
this.id = id;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
......@@ -78,6 +78,11 @@ public class Struct implements IStruct {
}
@Override
public String toShortString() {
return String.format("struct[type=%s]", typeName);
}
@Override
public int hashCode() {
int result = typeName.hashCode();
result = 31 * result + values.hashCode();
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
......@@ -70,6 +70,11 @@ public class DownCastStructInstance implements IStruct {
}
return m;
}
@Override
public String toShortString() {
return toString();
}
}
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
......@@ -58,20 +58,20 @@ public class Id implements ITypedReferenceableInstance {
}
}
public Id(String id, int version, String className) {
this(id, version, className, null);
public Id(String id, int version, String typeName) {
this(id, version, typeName, null);
}
public Id(long id, int version, String className) {
this("" + id, version, className);
public Id(long id, int version, String typeName) {
this("" + id, version, typeName);
}
public Id(long id, int version, String className, String state) {
this("" + id, version, className, state);
public Id(long id, int version, String typeName, String state) {
this("" + id, version, typeName, state);
}
public Id(String className) {
this("" + (-System.nanoTime()), 0, className);
public Id(String typeName) {
this("" + (-System.nanoTime()), 0, typeName);
}
public boolean isUnassigned() {
......@@ -93,10 +93,16 @@ public class Id implements ITypedReferenceableInstance {
return true;
}
@Override
public String toString() {
return String.format("(type: %s, id: %s)", typeName, isUnassigned() ? "<unassigned>" : "" + id);
}
@Override
public String toShortString() {
return String.format("id[type=%s guid=%s state=%s]", typeName, id, state);
}
public String getClassName() {
return typeName;
}
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
......@@ -98,6 +98,19 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
}
@Override
public String toShortString() {
String name = null;
if (fieldMapping().fields.containsKey("name")) {
try {
name = getString("name");
} catch (AtlasException e) {
//ignore if there is no field name
}
}
return String.format("entity[type=%s guid=%s name=%s]", getTypeName(), getId()._getId(), name);
}
@Override
public String getSignatureHash(MessageDigest digester) throws AtlasException {
ClassType classType = TypeSystem.getInstance().getDataType(ClassType.class, getTypeName());
classType.updateSignatureHash(digester, this);
......
typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
......@@ -764,4 +764,9 @@ public class StructInstance implements ITypedStruct {
byte[] digest = digester.digest();
return MD5Utils.toString(digest);
}
@Override
public String toShortString() {
return String.format("struct[type=%s]", dataTypeName);
}
}
typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
......@@ -17,10 +17,12 @@
package org.apache.atlas;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.configuration.Configuration;
import org.testng.Assert;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
public class ApplicationPropertiesTest {
@Test
......@@ -28,17 +30,17 @@ public class ApplicationPropertiesTest {
Configuration properties = ApplicationProperties.get(ApplicationProperties.APPLICATION_PROPERTIES);
//plain property without variables
Assert.assertEquals(properties.getString("atlas.service"), "atlas");
assertEquals(properties.getString("atlas.service"), "atlas");
//property containing system property
String data = "/var/data/" + System.getProperty("user.name") + "/atlas";
Assert.assertEquals(properties.getString("atlas.data"), data);
assertEquals(properties.getString("atlas.data"), data);
//property referencing other property
Assert.assertEquals(properties.getString("atlas.graph.data"), data + "/graph");
assertEquals(properties.getString("atlas.graph.data"), data + "/graph");
//invalid system property - not substituted
Assert.assertEquals(properties.getString("atlas.db"), "${atlasdb}");
assertEquals(properties.getString("atlas.db"), "${atlasdb}");
}
@Test
......@@ -47,9 +49,20 @@ public class ApplicationPropertiesTest {
Configuration configuration = ApplicationProperties.get(ApplicationProperties.APPLICATION_PROPERTIES);
Configuration subConfiguration = configuration.subset("atlas");
Assert.assertEquals(subConfiguration.getString("service"), "atlas");
assertEquals(subConfiguration.getString("service"), "atlas");
String data = "/var/data/" + System.getProperty("user.name") + "/atlas";
Assert.assertEquals(subConfiguration.getString("data"), data);
Assert.assertEquals(subConfiguration.getString("graph.data"), data + "/graph");
assertEquals(subConfiguration.getString("data"), data);
assertEquals(subConfiguration.getString("graph.data"), data + "/graph");
}
@Test
public void testGetClass() throws Exception {
//read from atlas-application.properties
Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName());
assertEquals(cls.getName(), TypeSystem.class.getName());
//default value
cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName());
assertEquals(cls.getName(), TypeSystem.class.getName());
}
}
typesystem/src/test/resources/atlas-application.properties
......@@ -26,4 +26,6 @@ atlas.graph.data=${atlas.data}/graph
atlas.service=atlas
#invalid system property
atlas.db=${atlasdb}
\ No newline at end of file
atlas.db=${atlasdb}
atlas.TypeSystem.impl=org.apache.atlas.typesystem.types.TypeSystem
\ No newline at end of file
webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
......@@ -21,6 +21,7 @@ package org.apache.atlas.notification;
import com.google.inject.Inject;
import org.apache.atlas.notification.hook.HookNotification;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.web.resources.BaseResourceIT;
import org.codehaus.jettison.json.JSONArray;
import org.testng.annotations.AfterClass;
......@@ -127,20 +128,19 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testDeleteByQualifiedName() throws Exception {
final Referenceable entity = new Referenceable(DATABASE_TYPE);
Referenceable entity = new Referenceable(DATABASE_TYPE);
final String dbName = "db" + randomString();
entity.set("name", dbName);
entity.set("description", randomString());
serviceClient.createEntity(entity);
final String dbId = serviceClient.createEntity(entity).getString(0);
sendHookMessage(
new HookNotification.EntityDeleteRequest(TEST_USER, DATABASE_TYPE, "name", dbName));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = serviceClient.searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE,
dbName));
return results.length() == 0;
Referenceable getEntity = serviceClient.getEntity(dbId);
return getEntity.getId().getState() == Id.EntityState.DELETED;
}
});
}
......
webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
......@@ -810,15 +810,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
// Verify entities were deleted from the repository.
for (String guid : deletedGuidsList) {
try {
serviceClient.getEntity(guid);
Assert.fail(AtlasServiceException.class.getSimpleName() +
" was expected but not thrown. The entity with guid " + guid +
" still exists in the repository after being deleted.");
}
catch (AtlasServiceException e) {
Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode())));
}
Referenceable entity = serviceClient.getEntity(guid);
assertEquals(entity.getId().getState(), Id.EntityState.DELETED);
}
}
......@@ -844,15 +837,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
// Verify entities were deleted from the repository.
for (String guid : deletedGuidsList) {
try {
serviceClient.getEntity(guid);
Assert.fail(AtlasServiceException.class.getSimpleName() +
" was expected but not thrown. The entity with guid " + guid +
" still exists in the repository after being deleted.");
}
catch (AtlasServiceException e) {
Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode())));
}
Referenceable entity = serviceClient.getEntity(guid);
assertEquals(entity.getId().getState(), Id.EntityState.DELETED);
}
}
......@@ -874,15 +860,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
// Verify entities were deleted from the repository.
for (String guid : deletedGuidsList) {
try {
serviceClient.getEntity(guid);
Assert.fail(AtlasServiceException.class.getSimpleName() +
" was expected but not thrown. The entity with guid " + guid +
" still exists in the repository after being deleted.");
}
catch (AtlasServiceException e) {
Assert.assertTrue(e.getMessage().contains(Integer.toString(Response.Status.NOT_FOUND.getStatusCode())));
}
Referenceable entity = serviceClient.getEntity(guid);
assertEquals(entity.getId().getState(), Id.EntityState.DELETED);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment