Skip to content

A
atlas
Commit 6ccba52c by Suma Shivaprasad Committed by Madhan Neethiraj
Options

ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes…

ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes in entity create/update Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 19a5f65c
Showing with 790 additions and 203 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -60,7 +60,7 @@ public enum AtlasErrorCode {
TYPE_NAME_NOT_FOUND(404, "ATLAS4041E", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS4042E", "Given type guid {0} was invalid"),
EMPTY_RESULTS(404, "ATLAS4044E", "No result found for {0}"),
INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid"),
INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid/not found"),
INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"),
INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
......@@ -146,6 +146,7 @@ public class AtlasStruct implements Serializable {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AtlasStruct that = (AtlasStruct) o;
return Objects.equals(typeName, that.typeName) &&
Objects.equals(attributes, that.attributes);
}
......
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -18,9 +18,11 @@
package org.apache.atlas.model.instance;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
......@@ -43,7 +45,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@XmlAccessorType(XmlAccessType.PROPERTY)
public class EntityMutationResponse {
Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated = new HashMap<>();
Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated;
public EntityMutationResponse() {
}
......@@ -67,15 +69,39 @@ public class EntityMutationResponse {
return null;
}
@JsonIgnore
public AtlasEntityHeader getFirstEntityCreated() {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
return entitiesByOperation.get(0);
}
return null;
}
@JsonIgnore
public AtlasEntityHeader getFirstEntityUpdated() {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
return entitiesByOperation.get(0);
}
return null;
}
public void addEntity(EntityMutations.EntityOperation op, AtlasEntityHeader header) {
if (entitiesMutated == null) {
entitiesMutated = new HashMap<>();
}
if (entitiesMutated != null && entitiesMutated.get(op) == null) {
entitiesMutated.put(op, new ArrayList<AtlasEntityHeader>());
List<AtlasEntityHeader> opEntities = entitiesMutated.get(op);
if (opEntities == null) {
opEntities = new ArrayList<>();
entitiesMutated.put(op, opEntities);
}
entitiesMutated.get(op).add(header);
opEntities.add(header);
}
......@@ -84,24 +110,7 @@ public class EntityMutationResponse {
sb = new StringBuilder();
}
if (MapUtils.isNotEmpty(entitiesMutated)) {
int i = 0;
for (Map.Entry<EntityMutations.EntityOperation, List<AtlasEntityHeader>> e : entitiesMutated.entrySet()) {
if (i > 0) {
sb.append(",");
}
sb.append(e.getKey()).append(":");
if (CollectionUtils.isNotEmpty(e.getValue())) {
for (int j = 0; i < e.getValue().size(); j++) {
if (j > 0) {
sb.append(",");
}
e.getValue().get(i).toString(sb);
}
}
i++;
}
}
AtlasBaseTypeDef.dumpObjects(entitiesMutated, sb);
return sb;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
......@@ -477,6 +477,7 @@ public class AtlasBuiltInTypes {
*/
public static class AtlasStringType extends AtlasType {
private static final String DEFAULT_VALUE = "";
private static final String OPTIONAL_DEFAULT_VALUE = null;
public AtlasStringType() {
super(AtlasBaseTypeDef.ATLAS_TYPE_STRING, TypeCategory.PRIMITIVE);
......@@ -488,6 +489,11 @@ public class AtlasBuiltInTypes {
}
@Override
public Object createOptionalDefaultValue() {
return OPTIONAL_DEFAULT_VALUE;
}
@Override
public boolean isValidValue(Object obj) {
return true;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
......@@ -95,7 +95,11 @@ public class AtlasMapType extends AtlasType {
public Map<Object, Object> createDefaultValue() {
Map<Object, Object> ret = new HashMap<>();
ret.put(keyType.createDefaultValue(), valueType.createDefaultValue());
Object key = keyType.createDefaultValue();
if ( key != null) {
ret.put(key, valueType.createDefaultValue());
}
return ret;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -328,6 +328,7 @@ public class AtlasStructType extends AtlasType {
if (attribute != null) {
AtlasType dataType = attribute.getAttributeType();
ret = dataType.createDefaultValue();
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasType.java
......@@ -33,7 +33,7 @@ import java.util.List;
public abstract class AtlasType {
private static final Gson GSON =
new GsonBuilder().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create();
new GsonBuilder().serializeNulls().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create();
private final String typeName;
private final TypeCategory typeCategory;
......@@ -59,6 +59,10 @@ public abstract class AtlasType {
public abstract Object createDefaultValue();
public Object createOptionalDefaultValue() {
return createDefaultValue();
}
public abstract boolean isValidValue(Object obj);
public abstract Object getNormalizedValue(Object obj);
......
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -71,7 +71,7 @@ public final class TestUtilsV2 {
new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0",
Arrays.asList(
new AtlasEnumElementDef("L1", "Element"+_description, 1),
new AtlasEnumElementDef("L1", "Element"+_description, 2)
new AtlasEnumElementDef("L2", "Element"+_description, 2)
));
AtlasStructDef addressDetails =
......@@ -149,7 +149,7 @@ public final class TestUtilsV2 {
new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0",
Arrays.asList(
new AtlasEnumElementDef("L1", "Element"+ _description, 1),
new AtlasEnumElementDef("L1", "Element"+ _description, 2)
new AtlasEnumElementDef("L2", "Element"+ _description, 2)
));
AtlasStructDef addressDetails =
......@@ -443,6 +443,7 @@ public final class TestUtilsV2 {
AtlasEntityDef databaseTypeDefinition =
AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME),
AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"),
AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("created", "string"),
AtlasTypeUtil.createRequiredAttrDef("description", "string"));
......@@ -543,7 +544,7 @@ public final class TestUtilsV2 {
AtlasEntityDef tableTypeDefinition =
AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("description", "string"),
AtlasTypeUtil.createOptionalAttrDef("description", "string"),
AtlasTypeUtil.createRequiredAttrDef("type", "string"),
AtlasTypeUtil.createOptionalAttrDef("created", "date"),
// enum
......@@ -582,6 +583,11 @@ public final class TestUtilsV2 {
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()),
// new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
//add(new AtlasStructDef.AtlasConstraintDef(
// AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, new HashMap<String, Object>()));
//}}),
//map of structs
new AtlasAttributeDef("partitionsMap",
String.format("map<%s,%s>", "string", "partition_struct_type"),
......@@ -646,7 +652,7 @@ public final class TestUtilsV2 {
entity.setAttribute("description", "random table");
entity.setAttribute("type", "type");
entity.setAttribute("tableType", "MANAGED");
entity.setAttribute("database", dbId);
entity.setAttribute("database", new AtlasObjectId(DATABASE_TYPE, dbId));
entity.setAttribute("created", new Date());
Map<String, Object> partAttributes = new HashMap<String, Object>() {{
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj)
ATLAS-1114 Performance improvements for create/update entity (jnhagelb)
ATLAS-1403 Perf and stability improvements to DSL search and lineage query execution (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1425 Integrate Discovery/Search API in Atlas UI (kevalbhatt via mneethiraj)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java
......@@ -82,7 +82,7 @@ public class ArrayVertexMapper implements InstanceGraphMapper<List> {
Optional<AtlasEdge> existingEdge = getEdgeAt(currentElements, index, arrType.getElementType());
GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getAttribute(),
GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(),
arrType.getElementType(), newElements.get(index))
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
......@@ -18,8 +18,8 @@
package org.apache.atlas.repository.store.graph.v1;
import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Provider;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
......@@ -112,7 +112,11 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
protected void discover(final List<AtlasEntity> entities) throws AtlasBaseException {
for (AtlasEntity entity : entities) {
AtlasType type = typeRegistry.getType(entity.getTypeName());
AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (type == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
discoveredEntities.addRootEntity(entity);
walkEntityGraph(type, entity);
......@@ -120,27 +124,24 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
private void visitReference(AtlasEntityType type, Object entity, boolean isManagedEntity) throws AtlasBaseException {
if ( entity != null) {
if ( entity instanceof String ) {
String guid = (String) entity;
discoveredEntities.addUnResolvedIdReference(type, guid);
} else if ( entity instanceof AtlasObjectId ) {
if (entity != null) {
if (entity instanceof AtlasObjectId) {
final String guid = ((AtlasObjectId) entity).getGuid();
discoveredEntities.addUnResolvedIdReference(type, guid);
} else if ( entity instanceof AtlasEntity ) {
AtlasEntity entityObj = ( AtlasEntity ) entity;
} else if (entity instanceof AtlasEntity) {
AtlasEntity entityObj = (AtlasEntity) entity;
if (isManagedEntity) {
if (!processedIds.contains(entityObj.getGuid())) {
processedIds.add(entityObj.getGuid());
if ( isManagedEntity ) {
discoveredEntities.addRootEntity(entityObj);
visitStruct(type, entityObj);
} else if ( entity instanceof AtlasObjectId) {
discoveredEntities.addUnResolvedIdReference(type, ((AtlasObjectId) entity).getGuid());
}
} else {
discoveredEntities.addUnResolvedEntityReference(entityObj);
}
}
} else {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, "Invalid object type " + entity.getClass());
}
}
}
......@@ -161,12 +162,14 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
} else if (attrType.getTypeCategory() == TypeCategory.STRUCT) {
visitStruct(attrType, val);
} else if (attrType.getTypeCategory() == TypeCategory.ENTITY) {
if ( val instanceof AtlasObjectId || val instanceof String) {
if ( val instanceof AtlasObjectId) {
visitReference((AtlasEntityType) attrType, val, false);
} else if ( val instanceof AtlasEntity ) {
//TODO - Change this to foreign key checks after changes in the model
if ( parentType.isMappedFromRefAttribute(attrDef.getName())) {
visitReference((AtlasEntityType) attrType, val, true);
} else {
visitReference((AtlasEntityType) attrType, val, false);
}
}
}
......@@ -179,10 +182,9 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
if (val != null) {
Iterator<Map.Entry> it = null;
if (Map.class.isAssignableFrom(val.getClass())) {
it = ((Map) val).entrySet().iterator();
ImmutableMap.Builder b = ImmutableMap.builder();
Iterator<Map.Entry> it = ((Map) val).entrySet().iterator();
while (it.hasNext()) {
Map.Entry e = it.next();
visitAttribute(parentType, keyType, attrDef, e.getKey());
......@@ -224,11 +226,10 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
AtlasStructType structType = (AtlasStructType) type;
for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) {
String attrName = attributeDef.getName();
AtlasType attrType = structType.getAttributeType(attrName);
Object attrVal = ((AtlasStruct) val).getAttribute(attrName);
visitAttribute(structType, attrType, attributeDef, attrVal);
for (AtlasStructType.AtlasAttribute attribute : structType.getAllAttributes().values()) {
AtlasType attrType = attribute.getAttributeType();
Object attrVal = ((AtlasStruct) val).getAttribute(attribute.getAttributeDef().getName());
visitAttribute(structType, attrType, attribute.getAttributeDef(), attrVal);
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
......@@ -70,43 +70,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return createOrUpdate(new ArrayList<AtlasEntity>() {{ add(entity); }});
}
public EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
EntityMutationContext context = new EntityMutationContext(discoveredEntities);
for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
AtlasVertex vertex = null;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
}
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
if ( discoveredEntities.isResolved(entity.getGuid()) ) {
vertex = discoveredEntities.getResolvedReference(entity.getGuid());
context.addUpdated(entity, entityType, vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityUpdate(guid);
} else {
//Create vertices which do not exist in the repository
vertex = graphMapper.createVertexTemplate(entity, entityType);
context.addCreated(entity, entityType, vertex);
discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityCreate(guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
}
}
return context;
}
@Override
public EntityMutationResponse updateById(final String guid, final AtlasEntity entity) {
return null;
......@@ -205,15 +168,56 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return null;
}
private EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
EntityMutationContext context = new EntityMutationContext(discoveredEntities);
for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
AtlasVertex vertex = null;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
}
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if ( entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
if ( discoveredEntities.isResolved(entity.getGuid()) ) {
vertex = discoveredEntities.getResolvedReference(entity.getGuid());
context.addUpdated(entity, entityType, vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityUpdate(guid);
} else {
//Create vertices which do not exist in the repository
vertex = graphMapper.createVertexTemplate(entity, entityType);
context.addCreated(entity, entityType, vertex);
discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityCreate(guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
}
}
return context;
}
private List<AtlasEntity> validateAndNormalize(final List<AtlasEntity> entities) throws AtlasBaseException {
List<AtlasEntity> normalizedEntities = new ArrayList<>();
List<String> messages = new ArrayList<>();
for (AtlasEntity entity : entities) {
AtlasType type = typeRegistry.getType(entity.getTypeName());
if (type.getTypeCategory() != TypeCategory.ENTITY) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, type.getTypeCategory().name(), TypeCategory.ENTITY.name());
AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (type == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
type.validateValue(entity, entity.getTypeName(), messages);
......@@ -232,16 +236,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return normalizedEntities;
}
@VisibleForTesting
EntityGraphDiscovery getGraphDiscoverer() {
return graphDiscoverer;
}
@VisibleForTesting
void setGraphDiscoverer(EntityGraphDiscovery discoverer) {
this.graphDiscoverer = discoverer;
}
public void cleanUp() throws AtlasBaseException {
this.graphDiscoverer.cleanUp();
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -94,8 +94,12 @@ public class AtlasGraphUtilsV1 {
return PROPERTY_PREFIX + "edge." + fromNode + "." + toNode;
}
public static String getEdgeLabel(String property) {
return GraphHelper.EDGE_LABEL_PREFIX + property;
}
public static String getAttributeEdgeLabel(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedAttributePropertyKey(fromType, attributeName);
return getEdgeLabel(getQualifiedAttributePropertyKey(fromType, attributeName));
}
public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
import org.apache.atlas.repository.util.FilterUtil;
......@@ -462,7 +463,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(ret, attrName);
String attribJson = vertex.getProperty(propertyKey, String.class);
String attribJson = vertex.getProperty(GraphHelper.encodePropertyKey(propertyKey), String.class);
attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attribJson, Map.class),
typeDefStore));
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
......@@ -129,7 +130,12 @@ public abstract class DeleteHandlerV1 {
continue;
}
result.add(new GraphHelper.VertexInfo(guid, vertex, typeName));
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(typeName);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName);
}
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
if (!entityType.isMappedFromRefAttribute(attributeInfo.getAttributeDef().getName())) {
continue;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
......@@ -22,6 +22,7 @@ import com.google.inject.Inject;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
......@@ -36,9 +37,11 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.UUID;
public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
......@@ -86,13 +89,12 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
AtlasEdge result = null;
String guid = getId(ctx.getValue());
AtlasVertex entityVertex = context.getDiscoveryContext().getResolvedReference(guid);
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName());
if ( ctx.getCurrentEdge().isPresent() ) {
updateEdge(ctx.getAttributeDef(), ctx.getValue(), ctx.getCurrentEdge().get(), entityVertex);
result = ctx.getCurrentEdge().get();
} else {
} else if (ctx.getValue() != null) {
String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey());
try {
result = graphHelper.getOrCreateEdge(ctx.getReferringVertex(), entityVertex, edgeLabel);
} catch (RepositoryException e) {
......@@ -112,7 +114,7 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
LOG.debug("Updating entity reference {} for reference attribute {}", attributeDef.getName());
// Update edge if it exists
AtlasVertex currentVertex = currentEdge.getOutVertex();
AtlasVertex currentVertex = currentEdge.getInVertex();
String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentVertex);
String newEntityId = getId(value);
AtlasEdge newEdge = currentEdge;
......@@ -140,17 +142,17 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
if (ctx.getCreatedEntities() != null) {
for (AtlasEntity createdEntity : ctx.getCreatedEntities()) {
AtlasVertex vertex = ctx.getVertex(createdEntity);
structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(createdEntity), createdEntity, vertex);
resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, vertex));
structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, ctx.getType(createdEntity), createdEntity, vertex);
resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, ctx.getType(createdEntity), vertex));
}
}
if (ctx.getUpdatedEntities() != null) {
for (AtlasEntity updated : ctx.getUpdatedEntities()) {
AtlasVertex vertex = ctx.getVertex(updated);
structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(updated), updated, vertex);
structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.UPDATE, ctx.getType(updated), updated, vertex);
resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, vertex));
resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, ctx.getType(updated), vertex));
}
}
......@@ -165,13 +167,30 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
} else if (value instanceof AtlasEntity) {
return ((AtlasEntity) value).getGuid();
}
}
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value);
}
private AtlasEntityHeader constructHeader(AtlasEntity entity, AtlasVertex vertex) {
return null;
}
private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) {
//TODO - enhance to return only selective attributes
return new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
final Map<String, AtlasStructType.AtlasAttribute> allAttributes = type.getAllAttributes();
for (String attribute : allAttributes.keySet()) {
AtlasType attributeType = allAttributes.get(attribute).getAttributeType();
AtlasStructDef.AtlasAttributeDef attributeDef = allAttributes.get(attribute).getAttributeDef();
if ( header.getAttribute(attribute) == null && (TypeCategory.PRIMITIVE == attributeType.getTypeCategory())) {
if ( attributeDef.getIsOptional()) {
header.setAttribute(attribute, attributeType.createOptionalDefaultValue());
} else {
header.setAttribute(attribute, attributeType.createDefaultValue());
}
}
}
return header;
}
public EntityMutationContext getContext() {
......@@ -180,6 +199,11 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
public AtlasEntityType getInstanceType(Object val) throws AtlasBaseException {
String guid = getId(val);
if ( guid != null) {
return (AtlasEntityType) getContext().getType(guid);
}
return null;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java
......@@ -21,6 +21,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import java.util.ArrayList;
......@@ -35,20 +36,20 @@ public class EntityMutationContext {
private List<AtlasEntity> entitiesUpdated = new ArrayList<>();
private EntityGraphDiscoveryContext context;
private Map<String, AtlasType> entityVsType = new HashMap<>();
private Map<String, AtlasEntityType> entityVsType = new HashMap<>();
private Map<String, AtlasVertex> entityVsVertex = new HashMap<>();
public EntityMutationContext(final EntityGraphDiscoveryContext context) {
this.context = context;
}
public void addCreated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
public void addCreated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
entitiesCreated.add(entity);
entityVsVertex.put(entity.getGuid(), atlasVertex);
entityVsType.put(entity.getGuid(), type);
}
public void addUpdated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
public void addUpdated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
entitiesUpdated.add(entity);
entityVsVertex.put(entity.getGuid(), atlasVertex);
entityVsType.put(entity.getGuid(), type);
......@@ -62,7 +63,7 @@ public class EntityMutationContext {
return entitiesUpdated;
}
public AtlasType getType(AtlasEntity entity) {
public AtlasEntityType getType(AtlasEntity entity) {
return entityVsType.get(entity.getGuid());
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
......@@ -30,6 +31,7 @@ import java.util.Objects;
public class GraphMutationContext {
private EntityMutations.EntityOperation op;
/**
* Atlas Attribute
*/
......@@ -65,6 +67,7 @@ public class GraphMutationContext {
private GraphMutationContext(final Builder builder) {
this.op = builder.op;
this.attribute = builder.attribute;
this.currentElementType = builder.elementType;
this.existingEdge = builder.currentEdge;
......@@ -79,7 +82,7 @@ public class GraphMutationContext {
@Override
public int hashCode() {
return Objects.hash(attribute, value, referringVertex, vertexPropertyKey, existingEdge);
return Objects.hash(op, attribute, value, referringVertex, vertexPropertyKey, existingEdge);
}
@Override
......@@ -96,13 +99,16 @@ public class GraphMutationContext {
&& Objects.equals(value, rhs.getValue())
&& Objects.equals(referringVertex, rhs.getReferringVertex())
&& Objects.equals(vertexPropertyKey, rhs.getReferringVertex())
&& Objects.equals(existingEdge, rhs.getCurrentEdge());
&& Objects.equals(existingEdge, rhs.getCurrentEdge())
&& Objects.equals(op, rhs.getOp());
}
}
public static final class Builder {
private final EntityMutations.EntityOperation op;
private final AtlasStructType.AtlasAttribute attribute;
private final AtlasType elementType;
......@@ -116,16 +122,15 @@ public class GraphMutationContext {
private String vertexPropertyKey;
public Builder(AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
this.op = op;
this.attribute = attribute;
this.elementType = currentElementType;
this.currentValue = currentValue;
}
public Builder(AtlasStructType.AtlasAttribute attribute, Object currentValue) {
this.attribute = attribute;
this.elementType = null;
this.currentValue = currentValue;
public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, Object currentValue) {
this(op, attribute, null, currentValue);
}
Builder referringVertex(AtlasVertex referringVertex) {
......@@ -192,4 +197,12 @@ public class GraphMutationContext {
public AtlasStructType.AtlasAttribute getAttribute() {
return attribute;
}
public EntityMutations.EntityOperation getOp() {
return op;
}
public void setExistingEdge(final Optional<AtlasEdge> existingEdge) {
this.existingEdge = existingEdge;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java
......@@ -105,7 +105,7 @@ public class IDBasedEntityResolver implements EntityResolver {
if ( vertex != null ) {
return Optional.of(vertex);
} else {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, "Could not find an entity with the specified guid " + typeIdPair.getGuid() + " in Atlas respository");
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, typeIdPair.getGuid());
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java
......@@ -97,13 +97,16 @@ public class MapVertexMapper implements InstanceGraphMapper<Map> {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), keyStr);
Optional<AtlasEdge> existingEdge = getEdgeIfExists(mapType, currentMap, keyStr);
GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getAttribute(), mapType.getValueType(), entry.getValue())
GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(), mapType.getValueType(), entry.getValue())
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
.vertexProperty(propertyNameForKey).build();
//Add/Update/Remove property value
Object newEntry = structVertexMapper.mapCollectionElementsToVertex(mapCtx);
MapVertexMapper.setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey, newEntry);
newMap.put(keyStr, newEntry);
}
}
......@@ -192,8 +195,10 @@ public class MapVertexMapper implements InstanceGraphMapper<Map> {
private Optional<AtlasEdge> getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) {
Optional<AtlasEdge> existingEdge = Optional.absent();
if ( AtlasGraphUtilsV1.isReference(mapType.getValueType()) ) {
if ( currentMap.get(keyStr) != null) {
existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
}
}
return existingEdge;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/StructVertexMapper.java
......@@ -17,27 +17,31 @@
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.AtlasEdgeLabel;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
private final AtlasGraph graph;
......@@ -64,18 +68,17 @@ public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
@Override
public AtlasEdge toGraph(GraphMutationContext ctx) throws AtlasBaseException {
AtlasEdge result = null;
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName());
AtlasEdge ret = null;
if ( ctx.getCurrentEdge().isPresent() ) {
updateVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getCurrentEdge().get().getOutVertex());
result = ctx.getCurrentEdge().get();
} else {
result = createVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getReferringVertex(), edgeLabel);
updateVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getCurrentEdge().get().getInVertex());
ret = ctx.getCurrentEdge().get();
} else if (ctx.getValue() != null) {
String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey());
ret = createVertex(ctx.getParentType(), (AtlasStructType) ctx.getAttrType(), ctx.getAttributeDef(), (AtlasStruct) ctx.getValue(), ctx.getReferringVertex(), edgeLabel);
}
return result;
return ret;
}
@Override
......@@ -88,44 +91,86 @@ public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
/**
* Map attributes for entity, struct or trait
*
* @param op
* @param structType
* @param struct
* @param vertex
* @return
* @throws AtlasBaseException
*/
public AtlasVertex mapAttributestoVertex(AtlasStructType structType, AtlasStruct struct, AtlasVertex vertex) throws AtlasBaseException {
public AtlasVertex mapAttributestoVertex(final EntityMutations.EntityOperation op, AtlasStructType structType, AtlasStruct struct, AtlasVertex vertex) throws AtlasBaseException {
if (struct.getAttributes() != null) {
if (op.equals(EntityMutations.EntityOperation.CREATE)) {
final Map<String, AtlasStructType.AtlasAttribute> allAttributes = structType.getAllAttributes();
for (String attrName : allAttributes.keySet()) {
Object value = struct.getAttribute(attrName);
mapAttribute(op, structType, attrName, value, vertex);
}
} else if (op.equals(EntityMutations.EntityOperation.UPDATE)) {
for (String attrName : struct.getAttributes().keySet()) {
Object value = struct.getAttribute(attrName);
mapAttribute(op, structType, attrName, value, vertex);
}
}
updateModificationMetadata(vertex);
}
return vertex;
}
private void mapAttribute(final EntityMutations.EntityOperation op, AtlasStructType structType, String attrName, Object value, AtlasVertex vertex) throws AtlasBaseException {
AtlasType attributeType = structType.getAttributeType(attrName);
if ( attributeType != null) {
if (attributeType != null) {
final AtlasStructType.AtlasAttribute attribute = structType.getAttribute(attrName);
GraphMutationContext ctx = new GraphMutationContext.Builder(attribute, value)
if (value == null) {
if ( attribute.getAttributeType().getTypeCategory() == TypeCategory.PRIMITIVE) {
if ( attribute.getAttributeDef().getIsOptional()) {
value = attribute.getAttributeType().createOptionalDefaultValue();
} else {
value = attribute.getAttributeType().createDefaultValue();
}
}
}
final String vertexProperty = structType.getQualifiedAttributeName(attrName);
GraphMutationContext ctx = new GraphMutationContext.Builder(op, attribute, value)
.referringVertex(vertex)
.vertexProperty(AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attrName)).build();
.vertexProperty(GraphHelper.encodePropertyKey(vertexProperty)).build();
mapToVertexByTypeCategory(ctx);
}
}
private void updateModificationMetadata(AtlasVertex vertex) {
//Set updated timestamp
AtlasGraphUtilsV1.setProperty(vertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContextV1.get().getRequestTime());
GraphHelper.setProperty(vertex, Constants.MODIFIED_BY_KEY, RequestContextV1.get().getUser());
}
return vertex;
}
protected Object mapToVertexByTypeCategory(GraphMutationContext ctx) throws AtlasBaseException {
switch(ctx.getAttrType().getTypeCategory()) {
if (ctx.getOp() == EntityMutations.EntityOperation.CREATE && ctx.getValue() == null) {
return null;
}
switch (ctx.getAttrType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return primitivesToVertex(ctx);
case STRUCT:
String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey());
AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel);
Optional<AtlasEdge> edge = currentEdge != null ? Optional.of(currentEdge) : Optional.<AtlasEdge>absent();
ctx.setExistingEdge(edge);
return toGraph(ctx);
case ENTITY:
edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey());
currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel);
AtlasEntityType instanceType = entityVertexMapper.getInstanceType(ctx.getValue());
edge = currentEdge != null ? Optional.of(currentEdge) : Optional.<AtlasEdge>absent();
ctx.setElementType(instanceType);
ctx.setExistingEdge(edge);
return entityVertexMapper.toGraph(ctx);
case MAP:
return mapVertexMapper.toGraph(ctx);
......@@ -137,17 +182,13 @@ public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
}
protected Object primitivesToVertex(GraphMutationContext ctx) {
if ( ctx.getAttrType().getTypeCategory() == TypeCategory.MAP ) {
MapVertexMapper.setMapValueProperty(((AtlasMapType) ctx.getAttrType()).getValueType(), ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue());
} else {
AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey(), ctx.getValue());
}
return ctx.getValue();
}
private AtlasEdge createVertex(AtlasStructType parentType, AtlasStructType attrType, AtlasStructDef.AtlasAttributeDef attributeDef, AtlasStruct struct, AtlasVertex referringVertex, String edgeLabel) throws AtlasBaseException {
AtlasVertex vertex = createVertexTemplate(struct, attrType);
mapAttributestoVertex(attrType, struct, vertex);
mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, attrType, struct, vertex);
try {
//TODO - Map directly in AtlasGraphUtilsV1
......@@ -158,7 +199,7 @@ public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
}
private void updateVertex(AtlasStructType parentType, AtlasStructType structAttributeType, AtlasStructDef.AtlasAttributeDef attributeDef, AtlasStruct value, AtlasVertex structVertex) throws AtlasBaseException {
mapAttributestoVertex(structAttributeType, value, structVertex);
mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, structAttributeType, value, structVertex);
}
protected AtlasVertex createVertexTemplate(final AtlasStruct instance, final AtlasStructType structType) {
......@@ -187,7 +228,7 @@ public class StructVertexMapper implements InstanceGraphMapper<AtlasEdge> {
switch(ctx.getAttrType().getTypeCategory()) {
case PRIMITIVE:
case ENUM:
return primitivesToVertex(ctx);
return ctx.getValue();
case STRUCT:
return toGraph(ctx);
case ENTITY:
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasStructDef;
......@@ -98,7 +99,12 @@ public class UniqAttrBasedEntityResolver implements EntityResolver {
}
Optional<AtlasVertex> resolveByUniqueAttribute(AtlasEntity entity) throws AtlasBaseException {
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
for (AtlasStructType.AtlasAttribute attr : entityType.getAllAttributes().values()) {
if (attr.getAttributeDef().getIsUnique()) {
Object attrVal = entity.getAttribute(attr.getAttributeDef().getName());
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
......@@ -17,7 +17,7 @@
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasClient;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
......@@ -25,13 +25,13 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
......@@ -39,24 +39,23 @@ import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.typesystem.IInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.persistence.StructInstance;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.lang.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
......@@ -64,11 +63,14 @@ import org.testng.annotations.Test;
import javax.inject.Inject;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtils.randomString;
import static org.testng.Assert.assertEquals;
@Guice(modules = RepositoryMetadataModule.class)
public class AtlasEntityStoreV1Test {
......@@ -85,15 +87,22 @@ public class AtlasEntityStoreV1Test {
@Inject
MetadataService metadataService;
private AtlasEntity entityCreated;
private AtlasEntity deptEntity;
private AtlasEntity dbEntity;
private AtlasEntity tableEntity;
@BeforeClass
public void setUp() throws Exception {
new GraphBackedSearchIndexer(typeRegistry);
final AtlasTypesDef atlasTypesDef = TestUtilsV2.defineDeptEmployeeTypes();
typeDefStore.createTypesDef(atlasTypesDef);
final AtlasTypesDef deptTypesDef = TestUtilsV2.defineDeptEmployeeTypes();
typeDefStore.createTypesDef(deptTypesDef);
final AtlasTypesDef hiveTypesDef = TestUtilsV2.defineHiveTypes();
typeDefStore.createTypesDef(hiveTypesDef);
entityCreated = TestUtilsV2.createDeptEg1();
deptEntity = TestUtilsV2.createDeptEg1();
dbEntity = TestUtilsV2.createDBEntity();
tableEntity = TestUtilsV2.createTableEntity(dbEntity.getGuid());
}
@AfterClass
......@@ -121,17 +130,363 @@ public class AtlasEntityStoreV1Test {
@Test
public void testCreate() throws Exception {
EntityMutationResponse response = entityStore.createOrUpdate(entityCreated);
List<AtlasEntityHeader> entitiesCreated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(entitiesCreated);
Assert.assertEquals(entitiesCreated.size(), 5);
EntityMutationResponse response = entityStore.createOrUpdate(deptEntity);
validateMutationResponse(response, EntityMutations.EntityOperation.CREATE, 5);
AtlasEntityHeader deptEntity = response.getFirstEntityCreated();
validateAttributes(deptEntity);
//Create DB
EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(dbEntity);
validateMutationResponse(dbCreationResponse, EntityMutations.EntityOperation.CREATE, 1);
AtlasEntityHeader dbEntity = dbCreationResponse.getFirstEntityCreated();
validateAttributes(dbEntity);
//Create Table
//Update DB guid
AtlasObjectId dbId = (AtlasObjectId) tableEntity.getAttribute("database");
dbId.setGuid(dbEntity.getGuid());
tableEntity.setAttribute("database", dbId);
EntityMutationResponse tableCreationResponse = entityStore.createOrUpdate(tableEntity);
validateMutationResponse(tableCreationResponse, EntityMutations.EntityOperation.CREATE, 1);
AtlasEntityHeader tableEntity = tableCreationResponse.getFirstEntityCreated();
validateAttributes(tableEntity);
}
@Test(dependsOnMethods = "testCreate")
public void testArrayOfEntityUpdate() throws Exception {
//clear state
init();
AtlasEntity entityClone = new AtlasEntity(deptEntity);
List<AtlasEntity> employees = (List<AtlasEntity>) entityClone.getAttribute("employees");
AtlasEntity entityRemoved = clearSubOrdinates(employees, 1);
entityClone.setAttribute("employees", employees);
EntityMutationResponse response = entityStore.createOrUpdate(entityClone);
validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 5);
AtlasEntityHeader deptEntity = response.getFirstEntityUpdated();
validateAttributes(deptEntity);
init();
//add entity back
addSubordinate(employees.get(1), entityRemoved);
response = entityStore.createOrUpdate(entityClone);
validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 5);
deptEntity = response.getFirstEntityUpdated();
validateAttributes(deptEntity);
//test array of class with id
final List<AtlasEntity> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
values.put(TestUtilsV2.NAME, "col1");
values.put("type", "type");
AtlasEntity col1 = new AtlasEntity(TestUtilsV2.COLUMN_TYPE, values);
columns.add(col1);
AtlasEntity tableUpdated = new AtlasEntity(tableEntity);
tableUpdated.setAttribute(TestUtilsV2.COLUMNS_ATTR_NAME, columns);
init();
entityStore.createOrUpdate(col1);
init();
response = entityStore.createOrUpdate(tableUpdated);
final AtlasEntityHeader updateTable = response.getFirstEntityUpdated();
validateAttributes(updateTable);
}
@Test(dependsOnMethods = "testCreate")
public void testUpdateEntityWithMap() throws Exception {
AtlasEntity tableClone = new AtlasEntity(tableEntity);
final Map<String, AtlasStruct> partsMap = new HashMap<>();
partsMap.put("part0", new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE,
new HashMap<String, Object>() {{
put(TestUtilsV2.NAME, "test");
}}));
tableClone.setAttribute("partitionsMap", partsMap);
init();
EntityMutationResponse response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition1 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition1);
Assert.assertTrue(partsMap.get("part0").equals(((Map<String, AtlasStruct>) tableDefinition1.getAttribute("partitionsMap")).get("part0")));
//update map - add a map key
partsMap.put("part1", new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE,
new HashMap<String, Object>() {{
put(TestUtilsV2.NAME, "test1");
}}));
tableClone.setAttribute("partitionsMap", partsMap);
init();
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition2 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition2);
assertEquals(((Map<String, AtlasStruct>) tableDefinition2.getAttribute("partitionsMap")).size(), 2);
Assert.assertTrue(partsMap.get("part1").equals(((Map<String, AtlasStruct>) tableDefinition2.getAttribute("partitionsMap")).get("part1")));
//update map - remove a key and add another key
partsMap.remove("part0");
partsMap.put("part2", new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE,
new HashMap<String, Object>() {{
put(TestUtilsV2.NAME, "test2");
}}));
tableClone.setAttribute("partitionsMap", partsMap);
init();
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition3 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition3);
assertEquals(((Map<String, AtlasStruct>) tableDefinition3.getAttribute("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, AtlasStruct>) tableDefinition3.getAttribute("partitionsMap")).get("part0"));
Assert.assertTrue(partsMap.get("part2").equals(((Map<String, AtlasStruct>) tableDefinition3.getAttribute("partitionsMap")).get("part2")));
//update struct value for existing map key
init();
AtlasStruct partition2 = partsMap.get("part2");
partition2.setAttribute(TestUtilsV2.NAME, "test2Updated");
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition4 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition4);
assertEquals(((Map<String, AtlasStruct>) tableDefinition4.getAttribute("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, AtlasStruct>) tableDefinition4.getAttribute("partitionsMap")).get("part0"));
assertEquals(((Map<String, AtlasStruct>) tableDefinition4.getAttribute("partitionsMap")).size(), 2);
Assert.assertNull(((Map<String, AtlasStruct>) tableDefinition4.getAttribute("partitionsMap")).get("part0"));
Assert.assertTrue(partsMap.get("part2").equals(((Map<String, AtlasStruct>) tableDefinition4.getAttribute("partitionsMap")).get("part2")));
//Test map pointing to a class
final Map<String, AtlasEntity> columnsMap = new HashMap<>();
AtlasEntity col0Type = new AtlasEntity(TestUtilsV2.COLUMN_TYPE,
new HashMap<String, Object>() {{
put(TestUtilsV2.NAME, "test1");
put("type", "string");
}});
init();
entityStore.createOrUpdate(col0Type);
AtlasEntity col1Type = new AtlasEntity(TestUtils.COLUMN_TYPE,
new HashMap<String, Object>() {{
put(TestUtilsV2.NAME, "test2");
put("type", "string");
}});
init();
entityStore.createOrUpdate(col1Type);
columnsMap.put("col0", col0Type);
columnsMap.put("col1", col1Type);
tableClone.setAttribute(TestUtils.COLUMNS_MAP, columnsMap);
init();
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition5 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition5);
//Swap elements
columnsMap.clear();
columnsMap.put("col0", col1Type);
columnsMap.put("col1", col0Type);
tableClone.setAttribute(TestUtils.COLUMNS_MAP, columnsMap);
init();
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition6 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition6);
//Drop the first key and change the class type as well to col0
columnsMap.clear();
columnsMap.put("col0", col0Type);
init();
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition7 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition7);
//Clear state
tableClone.setAttribute(TestUtils.COLUMNS_MAP, null);
init();
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition8 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition8);
}
AtlasEntityHeader deptEntity = entitiesCreated.get(0);
@Test(dependsOnMethods = "testCreate")
public void testMapOfPrimitivesUpdate() throws Exception {
//clear state
init();
AtlasEntity entityClone = new AtlasEntity(tableEntity);
//Add a new entry
Map<String, String> paramsMap = (Map<String, String>) entityClone.getAttribute("parametersMap");
paramsMap.put("newParam", "value");
entityClone.setAttribute("parametersMap", paramsMap);
EntityMutationResponse response = entityStore.createOrUpdate(entityClone);
validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1);
AtlasEntityHeader tableEntity = response.getFirstEntityUpdated();
validateAttributes(tableEntity);
//clear state
init();
//Remove an entry
paramsMap.remove("key1");
entityClone.setAttribute("parametersMap", paramsMap);
response = entityStore.createOrUpdate(entityClone);
validateMutationResponse(response, EntityMutations.EntityOperation.UPDATE, 1);
tableEntity = response.getFirstEntityUpdated();
validateAttributes(tableEntity);
}
@Test(dependsOnMethods = "testCreate")
public void testArrayOfStructs() throws Exception {
//Modify array of structs
// TestUtils.dumpGraph(TestUtils.getGraph());
init();
final AtlasStruct partition1 = new AtlasStruct(TestUtilsV2.PARTITION_STRUCT_TYPE);
partition1.setAttribute(TestUtilsV2.NAME, "part1");
final AtlasStruct partition2 = new AtlasStruct(TestUtilsV2.PARTITION_STRUCT_TYPE);
partition2.setAttribute(TestUtilsV2.NAME, "part2");
List<AtlasStruct> partitions = new ArrayList<AtlasStruct>(){{ add(partition1); add(partition2); }};
tableEntity.setAttribute("partitions", partitions);
EntityMutationResponse response = entityStore.createOrUpdate(tableEntity);
AtlasEntityHeader tableDefinition = response.getFirstEntityUpdated();
//TODO : Use the older API for get until new instance API is ready.
ITypedReferenceableInstance instance = metadataService.getEntityDefinition(deptEntity.getGuid());
assertAttributes(deptEntity, instance);
validateAttributes(tableDefinition);
//add a new element to array of struct
init();
final AtlasStruct partition3 = new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE);
partition3.setAttribute(TestUtilsV2.NAME, "part3");
partitions.add(partition3);
tableEntity.setAttribute("partitions", partitions);
response = entityStore.createOrUpdate(tableEntity);
tableDefinition = response.getFirstEntityUpdated();
validateAttributes(tableDefinition);
//remove one of the struct values
init();
partitions.remove(1);
tableEntity.setAttribute("partitions", partitions);
response = entityStore.createOrUpdate(tableEntity);
tableDefinition = response.getFirstEntityUpdated();
validateAttributes(tableDefinition);
//Update struct value within array of struct
init();
partitions.get(0).setAttribute(TestUtilsV2.NAME, "part4");
tableEntity.setAttribute("partitions", partitions);
response = entityStore.createOrUpdate(tableEntity);
tableDefinition = response.getFirstEntityUpdated();
validateAttributes(tableDefinition);
//add a repeated element to array of struct
init();
final AtlasStruct partition4 = new AtlasStruct(TestUtils.PARTITION_STRUCT_TYPE);
partition4.setAttribute(TestUtilsV2.NAME, "part4");
partitions.add(partition4);
tableEntity.setAttribute("partitions", partitions);
response = entityStore.createOrUpdate(tableEntity);
tableDefinition = response.getFirstEntityUpdated();
validateAttributes(tableDefinition);
// Remove all elements. Should set array attribute to null
init();
partitions.clear();
tableEntity.setAttribute("partitions", partitions);
response = entityStore.createOrUpdate(tableEntity);
tableDefinition = response.getFirstEntityUpdated();
validateAttributes(tableDefinition);
}
@Test(dependsOnMethods = "testCreate")
public void testStructs() throws Exception {
init();
AtlasEntity tableClone = new AtlasEntity(tableEntity);
AtlasStruct serdeInstance = new AtlasStruct(TestUtils.SERDE_TYPE);
serdeInstance.setAttribute(TestUtilsV2.NAME, "serde1Name");
serdeInstance.setAttribute("serde", "test");
serdeInstance.setAttribute("description", "testDesc");
tableClone.setAttribute("serde1", serdeInstance);
tableClone.setAttribute("database", dbEntity);
EntityMutationResponse response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition1 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition1);
//update struct attribute
init();
serdeInstance.setAttribute("serde", "testUpdated");
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition2 = response.getFirstEntityUpdated();
validateAttributes(tableDefinition2);
//set to null
init();
tableClone.setAttribute("description", null);
response = entityStore.createOrUpdate(tableClone);
AtlasEntityHeader tableDefinition3 = response.getFirstEntityUpdated();
Assert.assertNull(tableDefinition3.getAttribute("description"));
validateAttributes(tableDefinition3);
}
private AtlasEntity clearSubOrdinates(List<AtlasEntity> employees, int index) {
AtlasEntity ret = null;
List<AtlasEntity> subOrdinates = (List<AtlasEntity>) employees.get(index).getAttribute("subordinates");
List<AtlasEntity> subOrdClone = new ArrayList<>(subOrdinates);
ret = subOrdClone.remove(index);
employees.get(index).setAttribute("subordinates", subOrdClone);
return ret;
}
private int addSubordinate(AtlasEntity manager, AtlasEntity employee) {
List<AtlasEntity> subOrdinates = (List<AtlasEntity>) manager.getAttribute("subordinates");
subOrdinates.add(employee);
manager.setAttribute("subordinates", subOrdinates);
return subOrdinates.size() - 1;
}
private void validateMutationResponse(EntityMutationResponse response, EntityMutations.EntityOperation op, int expectedNumCreated) {
List<AtlasEntityHeader> entitiesCreated = response.getEntitiesByOperation(op);
Assert.assertNotNull(entitiesCreated);
Assert.assertEquals(entitiesCreated.size(), expectedNumCreated);
}
private void validateAttributes(AtlasEntityHeader entity) throws AtlasBaseException, AtlasException {
//TODO : Use the older API for get until new instance API is ready and validated
ITypedReferenceableInstance instance = metadataService.getEntityDefinition(entity.getGuid());
assertAttributes(entity, instance);
}
private void assertAttributes(AtlasStruct entity, IInstance instance) throws AtlasBaseException, AtlasException {
......@@ -157,13 +512,20 @@ public class AtlasEntityStoreV1Test {
} else {
ReferenceableInstance expectedInstance = (ReferenceableInstance) expected;
AtlasEntity actualInstance = (AtlasEntity) actual;
if ( actualInstance != null) {
assertAttributes(actualInstance, expectedInstance);
}
}
break;
case PRIMITIVE:
case ENUM:
Assert.assertEquals(actual, expected);
break;
case ENUM:
EnumValue expectedEnumVal = (EnumValue) expected;
if ( actual != null) {
Assert.assertEquals(actual, expectedEnumVal.value);
}
break;
case MAP:
AtlasMapType mapType = (AtlasMapType) attributeType;
AtlasType keyType = mapType.getKeyType();
......@@ -171,10 +533,12 @@ public class AtlasEntityStoreV1Test {
Map actualMap = (Map) actual;
Map expectedMap = (Map) expected;
if (expectedMap != null && actualMap != null) {
Assert.assertEquals(actualMap.size(), expectedMap.size());
for (Object key : actualMap.keySet()) {
assertAttribute(actualMap.get(key), expectedMap.get(key), valueType, attrName);
}
}
break;
case ARRAY:
AtlasArrayType arrType = (AtlasArrayType) attributeType;
......@@ -200,37 +564,120 @@ public class AtlasEntityStoreV1Test {
}
@Test(dependsOnMethods = "testCreate")
public void testArrayUpdate() throws Exception {
//clear state
public void testClassUpdate() throws Exception {
init();
//Create new db instance
final AtlasEntity databaseInstance = TestUtilsV2.createDBEntity();
AtlasEntity entityClone = new AtlasEntity(entityCreated);
EntityMutationResponse response = entityStore.createOrUpdate(databaseInstance);
final AtlasEntityHeader dbCreated = response.getFirstEntityCreated();
List<AtlasEntity> employees = (List<AtlasEntity>) entityClone.getAttribute("employees");
init();
AtlasEntity tableClone = new AtlasEntity(tableEntity);
tableClone.setAttribute("database", new AtlasObjectId(TestUtils.DATABASE_TYPE, dbCreated.getGuid()));
response = entityStore.createOrUpdate(tableClone);
final AtlasEntityHeader tableDefinition = response.getFirstEntityUpdated();
List<AtlasEntity> updatedEmployees = new ArrayList<>(employees);
clearSubOrdinates(updatedEmployees, 1);
entityClone.setAttribute("employees", updatedEmployees);
Assert.assertNotNull(tableDefinition.getAttribute("database"));
Assert.assertEquals(((AtlasObjectId) tableDefinition.getAttribute("database")).getGuid(), dbCreated.getGuid());
}
EntityMutationResponse response = entityStore.createOrUpdate(entityClone);
@Test
public void testCheckOptionalAttrValueRetention() throws Exception {
List<AtlasEntityHeader> entitiesUpdated = response.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE);
Assert.assertNotNull(entitiesUpdated);
Assert.assertEquals(entitiesUpdated.size(), 5);
AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
EntityMutationResponse response = entityStore.createOrUpdate(dbEntity);
AtlasEntityHeader firstEntityCreated = response.getFirstEntityCreated();
AtlasEntityHeader deptEntity = entitiesUpdated.get(0);
//The optional boolean attribute should have a non-null value
final String isReplicatedAttr = "isReplicated";
final String paramsAttr = "parameters";
Assert.assertNotNull(firstEntityCreated.getAttribute(isReplicatedAttr));
Assert.assertEquals(firstEntityCreated.getAttribute(isReplicatedAttr), Boolean.FALSE);
Assert.assertNull(firstEntityCreated.getAttribute(paramsAttr));
//TODO : Change to new API after new instance GET API is ready.
ITypedReferenceableInstance instance = metadataService.getEntityDefinition(deptEntity.getGuid());
assertAttributes(deptEntity, instance);
//Update to true
init();
dbEntity.setAttribute(isReplicatedAttr, Boolean.TRUE);
//Update array
final HashMap<String, String> params = new HashMap<String, String>() {{ put("param1", "val1"); put("param2", "val2"); }};
dbEntity.setAttribute(paramsAttr, params);
//Complete update
response = entityStore.createOrUpdate(dbEntity);
AtlasEntityHeader firstEntityUpdated = response.getFirstEntityUpdated();
Assert.assertNotNull(firstEntityUpdated.getAttribute(isReplicatedAttr));
Assert.assertEquals(firstEntityUpdated.getAttribute(isReplicatedAttr), Boolean.TRUE);
Assert.assertEquals(firstEntityUpdated.getAttribute(paramsAttr), params);
//TODO - enable test after GET API is ready
// init();
// //Complete update without setting the attribute
// AtlasEntity newEntity = TestUtilsV2.createDBEntity();
// //Reset name to the current DB name
// newEntity.setAttribute(AtlasClient.NAME, firstEntityCreated.getAttribute(AtlasClient.NAME));
// response = entityStore.createOrUpdate(newEntity);
//
// firstEntityUpdated = response.getFirstEntityUpdated();
// Assert.assertNotNull(firstEntityUpdated.getAttribute(isReplicatedAttr));
// Assert.assertEquals(firstEntityUpdated.getAttribute(isReplicatedAttr), Boolean.TRUE);
// Assert.assertEquals(firstEntityUpdated.getAttribute(paramsAttr), params);
}
@Test(enabled = false)
//Titan doesn't allow some reserved chars in property keys. Verify that atlas encodes these
//See GraphHelper.encodePropertyKey()
//TODO : Failing in typedef creation
public void testSpecialCharacters() throws Exception {
//Verify that type can be created with reserved characters in typename, attribute name
final String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10);
String strAttrName = randomStrWithReservedChars();
String arrayAttrName = randomStrWithReservedChars();
String mapAttrName = randomStrWithReservedChars();
AtlasEntityDef typeDefinition =
AtlasTypeUtil.createClassTypeDef(typeName, "Special chars test type", ImmutableSet.<String>of(),
AtlasTypeUtil.createOptionalAttrDef(strAttrName, "string"),
AtlasTypeUtil.createOptionalAttrDef(arrayAttrName, "array<string>"),
AtlasTypeUtil.createOptionalAttrDef(mapAttrName, "map<string,string>"));
typeDefStore.createEntityDef(typeDefinition);
//verify that entity can be created with reserved characters in string value, array value and map key and value
AtlasEntity entity = new AtlasEntity();
entity.setAttribute(strAttrName, randomStrWithReservedChars());
entity.setAttribute(arrayAttrName, new String[]{randomStrWithReservedChars()});
entity.setAttribute(mapAttrName, new HashMap<String, String>() {{
put(randomStrWithReservedChars(), randomStrWithReservedChars());
}});
final EntityMutationResponse response = entityStore.createOrUpdate(entity);
final AtlasEntityHeader firstEntityCreated = response.getFirstEntityCreated();
validateAttributes(firstEntityCreated);
//Verify that search with reserved characters works - for string attribute
// String query =
// String.format("`%s` where `%s` = '%s'", typeName, strAttrName, entity.getAttribute(strAttrName));
// String responseJson = discoveryService.searchByDSL(query, new QueryParams(1, 0));
// JSONObject response = new JSONObject(responseJson);
// assertEquals(response.getJSONArray("rows").length(), 1);
}
private void clearSubOrdinates(List<AtlasEntity> updatedEmployees, int index) {
List<AtlasEntity> subOrdinates = (List<AtlasEntity>) updatedEmployees.get(index).getAttribute("subordinates");
List<AtlasEntity> subOrdClone = new ArrayList<>(subOrdinates);
subOrdClone.remove(index);
private String randomStrWithReservedChars() {
return randomString() + "\"${}%";
}
@Test(expectedExceptions = AtlasBaseException.class)
public void testCreateRequiredAttrNull() throws Exception {
//Update required attribute
updatedEmployees.get(index).setAttribute("subordinates", subOrdClone);
AtlasEntity tableEntity = new AtlasEntity(TestUtilsV2.TABLE_TYPE);
tableEntity.setAttribute(TestUtilsV2.NAME, "table_" + TestUtils.randomString());
entityStore.createOrUpdate(tableEntity);
Assert.fail("Expected exception while creating with required attribute null");
}
}
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java
......@@ -145,8 +145,8 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
ret = new Referenceable(entity.getGuid(), entity.getTypeName(),
fromV2ToV1(entityType, entity.getAttributes()));
} else if (v2Obj instanceof String) { // transient-id
ret = new Referenceable((String) v2Obj, type.getTypeName(), null);
} else if (v2Obj instanceof AtlasObjectId) { // transient-id
ret = new Referenceable(((AtlasObjectId) v2Obj).getGuid(), type.getTypeName(), null);
} else {
throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "Map or AtlasEntity or String",
v2Obj.getClass().getCanonicalName());
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java
......@@ -31,6 +31,8 @@ import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IReferenceableInstance;
......@@ -106,7 +108,10 @@ public class AtlasInstanceRestAdapters {
public AtlasClassification getClassification(IStruct classification) throws AtlasBaseException {
AtlasFormatConverter converter = instanceFormatters.getConverter(TypeCategory.CLASSIFICATION);
AtlasType classificationType = typeRegistry.getType(classification.getTypeName());
AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification.getTypeName());
if (classificationType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.CLASSIFICATION.name(), classification.getTypeName());
}
AtlasClassification ret = (AtlasClassification)converter.fromV1ToV2(classification, classificationType);
return ret;
......@@ -114,7 +119,11 @@ public class AtlasInstanceRestAdapters {
public AtlasEntityWithAssociations getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
AtlasFormatConverter converter = instanceFormatters.getConverter(TypeCategory.ENTITY);
AtlasType entityType = typeRegistry.getType(referenceable.getTypeName());
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(referenceable.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), referenceable.getTypeName());
}
AtlasEntityWithAssociations ret = (AtlasEntityWithAssociations)converter.fromV1ToV2(referenceable, entityType);
return ret;
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -190,12 +190,12 @@ public class TestEntitiesREST {
if ( retrievedDBEntity != null) {
LOG.info("verifying entity of type {} ", dbEntity.getTypeName());
verifyAttributes(dbEntity.getAttributes(), retrievedDBEntity.getAttributes());
verifyAttributes(retrievedDBEntity.getAttributes(), dbEntity.getAttributes());
}
if ( retrievedColumnEntity != null) {
LOG.info("verifying entity of type {} ", columns.get(0).getTypeName());
verifyAttributes(columns.get(0).getAttributes(), retrievedColumnEntity.getAttributes());
verifyAttributes(retrievedColumnEntity.getAttributes(), columns.get(0).getAttributes());
}
if ( retrievedTableEntity != null) {
......@@ -216,10 +216,13 @@ public class TestEntitiesREST {
}
}
public static void verifyAttributes(Map<String, Object> sourceAttrs, Map<String, Object> targetAttributes) throws Exception {
for (String name : sourceAttrs.keySet() ) {
public static void verifyAttributes(Map<String, Object> actual, Map<String, Object> expected) throws Exception {
for (String name : actual.keySet() ) {
LOG.info("verifying attribute {} ", name);
Assert.assertEquals(targetAttributes.get(name), sourceAttrs.get(name));
if ( expected.get(name) != null) {
Assert.assertEquals(actual.get(name), expected.get(name));
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment