Skip to content

A
atlas
Commit 6ccba52c by Suma Shivaprasad Committed by Madhan Neethiraj
Options

ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes…

ATLAS-1498: added unit-tests to validate handling of array/map/struct attributes in entity create/update Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent 19a5f65c
Showing with 318 additions and 178 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -60,7 +60,7 @@ public enum AtlasErrorCode {
TYPE_NAME_NOT_FOUND(404, "ATLAS4041E", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS4042E", "Given type guid {0} was invalid"),
EMPTY_RESULTS(404, "ATLAS4044E", "No result found for {0}"),
INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid"),
INSTANCE_GUID_NOT_FOUND(404, "ATLAS4045E", "Given instance guid {0} is invalid/not found"),
INSTANCE_LINEAGE_INVALID_PARAMS(404, "ATLAS4046E", "Invalid lineage query parameters passed {0}: {1}"),
INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
......@@ -146,8 +146,9 @@ public class AtlasStruct implements Serializable {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AtlasStruct that = (AtlasStruct) o;
return Objects.equals(typeName, that.typeName) &&
Objects.equals(attributes, that.attributes);
Objects.equals(attributes, that.attributes);
}
@Override
......
intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
......@@ -18,9 +18,11 @@
package org.apache.atlas.model.instance;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
......@@ -43,7 +45,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@XmlAccessorType(XmlAccessType.PROPERTY)
public class EntityMutationResponse {
Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated = new HashMap<>();
Map<EntityMutations.EntityOperation, List<AtlasEntityHeader>> entitiesMutated;
public EntityMutationResponse() {
}
......@@ -67,15 +69,39 @@ public class EntityMutationResponse {
return null;
}
@JsonIgnore
public AtlasEntityHeader getFirstEntityCreated() {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
return entitiesByOperation.get(0);
}
return null;
}
@JsonIgnore
public AtlasEntityHeader getFirstEntityUpdated() {
final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
return entitiesByOperation.get(0);
}
return null;
}
public void addEntity(EntityMutations.EntityOperation op, AtlasEntityHeader header) {
if (entitiesMutated == null) {
entitiesMutated = new HashMap<>();
}
if (entitiesMutated != null && entitiesMutated.get(op) == null) {
entitiesMutated.put(op, new ArrayList<AtlasEntityHeader>());
List<AtlasEntityHeader> opEntities = entitiesMutated.get(op);
if (opEntities == null) {
opEntities = new ArrayList<>();
entitiesMutated.put(op, opEntities);
}
entitiesMutated.get(op).add(header);
opEntities.add(header);
}
......@@ -84,24 +110,7 @@ public class EntityMutationResponse {
sb = new StringBuilder();
}
if (MapUtils.isNotEmpty(entitiesMutated)) {
int i = 0;
for (Map.Entry<EntityMutations.EntityOperation, List<AtlasEntityHeader>> e : entitiesMutated.entrySet()) {
if (i > 0) {
sb.append(",");
}
sb.append(e.getKey()).append(":");
if (CollectionUtils.isNotEmpty(e.getValue())) {
for (int j = 0; i < e.getValue().size(); j++) {
if (j > 0) {
sb.append(",");
}
e.getValue().get(i).toString(sb);
}
}
i++;
}
}
AtlasBaseTypeDef.dumpObjects(entitiesMutated, sb);
return sb;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
......@@ -477,6 +477,7 @@ public class AtlasBuiltInTypes {
*/
public static class AtlasStringType extends AtlasType {
private static final String DEFAULT_VALUE = "";
private static final String OPTIONAL_DEFAULT_VALUE = null;
public AtlasStringType() {
super(AtlasBaseTypeDef.ATLAS_TYPE_STRING, TypeCategory.PRIMITIVE);
......@@ -488,6 +489,11 @@ public class AtlasBuiltInTypes {
}
@Override
public Object createOptionalDefaultValue() {
return OPTIONAL_DEFAULT_VALUE;
}
@Override
public boolean isValidValue(Object obj) {
return true;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasMapType.java
......@@ -95,7 +95,11 @@ public class AtlasMapType extends AtlasType {
public Map<Object, Object> createDefaultValue() {
Map<Object, Object> ret = new HashMap<>();
ret.put(keyType.createDefaultValue(), valueType.createDefaultValue());
Object key = keyType.createDefaultValue();
if ( key != null) {
ret.put(key, valueType.createDefaultValue());
}
return ret;
}
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -328,6 +328,7 @@ public class AtlasStructType extends AtlasType {
if (attribute != null) {
AtlasType dataType = attribute.getAttributeType();
ret = dataType.createDefaultValue();
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasType.java
......@@ -33,7 +33,7 @@ import java.util.List;
public abstract class AtlasType {
private static final Gson GSON =
new GsonBuilder().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create();
new GsonBuilder().serializeNulls().setDateFormat(AtlasBaseTypeDef.SERIALIZED_DATE_FORMAT_STR).create();
private final String typeName;
private final TypeCategory typeCategory;
......@@ -59,6 +59,10 @@ public abstract class AtlasType {
public abstract Object createDefaultValue();
public Object createOptionalDefaultValue() {
return createDefaultValue();
}
public abstract boolean isValidValue(Object obj);
public abstract Object getNormalizedValue(Object obj);
......
intg/src/test/java/org/apache/atlas/TestUtilsV2.java
......@@ -71,7 +71,7 @@ public final class TestUtilsV2 {
new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0",
Arrays.asList(
new AtlasEnumElementDef("L1", "Element"+_description, 1),
new AtlasEnumElementDef("L1", "Element"+_description, 2)
new AtlasEnumElementDef("L2", "Element"+_description, 2)
));
AtlasStructDef addressDetails =
......@@ -149,7 +149,7 @@ public final class TestUtilsV2 {
new AtlasEnumDef("OrgLevel", "OrgLevel"+_description, "1.0",
Arrays.asList(
new AtlasEnumElementDef("L1", "Element"+ _description, 1),
new AtlasEnumElementDef("L1", "Element"+ _description, 2)
new AtlasEnumElementDef("L2", "Element"+ _description, 2)
));
AtlasStructDef addressDetails =
......@@ -443,6 +443,7 @@ public final class TestUtilsV2 {
AtlasEntityDef databaseTypeDefinition =
AtlasTypeUtil.createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME),
AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"),
AtlasTypeUtil.createOptionalAttrDef("isReplicated", "boolean"),
AtlasTypeUtil.createOptionalAttrDef("created", "string"),
AtlasTypeUtil.createRequiredAttrDef("description", "string"));
......@@ -543,7 +544,7 @@ public final class TestUtilsV2 {
AtlasEntityDef tableTypeDefinition =
AtlasTypeUtil.createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
AtlasTypeUtil.createUniqueRequiredAttrDef("name", "string"),
AtlasTypeUtil.createRequiredAttrDef("description", "string"),
AtlasTypeUtil.createOptionalAttrDef("description", "string"),
AtlasTypeUtil.createRequiredAttrDef("type", "string"),
AtlasTypeUtil.createOptionalAttrDef("created", "date"),
// enum
......@@ -582,6 +583,11 @@ public final class TestUtilsV2 {
AtlasAttributeDef.Cardinality.SINGLE, 0, 1,
false, false,
Collections.<AtlasStructDef.AtlasConstraintDef>emptyList()),
// new ArrayList<AtlasStructDef.AtlasConstraintDef>() {{
//add(new AtlasStructDef.AtlasConstraintDef(
// AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, new HashMap<String, Object>()));
//}}),
//map of structs
new AtlasAttributeDef("partitionsMap",
String.format("map<%s,%s>", "string", "partition_struct_type"),
......@@ -646,7 +652,7 @@ public final class TestUtilsV2 {
entity.setAttribute("description", "random table");
entity.setAttribute("type", "type");
entity.setAttribute("tableType", "MANAGED");
entity.setAttribute("database", dbId);
entity.setAttribute("database", new AtlasObjectId(DATABASE_TYPE, dbId));
entity.setAttribute("created", new Date());
Map<String, Object> partAttributes = new HashMap<String, Object>() {{
......
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj)
ATLAS-1114 Performance improvements for create/update entity (jnhagelb)
ATLAS-1403 Perf and stability improvements to DSL search and lineage query execution (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1425 Integrate Discovery/Search API in Atlas UI (kevalbhatt via mneethiraj)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java
......@@ -82,7 +82,7 @@ public class ArrayVertexMapper implements InstanceGraphMapper<List> {
Optional<AtlasEdge> existingEdge = getEdgeAt(currentElements, index, arrType.getElementType());
GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getAttribute(),
GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(),
arrType.getElementType(), newElements.get(index))
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
......@@ -18,8 +18,8 @@
package org.apache.atlas.repository.store.graph.v1;
import atlas.shaded.hbase.guava.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Provider;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
......@@ -112,7 +112,11 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
protected void discover(final List<AtlasEntity> entities) throws AtlasBaseException {
for (AtlasEntity entity : entities) {
AtlasType type = typeRegistry.getType(entity.getTypeName());
AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (type == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
discoveredEntities.addRootEntity(entity);
walkEntityGraph(type, entity);
......@@ -120,27 +124,24 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
private void visitReference(AtlasEntityType type, Object entity, boolean isManagedEntity) throws AtlasBaseException {
if ( entity != null) {
if ( entity instanceof String ) {
String guid = (String) entity;
discoveredEntities.addUnResolvedIdReference(type, guid);
} else if ( entity instanceof AtlasObjectId ) {
if (entity != null) {
if (entity instanceof AtlasObjectId) {
final String guid = ((AtlasObjectId) entity).getGuid();
discoveredEntities.addUnResolvedIdReference(type, guid);
} else if ( entity instanceof AtlasEntity ) {
AtlasEntity entityObj = ( AtlasEntity ) entity;
if (!processedIds.contains(entityObj.getGuid())) {
processedIds.add(entityObj.getGuid());
} else if (entity instanceof AtlasEntity) {
AtlasEntity entityObj = (AtlasEntity) entity;
if (isManagedEntity) {
if (!processedIds.contains(entityObj.getGuid())) {
processedIds.add(entityObj.getGuid());
if ( isManagedEntity ) {
discoveredEntities.addRootEntity(entityObj);
visitStruct(type, entityObj);
} else if ( entity instanceof AtlasObjectId) {
discoveredEntities.addUnResolvedIdReference(type, ((AtlasObjectId) entity).getGuid());
} else {
discoveredEntities.addUnResolvedEntityReference(entityObj);
}
} else {
discoveredEntities.addUnResolvedEntityReference(entityObj);
}
} else {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS, "Invalid object type " + entity.getClass());
}
}
}
......@@ -161,12 +162,14 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
} else if (attrType.getTypeCategory() == TypeCategory.STRUCT) {
visitStruct(attrType, val);
} else if (attrType.getTypeCategory() == TypeCategory.ENTITY) {
if ( val instanceof AtlasObjectId || val instanceof String) {
if ( val instanceof AtlasObjectId) {
visitReference((AtlasEntityType) attrType, val, false);
} else if ( val instanceof AtlasEntity ) {
//TODO - Change this to foreign key checks after changes in the model
if ( parentType.isMappedFromRefAttribute(attrDef.getName())) {
visitReference((AtlasEntityType) attrType, val, true);
} else {
visitReference((AtlasEntityType) attrType, val, false);
}
}
}
......@@ -179,10 +182,9 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
if (val != null) {
Iterator<Map.Entry> it = null;
if (Map.class.isAssignableFrom(val.getClass())) {
it = ((Map) val).entrySet().iterator();
ImmutableMap.Builder b = ImmutableMap.builder();
Iterator<Map.Entry> it = ((Map) val).entrySet().iterator();
while (it.hasNext()) {
Map.Entry e = it.next();
visitAttribute(parentType, keyType, attrDef, e.getKey());
......@@ -224,11 +226,10 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
AtlasStructType structType = (AtlasStructType) type;
for (AtlasStructDef.AtlasAttributeDef attributeDef : structType.getStructDef().getAttributeDefs()) {
String attrName = attributeDef.getName();
AtlasType attrType = structType.getAttributeType(attrName);
Object attrVal = ((AtlasStruct) val).getAttribute(attrName);
visitAttribute(structType, attrType, attributeDef, attrVal);
for (AtlasStructType.AtlasAttribute attribute : structType.getAllAttributes().values()) {
AtlasType attrType = attribute.getAttributeType();
Object attrVal = ((AtlasStruct) val).getAttribute(attribute.getAttributeDef().getName());
visitAttribute(structType, attrType, attribute.getAttributeDef(), attrVal);
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
......@@ -70,43 +70,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return createOrUpdate(new ArrayList<AtlasEntity>() {{ add(entity); }});
}
public EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
EntityMutationContext context = new EntityMutationContext(discoveredEntities);
for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
AtlasVertex vertex = null;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
}
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
if ( discoveredEntities.isResolved(entity.getGuid()) ) {
vertex = discoveredEntities.getResolvedReference(entity.getGuid());
context.addUpdated(entity, entityType, vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityUpdate(guid);
} else {
//Create vertices which do not exist in the repository
vertex = graphMapper.createVertexTemplate(entity, entityType);
context.addCreated(entity, entityType, vertex);
discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityCreate(guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
}
}
return context;
}
@Override
public EntityMutationResponse updateById(final String guid, final AtlasEntity entity) {
return null;
......@@ -205,15 +168,56 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return null;
}
private EntityMutationContext preCreateOrUpdate(final List<AtlasEntity> atlasEntities) throws AtlasBaseException {
EntityGraphDiscoveryContext discoveredEntities = graphDiscoverer.discoverEntities(atlasEntities);
EntityMutationContext context = new EntityMutationContext(discoveredEntities);
for (AtlasEntity entity : discoveredEntities.getRootEntities()) {
AtlasVertex vertex = null;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity);
}
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if ( entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
if ( discoveredEntities.isResolved(entity.getGuid()) ) {
vertex = discoveredEntities.getResolvedReference(entity.getGuid());
context.addUpdated(entity, entityType, vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityUpdate(guid);
} else {
//Create vertices which do not exist in the repository
vertex = graphMapper.createVertexTemplate(entity, entityType);
context.addCreated(entity, entityType, vertex);
discoveredEntities.addRepositoryResolvedReference(new AtlasObjectId(entityType.getTypeName(), entity.getGuid()), vertex);
String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
RequestContextV1.get().recordEntityCreate(guid);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasEntityStoreV1.preCreateOrUpdate({}): {}", entity, vertex);
}
}
return context;
}
private List<AtlasEntity> validateAndNormalize(final List<AtlasEntity> entities) throws AtlasBaseException {
List<AtlasEntity> normalizedEntities = new ArrayList<>();
List<String> messages = new ArrayList<>();
for (AtlasEntity entity : entities) {
AtlasType type = typeRegistry.getType(entity.getTypeName());
if (type.getTypeCategory() != TypeCategory.ENTITY) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_MATCH_FAILED, type.getTypeCategory().name(), TypeCategory.ENTITY.name());
AtlasEntityType type = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (type == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
type.validateValue(entity, entity.getTypeName(), messages);
......@@ -232,16 +236,6 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return normalizedEntities;
}
@VisibleForTesting
EntityGraphDiscovery getGraphDiscoverer() {
return graphDiscoverer;
}
@VisibleForTesting
void setGraphDiscoverer(EntityGraphDiscovery discoverer) {
this.graphDiscoverer = discoverer;
}
public void cleanUp() throws AtlasBaseException {
this.graphDiscoverer.cleanUp();
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -94,8 +94,12 @@ public class AtlasGraphUtilsV1 {
return PROPERTY_PREFIX + "edge." + fromNode + "." + toNode;
}
public static String getEdgeLabel(String property) {
return GraphHelper.EDGE_LABEL_PREFIX + property;
}
public static String getAttributeEdgeLabel(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedAttributePropertyKey(fromType, attributeName);
return getEdgeLabel(getQualifiedAttributePropertyKey(fromType, attributeName));
}
public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
import org.apache.atlas.repository.util.FilterUtil;
......@@ -462,7 +463,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String propertyKey = AtlasGraphUtilsV1.getTypeDefPropertyKey(ret, attrName);
String attribJson = vertex.getProperty(propertyKey, String.class);
String attribJson = vertex.getProperty(GraphHelper.encodePropertyKey(propertyKey), String.class);
attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attribJson, Map.class),
typeDefStore));
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -18,6 +18,7 @@
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
......@@ -129,7 +130,12 @@ public abstract class DeleteHandlerV1 {
continue;
}
result.add(new GraphHelper.VertexInfo(guid, vertex, typeName));
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(typeName);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName);
}
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
if (!entityType.isMappedFromRefAttribute(attributeInfo.getAttributeDef().getName())) {
continue;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
......@@ -22,6 +22,7 @@ import com.google.inject.Inject;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
......@@ -36,9 +37,11 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.UUID;
public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
......@@ -86,13 +89,12 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
AtlasEdge result = null;
String guid = getId(ctx.getValue());
AtlasVertex entityVertex = context.getDiscoveryContext().getResolvedReference(guid);
String edgeLabel = AtlasGraphUtilsV1.getAttributeEdgeLabel(ctx.getParentType(), ctx.getAttributeDef().getName());
if ( ctx.getCurrentEdge().isPresent() ) {
updateEdge(ctx.getAttributeDef(), ctx.getValue(), ctx.getCurrentEdge().get(), entityVertex);
result = ctx.getCurrentEdge().get();
} else {
} else if (ctx.getValue() != null) {
String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexPropertyKey());
try {
result = graphHelper.getOrCreateEdge(ctx.getReferringVertex(), entityVertex, edgeLabel);
} catch (RepositoryException e) {
......@@ -112,7 +114,7 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
LOG.debug("Updating entity reference {} for reference attribute {}", attributeDef.getName());
// Update edge if it exists
AtlasVertex currentVertex = currentEdge.getOutVertex();
AtlasVertex currentVertex = currentEdge.getInVertex();
String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentVertex);
String newEntityId = getId(value);
AtlasEdge newEdge = currentEdge;
......@@ -140,17 +142,17 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
if (ctx.getCreatedEntities() != null) {
for (AtlasEntity createdEntity : ctx.getCreatedEntities()) {
AtlasVertex vertex = ctx.getVertex(createdEntity);
structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(createdEntity), createdEntity, vertex);
resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, vertex));
structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.CREATE, ctx.getType(createdEntity), createdEntity, vertex);
resp.addEntity(EntityMutations.EntityOperation.CREATE, constructHeader(createdEntity, ctx.getType(createdEntity), vertex));
}
}
if (ctx.getUpdatedEntities() != null) {
for (AtlasEntity updated : ctx.getUpdatedEntities()) {
AtlasVertex vertex = ctx.getVertex(updated);
structVertexMapper.mapAttributestoVertex((AtlasStructType) ctx.getType(updated), updated, vertex);
structVertexMapper.mapAttributestoVertex(EntityMutations.EntityOperation.UPDATE, ctx.getType(updated), updated, vertex);
resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, vertex));
resp.addEntity(EntityMutations.EntityOperation.UPDATE, constructHeader(updated, ctx.getType(updated), vertex));
}
}
......@@ -165,13 +167,30 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
} else if (value instanceof AtlasEntity) {
return ((AtlasEntity) value).getGuid();
}
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value);
}
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, (String) value);
return null;
}
private AtlasEntityHeader constructHeader(AtlasEntity entity, AtlasVertex vertex) {
private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) {
//TODO - enhance to return only selective attributes
return new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName(), AtlasGraphUtilsV1.getIdFromVertex(vertex), entity.getAttributes());
final Map<String, AtlasStructType.AtlasAttribute> allAttributes = type.getAllAttributes();
for (String attribute : allAttributes.keySet()) {
AtlasType attributeType = allAttributes.get(attribute).getAttributeType();
AtlasStructDef.AtlasAttributeDef attributeDef = allAttributes.get(attribute).getAttributeDef();
if ( header.getAttribute(attribute) == null && (TypeCategory.PRIMITIVE == attributeType.getTypeCategory())) {
if ( attributeDef.getIsOptional()) {
header.setAttribute(attribute, attributeType.createOptionalDefaultValue());
} else {
header.setAttribute(attribute, attributeType.createDefaultValue());
}
}
}
return header;
}
public EntityMutationContext getContext() {
......@@ -180,6 +199,11 @@ public class EntityGraphMapper implements InstanceGraphMapper<AtlasEdge> {
public AtlasEntityType getInstanceType(Object val) throws AtlasBaseException {
String guid = getId(val);
return (AtlasEntityType) getContext().getType(guid);
if ( guid != null) {
return (AtlasEntityType) getContext().getType(guid);
}
return null;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java
......@@ -21,6 +21,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import java.util.ArrayList;
......@@ -35,20 +36,20 @@ public class EntityMutationContext {
private List<AtlasEntity> entitiesUpdated = new ArrayList<>();
private EntityGraphDiscoveryContext context;
private Map<String, AtlasType> entityVsType = new HashMap<>();
private Map<String, AtlasEntityType> entityVsType = new HashMap<>();
private Map<String, AtlasVertex> entityVsVertex = new HashMap<>();
public EntityMutationContext(final EntityGraphDiscoveryContext context) {
this.context = context;
}
public void addCreated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
public void addCreated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
entitiesCreated.add(entity);
entityVsVertex.put(entity.getGuid(), atlasVertex);
entityVsType.put(entity.getGuid(), type);
}
public void addUpdated(AtlasEntity entity, AtlasType type, AtlasVertex atlasVertex) {
public void addUpdated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
entitiesUpdated.add(entity);
entityVsVertex.put(entity.getGuid(), atlasVertex);
entityVsType.put(entity.getGuid(), type);
......@@ -62,7 +63,7 @@ public class EntityMutationContext {
return entitiesUpdated;
}
public AtlasType getType(AtlasEntity entity) {
public AtlasEntityType getType(AtlasEntity entity) {
return entityVsType.get(entity.getGuid());
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java
......@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
......@@ -30,6 +31,7 @@ import java.util.Objects;
public class GraphMutationContext {
private EntityMutations.EntityOperation op;
/**
* Atlas Attribute
*/
......@@ -65,6 +67,7 @@ public class GraphMutationContext {
private GraphMutationContext(final Builder builder) {
this.op = builder.op;
this.attribute = builder.attribute;
this.currentElementType = builder.elementType;
this.existingEdge = builder.currentEdge;
......@@ -79,7 +82,7 @@ public class GraphMutationContext {
@Override
public int hashCode() {
return Objects.hash(attribute, value, referringVertex, vertexPropertyKey, existingEdge);
return Objects.hash(op, attribute, value, referringVertex, vertexPropertyKey, existingEdge);
}
@Override
......@@ -96,13 +99,16 @@ public class GraphMutationContext {
&& Objects.equals(value, rhs.getValue())
&& Objects.equals(referringVertex, rhs.getReferringVertex())
&& Objects.equals(vertexPropertyKey, rhs.getReferringVertex())
&& Objects.equals(existingEdge, rhs.getCurrentEdge());
&& Objects.equals(existingEdge, rhs.getCurrentEdge())
&& Objects.equals(op, rhs.getOp());
}
}
public static final class Builder {
private final EntityMutations.EntityOperation op;
private final AtlasStructType.AtlasAttribute attribute;
private final AtlasType elementType;
......@@ -116,16 +122,15 @@ public class GraphMutationContext {
private String vertexPropertyKey;
public Builder(AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
this.op = op;
this.attribute = attribute;
this.elementType = currentElementType;
this.currentValue = currentValue;
}
public Builder(AtlasStructType.AtlasAttribute attribute, Object currentValue) {
this.attribute = attribute;
this.elementType = null;
this.currentValue = currentValue;
public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, Object currentValue) {
this(op, attribute, null, currentValue);
}
Builder referringVertex(AtlasVertex referringVertex) {
......@@ -192,4 +197,12 @@ public class GraphMutationContext {
public AtlasStructType.AtlasAttribute getAttribute() {
return attribute;
}
public EntityMutations.EntityOperation getOp() {
return op;
}
public void setExistingEdge(final Optional<AtlasEdge> existingEdge) {
this.existingEdge = existingEdge;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java
......@@ -105,7 +105,7 @@ public class IDBasedEntityResolver implements EntityResolver {
if ( vertex != null ) {
return Optional.of(vertex);
} else {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, "Could not find an entity with the specified guid " + typeIdPair.getGuid() + " in Atlas respository");
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, typeIdPair.getGuid());
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java
......@@ -97,13 +97,16 @@ public class MapVertexMapper implements InstanceGraphMapper<Map> {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), keyStr);
Optional<AtlasEdge> existingEdge = getEdgeIfExists(mapType, currentMap, keyStr);
GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getAttribute(), mapType.getValueType(), entry.getValue())
GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(), mapType.getValueType(), entry.getValue())
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
.vertexProperty(propertyNameForKey).build();
//Add/Update/Remove property value
Object newEntry = structVertexMapper.mapCollectionElementsToVertex(mapCtx);
MapVertexMapper.setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey, newEntry);
newMap.put(keyStr, newEntry);
}
}
......@@ -192,7 +195,9 @@ public class MapVertexMapper implements InstanceGraphMapper<Map> {
private Optional<AtlasEdge> getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) {
Optional<AtlasEdge> existingEdge = Optional.absent();
if ( AtlasGraphUtilsV1.isReference(mapType.getValueType()) ) {
existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
if ( currentMap.get(keyStr) != null) {
existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
}
}
return existingEdge;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/UniqAttrBasedEntityResolver.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasStructDef;
......@@ -98,7 +99,12 @@ public class UniqAttrBasedEntityResolver implements EntityResolver {
}
Optional<AtlasVertex> resolveByUniqueAttribute(AtlasEntity entity) throws AtlasBaseException {
AtlasEntityType entityType = (AtlasEntityType) typeRegistry.getType(entity.getTypeName());
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
for (AtlasStructType.AtlasAttribute attr : entityType.getAllAttributes().values()) {
if (attr.getAttributeDef().getIsUnique()) {
Object attrVal = entity.getAttribute(attr.getAttributeDef().getName());
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java
......@@ -145,8 +145,8 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
ret = new Referenceable(entity.getGuid(), entity.getTypeName(),
fromV2ToV1(entityType, entity.getAttributes()));
} else if (v2Obj instanceof String) { // transient-id
ret = new Referenceable((String) v2Obj, type.getTypeName(), null);
} else if (v2Obj instanceof AtlasObjectId) { // transient-id
ret = new Referenceable(((AtlasObjectId) v2Obj).getGuid(), type.getTypeName(), null);
} else {
throw new AtlasBaseException(AtlasErrorCode.UNEXPECTED_TYPE, "Map or AtlasEntity or String",
v2Obj.getClass().getCanonicalName());
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasInstanceRestAdapters.java
......@@ -31,6 +31,8 @@ import org.apache.atlas.model.instance.AtlasEntityWithAssociations;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IReferenceableInstance;
......@@ -106,7 +108,10 @@ public class AtlasInstanceRestAdapters {
public AtlasClassification getClassification(IStruct classification) throws AtlasBaseException {
AtlasFormatConverter converter = instanceFormatters.getConverter(TypeCategory.CLASSIFICATION);
AtlasType classificationType = typeRegistry.getType(classification.getTypeName());
AtlasClassificationType classificationType = typeRegistry.getClassificationTypeByName(classification.getTypeName());
if (classificationType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.CLASSIFICATION.name(), classification.getTypeName());
}
AtlasClassification ret = (AtlasClassification)converter.fromV1ToV2(classification, classificationType);
return ret;
......@@ -114,7 +119,11 @@ public class AtlasInstanceRestAdapters {
public AtlasEntityWithAssociations getAtlasEntity(IReferenceableInstance referenceable) throws AtlasBaseException {
AtlasFormatConverter converter = instanceFormatters.getConverter(TypeCategory.ENTITY);
AtlasType entityType = typeRegistry.getType(referenceable.getTypeName());
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(referenceable.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), referenceable.getTypeName());
}
AtlasEntityWithAssociations ret = (AtlasEntityWithAssociations)converter.fromV1ToV2(referenceable, entityType);
return ret;
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -190,12 +190,12 @@ public class TestEntitiesREST {
if ( retrievedDBEntity != null) {
LOG.info("verifying entity of type {} ", dbEntity.getTypeName());
verifyAttributes(dbEntity.getAttributes(), retrievedDBEntity.getAttributes());
verifyAttributes(retrievedDBEntity.getAttributes(), dbEntity.getAttributes());
}
if ( retrievedColumnEntity != null) {
LOG.info("verifying entity of type {} ", columns.get(0).getTypeName());
verifyAttributes(columns.get(0).getAttributes(), retrievedColumnEntity.getAttributes());
verifyAttributes(retrievedColumnEntity.getAttributes(), columns.get(0).getAttributes());
}
if ( retrievedTableEntity != null) {
......@@ -216,10 +216,13 @@ public class TestEntitiesREST {
}
}
public static void verifyAttributes(Map<String, Object> sourceAttrs, Map<String, Object> targetAttributes) throws Exception {
for (String name : sourceAttrs.keySet() ) {
public static void verifyAttributes(Map<String, Object> actual, Map<String, Object> expected) throws Exception {
for (String name : actual.keySet() ) {
LOG.info("verifying attribute {} ", name);
Assert.assertEquals(targetAttributes.get(name), sourceAttrs.get(name));
if ( expected.get(name) != null) {
Assert.assertEquals(actual.get(name), expected.get(name));
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment