Skip to content

A
atlas
Commit ce20d6f5 by Madhan Neethiraj
Options

ATLAS-1544: implementation of REST endpoints for entity create/update/bulk-get

parent 414b7bbc
Showing with 758 additions and 1096 deletions
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -64,6 +64,7 @@ public enum AtlasErrorCode {
UNRESOLVED_REFERENCES_FOUND(400, "ATLAS40033E", "Unresolved references: byId={0}; byUniqueAttributes={1}"),
UNKNOWN_ATTRIBUTE(400, "ATLAS40034E", "Attribute {0} not found for type {1}"),
SYSTEM_TYPE(400, "ATLAS40035E", "{0} is a System-type"),
INVALID_STRUCT_VALUE(400, "ATLAS40036E", "not a valid struct value {0}"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS4041E", "Given typename {0} was invalid"),
......@@ -74,8 +75,8 @@ public enum AtlasErrorCode {
INSTANCE_LINEAGE_QUERY_FAILED(404, "ATLAS4047E", "Instance lineage query failed {0}"),
DISCOVERY_QUERY_FAILED(404, "ATLAS4048E", "Discovery query failed {0}"),
INSTANCE_CRUD_INVALID_PARAMS(404, "ATLAS4049E", "Invalid instance creation/updation parameters passed : {0}"),
INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS40410E", "Instance {0} with unique attribute {1}={2} does not exist"),
INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS40410E", "Instance {0} with unique attribute {1} does not exist"),
REFERENCED_ENTITY_NOT_FOUND(404, "ATLAS40411E", "Referenced entity {0} is not found"),
// All data conflict errors go here
TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
......@@ -34,6 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
......@@ -87,16 +88,16 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this(entityDef != null ? entityDef.getName() : null, null);
}
public AtlasEntity(String typeName, String attrName, Object attrValue) {
super(typeName, attrName, attrValue);
init();
}
public AtlasEntity(String typeName, Map<String, Object> attributes) {
super(typeName, attributes);
setGuid(nextInternalId());
setStatus(null);
setCreatedBy(null);
setUpdatedBy(null);
setCreateTime(null);
setUpdateTime(null);
setClassifications(null);
init();
}
public AtlasEntity(AtlasEntity other) {
......@@ -186,13 +187,15 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
@JsonIgnore
public static boolean isAssigned(String guid) {
try {
UUID.fromString(guid);
} catch (IllegalArgumentException e) {
return false;
if (guid != null) {
try {
UUID.fromString(guid);
return true;
} catch (IllegalArgumentException e) {
// ignore
}
}
return true;
return false;
}
@JsonIgnore
......@@ -200,13 +203,23 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
return guid != null && guid.length() > 0 && guid.charAt(0) == '-';
}
private void init() {
setGuid(nextInternalId());
setStatus(null);
setCreatedBy(null);
setUpdatedBy(null);
setCreateTime(null);
setUpdateTime(null);
setClassifications(null);
}
private static String nextInternalId() {
return "-" + Long.toString(s_nextId.getAndIncrement());
}
@JsonIgnore
public AtlasObjectId getAtlasObjectId() {
return new AtlasObjectId(getTypeName(), getGuid());
return new AtlasObjectId(getGuid(), getTypeName());
}
@Override
......@@ -279,6 +292,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setReferredEntities(null);
}
public AtlasEntityExtInfo(AtlasEntity referredEntity) {
addReferredEntity(referredEntity);
}
public AtlasEntityExtInfo(Map<String, AtlasEntity> referredEntities) {
setReferredEntities(referredEntities);
}
......@@ -294,6 +311,11 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
public void setReferredEntities(Map<String, AtlasEntity> referredEntities) { this.referredEntities = referredEntities; }
@JsonIgnore
public final void addReferredEntity(AtlasEntity entity) {
addReferredEntity(entity.getGuid(), entity);
}
@JsonIgnore
public final void addReferredEntity(String guid, AtlasEntity entity) {
Map<String, AtlasEntity> r = this.referredEntities;
......@@ -494,10 +516,17 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this(null, null);
}
public AtlasEntitiesWithExtInfo(AtlasEntity entity) { this(Arrays.asList(entity), null);
}
public AtlasEntitiesWithExtInfo(List<AtlasEntity> entities) {
this(entities, null);
}
public AtlasEntitiesWithExtInfo(AtlasEntityWithExtInfo entity) {
this(Arrays.asList(entity.getEntity()), entity);
}
public AtlasEntitiesWithExtInfo(List<AtlasEntity> entities, AtlasEntityExtInfo extInfo) {
super(extInfo);
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasObjectId.java
......@@ -20,6 +20,7 @@ package org.apache.atlas.model.instance;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Objects;
import javax.xml.bind.annotation.XmlAccessType;
......@@ -51,88 +52,83 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
public class AtlasObjectId implements Serializable {
private static final long serialVersionUID = 1L;
public static final String KEY_TYPENAME = "typeName";
public static final String KEY_GUID = "guid";
public static final String KEY_TYPENAME = "typeName";
public static final String KEY_UNIQUE_ATTRIBUTES = "uniqueAttributes";
private String typeName;
private String guid;
private String typeName;
private Map<String, Object> uniqueAttributes;
@JsonIgnore
private boolean isAssignedGuid = false;
@JsonIgnore
private boolean isUnAssignedGuid = false;
public AtlasObjectId() {
this(null, null, null);
this(null, null, (Map<String, Object>)null);
}
public AtlasObjectId(String typeName) {
this(typeName, null, null);
public AtlasObjectId(String guid) {
this(guid, null, (Map<String, Object>)null);
}
public AtlasObjectId(String typeName, String guid) {
this(typeName, guid, null);
public AtlasObjectId(String guid, String typeName) {
this(guid, typeName, (Map<String, Object>)null);
}
public AtlasObjectId(String typeName, Map<String, Object> uniqueAttributes) {
this(typeName, null, uniqueAttributes);
this(null, typeName, uniqueAttributes);
}
public AtlasObjectId(String typeName, String guid, Map<String, Object> uniqueAttributes) {
setTypeName(typeName);
public AtlasObjectId(String typeName, final String attrName, final Object attrValue) {
this(null, typeName, new HashMap<String, Object>() {{ put(attrName, attrValue); }});
}
public AtlasObjectId(String guid, String typeName, Map<String, Object> uniqueAttributes) {
setGuid(guid);
setTypeName(typeName);
setUniqueAttributes(uniqueAttributes);
}
public AtlasObjectId(AtlasObjectId other) {
if (other != null) {
setTypeName(other.getTypeName());
setGuid(other.getGuid());
setTypeName(other.getTypeName());
setUniqueAttributes(other.getUniqueAttributes());
}
}
public AtlasObjectId(Map objIdMap) {
if (objIdMap != null) {
Object t = objIdMap.get(KEY_TYPENAME);
Object g = objIdMap.get(KEY_GUID);
Object t = objIdMap.get(KEY_TYPENAME);
Object u = objIdMap.get(KEY_UNIQUE_ATTRIBUTES);
if (t != null) {
setTypeName(t.toString());
}
if (g != null) {
setGuid(g.toString());
}
if (t != null) {
setTypeName(t.toString());
}
if (u != null && u instanceof Map) {
setUniqueAttributes((Map)u);
}
}
}
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public String getGuid() {
return guid;
}
public void setGuid(String guid) {
this.guid = guid;
if ( guid != null) {
this.isAssignedGuid = AtlasEntity.isAssigned(guid);
this.isUnAssignedGuid = AtlasEntity.isUnAssigned(guid);
}
}
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public Map<String, Object> getUniqueAttributes() {
......@@ -144,28 +140,29 @@ public class AtlasObjectId implements Serializable {
}
@JsonIgnore
public boolean isValidGuid() {
return isAssignedGuid() || isUnAssignedGuid();
}
@JsonIgnore
public boolean isAssignedGuid() {
return isAssignedGuid;
return AtlasEntity.isAssigned(guid);
}
@JsonIgnore
public boolean isUnAssignedGuid() {
return isUnAssignedGuid;
return AtlasEntity.isUnAssigned(guid);
}
@JsonIgnore
public boolean isValid() {
if (StringUtils.isEmpty(typeName)) {
return false;
} else if (StringUtils.isNotEmpty(guid)) {
if (!isAssignedGuid && !isUnAssignedGuid) {
return false;
}
} else if (MapUtils.isEmpty(uniqueAttributes)) {
return false;
if (isAssignedGuid() || isUnAssignedGuid()) {
return true;
} else if (StringUtils.isNotEmpty(typeName) && MapUtils.isNotEmpty(uniqueAttributes)) {
return true;
}
return true;
return false;
}
public StringBuilder toString(StringBuilder sb) {
......@@ -174,8 +171,8 @@ public class AtlasObjectId implements Serializable {
}
sb.append("AtlasObjectId{");
sb.append("typeName='").append(typeName).append('\'');
sb.append(", guid='").append(guid).append('\'');
sb.append("guid='").append(guid).append('\'');
sb.append(", typeName='").append(typeName).append('\'');
sb.append(", uniqueAttributes={");
AtlasBaseTypeDef.dumpObjects(uniqueAttributes, sb);
sb.append('}');
......@@ -186,17 +183,27 @@ public class AtlasObjectId implements Serializable {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
AtlasObjectId that = (AtlasObjectId) o;
if (isValidGuid() && Objects.equals(guid, that.guid)) {
return true;
}
return Objects.equals(typeName, that.typeName) &&
Objects.equals(guid, that.guid) &&
Objects.equals(uniqueAttributes, that.uniqueAttributes);
Objects.equals(uniqueAttributes, that.uniqueAttributes);
}
@Override
public int hashCode() {
return Objects.hash(typeName, guid, uniqueAttributes);
return isValidGuid() ? Objects.hash(guid) : Objects.hash(typeName, uniqueAttributes);
}
@Override
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
......@@ -81,7 +81,7 @@ public class AtlasStruct implements Serializable {
public AtlasStruct(AtlasStruct other) {
if (other != null) {
setTypeName(other.getTypeName());
setAttributes(other.getAttributes());
setAttributes(new HashMap<>(other.getAttributes()));
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasArrayType.java
......@@ -233,7 +233,7 @@ public class AtlasArrayType extends AtlasType {
ret = false;
messages.add(objName + ": incorrect number of values. found=" + objList.size()
+ "; expected: minCount=" + minCount + ", maxCount=" + maxCount);
+ "; expected: minCount=" + minCount + ", maxCount=" + maxCount);
}
int idx = 0;
......@@ -248,7 +248,7 @@ public class AtlasArrayType extends AtlasType {
ret = false;
messages.add(objName + ": incorrect number of values. found=" + arrayLen
+ "; expected: minCount=" + minCount + ", maxCount=" + maxCount);
+ "; expected: minCount=" + minCount + ", maxCount=" + maxCount);
}
for (int i = 0; i < arrayLen; i++) {
......
intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
......@@ -519,7 +519,7 @@ public class AtlasBuiltInTypes {
@Override
public AtlasObjectId createDefaultValue() {
return new AtlasObjectId(AtlasBaseTypeDef.ATLAS_TYPE_ASSET, "test");
return new AtlasObjectId("-1", AtlasBaseTypeDef.ATLAS_TYPE_ASSET);
}
@Override
......@@ -551,13 +551,16 @@ public class AtlasBuiltInTypes {
}
private boolean isValidMap(Map map) {
if (map.containsKey(AtlasObjectId.KEY_TYPENAME)) {
if (map.containsKey(AtlasObjectId.KEY_GUID)) {
return true;
} else {
Object guid = map.get(AtlasObjectId.KEY_GUID);
if (guid != null && StringUtils.isNotEmpty(guid.toString())) {
return true;
} else {
Object typeName = map.get(AtlasObjectId.KEY_TYPENAME);
if (typeName != null && StringUtils.isNotEmpty(typeName.toString())) {
Object uniqueAttributes = map.get(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES);
if (uniqueAttributes instanceof Map && MapUtils.isNotEmpty((Map)uniqueAttributes)) {
if (uniqueAttributes instanceof Map && MapUtils.isNotEmpty((Map) uniqueAttributes)) {
return true;
}
}
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -190,6 +190,9 @@ public class AtlasEntityType extends AtlasStructType {
if (obj instanceof AtlasObjectId) {
AtlasObjectId objId = (AtlasObjectId ) obj;
return isAssignableFrom(objId);
} else if (obj instanceof Map) {
AtlasObjectId objId = new AtlasObjectId((Map)obj);
return isAssignableFrom(objId);
}
for (AtlasEntityType superType : superTypes) {
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -254,7 +254,7 @@ public class AtlasStructType extends AtlasType {
}
}
} else if (obj instanceof Map) {
Map map = (Map) obj;
Map attributes = AtlasTypeUtil.toStructAttributes((Map)obj);
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
String attrName = attributeDef.getName();
......@@ -262,7 +262,7 @@ public class AtlasStructType extends AtlasType {
if (attribute != null) {
AtlasType dataType = attribute.getAttributeType();
Object value = map.get(attrName);
Object value = attributes.get(attrName);
String fieldName = objName + "." + attrName;
if (value != null) {
......@@ -408,14 +408,16 @@ public class AtlasStructType extends AtlasType {
private final AtlasType attributeType;
private final AtlasAttributeDef attributeDef;
private final String qualifiedName;
private final String vertexPropertyName;
private final boolean isOwnedRef;
private final String inverseRefAttribute;
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
this.definedInType = definedInType;
this.attributeDef = attrDef;
this.attributeType = attributeType;
this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
this.definedInType = definedInType;
this.attributeDef = attrDef;
this.attributeType = attributeType;
this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
this.vertexPropertyName = encodePropertyKey(this.qualifiedName);
boolean isOwnedRef = false;
String inverseRefAttribute = null;
......@@ -458,17 +460,47 @@ public class AtlasStructType extends AtlasType {
public String getQualifiedName() { return qualifiedName; }
public String getQualifiedAttributeName() {
return qualifiedName;
}
public String getVertexPropertyName() { return vertexPropertyName; }
public boolean isOwnedRef() { return isOwnedRef; }
public String getInverseRefAttribute() { return inverseRefAttribute; }
public static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
final String typeName = structDef.getName();
return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName);
}
private static String encodePropertyKey(String key) {
if (StringUtils.isBlank(key)) {
return key;
}
for (String[] strMap : RESERVED_CHAR_ENCODE_MAP) {
key = key.replace(strMap[0], strMap[1]);
}
return key;
}
private static String decodePropertyKey(String key) {
if (StringUtils.isBlank(key)) {
return key;
}
for (String[] strMap : RESERVED_CHAR_ENCODE_MAP) {
key = key.replace(strMap[1], strMap[0]);
}
return key;
}
private static String[][] RESERVED_CHAR_ENCODE_MAP = new String[][] {
new String[] { "{", "_o" },
new String[] { "}", "_c" },
new String[] { "\"", "_q" },
new String[] { "$", "_d" },
new String[] { "%", "_p" },
};
}
}
intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
......@@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
......@@ -35,14 +37,16 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
......@@ -298,4 +302,26 @@ public class AtlasTypeUtil {
return headerList;
}
public static Collection<AtlasObjectId> toObjectIds(Collection<AtlasEntity> entities) {
List<AtlasObjectId> ret = new ArrayList<>();
if (CollectionUtils.isNotEmpty(entities)) {
for (AtlasEntity entity : entities) {
if (entity != null) {
ret.add(entity.getAtlasObjectId());
}
}
}
return ret;
}
public static Map toStructAttributes(Map map) {
if (map != null && map.containsKey("typeName") && map.containsKey("attributes") && map.get("attributes") instanceof Map) {
return (Map)map.get("attributes");
}
return map;
}
}
\ No newline at end of file
intg/src/test/java/org/apache/atlas/type/TestAtlasObjectIdType.java
......@@ -45,37 +45,41 @@ public class TestAtlasObjectIdType {
Map<Object, Object> invalidObj2 = new HashMap<>();
Map<Object, Object> invalidObj3 = new HashMap<>();
Map<Object, Object> invalidObj4 = new HashMap<>();
Map<Object, Object> invalidObj5 = new HashMap<>();
validObj1.put(AtlasObjectId.KEY_TYPENAME, "testType");
validObj1.put(AtlasObjectId.KEY_GUID, "guid-1234");
validObj2.put(AtlasObjectId.KEY_TYPENAME, "testType");
validObj2.put(AtlasObjectId.KEY_GUID, 1234);
Map<String, Object> uniqAttribs = new HashMap<String, Object>();
uniqAttribs.put("name", "testTypeInstance-1");
// guid
validObj1.put(AtlasObjectId.KEY_GUID, "guid-1234");
// typeName & unique-attributes
validObj2.put(AtlasObjectId.KEY_TYPENAME, "testType");
validObj2.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, uniqAttribs);
// guid, typeName & unique-attributes
validObj3.put(AtlasObjectId.KEY_GUID, "guid-1234");
validObj3.put(AtlasObjectId.KEY_TYPENAME, "testType");
validObj3.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, uniqAttribs);
invalidObj1.put(AtlasObjectId.KEY_TYPENAME, "testType"); // no guid
invalidObj2.put(AtlasObjectId.KEY_GUID, "guid-1234"); // no typeName or uniqueAttribute
invalidObj2.put(AtlasObjectId.KEY_TYPENAME + "-invalid", "testType");
// no guid or typeName/unique-attributes
invalidObj1.put(AtlasObjectId.KEY_GUID + "-invalid", "guid-1234"); // no guid or typename or uniqueAttribute
invalidObj3.put(AtlasObjectId.KEY_GUID + "-invalid", "guid-1234"); // no guid or typename or uniqueAttribute
// no unique-attributes
invalidObj2.put(AtlasObjectId.KEY_TYPENAME, "testType"); // no guid
invalidObj4.put(AtlasObjectId.KEY_TYPENAME, "testType"); // empty uniqueAttribute
invalidObj4.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new HashMap<String, Object>());
// empty uniqueAttribute
invalidObj3.put(AtlasObjectId.KEY_TYPENAME, "testType");
invalidObj3.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new HashMap<String, Object>());
invalidObj5.put(AtlasObjectId.KEY_TYPENAME, "testType"); // non-map uniqueAttribute
invalidObj5.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new ArrayList<String>());
// non-map uniqueAttribute
invalidObj4.put(AtlasObjectId.KEY_TYPENAME, "testType");
invalidObj4.put(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES, new ArrayList<String>());
validValues = new Object[] {
null, validObj1, validObj2, validObj3, new AtlasObjectId(), new AtlasObjectId("testType", "guid-1234"), };
null, validObj1, validObj2, validObj3, new AtlasObjectId(), new AtlasObjectId("guid-1234", "testType"), };
invalidValues = new Object[] {
invalidObj1, invalidObj2, invalidObj3, invalidObj4, invalidObj5,
invalidObj1, invalidObj2, invalidObj3, invalidObj4,
Byte.valueOf((byte)1), Short.valueOf((short)1), Integer.valueOf(1),
Long.valueOf(1L), Float.valueOf(1), Double.valueOf(1), BigInteger.valueOf(1), BigDecimal.valueOf(1), "1",
"", "12ab", "abcd", "-12ab",
......@@ -116,12 +120,12 @@ public class TestAtlasObjectIdType {
if (value instanceof AtlasObjectId) {
assertEquals(normalizedValue, value, "value=" + value);
} else if (value instanceof Map) {
assertEquals(normalizedValue.getTypeName(), ((Map)value).get(AtlasObjectId.KEY_TYPENAME).toString(),
assertEquals(normalizedValue.getTypeName(), ((Map)value).get(AtlasObjectId.KEY_TYPENAME),
"value=" + value);
if (((Map)value).get(AtlasObjectId.KEY_GUID) == null) {
assertEquals(normalizedValue.getGuid(), ((Map)value).get(AtlasObjectId.KEY_GUID), "value=" + value);
} else {
assertEquals(normalizedValue.getGuid().toString(), ((Map) value).get(AtlasObjectId.KEY_GUID).toString(), "value=" + value);
assertEquals(normalizedValue.getGuid(), ((Map) value).get(AtlasObjectId.KEY_GUID), "value=" + value);
}
assertEquals(normalizedValue.getUniqueAttributes(), ((Map)value).get(AtlasObjectId.KEY_UNIQUE_ATTRIBUTES),
......
repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
......@@ -42,17 +42,10 @@ import org.apache.atlas.repository.graph.DeleteHandler;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.repository.store.graph.v1.ArrayVertexMapper;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityGraphDiscoveryV1;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasTypeDefGraphStoreV1;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
import org.apache.atlas.repository.store.graph.v1.IDBasedEntityResolver;
import org.apache.atlas.repository.store.graph.v1.MapVertexMapper;
import org.apache.atlas.repository.store.graph.v1.UniqAttrBasedEntityResolver;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service;
......@@ -121,15 +114,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bind(EntityGraphMapper.class);
bind(MapVertexMapper.class).asEagerSingleton();
bind(ArrayVertexMapper.class).asEagerSingleton();
Multibinder<EntityResolver> entityRefResolver =
Multibinder.newSetBinder(binder(), EntityResolver.class);
entityRefResolver.addBinding().to(IDBasedEntityResolver.class);
entityRefResolver.addBinding().to(UniqAttrBasedEntityResolver.class);
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
Multibinder.newSetBinder(binder(), EntityChangeListener.class);
......@@ -138,8 +122,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
MethodInterceptor interceptor = new GraphTransactionInterceptor();
requestInjection(interceptor);
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
bind(EntityGraphDiscovery.class).to(AtlasEntityGraphDiscoveryV1.class);
}
protected Configuration getConfiguration() {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -244,7 +244,7 @@ public final class GraphHelper {
/**
Iterable<AtlasEdge> edges = outVertex.getEdges(Direction.OUT, edgeLabel);
for (AtlasEdge edge : edges) {
if (edge.getId().toString().equals(edgeId)) {
if (edge.getObjectId().toString().equals(edgeId)) {
return edge;
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -86,39 +86,7 @@ public class AtlasTypeDefStoreInitializer {
continue;
}
AtlasTypesDef typesToCreate = new AtlasTypesDef();
if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
if (!typeRegistry.isRegisteredType(enumDef.getName())) {
typesToCreate.getEnumDefs().add(enumDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
if (!typeRegistry.isRegisteredType(structDef.getName())) {
typesToCreate.getStructDefs().add(structDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
if (!typeRegistry.isRegisteredType(classificationDef.getName())) {
typesToCreate.getClassificationDefs().add(classificationDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
if (!typeRegistry.isRegisteredType(entityDef.getName())) {
typesToCreate.getEntityDefs().add(entityDef);
}
}
}
AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, typeRegistry);
if (typesToCreate.isEmpty()) {
LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
......@@ -137,6 +105,44 @@ public class AtlasTypeDefStoreInitializer {
applyTypePatches(typeDefStore, typeRegistry, typesDirName);
}
public static AtlasTypesDef getTypesToCreate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) {
AtlasTypesDef typesToCreate = new AtlasTypesDef();
if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
if (!typeRegistry.isRegisteredType(enumDef.getName())) {
typesToCreate.getEnumDefs().add(enumDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
if (!typeRegistry.isRegisteredType(structDef.getName())) {
typesToCreate.getStructDefs().add(structDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
if (!typeRegistry.isRegisteredType(classificationDef.getName())) {
typesToCreate.getClassificationDefs().add(classificationDef);
}
}
}
if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
if (!typeRegistry.isRegisteredType(entityDef.getName())) {
typesToCreate.getEntityDefs().add(entityDef);
}
}
}
return typesToCreate;
}
private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
String typePatchesDirName = typesDirName + File.separator + "patches";
File typePatchesDir = new File(typePatchesDirName);
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
......@@ -24,8 +24,8 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.store.graph.v1.EntityStream;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import java.util.List;
import java.util.Map;
......@@ -34,13 +34,6 @@ import java.util.Map;
* Persistence/Retrieval API for AtlasEntity
*/
public interface AtlasEntityStore {
/**
* Initialization
*/
void init(AtlasTypeRegistry typeRegistry) throws AtlasBaseException;
/**
*
* Get entity definition by its guid
......@@ -50,22 +43,6 @@ public interface AtlasEntityStore {
AtlasEntityWithExtInfo getById(String guid) throws AtlasBaseException;
/**
* Delete an entity by its guid
* @param guid
* @return
*/
EntityMutationResponse deleteById(String guid);
/**
* Create or update entities
* @param entities Map of the entity Id(guid or transient Id) to AtlasEntity objects that need to be created
* @return EntityMutationResponse Entity mutations operations with the correspomding set of entities on which these operations were performed
* @throws AtlasBaseException
*/
EntityMutationResponse createOrUpdate(Map<String, AtlasEntity> entities) throws AtlasBaseException;
/**
* Batch GET to retrieve entities by their ID
* @param guid
* @return
......@@ -73,43 +50,57 @@ public interface AtlasEntityStore {
*/
AtlasEntitiesWithExtInfo getByIds(List<String> guid) throws AtlasBaseException;
/*
* Return list of deleted entity guids
*/
EntityMutationResponse deleteByIds(List<String> guid) throws AtlasBaseException;
/**
*
* Get an eneity by its unique attribute
* @param entityType
* @param uniqAttributes
* @return AtlasEntity
* @param entityType type of the entity
* @param uniqAttributes Attributes that uniquely identify the entity
* @return EntityMutationResponse details of the updates performed by this call
*/
AtlasEntityWithExtInfo getByUniqueAttribute(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException;
AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException;
/**
* Create or update entities in the stream
* @param entityStream AtlasEntityStream
* @return EntityMutationResponse Entity mutations operations with the corresponding set of entities on which these operations were performed
* @throws AtlasBaseException
*/
EntityMutationResponse createOrUpdate(EntityStream entityStream) throws AtlasBaseException;
/**
* @deprecated
* Create or update a single entity
* @param typeName The entity's type
* @param attributeName Attribute that uniquely identifies the entity
* @param attributeValue The unqiue attribute's value
* @return EntityMutationResponse Entity mutations operations with the correspomding set of entities on which these operations were performed
* Update a single entity
* @param entityType type of the entity
* @param uniqAttributes Attributes that uniquely identify the entity
* @return EntityMutationResponse details of the updates performed by this call
* @throws AtlasBaseException
*
*/
EntityMutationResponse updateByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes,
AtlasEntity entity) throws AtlasBaseException;
EntityMutationResponse updateByUniqueAttribute(String typeName, String attributeName, String attributeValue, AtlasEntity entity) throws AtlasBaseException;
/**
* Delete an entity by its guid
* @param guid
* @return
*/
EntityMutationResponse deleteById(String guid) throws AtlasBaseException;
/**
* @deprecated
* @param typeName
* @param attributeName
* @param attributeValue
* @return
* @param entityType type of the entity
* @param uniqAttributes Attributes that uniquely identify the entity
* @return EntityMutationResponse details of the updates performed by this call
* @throws AtlasBaseException
*/
EntityMutationResponse deleteByUniqueAttribute(String typeName, String attributeName, String attributeValue) throws AtlasBaseException;
EntityMutationResponse deleteByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
throws AtlasBaseException;
/*
* Return list of deleted entity guids
*/
EntityMutationResponse deleteByIds(List<String> guid) throws AtlasBaseException;
/**
* Add classification(s)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscovery.java
......@@ -20,8 +20,10 @@ package org.apache.atlas.repository.store.graph;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
import java.util.List;
import java.util.Map;
public interface EntityGraphDiscovery {
......@@ -33,7 +35,7 @@ public interface EntityGraphDiscovery {
* Resolved references already exist in the ATLAS repository and have an assigned unique GUID
* Unresolved attribute references result in an error if they are not composite (managed by a parent entity)
*/
EntityGraphDiscoveryContext discoverEntities(List<AtlasEntity> entities) throws AtlasBaseException;
EntityGraphDiscoveryContext discoverEntities() throws AtlasBaseException;
void cleanUp() throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java
......@@ -17,179 +17,117 @@
*/
package org.apache.atlas.repository.store.graph;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.EntityStream;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public final class EntityGraphDiscoveryContext {
private static final Logger LOG = LoggerFactory.getLogger(EntityGraphDiscoveryContext.class);
private final AtlasTypeRegistry typeRegistry;
private List<AtlasEntity> rootEntities = new ArrayList<>();
private Map<AtlasObjectId, AtlasVertex> resolvedIds = new LinkedHashMap<>();
private Set<AtlasObjectId> unresolvedIds = new HashSet<>();
private List<AtlasObjectId> unresolvedIdsByUniqAttribs = new ArrayList<>();
private final AtlasTypeRegistry typeRegistry;
private final EntityStream entityStream;
private final List<String> referencedGuids = new ArrayList<>();
private final Set<AtlasObjectId> referencedByUniqAttribs = new HashSet<>();
private final Map<String, AtlasVertex> resolvedGuids = new HashMap<>();
private final Map<AtlasObjectId, AtlasVertex> resolvedIdsByUniqAttribs = new HashMap<>();
private final Set<String> localGuids = new HashSet<>();
public EntityGraphDiscoveryContext(AtlasTypeRegistry typeRegistry) {
public EntityGraphDiscoveryContext(AtlasTypeRegistry typeRegistry, EntityStream entityStream) {
this.typeRegistry = typeRegistry;
this.entityStream = entityStream;
}
public Collection<AtlasEntity> getRootEntities() {
return rootEntities;
public EntityStream getEntityStream() {
return entityStream;
}
public Map<AtlasObjectId, AtlasVertex> getResolvedIds() {
return resolvedIds;
}
public List<String> getReferencedGuids() { return referencedGuids; }
public Set<AtlasObjectId> getUnresolvedIds() {
return unresolvedIds;
}
public Set<AtlasObjectId> getReferencedByUniqAttribs() { return referencedByUniqAttribs; }
public List<AtlasObjectId> getUnresolvedIdsByUniqAttribs() {
return unresolvedIdsByUniqAttribs;
public Map<String, AtlasVertex> getResolvedGuids() {
return resolvedGuids;
}
public void addRootEntity(AtlasEntity rootEntity) {
this.rootEntities.add(rootEntity);
public Map<AtlasObjectId, AtlasVertex> getResolvedIdsByUniqAttribs() {
return resolvedIdsByUniqAttribs;
}
public Set<String> getLocalGuids() { return localGuids; }
public void addResolvedId(AtlasObjectId objId, AtlasVertex vertex) {
if (LOG.isDebugEnabled()) {
LOG.debug("addResolvedId({})", objId);
}
resolvedIds.put(objId, vertex);
}
public boolean removeUnResolvedId(AtlasObjectId objId) {
if (LOG.isDebugEnabled()) {
LOG.debug("removeUnResolvedId({})", objId);
public void addReferencedGuid(String guid) {
if (! referencedGuids.contains(guid)) {
referencedGuids.add(guid);
}
return unresolvedIds.remove(objId);
}
public void addReferencedByUniqAttribs(AtlasObjectId objId) { referencedByUniqAttribs.add(objId); }
public void addUnResolvedId(AtlasObjectId objId) {
if (LOG.isDebugEnabled()) {
LOG.debug("addUnResolvedId({})", objId);
}
this.unresolvedIds.add(objId);
}
public void addResolvedGuid(String guid, AtlasVertex vertex) { resolvedGuids.put(guid, vertex); }
public boolean removeUnResolvedIds(List<AtlasObjectId> objIds) {
if (LOG.isDebugEnabled()) {
LOG.debug("removeUnResolvedIds({})", objIds);
}
public void addResolvedIdByUniqAttribs(AtlasObjectId objId, AtlasVertex vertex) { resolvedIdsByUniqAttribs.put(objId, vertex); }
return unresolvedIds.removeAll(objIds);
}
public void addLocalGuidReference(String guid) { localGuids.add(guid); }
public boolean isResolvedGuid(String guid) { return resolvedGuids.containsKey(guid); }
public void addUnresolvedIdByUniqAttribs(AtlasObjectId objId) {
if (LOG.isDebugEnabled()) {
LOG.debug("addUnresolvedIdByUniqAttribs({})", objId);
}
public boolean isResolvedIdByUniqAttrib(AtlasObjectId objId) { return resolvedIdsByUniqAttribs.containsKey(objId); }
this.unresolvedIdsByUniqAttribs.add(objId);
}
public boolean removeUnresolvedIdsByUniqAttribs(List<AtlasObjectId> objIds) {
if (LOG.isDebugEnabled()) {
LOG.debug("removeUnresolvedIdsByUniqAttribs({})", objIds);
}
public AtlasVertex getResolvedEntityVertex(String guid) throws AtlasBaseException {
AtlasVertex ret = resolvedGuids.get(guid);
return unresolvedIdsByUniqAttribs.removeAll(objIds);
return ret;
}
public boolean hasUnresolvedReferences() {
return unresolvedIdsByUniqAttribs.size() > 0 || unresolvedIds.size() > 0;
}
public boolean isResolvedId(AtlasObjectId id) {
return resolvedIds.containsKey(id);
}
public AtlasVertex getResolvedEntityVertex(AtlasObjectId ref) throws AtlasBaseException {
AtlasVertex vertex = resolvedIds.get(ref);
public AtlasVertex getResolvedEntityVertex(AtlasObjectId objId) {
AtlasVertex vertex = resolvedIdsByUniqAttribs.get(objId);
// check also for sub-types; ref={typeName=Asset; guid=abcd} should match {typeName=hive_table; guid=abcd}
if (vertex == null) {
final AtlasEntityType entityType = typeRegistry.getEntityTypeByName(ref.getTypeName());
final AtlasEntityType entityType = typeRegistry.getEntityTypeByName(objId.getTypeName());
final Set<String> allSubTypes = entityType.getAllSubTypes();
for (String subType : allSubTypes) {
AtlasObjectId subTypeObjId = new AtlasObjectId(subType, ref.getGuid(), ref.getUniqueAttributes());
AtlasObjectId subTypeObjId = new AtlasObjectId(objId.getGuid(), subType, objId.getUniqueAttributes());
vertex = resolvedIds.get(subTypeObjId);
vertex = resolvedIdsByUniqAttribs.get(subTypeObjId);
if (vertex != null) {
resolvedIds.put(ref, vertex);
resolvedIdsByUniqAttribs.put(objId, vertex);
break;
}
}
}
if (vertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_CRUD_INVALID_PARAMS,
" : Could not find an entity with " + ref.toString());
}
return vertex;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
} else if (obj == this) {
return true;
} else if (obj.getClass() != getClass()) {
return false;
} else {
EntityGraphDiscoveryContext ctx = (EntityGraphDiscoveryContext) obj;
return Objects.equals(rootEntities, ctx.getRootEntities()) &&
Objects.equals(resolvedIds, ctx.getResolvedIds()) &&
Objects.equals(unresolvedIdsByUniqAttribs, ctx.getUnresolvedIdsByUniqAttribs()) &&
Objects.equals(unresolvedIds, ctx.getUnresolvedIds());
}
}
@Override
public int hashCode() {
return Objects.hash(rootEntities, resolvedIds, unresolvedIdsByUniqAttribs, unresolvedIds);
}
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
}
sb.append("EntityGraphDiscoveryCtx{");
sb.append("rootEntities='").append(rootEntities).append('\'');
sb.append(", resolvedIds=").append(resolvedIds);
sb.append(", unresolvedIdsByUniqAttribs='").append(unresolvedIdsByUniqAttribs).append('\'');
sb.append(", unresolvedIds='").append(unresolvedIds).append('\'');
sb.append("referencedGuids=").append(referencedGuids);
sb.append(", referencedByUniqAttribs=").append(referencedByUniqAttribs);
sb.append(", resolvedGuids='").append(resolvedGuids);
sb.append(", resolvedIdsByUniqAttribs='").append(resolvedIdsByUniqAttribs);
sb.append(", localGuids='").append(localGuids);
sb.append('}');
return sb;
......@@ -201,9 +139,10 @@ public final class EntityGraphDiscoveryContext {
}
public void cleanUp() {
rootEntities.clear();
unresolvedIdsByUniqAttribs.clear();
resolvedIds.clear();
unresolvedIds.clear();
referencedGuids.clear();
referencedByUniqAttribs.clear();
resolvedGuids.clear();
resolvedIdsByUniqAttribs.clear();
localGuids.clear();
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/EntityResolver.java
......@@ -21,10 +21,5 @@ import org.apache.atlas.exception.AtlasBaseException;
public interface EntityResolver {
void init(EntityGraphDiscoveryContext entities) throws AtlasBaseException;
EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException;
void cleanUp() throws AtlasBaseException;
EntityGraphDiscoveryContext resolveEntityReferences(EntityGraphDiscoveryContext entities) throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/ArrayVertexMapper.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import static org.apache.atlas.repository.graph.GraphHelper.string;
@Singleton
public class ArrayVertexMapper implements InstanceGraphMapper<List> {
private static final Logger LOG = LoggerFactory.getLogger(ArrayVertexMapper.class);
protected final DeleteHandlerV1 deleteHandler;
protected StructVertexMapper structVertexMapper;
@Inject
public ArrayVertexMapper(DeleteHandlerV1 deleteHandler) {
this.deleteHandler = deleteHandler;
}
void init(StructVertexMapper structVertexMapper) {
this.structVertexMapper = structVertexMapper;
}
@Override
public List toGraph(GraphMutationContext ctx) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping instance to vertex {} for array attribute {}", string(ctx.getReferringVertex()), ctx.getAttrType().getTypeName());
}
List newElements = (List) ctx.getValue();
boolean newAttributeEmpty = (newElements == null || newElements.isEmpty());
AtlasArrayType arrType = (AtlasArrayType) ctx.getAttrType();
AtlasType elementType = arrType.getElementType();
List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexPropertyKey());
List<Object> newElementsCreated = new ArrayList<>();
if (!newAttributeEmpty) {
for (int index = 0; index < newElements.size(); index++) {
LOG.debug("Adding/updating element at position {}, current element {}, new element {}", index,
(currentElements != null && index < currentElements.size()) ? currentElements.get(index) : null, newElements.get(index));
Optional<AtlasEdge> existingEdge = getEdgeAt(currentElements, index, arrType.getElementType());
GraphMutationContext arrCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(),
arrType.getElementType(), newElements.get(index))
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
.vertexProperty(ctx.getVertexPropertyKey()).build();
Object newEntry = structVertexMapper.mapCollectionElementsToVertex(arrCtx);
newElementsCreated.add(newEntry);
}
}
if (AtlasGraphUtilsV1.isReference(elementType)) {
List<AtlasEdge> additionalEdges = removeUnusedArrayEntries(ctx.getParentType(), ctx.getAttributeDef(), (List) currentElements, (List) newElementsCreated, elementType);
newElementsCreated.addAll(additionalEdges);
}
// for dereference on way out
setArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexPropertyKey(), newElementsCreated);
return newElementsCreated;
}
@Override
public void cleanUp() throws AtlasBaseException {
}
//Removes unused edges from the old collection, compared to the new collection
private List<AtlasEdge> removeUnusedArrayEntries(
AtlasStructType entityType,
AtlasAttributeDef attributeDef,
List<AtlasEdge> currentEntries,
List<AtlasEdge> newEntries,
AtlasType entryType) throws AtlasBaseException {
if (currentEntries != null && !currentEntries.isEmpty()) {
LOG.debug("Removing unused entries from the old collection");
if (AtlasGraphUtilsV1.isReference(entryType)) {
Collection<AtlasEdge> edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries);
LOG.debug("Removing unused entries from the old collection - {}", edgesToRemove);
if (!edgesToRemove.isEmpty()) {
//Remove the edges for (current edges - new edges)
List<AtlasEdge> additionalElements = new ArrayList<>();
for (AtlasEdge edge : edgesToRemove) {
boolean deleteChildReferences = StructVertexMapper.shouldManageChildReferences(entityType, attributeDef.getName());
boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(),
deleteChildReferences, true);
if (!deleted) {
additionalElements.add(edge);
}
}
return additionalElements;
}
}
}
return Collections.emptyList();
}
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
return (List)instanceVertex.getListProperty(actualPropertyName, AtlasEdge.class);
}
else {
return (List)instanceVertex.getListProperty(actualPropertyName);
}
}
private Optional<AtlasEdge> getEdgeAt(List<Object> currentElements, int index, AtlasType elemType) {
Optional<AtlasEdge> existingEdge = Optional.absent();
if ( AtlasGraphUtilsV1.isReference(elemType) ) {
Object currentElement = (currentElements != null && index < currentElements.size()) ?
currentElements.get(index) : null;
if ( currentElement != null) {
existingEdge = Optional.of((AtlasEdge) currentElement);
}
}
return existingEdge;
}
private void setArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
GraphHelper.setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List) values);
}
else {
GraphHelper.setProperty(instanceVertex, actualPropertyName, values);
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStream.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import java.util.Iterator;
public class AtlasEntityStream implements EntityStream {
private AtlasEntitiesWithExtInfo entitiesWithExtInfo = new AtlasEntitiesWithExtInfo();
private Iterator<AtlasEntity> iterator;
public AtlasEntityStream() {
}
public AtlasEntityStream(AtlasEntity entity) {
this(new AtlasEntitiesWithExtInfo(entity));
}
public AtlasEntityStream(AtlasEntityWithExtInfo entityWithExtInfo) {
this(new AtlasEntitiesWithExtInfo(entityWithExtInfo));
}
public AtlasEntityStream(AtlasEntitiesWithExtInfo entitiesWithExtInfo) {
this.entitiesWithExtInfo = entitiesWithExtInfo;
this.iterator = this.entitiesWithExtInfo.getEntities().iterator();
}
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public AtlasEntity next() {
return iterator.hasNext() ? iterator.next() : null;
}
@Override
public void reset() {
this.iterator = entitiesWithExtInfo.getEntities().iterator();
}
@Override
public AtlasEntity getByGuid(String guid) {
return entitiesWithExtInfo.getEntity(guid);
}
@Override
public String toString() {
StringBuffer sb = new StringBuffer("AtlasEntityStream{");
sb.append("entitiesWithExtInfo=").append(entitiesWithExtInfo);
sb.append(", iterator=").append(iterator);
sb.append('}');
return sb.toString();
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -217,7 +217,7 @@ public class AtlasGraphUtilsV1 {
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(),
uniqAttributes.keySet().toString(), uniqAttributes.values().toString());
uniqAttributes.toString());
}
return entityVertex;
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/GraphMutationContext.java → repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java
......@@ -18,26 +18,25 @@
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import java.util.Objects;
public class GraphMutationContext {
import java.util.Objects;
private EntityMutations.EntityOperation op;
public class AttributeMutationContext {
private EntityOperation op;
/**
* Atlas Attribute
*/
private AtlasStructType.AtlasAttribute attribute;
private AtlasAttribute attribute;
/**
* Overriding type for which elements are being mapped
......@@ -49,6 +48,8 @@ public class GraphMutationContext {
*/
private Object value;
private String vertexProperty;
/**
*
* The vertex which corresponds to the entity/struct for which we are mapping a complex attributes like struct, traits
......@@ -56,34 +57,28 @@ public class GraphMutationContext {
AtlasVertex referringVertex;
/**
* the vertex property that we are updating
*/
String vertexPropertyKey;
/**
* The current edge(in case of updates) from the parent entity/struct to the complex attribute like struct, trait
*/
Optional<AtlasEdge> existingEdge;
AtlasEdge existingEdge;
private GraphMutationContext(final Builder builder) {
this.op = builder.op;
this.attribute = builder.attribute;
this.currentElementType = builder.elementType;
this.existingEdge = builder.currentEdge;
this.value = builder.currentValue;
this.referringVertex = builder.referringVertex;
this.vertexPropertyKey = builder.vertexPropertyKey;
public AttributeMutationContext(EntityOperation op, AtlasVertex referringVertex, AtlasAttribute attribute, Object value) {
this(op, referringVertex, attribute, value, attribute.getVertexPropertyName(), null, null);
}
public String getVertexPropertyKey() {
return vertexPropertyKey;
public AttributeMutationContext(EntityOperation op, AtlasVertex referringVertex, AtlasAttribute attribute, Object value,
String vertexProperty, AtlasType currentElementType, AtlasEdge currentEdge) {
this.op = op;
this.referringVertex = referringVertex;
this.attribute = attribute;
this.value = value;
this.vertexProperty = vertexProperty;
this.currentElementType = currentElementType;
this.existingEdge = currentEdge;
}
@Override
public int hashCode() {
return Objects.hash(op, attribute, value, referringVertex, vertexPropertyKey, existingEdge);
return Objects.hash(op, referringVertex, attribute, value, vertexProperty, currentElementType, existingEdge);
}
@Override
......@@ -95,70 +90,18 @@ public class GraphMutationContext {
} else if (obj.getClass() != getClass()) {
return false;
} else {
GraphMutationContext rhs = (GraphMutationContext) obj;
return Objects.equals(attribute, rhs.getAttribute())
&& Objects.equals(value, rhs.getValue())
&& Objects.equals(referringVertex, rhs.getReferringVertex())
&& Objects.equals(vertexPropertyKey, rhs.getReferringVertex())
&& Objects.equals(existingEdge, rhs.getCurrentEdge())
&& Objects.equals(op, rhs.getOp());
AttributeMutationContext rhs = (AttributeMutationContext) obj;
return Objects.equals(op, rhs.op)
&& Objects.equals(referringVertex, rhs.referringVertex)
&& Objects.equals(attribute, rhs.attribute)
&& Objects.equals(value, rhs.value)
&& Objects.equals(vertexProperty, rhs.vertexProperty)
&& Objects.equals(currentElementType, rhs.currentElementType)
&& Objects.equals(existingEdge, rhs.existingEdge);
}
}
public static final class Builder {
private final EntityMutations.EntityOperation op;
private final AtlasStructType.AtlasAttribute attribute;
private final AtlasType elementType;
private final Object currentValue;
private AtlasVertex referringVertex;
private Optional<AtlasEdge> currentEdge = Optional.absent();
private String vertexPropertyKey;
public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, AtlasType currentElementType, Object currentValue) {
this.op = op;
this.attribute = attribute;
this.elementType = currentElementType;
this.currentValue = currentValue;
}
public Builder(EntityMutations.EntityOperation op, AtlasStructType.AtlasAttribute attribute, Object currentValue) {
this(op, attribute, null, currentValue);
}
Builder referringVertex(AtlasVertex referringVertex) {
this.referringVertex = referringVertex;
return this;
}
Builder edge(AtlasEdge edge) {
this.currentEdge = Optional.of(edge);
return this;
}
Builder edge(Optional<AtlasEdge> edge) {
this.currentEdge = edge;
return this;
}
Builder vertexProperty(String propertyKey) {
this.vertexPropertyKey = propertyKey;
return this;
}
GraphMutationContext build() {
return new GraphMutationContext(this);
}
}
public AtlasStructType getParentType() {
return attribute.getDefinedInType();
}
......@@ -183,11 +126,11 @@ public class GraphMutationContext {
return value;
}
public AtlasVertex getReferringVertex() {
return referringVertex;
}
public String getVertexProperty() { return vertexProperty; }
public Optional<AtlasEdge> getCurrentEdge() {
public AtlasVertex getReferringVertex() { return referringVertex; }
public AtlasEdge getCurrentEdge() {
return existingEdge;
}
......@@ -195,15 +138,13 @@ public class GraphMutationContext {
this.currentElementType = attrType;
}
public AtlasStructType.AtlasAttribute getAttribute() {
public AtlasAttribute getAttribute() {
return attribute;
}
public EntityMutations.EntityOperation getOp() {
public EntityOperation getOp() {
return op;
}
public void setExistingEdge(final Optional<AtlasEdge> existingEdge) {
this.existingEdge = existingEdge;
}
public void setExistingEdge(AtlasEdge existingEdge) { this.existingEdge = existingEdge; }
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -98,7 +98,7 @@ public abstract class DeleteHandlerV1 {
// Record all deletion candidate GUIDs in RequestContext
// and gather deletion candidate vertices.
for (GraphHelper.VertexInfo vertexInfo : compositeVertices) {
requestContext.recordEntityDelete(new AtlasObjectId(vertexInfo.getTypeName(), vertexInfo.getGuid()));
requestContext.recordEntityDelete(new AtlasObjectId(vertexInfo.getGuid(), vertexInfo.getTypeName()));
deletionCandidateVertices.add(vertexInfo.getVertex());
}
}
......@@ -324,7 +324,7 @@ public abstract class DeleteHandlerV1 {
String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(structType, attributeInfo.getName());
if (AtlasGraphUtilsV1.isReference(valueTypeCategory)) {
List<Object> keys = ArrayVertexMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName);
List<Object> keys = EntityGraphMapper.getArrayElementsProperty(keyType, instanceVertex, propertyName);
if (keys != null) {
for (Object key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, (String) key);
......@@ -513,7 +513,7 @@ public abstract class DeleteHandlerV1 {
GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime());
GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
requestContext.recordEntityUpdate(new AtlasObjectId(typeName, outId));
requestContext.recordEntityUpdate(new AtlasObjectId(outId, typeName));
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -22,6 +22,7 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
......@@ -100,6 +101,22 @@ public final class EntityGraphRetriever {
return ret;
}
public AtlasEntitiesWithExtInfo toAtlasEntitiesWithExtInfo(List<String> guids) throws AtlasBaseException {
AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo();
for (String guid : guids) {
AtlasVertex vertex = getEntityVertex(guid);
AtlasEntity entity = mapVertexToAtlasEntity(vertex, ret);
ret.addEntity(entity);
}
ret.compact();
return ret;
}
private AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
try {
return graphHelper.getVertexForGUID(guid);
......@@ -405,7 +422,7 @@ public final class EntityGraphRetriever {
ret = entity.getAtlasObjectId();
}
} else {
ret = new AtlasObjectId(GraphHelper.getTypeName(referenceVertex), GraphHelper.getGuid(referenceVertex));
ret = new AtlasObjectId(GraphHelper.getGuid(referenceVertex), GraphHelper.getTypeName(referenceVertex));
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityMutationContext.java
......@@ -24,32 +24,37 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
import java.util.*;
public class EntityMutationContext {
private final EntityGraphDiscoveryContext context;
private final List<AtlasEntity> entitiesCreated = new ArrayList<>();
private final List<AtlasEntity> entitiesUpdated = new ArrayList<>();
private final Map<AtlasObjectId, AtlasEntityType> entityVsType = new HashMap<>();
private final Map<AtlasObjectId, AtlasVertex> entityVsVertex = new HashMap<>();
private final EntityGraphDiscoveryContext context;
private final List<AtlasEntity> entitiesCreated = new ArrayList<>();
private final List<AtlasEntity> entitiesUpdated = new ArrayList<>();
private final Map<String, AtlasEntityType> entityVsType = new HashMap<>();
private final Map<String, AtlasVertex> entityVsVertex = new HashMap<>();
private final Map<String, String> guidAssignments = new HashMap<>();
public EntityMutationContext(final EntityGraphDiscoveryContext context) {
this.context = context;
}
public void addCreated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
AtlasObjectId objId = entity.getAtlasObjectId();
public void addCreated(String internalGuid, AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
entitiesCreated.add(entity);
entityVsType.put(objId, type);
entityVsVertex.put(objId, atlasVertex);
entityVsType.put(entity.getGuid(), type);
entityVsVertex.put(entity.getGuid(), atlasVertex);
if (!StringUtils.equals(internalGuid, entity.getGuid())) {
guidAssignments.put(internalGuid, entity.getGuid());
entityVsVertex.put(internalGuid, atlasVertex);
}
}
public void addUpdated(AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) {
AtlasObjectId objId = entity.getAtlasObjectId();
entitiesUpdated.add(entity);
entityVsType.put(objId, type);
entityVsVertex.put(objId, atlasVertex);
entityVsType.put(entity.getGuid(), type);
entityVsVertex.put(entity.getGuid(), atlasVertex);
}
public EntityGraphDiscoveryContext getDiscoveryContext() {
......@@ -64,21 +69,15 @@ public class EntityMutationContext {
return entitiesUpdated;
}
public AtlasEntityType getType(AtlasEntity entity) {
return entityVsType.get(entity.getAtlasObjectId());
}
public AtlasType getType(AtlasObjectId entityId) {
return entityVsType.get(entityId);
public Map<String, String> getGuidAssignments() {
return guidAssignments;
}
public AtlasVertex getVertex(AtlasEntity entity) {
return entityVsVertex.get(entity.getAtlasObjectId());
public AtlasEntityType getType(String guid) {
return entityVsType.get(guid);
}
public AtlasVertex getVertex(AtlasObjectId entityId) {
return entityVsVertex.get(entityId);
}
public AtlasVertex getVertex(String guid) { return entityVsVertex.get(guid); }
@Override
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityStream.java
......@@ -28,5 +28,5 @@ public interface EntityStream {
void reset();
AtlasEntity getById(AtlasObjectId id);
AtlasEntity getByGuid(String guid);
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/IDBasedEntityResolver.java
......@@ -17,11 +17,9 @@
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
......@@ -29,93 +27,53 @@ import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class IDBasedEntityResolver implements EntityResolver {
private static final Logger LOG = LoggerFactory.getLogger(IDBasedEntityResolver.class);
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final Map<String, AtlasEntity> idToEntityMap = new HashMap<>();
private EntityGraphDiscoveryContext context;
@Override
public void init(EntityGraphDiscoveryContext context) throws AtlasBaseException {
this.context = context;
private final GraphHelper graphHelper = GraphHelper.getInstance();
for (AtlasEntity entity : context.getRootEntities()) {
idToEntityMap.put(entity.getGuid(), entity);
}
}
public EntityGraphDiscoveryContext resolveEntityReferences() throws AtlasBaseException {
public EntityGraphDiscoveryContext resolveEntityReferences(EntityGraphDiscoveryContext context) throws AtlasBaseException {
if (context == null) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "Entity resolver not initialized");
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "IDBasedEntityResolver.resolveEntityReferences(): context is null");
}
List<AtlasObjectId> resolvedReferences = new ArrayList<>();
EntityStream entityStream = context.getEntityStream();
for (AtlasObjectId objId : context.getUnresolvedIds()) {
if (objId.isAssignedGuid()) {
//validate in graph repo that given guid, typename exists
Optional<AtlasVertex> vertex = resolveGuid(objId);
for (String guid : context.getReferencedGuids()) {
if (AtlasEntity.isAssigned(guid)) { // validate in graph repo that given guid exists
AtlasVertex vertex = resolveGuid(guid);
if (vertex.isPresent()) {
context.addResolvedId(objId, vertex.get());
resolvedReferences.add(objId);
}
context.addResolvedGuid(guid, vertex);
} else if (entityStream.getByGuid(guid) != null) { //check if entity stream have this reference id
context.addLocalGuidReference(guid);
} else {
//check if root references have this temporary id
if (!idToEntityMap.containsKey(objId.getGuid()) ) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, objId.toString());
}
resolvedReferences.add(objId);
}
}
context.removeUnResolvedIds(resolvedReferences);
//Resolve root references
for (AtlasEntity entity : context.getRootEntities()) {
AtlasObjectId objId = entity.getAtlasObjectId();
if (!context.isResolvedId(objId) && AtlasEntity.isAssigned(entity.getGuid())) {
Optional<AtlasVertex> vertex = resolveGuid(objId);
if (vertex.isPresent()) {
context.addResolvedId(objId, vertex.get());
context.removeUnResolvedId(objId);
}
throw new AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
}
}
return context;
}
private Optional<AtlasVertex> resolveGuid(AtlasObjectId objId) throws AtlasBaseException {
private AtlasVertex resolveGuid(String guid) throws AtlasBaseException {
//validate in graph repo that given guid, typename exists
AtlasVertex vertex = null;
try {
vertex = graphHelper.findVertex(Constants.GUID_PROPERTY_KEY, objId.getGuid(),
Constants.TYPE_NAME_PROPERTY_KEY, objId.getTypeName(),
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
vertex = graphHelper.findVertex(Constants.GUID_PROPERTY_KEY, guid,
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
} catch (EntityNotFoundException e) {
//Ignore
}
if ( vertex != null ) {
return Optional.of(vertex);
if (vertex != null) {
return vertex;
} else {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, objId.getGuid());
throw new AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
}
}
@Override
public void cleanUp() throws AtlasBaseException {
idToEntityMap.clear();
this.context = null;
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/InMemoryMapEntityStream.java
......@@ -27,14 +27,11 @@ import java.util.Map;
public class InMemoryMapEntityStream implements EntityStream {
private final Map<AtlasObjectId, AtlasEntity> entities = new HashMap<>();
private Iterator<Map.Entry<AtlasObjectId, AtlasEntity>> iterator;
public InMemoryMapEntityStream(Map<String, AtlasEntity> entityMap) {
for (AtlasEntity entity : entityMap.values()) {
entities.put(entity.getAtlasObjectId(), entity);
}
private final Map<String, AtlasEntity> entities;
private Iterator<Map.Entry<String, AtlasEntity>> iterator;
public InMemoryMapEntityStream(Map<String, AtlasEntity> entities) {
this.entities = entities;
this.iterator = entities.entrySet().iterator();
}
......@@ -54,7 +51,7 @@ public class InMemoryMapEntityStream implements EntityStream {
}
@Override
public AtlasEntity getById(final AtlasObjectId id) {
return entities.get(id);
public AtlasEntity getByGuid(final String guid) {
return entities.get(guid);
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/InstanceGraphMapper.java
......@@ -18,9 +18,6 @@
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
public interface InstanceGraphMapper<T> {
......@@ -31,9 +28,5 @@ public interface InstanceGraphMapper<T> {
* @return the value that was mapped to the vertex
* @throws AtlasBaseException
*/
T toGraph(GraphMutationContext ctx) throws AtlasBaseException;
void cleanUp() throws AtlasBaseException;
T toGraph(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException;
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/MapVertexMapper.java deleted 100644 → 0
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import com.google.common.base.Optional;
import com.google.inject.Inject;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.repository.graph.GraphHelper.string;
public class MapVertexMapper implements InstanceGraphMapper<Map> {
private DeleteHandlerV1 deleteHandler;
private static final Logger LOG = LoggerFactory.getLogger(MapVertexMapper.class);
private StructVertexMapper structVertexMapper;
@Inject
public MapVertexMapper(DeleteHandlerV1 deleteHandler) {
this.deleteHandler = deleteHandler;
}
void init(StructVertexMapper structVertexMapper) {
this.structVertexMapper = structVertexMapper;
}
@Override
public Map<String, Object> toGraph(GraphMutationContext ctx) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping instance to vertex {} for map type {}", string(ctx.getReferringVertex()), ctx.getAttrType().getTypeName());
}
@SuppressWarnings("unchecked") Map<Object, Object> newVal =
(Map<Object, Object>) ctx.getValue();
boolean newAttributeEmpty = MapUtils.isEmpty(newVal);
Map<String, Object> currentMap = new HashMap<>();
Map<String, Object> newMap = new HashMap<>();
AtlasMapType mapType = (AtlasMapType) ctx.getAttrType();
try {
List<String> currentKeys = GraphHelper.getListProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey());
if (currentKeys != null && !currentKeys.isEmpty()) {
for (String key : currentKeys) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), key);
Object propertyValueForKey = getMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey);
currentMap.put(key, propertyValueForKey);
}
}
if (!newAttributeEmpty) {
for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
String keyStr = entry.getKey().toString();
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexPropertyKey(), keyStr);
Optional<AtlasEdge> existingEdge = getEdgeIfExists(mapType, currentMap, keyStr);
GraphMutationContext mapCtx = new GraphMutationContext.Builder(ctx.getOp(), ctx.getAttribute(), mapType.getValueType(), entry.getValue())
.referringVertex(ctx.getReferringVertex())
.edge(existingEdge)
.vertexProperty(propertyNameForKey).build();
//Add/Update/Remove property value
Object newEntry = structVertexMapper.mapCollectionElementsToVertex(mapCtx);
MapVertexMapper.setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyNameForKey, newEntry);
newMap.put(keyStr, newEntry);
}
}
Map<String, Object> finalMap =
removeUnusedMapEntries(ctx.getParentType(), mapType, ctx.getAttributeDef(), ctx.getReferringVertex(), ctx.getVertexPropertyKey(), currentMap, newMap);
Set<String> newKeys = new HashSet<>(newMap.keySet());
newKeys.addAll(finalMap.keySet());
// for dereference on way out
GraphHelper.setListProperty(ctx.getReferringVertex(), ctx.getVertexPropertyKey(), new ArrayList<>(newKeys));
} catch (AtlasException e) {
throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Map values set in vertex {} {}", mapType.getTypeName(), newMap);
}
return newMap;
}
@Override
public void cleanUp() throws AtlasBaseException {
}
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(actualPropertyName, AtlasEdge.class);
}
else {
return instanceVertex.getProperty(actualPropertyName, String.class).toString();
}
}
public static void setMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value);
}
else {
instanceVertex.setProperty(actualPropertyName, value);
}
}
//Remove unused entries from map
private Map<String, Object> removeUnusedMapEntries(
AtlasStructType entityType,
AtlasMapType mapType, AtlasAttributeDef attributeDef,
AtlasVertex instanceVertex, String propertyName,
Map<String, Object> currentMap,
Map<String, Object> newMap)
throws AtlasException, AtlasBaseException {
Map<String, Object> additionalMap = new HashMap<>();
for (String currentKey : currentMap.keySet()) {
boolean shouldDeleteKey = !newMap.containsKey(currentKey);
if (AtlasGraphUtilsV1.isReference(mapType.getValueType())) {
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey);
if (!newMap.values().contains(currentEdge)) {
boolean deleteChildReferences = StructVertexMapper.shouldManageChildReferences(entityType, attributeDef.getName());
boolean deleted =
deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), deleteChildReferences, true);
if (!deleted) {
additionalMap.put(currentKey, currentEdge);
shouldDeleteKey = false;
}
}
}
if (shouldDeleteKey) {
String propertyNameForKey = GraphHelper.getQualifiedNameForMapKey(propertyName, currentKey);
GraphHelper.setProperty(instanceVertex, propertyNameForKey, null);
}
}
return additionalMap;
}
private Optional<AtlasEdge> getEdgeIfExists(AtlasMapType mapType, Map<String, Object> currentMap, String keyStr) {
Optional<AtlasEdge> existingEdge = Optional.absent();
if ( AtlasGraphUtilsV1.isReference(mapType.getValueType()) ) {
if ( currentMap.get(keyStr) != null) {
existingEdge = Optional.of((AtlasEdge) currentMap.get(keyStr));
}
}
return existingEdge;
}
}
repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
......@@ -765,7 +765,7 @@ public class DefaultMetadataServiceTest {
Referenceable dbDef = InstanceSerialization.fromJsonReferenceable(dbDefJson, true);
Assert.assertNotEquals(dbId, (((Id) tableDefinitionActual.get("database"))._getId()));
Assert.assertEquals(dbDef.getId()._getId(), (((Id) tableDefinitionActual.get("database"))._getId())); */
Assert.assertEquals(dbDef.getObjectId()._getId(), (((Id) tableDefinitionActual.get("database"))._getId())); */
}
......
webapp/src/main/java/org/apache/atlas/examples/QuickStartV2.java
......@@ -341,7 +341,7 @@ public class QuickStartV2 {
private AtlasEntity createInstance(AtlasEntity entity, String[] traitNames) throws Exception {
AtlasEntity ret = null;
EntityMutationResponse response = entitiesClient.createEntity(entity);
EntityMutationResponse response = entitiesClient.createEntity(new AtlasEntityWithExtInfo(entity));
List<AtlasEntityHeader> entities = response.getEntitiesByOperation(EntityOperation.CREATE);
if (CollectionUtils.isNotEmpty(entities)) {
......@@ -422,7 +422,7 @@ public class QuickStartV2 {
entity.setAttribute("retention", System.currentTimeMillis());
entity.setAttribute("db", db.getAtlasObjectId());
entity.setAttribute("sd", sd.getAtlasObjectId());
entity.setAttribute("columns", getObjectIds(columns));
entity.setAttribute("columns", AtlasTypeUtil.toObjectIds(columns));
return createInstance(entity, traitNames);
}
......@@ -563,18 +563,4 @@ public class QuickStartV2 {
AtlasEntity tableEntity = entitiesClient.getEntityByAttribute(TABLE_TYPE, attributes).getEntity();
return tableEntity.getGuid();
}
private Collection<AtlasObjectId> getObjectIds(Collection<AtlasEntity> entities) {
List<AtlasObjectId> ret = new ArrayList<>();
if (CollectionUtils.isNotEmpty(entities)) {
for (AtlasEntity entity : entities) {
if (entity != null) {
ret.add(entity.getAtlasObjectId());
}
}
}
return ret;
}
}
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasEntityFormatConverter.java
......@@ -60,11 +60,11 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
if (v1Obj instanceof Id) {
Id id = (Id) v1Obj;
ret = new AtlasObjectId(id.getTypeName(), id._getId());
ret = new AtlasObjectId(id._getId(), id.getTypeName());
} else if (v1Obj instanceof IReferenceableInstance) {
IReferenceableInstance entRef = (IReferenceableInstance) v1Obj;
ret = new AtlasObjectId(entRef.getTypeName(), entRef.getId()._getId());
ret = new AtlasObjectId(entRef.getId()._getId(), entRef.getTypeName());
if (!context.entityExists(ret.getGuid())) {
Map<String, Object> v1Attribs = null;
......
webapp/src/main/java/org/apache/atlas/web/adapters/AtlasObjectIdConverter.java
......@@ -51,10 +51,10 @@ AtlasObjectIdConverter extends AtlasAbstractFormatConverter {
if (v1Obj != null) {
if (v1Obj instanceof Id) {
Id id = (Id) v1Obj;
ret = new AtlasObjectId(id.getTypeName(), id._getId());
ret = new AtlasObjectId(id._getId(), id.getTypeName());
} else if (v1Obj instanceof IReferenceableInstance) {
IReferenceableInstance entity = (IReferenceableInstance) v1Obj;
ret = new AtlasObjectId(entity.getTypeName(), entity.getId()._getId());
ret = new AtlasObjectId(entity.getId()._getId(), entity.getTypeName());
}
}
return ret;
......
webapp/src/main/java/org/apache/atlas/web/rest/EntitiesREST.java
......@@ -20,15 +20,16 @@ package org.apache.atlas.web.rest;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.ClassificationAssociateRequest;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v1.EntityStream;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.web.adapters.AtlasInstanceRestAdapters;
import org.apache.atlas.web.util.Servlets;
......@@ -50,13 +51,9 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.web.adapters.AtlasInstanceRestAdapters.toAtlasBaseException;
import static org.apache.atlas.web.adapters.AtlasInstanceRestAdapters.toEntityMutationResponse;
@Path("v2/entities")
......@@ -81,101 +78,6 @@ public class EntitiesREST {
this.restAdapters = restAdapters;
}
/*******
* Entity Creation/Updation if it already exists in ATLAS
* An existing entity is matched by its guid if supplied or by its unique attribute eg: qualifiedName
* Any associations like Classifications, Business Terms will have to be handled through the respective APIs
*******/
@POST
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public EntityMutationResponse createOrUpdate(Map<String, AtlasEntity> entities) throws AtlasBaseException {
EntityMutationResponse response = null;
ITypedReferenceableInstance[] entitiesInOldFormat = restAdapters.getITypedReferenceables(entities.values());
try {
final CreateUpdateEntitiesResult result = metadataService.updateEntities(entitiesInOldFormat);
response = toEntityMutationResponse(result);
} catch (AtlasException e) {
LOG.error("Exception while getting a typed reference for the entity ", e);
throw AtlasInstanceRestAdapters.toAtlasBaseException(e);
}
return response;
}
/*******
* Entity Updation - Allows full update of the specified entities.
* Any associations like Classifications, Business Terms will have to be handled through the respective APIs
* Null updates are supported i.e Set an attribute value to Null if its an optional attribute
*******/
@PUT
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public EntityMutationResponse update(Map<String, AtlasEntity> entities) throws AtlasBaseException {
return createOrUpdate(entities);
}
@GET
@Path("/guids")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public AtlasEntity.AtlasEntities getById(@QueryParam("guid") List<String> guids) throws AtlasBaseException {
if (CollectionUtils.isEmpty(guids)) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guids);
}
AtlasEntity.AtlasEntities entities = new AtlasEntity.AtlasEntities();
List<AtlasEntity> entityList = new ArrayList<>();
for (String guid : guids) {
try {
ITypedReferenceableInstance ref = metadataService.getEntityDefinition(guid);
Map<String, AtlasEntity> entityRet = restAdapters.getAtlasEntity(ref);
addToEntityList(entityList, entityRet.values());
} catch (AtlasException e) {
throw toAtlasBaseException(e);
}
}
entities.setList(entityList);
return entities;
}
private void addToEntityList(final List<AtlasEntity> entityList, final Collection<AtlasEntity> values) {
for (AtlasEntity val : values) {
if ( !entityList.contains(val)) {
entityList.add(val);
}
}
}
/*******
* Entity Delete
*******/
@DELETE
@Path("/guids")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
public EntityMutationResponse deleteById(@QueryParam("guid") final List<String> guids) throws AtlasBaseException {
if (CollectionUtils.isEmpty(guids)) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guids);
}
try {
AtlasClient.EntityResult result = metadataService.deleteEntities(guids);
return toEntityMutationResponse(result);
} catch (AtlasException e) {
throw toAtlasBaseException(e);
}
}
/**
* Bulk API to associate a tag to multiple entities
*
......
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
......@@ -25,6 +25,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
......@@ -33,7 +34,9 @@ import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.web.rest.EntitiesREST;
import org.apache.atlas.web.rest.EntityREST;
......@@ -49,16 +52,21 @@ import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Guice(modules = {RepositoryMetadataModule.class})
public class TestEntitiesREST {
private static final Logger LOG = LoggerFactory.getLogger(TestEntitiesREST.class);
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStore typeStore;
@Inject
......@@ -69,10 +77,6 @@ public class TestEntitiesREST {
private List<String> createdGuids = new ArrayList<>();
private Map<String, AtlasEntity> dbEntityMap;
private Map<String, AtlasEntity> tableEntityMap;
private AtlasEntity dbEntity;
private AtlasEntity tableEntity;
......@@ -81,17 +85,22 @@ public class TestEntitiesREST {
@BeforeClass
public void setUp() throws Exception {
AtlasTypesDef typesDef = TestUtilsV2.defineHiveTypes();
typeStore.createTypesDef(typesDef);
dbEntityMap = TestUtilsV2.createDBEntity();
dbEntity = dbEntityMap.values().iterator().next();
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineHiveTypes() };
tableEntityMap = TestUtilsV2.createTableEntity(dbEntity.getGuid());
tableEntity = tableEntityMap.values().iterator().next();
for (AtlasTypesDef typesDef : testTypesDefs) {
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry);
final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity.getGuid());
if (!typesToCreate.isEmpty()) {
typeStore.createTypesDef(typesToCreate);
}
}
dbEntity = TestUtilsV2.createDBEntity();
tableEntity = TestUtilsV2.createTableEntity(dbEntity);
final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
tableEntity.setAttribute("columns", columns);
tableEntity.setAttribute("columns", getObjIdList(columns));
}
@AfterMethod
......@@ -106,11 +115,15 @@ public class TestEntitiesREST {
@Test
public void testCreateOrUpdateEntities() throws Exception {
Map<String, AtlasEntity> entities = new HashMap<>();
entities.put(dbEntity.getGuid(), dbEntity);
entities.put(tableEntity.getGuid(), tableEntity);
AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo();
EntityMutationResponse response = entitiesREST.createOrUpdate(entities);
entities.addEntity(dbEntity);
entities.addEntity(tableEntity);
for (AtlasEntity column : columns) {
entities.addReferredEntity(column);
}
EntityMutationResponse response = entityREST.createOrUpdate(entities);
List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(guids);
......@@ -137,23 +150,24 @@ public class TestEntitiesREST {
public void testUpdateWithSerializedEntities() throws Exception {
//Check with serialization and deserialization of entity attributes for the case
// where attributes which are de-serialized into a map
Map<String, AtlasEntity> dbEntityMap = TestUtilsV2.createDBEntity();
AtlasEntity dbEntity = dbEntityMap.values().iterator().next();
Map<String, AtlasEntity> tableEntityMap = TestUtilsV2.createTableEntity(dbEntity.getGuid());
AtlasEntity tableEntity = tableEntityMap.values().iterator().next();
AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity);
final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity.getGuid());
final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
List<AtlasEntity> columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
tableEntity.setAttribute("columns", columns);
tableEntity.setAttribute("columns", getObjIdList(columns));
AtlasEntity newDBEntity = serDeserEntity(dbEntity);
AtlasEntity newTableEntity = serDeserEntity(tableEntity);
Map<String, AtlasEntity> newEntities = new HashMap<>();
newEntities.put(newDBEntity.getGuid(), newDBEntity);
newEntities.put(newTableEntity.getGuid(), newTableEntity);
EntityMutationResponse response2 = entitiesREST.createOrUpdate(newEntities);
AtlasEntitiesWithExtInfo newEntities = new AtlasEntitiesWithExtInfo();
newEntities.addEntity(newDBEntity);
newEntities.addEntity(newTableEntity);
for (AtlasEntity column : columns) {
newEntities.addReferredEntity(serDeserEntity(column));
}
EntityMutationResponse response2 = entityREST.createOrUpdate(newEntities);
List<AtlasEntityHeader> newGuids = response2.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(newGuids);
......@@ -163,8 +177,8 @@ public class TestEntitiesREST {
@Test(dependsOnMethods = "testCreateOrUpdateEntities")
public void testGetEntities() throws Exception {
final AtlasEntity.AtlasEntities response = entitiesREST.getById(createdGuids);
final List<AtlasEntity> entities = response.getList();
final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids);
final List<AtlasEntity> entities = response.getEntities();
Assert.assertNotNull(entities);
Assert.assertEquals(entities.size(), 3);
......@@ -174,7 +188,7 @@ public class TestEntitiesREST {
@Test(dependsOnMethods = "testGetEntities")
public void testDeleteEntities() throws Exception {
final EntityMutationResponse response = entitiesREST.deleteById(createdGuids);
final EntityMutationResponse response = entityREST.deleteByGuids(createdGuids);
final List<AtlasEntityHeader> entities = response.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE);
Assert.assertNotNull(entities);
......@@ -247,4 +261,14 @@ public class TestEntitiesREST {
AtlasEntity newEntity = mapper.readValue(entityJson, AtlasEntity.class);
return newEntity;
}
private List<AtlasObjectId> getObjIdList(Collection<AtlasEntity> entities) {
List<AtlasObjectId> ret = new ArrayList<>();
for (AtlasEntity entity : entities) {
ret.add(entity.getAtlasObjectId());
}
return ret;
}
}
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java
......@@ -22,6 +22,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
......@@ -29,8 +30,8 @@ import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.web.rest.EntitiesREST;
import org.apache.atlas.web.rest.EntityREST;
import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
......@@ -39,6 +40,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -53,9 +55,6 @@ public class TestEntityREST {
@Inject
private EntityREST entityREST;
@Inject
private EntitiesREST entitiesREST;
private AtlasEntity dbEntity;
private AtlasClassification testClassification;
......@@ -77,12 +76,9 @@ public class TestEntityREST {
}
private void createTestEntity() throws Exception {
Map<String, AtlasEntity> dbEntityMap = TestUtilsV2.createDBEntity();
AtlasEntity dbEntity = dbEntityMap.values().iterator().next();
AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
dbEntityMap.put(dbEntity.getGuid(), dbEntity);
final EntityMutationResponse response = entitiesREST.createOrUpdate(dbEntityMap);
final EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity));
Assert.assertNotNull(response);
List<AtlasEntityHeader> entitiesMutated = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
......@@ -158,10 +154,9 @@ public class TestEntityREST {
@Test
public void testUpdateGetDeleteEntityByUniqueAttribute() throws Exception {
Map<String, AtlasEntity> dbEntityMap = TestUtilsV2.createDBEntity();
AtlasEntity dbEntity = dbEntityMap.values().iterator().next();
EntityMutationResponse response = entitiesREST.createOrUpdate(dbEntityMap);
String dbGuid = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0).getGuid();
AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
EntityMutationResponse response = entityREST.createOrUpdate(new AtlasEntitiesWithExtInfo(dbEntity));
String dbGuid = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0).getGuid();
Assert.assertTrue(AtlasEntity.isAssigned(dbGuid));
......@@ -170,21 +165,35 @@ public class TestEntityREST {
dbEntity.setAttribute(TestUtilsV2.NAME, updatedDBName);
response = entityREST.partialUpdateByUniqueAttribute(TestUtilsV2.DATABASE_TYPE, TestUtilsV2.NAME, prevDBName, dbEntity);
response = entityREST.partialUpdateByUniqueAttribute(TestUtilsV2.DATABASE_TYPE, toHttpServletRequest(TestUtilsV2.NAME, prevDBName), dbEntity);
Assert.assertEquals(response.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0).getGuid(), dbGuid);
//Get By unique attribute
List<AtlasEntity> entities = entityREST.getByUniqueAttribute(TestUtilsV2.DATABASE_TYPE, TestUtilsV2.NAME, updatedDBName);
Assert.assertNotNull(entities);
Assert.assertNotNull(entities.get(0).getGuid());
Assert.assertEquals(entities.get(0).getGuid(), dbGuid);
TestEntitiesREST.verifyAttributes(entities.get(0).getAttributes(), dbEntity.getAttributes());
AtlasEntityWithExtInfo entity = entityREST.getByUniqueAttributes(TestUtilsV2.DATABASE_TYPE, toHttpServletRequest(TestUtilsV2.NAME, updatedDBName));
Assert.assertNotNull(entity);
Assert.assertNotNull(entity.getEntity().getGuid());
Assert.assertEquals(entity.getEntity().getGuid(), dbGuid);
TestEntitiesREST.verifyAttributes(entity.getEntity().getAttributes(), dbEntity.getAttributes());
final EntityMutationResponse deleteResponse = entityREST.deleteByUniqueAttribute(TestUtilsV2.DATABASE_TYPE, TestUtilsV2.NAME, (String) dbEntity.getAttribute(TestUtilsV2.NAME));
final EntityMutationResponse deleteResponse = entityREST.deleteByUniqueAttribute(TestUtilsV2.DATABASE_TYPE, toHttpServletRequest(TestUtilsV2.NAME, (String) dbEntity.getAttribute(TestUtilsV2.NAME)));
Assert.assertNotNull(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE));
Assert.assertEquals(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).size(), 1);
Assert.assertEquals(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).get(0).getGuid(), dbGuid);
}
private HttpServletRequest toHttpServletRequest(String attrName, String attrValue) {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
Map<String, String[]> paramsMap = toParametersMap(EntityREST.PREFIX_ATTR + attrName, attrValue);
Mockito.when(request.getParameterMap()).thenReturn(paramsMap);
return request;
}
private Map<String, String[]> toParametersMap(final String name, final String value) {
return new HashMap<String, String[]>() {{
put(name, new String[] { value });
}};
}
}
webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
......@@ -43,6 +43,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
......@@ -74,6 +75,7 @@ import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
......@@ -286,13 +288,13 @@ public abstract class BaseResourceIT {
EntityMutationResponse entity = null;
try {
if (!update) {
entity = entitiesClientV2.createEntity(atlasEntity);
entity = entitiesClientV2.createEntity(new AtlasEntityWithExtInfo(atlasEntity));
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size() > 0);
return entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0);
} else {
entity = entitiesClientV2.updateEntity(atlasEntity);
entity = entitiesClientV2.updateEntity(new AtlasEntityWithExtInfo(atlasEntity));
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment