Skip to content

A
atlas
Commit 7e4788ed by grahamwallis
Options

ATLAS-2936: Ability to store provenance type

parent 75947444
Showing with 134 additions and 72 deletions
common/src/main/java/org/apache/atlas/repository/Constants.java
......@@ -103,6 +103,14 @@ public final class Constants {
*/
public static final String IS_PROXY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isProxy");
/**
* The provenanceType field is used to record the provenance of an instance of an entity or relationship - this
* indicates how the instance was created. This corresponds to the InstanceProvenanceType enum defined in ODPi.
* To avoid creating a hard dependency on the ODPi class, the value is stored as an int corresponding to the
* ordinal in the ODPi enum.
*/
public static final String PROVENANCE_TYPE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "provenanceType");
public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp");
public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp");
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
......@@ -59,30 +59,32 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
public class AtlasEntity extends AtlasStruct implements Serializable {
private static final long serialVersionUID = 1L;
public static final String KEY_GUID = "guid";
public static final String KEY_HOME_ID = "homeId";
public static final String KEY_IS_PROXY = "isProxy";
public static final String KEY_STATUS = "status";
public static final String KEY_CREATED_BY = "createdBy";
public static final String KEY_UPDATED_BY = "updatedBy";
public static final String KEY_CREATE_TIME = "createTime";
public static final String KEY_UPDATE_TIME = "updateTime";
public static final String KEY_VERSION = "version";
public static final String KEY_GUID = "guid";
public static final String KEY_HOME_ID = "homeId";
public static final String KEY_IS_PROXY = "isProxy";
public static final String KEY_PROVENANCE_TYPE = "provenanceType";
public static final String KEY_STATUS = "status";
public static final String KEY_CREATED_BY = "createdBy";
public static final String KEY_UPDATED_BY = "updatedBy";
public static final String KEY_CREATE_TIME = "createTime";
public static final String KEY_UPDATE_TIME = "updateTime";
public static final String KEY_VERSION = "version";
/**
* Status of the entity - can be active or deleted. Deleted entities are not removed from Atlas store.
*/
public enum Status { ACTIVE, DELETED }
private String guid = null;
private String homeId = null;
private Boolean isProxy = Boolean.FALSE;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
private Date createTime = null;
private Date updateTime = null;
private Long version = 0L;
private String guid = null;
private String homeId = null;
private Boolean isProxy = Boolean.FALSE;
private Integer provenanceType = 0;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
private Date createTime = null;
private Date updateTime = null;
private Long version = 0L;
private Map<String, Object> relationshipAttributes;
private List<AtlasClassification> classifications;
......@@ -119,15 +121,16 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
super(map);
if (map != null) {
Object oGuid = map.get(KEY_GUID);
Object homeId = map.get(KEY_HOME_ID);
Object isProxy = map.get(KEY_IS_PROXY);
Object status = map.get(KEY_STATUS);
Object createdBy = map.get(KEY_CREATED_BY);
Object updatedBy = map.get(KEY_UPDATED_BY);
Object createTime = map.get(KEY_CREATE_TIME);
Object updateTime = map.get(KEY_UPDATE_TIME);
Object version = map.get(KEY_VERSION);
Object oGuid = map.get(KEY_GUID);
Object homeId = map.get(KEY_HOME_ID);
Object isProxy = map.get(KEY_IS_PROXY);
Object provenanceType = map.get(KEY_PROVENANCE_TYPE);
Object status = map.get(KEY_STATUS);
Object createdBy = map.get(KEY_CREATED_BY);
Object updatedBy = map.get(KEY_UPDATED_BY);
Object createTime = map.get(KEY_CREATE_TIME);
Object updateTime = map.get(KEY_UPDATE_TIME);
Object version = map.get(KEY_VERSION);
if (oGuid != null) {
setGuid(oGuid.toString());
......@@ -144,6 +147,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setIsProxy(Boolean.FALSE);
}
if (provenanceType instanceof Number) {
setProvenanceType(((Number) version).intValue());
}
if (status != null) {
setStatus(Status.valueOf(status.toString()));
}
......@@ -177,6 +184,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setGuid(other.getGuid());
setHomeId(other.getHomeId());
setIsProxy(other.isProxy());
setProvenanceType(other.getProvenanceType());
setStatus(other.getStatus());
setCreatedBy(other.getCreatedBy());
setUpdatedBy(other.getUpdatedBy());
......@@ -211,6 +219,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this.isProxy = isProxy;
}
public Integer getProvenanceType() {
return provenanceType;
}
public void setProvenanceType(Integer provenanceType) {
this.provenanceType = provenanceType;
}
public Status getStatus() {
return status;
}
......@@ -328,6 +344,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
setGuid(nextInternalId());
setHomeId(null);
setIsProxy(Boolean.FALSE);
setProvenanceType(0);
setStatus(null);
setCreatedBy(null);
setUpdatedBy(null);
......@@ -352,6 +369,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
sb.append("guid='").append(guid).append('\'');
sb.append(", homeId='").append(homeId).append('\'');
sb.append(", isProxy='").append(isProxy).append('\'');
sb.append(", provenanceType=").append(provenanceType);
sb.append(", status=").append(status);
sb.append(", createdBy='").append(createdBy).append('\'');
sb.append(", updatedBy='").append(updatedBy).append('\'');
......@@ -382,6 +400,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
return Objects.equals(guid, that.guid) &&
Objects.equals(homeId, that.homeId) &&
Objects.equals(isProxy, that.isProxy) &&
Objects.equals(provenanceType, that.provenanceType) &&
status == that.status &&
Objects.equals(createdBy, that.createdBy) &&
Objects.equals(updatedBy, that.updatedBy) &&
......@@ -394,7 +413,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, homeId, isProxy, status, createdBy, updatedBy, createTime, updateTime, version,
return Objects.hash(super.hashCode(), guid, homeId, isProxy, provenanceType, status, createdBy, updatedBy, createTime, updateTime, version,
relationshipAttributes, classifications);
}
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
......@@ -54,34 +54,36 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
public class AtlasRelationship extends AtlasStruct implements Serializable {
private static final long serialVersionUID = 1L;
public static final String KEY_GUID = "guid";
public static final String KEY_HOME_ID = "homeId";
public static final String KEY_STATUS = "status";
public static final String KEY_CREATED_BY = "createdBy";
public static final String KEY_UPDATED_BY = "updatedBy";
public static final String KEY_CREATE_TIME = "createTime";
public static final String KEY_UPDATE_TIME = "updateTime";
public static final String KEY_VERSION = "version";
public static final String KEY_END1 = "end1";
public static final String KEY_END2 = "end2";
public static final String KEY_LABEL = "label";
public static final String KEY_PROPAGATE_TAGS = "propagateTags";
public static final String KEY_GUID = "guid";
public static final String KEY_HOME_ID = "homeId";
public static final String KEY_PROVENANCE_TYPE = "provenanceType";
public static final String KEY_STATUS = "status";
public static final String KEY_CREATED_BY = "createdBy";
public static final String KEY_UPDATED_BY = "updatedBy";
public static final String KEY_CREATE_TIME = "createTime";
public static final String KEY_UPDATE_TIME = "updateTime";
public static final String KEY_VERSION = "version";
public static final String KEY_END1 = "end1";
public static final String KEY_END2 = "end2";
public static final String KEY_LABEL = "label";
public static final String KEY_PROPAGATE_TAGS = "propagateTags";
public static final String KEY_BLOCKED_PROPAGATED_CLASSIFICATIONS = "blockedPropagatedClassifications";
public static final String KEY_PROPAGATED_CLASSIFICATIONS = "propagatedClassifications";
private String guid = null;
private String homeId = null;
private AtlasObjectId end1 = null;
private AtlasObjectId end2 = null;
private String label = null;
private PropagateTags propagateTags = PropagateTags.NONE;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
private Date createTime = null;
private Date updateTime = null;
private Long version = 0L;
private String guid = null;
private String homeId = null;
private Integer provenanceType = null;
private AtlasObjectId end1 = null;
private AtlasObjectId end2 = null;
private String label = null;
private PropagateTags propagateTags = PropagateTags.NONE;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
private Date createTime = null;
private Date updateTime = null;
private Long version = 0L;
public enum Status { ACTIVE, DELETED }
......@@ -110,13 +112,13 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) {
super(typeName);
init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
init(nextInternalId(), null, 0, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
}
public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2, Map<String, Object> attributes) {
super(typeName, attributes);
init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
init(nextInternalId(), null, 0, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
}
public AtlasRelationship(String typeName, String attrName, Object attrValue) {
......@@ -133,18 +135,19 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
super(map);
if (map != null) {
Object oGuid = map.get(KEY_GUID);
Object homeId = map.get(KEY_HOME_ID);
Object oEnd1 = map.get(KEY_END1);
Object oEnd2 = map.get(KEY_END2);
Object label = map.get(KEY_LABEL);
Object propagateTags = map.get(KEY_PROPAGATE_TAGS);
Object status = map.get(KEY_STATUS);
Object createdBy = map.get(KEY_CREATED_BY);
Object updatedBy = map.get(KEY_UPDATED_BY);
Object createTime = map.get(KEY_CREATE_TIME);
Object updateTime = map.get(KEY_UPDATE_TIME);
Object version = map.get(KEY_VERSION);
Object oGuid = map.get(KEY_GUID);
Object homeId = map.get(KEY_HOME_ID);
Object provenanceType = map.get(KEY_PROVENANCE_TYPE);
Object oEnd1 = map.get(KEY_END1);
Object oEnd2 = map.get(KEY_END2);
Object label = map.get(KEY_LABEL);
Object propagateTags = map.get(KEY_PROPAGATE_TAGS);
Object status = map.get(KEY_STATUS);
Object createdBy = map.get(KEY_CREATED_BY);
Object updatedBy = map.get(KEY_UPDATED_BY);
Object createTime = map.get(KEY_CREATE_TIME);
Object updateTime = map.get(KEY_UPDATE_TIME);
Object version = map.get(KEY_VERSION);
Object propagatedClassifications = map.get(KEY_PROPAGATED_CLASSIFICATIONS);
Object blockedPropagatedClassifications = map.get(KEY_BLOCKED_PROPAGATED_CLASSIFICATIONS);
......@@ -157,6 +160,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
setHomeId(homeId.toString());
}
if (provenanceType instanceof Number) {
setProvenanceType(((Number) provenanceType).intValue());
}
if (oEnd1 != null) {
if (oEnd1 instanceof AtlasObjectId) {
setEnd1((AtlasObjectId) oEnd1);
......@@ -235,7 +242,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
super(other);
if (other != null) {
init(other.guid, other.homeId, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy,
init(other.guid, other.homeId, other.provenanceType, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy,
other.createTime, other.updateTime, other.version, other.propagatedClassifications, other.blockedPropagatedClassifications);
}
}
......@@ -256,6 +263,12 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
this.homeId = homeId;
}
public Integer getProvenanceType() {
return provenanceType;
}
public void setProvenanceType(Integer provenanceType) { this.provenanceType = provenanceType; }
public Status getStatus() {
return status;
}
......@@ -341,14 +354,15 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
}
private void init() {
init(nextInternalId(), null, null, null, null, null, null, null, null, null, null, 0L, null, null);
init(nextInternalId(), null, 0 ,null, null, null, null, null, null, null, null, null, 0L, null, null);
}
private void init(String guid, String homeId, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags,
private void init(String guid, String homeId, Integer provenanceType, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags,
Status status, String createdBy, String updatedBy, Date createTime, Date updateTime, Long version,
Set<AtlasClassification> propagatedClassifications, Set<AtlasClassification> blockedPropagatedClassifications) {
setGuid(guid);
setHomeId(homeId);
setProvenanceType(provenanceType);
setEnd1(end1);
setEnd2(end2);
setLabel(label);
......@@ -373,6 +387,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
super.toString(sb);
sb.append("guid='").append(guid).append('\'');
sb.append(", homeId='").append(homeId).append('\'');
sb.append(", provenanceType=").append(provenanceType);
sb.append(", end1=").append(end1);
sb.append(", end2=").append(end2);
sb.append(", label='").append(label).append('\'');
......@@ -403,6 +418,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
AtlasRelationship that = (AtlasRelationship) o;
return Objects.equals(guid, that.guid) &&
Objects.equals(homeId, that.homeId) &&
Objects.equals(provenanceType, that.provenanceType) &&
Objects.equals(end1, that.end1) &&
Objects.equals(end2, that.end2) &&
Objects.equals(label, that.label) &&
......@@ -419,7 +435,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, homeId, end1, end2, label, propagateTags, status, createdBy, updatedBy,
return Objects.hash(super.hashCode(), guid, homeId, provenanceType, end1, end2, label, propagateTags, status, createdBy, updatedBy,
createTime, updateTime, version, propagatedClassifications, blockedPropagatedClassifications);
}
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -1086,6 +1086,10 @@ public final class GraphHelper {
return element.getProperty(Constants.IS_PROXY_KEY, Boolean.class);
}
public static Integer getProvenanceType(AtlasElement element) {
return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class);
}
public static String getTypeName(AtlasElement element) {
return element.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class);
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
......@@ -74,6 +74,9 @@ import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
......@@ -748,6 +751,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, guid);
AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY, relationship.getProvenanceType());
AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
// blocked propagated classifications
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
......@@ -165,6 +165,10 @@ public class EntityGraphMapper {
if (entity.isProxy() != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, IS_PROXY_KEY, entity.isProxy());
}
if (entity.getProvenanceType() != null) {
AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType());
}
}
public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -525,6 +525,8 @@ public final class EntityGraphRetriever {
entity.setIsProxy(GraphHelper.isProxy(entityVertex));
entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex));
return entity;
}
......@@ -1166,12 +1168,17 @@ public final class EntityGraphRetriever {
relationship.setUpdateTime(new Date(GraphHelper.getModifiedTime(edge)));
Long version = GraphHelper.getVersion(edge);
if (version == null) {
version = Long.valueOf(1L);
}
relationship.setVersion(version);
Integer provenanceType = GraphHelper.getProvenanceType(edge);
if (provenanceType == null) {
provenanceType = Integer.valueOf(0);
}
relationship.setProvenanceType(provenanceType);
relationship.setStatus(GraphHelper.getEdgeStatus(edge));
AtlasVertex end1Vertex = edge.getOutVertex();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment