ATLAS-2936: Ability to store provenance type

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -103,6 +103,14 @@ public final class Constants {
      */
     public static final String IS_PROXY_KEY           = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isProxy");
 
+    /**
+     * The provenanceType field is used to record the provenance of an instance of an entity or relationship - this
+     * indicates how the instance was created. This corresponds to the InstanceProvenanceType enum defined in ODPi.
+     * To avoid creating a hard dependency on the ODPi class, the value is stored as an int corresponding to the
+     * ordinal in the ODPi enum.
+     */
+    public static final String PROVENANCE_TYPE_KEY    = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "provenanceType");
+
     public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp");
 
     public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp");

intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java

@@ -62,6 +62,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
     public static final String KEY_GUID               = "guid";
     public static final String KEY_HOME_ID            = "homeId";
     public static final String KEY_IS_PROXY           = "isProxy";
+    public static final String KEY_PROVENANCE_TYPE    = "provenanceType";
     public static final String KEY_STATUS             = "status";
     public static final String KEY_CREATED_BY         = "createdBy";
     public static final String KEY_UPDATED_BY         = "updatedBy";
@@ -77,6 +78,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
     private String  guid           = null;
     private String  homeId         = null;
     private Boolean isProxy        = Boolean.FALSE;
+    private Integer provenanceType = 0;
     private Status  status         = Status.ACTIVE;
     private String  createdBy      = null;
     private String  updatedBy      = null;
@@ -122,6 +124,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
             Object oGuid             = map.get(KEY_GUID);
             Object homeId            = map.get(KEY_HOME_ID);
             Object isProxy           = map.get(KEY_IS_PROXY);
+            Object provenanceType    = map.get(KEY_PROVENANCE_TYPE);
             Object status            = map.get(KEY_STATUS);
             Object createdBy         = map.get(KEY_CREATED_BY);
             Object updatedBy         = map.get(KEY_UPDATED_BY);
@@ -144,6 +147,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
                 setIsProxy(Boolean.FALSE);
             }
 
+            if (provenanceType instanceof Number) {
+                setProvenanceType(((Number) version).intValue());
+            }
+
             if (status != null) {
                 setStatus(Status.valueOf(status.toString()));
             }
@@ -177,6 +184,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
             setGuid(other.getGuid());
             setHomeId(other.getHomeId());
             setIsProxy(other.isProxy());
+            setProvenanceType(other.getProvenanceType());
             setStatus(other.getStatus());
             setCreatedBy(other.getCreatedBy());
             setUpdatedBy(other.getUpdatedBy());
@@ -211,6 +219,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         this.isProxy = isProxy;
     }
 
+    public Integer getProvenanceType() {
+        return provenanceType;
+    }
+
+    public void setProvenanceType(Integer provenanceType) {
+        this.provenanceType = provenanceType;
+    }
+
     public Status getStatus() {
         return status;
     }
@@ -328,6 +344,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         setGuid(nextInternalId());
         setHomeId(null);
         setIsProxy(Boolean.FALSE);
+        setProvenanceType(0);
         setStatus(null);
         setCreatedBy(null);
         setUpdatedBy(null);
@@ -352,6 +369,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         sb.append("guid='").append(guid).append('\'');
         sb.append(", homeId='").append(homeId).append('\'');
         sb.append(", isProxy='").append(isProxy).append('\'');
+        sb.append(", provenanceType=").append(provenanceType);
         sb.append(", status=").append(status);
         sb.append(", createdBy='").append(createdBy).append('\'');
         sb.append(", updatedBy='").append(updatedBy).append('\'');
@@ -382,6 +400,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         return Objects.equals(guid, that.guid) &&
                 Objects.equals(homeId, that.homeId) &&
                 Objects.equals(isProxy, that.isProxy) &&
+                Objects.equals(provenanceType, that.provenanceType) &&
                 status == that.status &&
                 Objects.equals(createdBy, that.createdBy) &&
                 Objects.equals(updatedBy, that.updatedBy) &&
@@ -394,7 +413,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), guid, homeId, isProxy, status, createdBy, updatedBy, createTime, updateTime, version,
+        return Objects.hash(super.hashCode(), guid, homeId, isProxy, provenanceType, status, createdBy, updatedBy, createTime, updateTime, version,
                             relationshipAttributes, classifications);
     }
 

intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java

@@ -56,6 +56,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
 
     public static final String KEY_GUID             = "guid";
     public static final String KEY_HOME_ID          = "homeId";
+    public static final String KEY_PROVENANCE_TYPE = "provenanceType";
     public static final String KEY_STATUS           = "status";
     public static final String KEY_CREATED_BY       = "createdBy";
     public static final String KEY_UPDATED_BY       = "updatedBy";
@@ -72,6 +73,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
 
     private String        guid           = null;
     private String        homeId         = null;
+    private Integer       provenanceType = null;
     private AtlasObjectId end1           = null;
     private AtlasObjectId end2           = null;
     private String        label          = null;
@@ -110,13 +112,13 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
     public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) {
         super(typeName);
 
-        init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
+        init(nextInternalId(), null, 0, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
     }
 
     public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2, Map<String, Object> attributes) {
         super(typeName, attributes);
 
-        init(nextInternalId(), null, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
+        init(nextInternalId(), null, 0, end1, end2, null, null, null, null, null, null, null, 0L, null, null);
     }
 
     public AtlasRelationship(String typeName, String attrName, Object attrValue) {
@@ -135,6 +137,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
         if (map != null) {
             Object oGuid          = map.get(KEY_GUID);
             Object homeId         = map.get(KEY_HOME_ID);
+            Object provenanceType = map.get(KEY_PROVENANCE_TYPE);
             Object oEnd1          = map.get(KEY_END1);
             Object oEnd2          = map.get(KEY_END2);
             Object label          = map.get(KEY_LABEL);
@@ -157,6 +160,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
                 setHomeId(homeId.toString());
             }
 
+            if (provenanceType instanceof Number) {
+                setProvenanceType(((Number) provenanceType).intValue());
+            }
+
             if (oEnd1 != null) {
                 if (oEnd1 instanceof AtlasObjectId) {
                     setEnd1((AtlasObjectId) oEnd1);
@@ -235,7 +242,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
         super(other);
 
         if (other != null) {
-            init(other.guid, other.homeId, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy,
+            init(other.guid, other.homeId, other.provenanceType, other.end1, other.end2, other.label, other.propagateTags, other.status, other.createdBy, other.updatedBy,
                  other.createTime, other.updateTime, other.version, other.propagatedClassifications, other.blockedPropagatedClassifications);
         }
     }
@@ -256,6 +263,12 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
         this.homeId = homeId;
     }
 
+    public Integer getProvenanceType() {
+        return provenanceType;
+    }
+
+    public void setProvenanceType(Integer provenanceType) { this.provenanceType = provenanceType; }
+
     public Status getStatus() {
         return status;
     }
@@ -341,14 +354,15 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
     }
 
     private void init() {
-        init(nextInternalId(), null, null, null, null, null, null,  null, null, null, null, 0L, null, null);
+        init(nextInternalId(), null, 0 ,null, null, null, null, null,  null, null, null, null, 0L, null, null);
     }
 
-    private void init(String guid, String homeId, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags,
+    private void init(String guid, String homeId, Integer provenanceType, AtlasObjectId end1, AtlasObjectId end2, String label, PropagateTags propagateTags,
                       Status status, String createdBy, String updatedBy, Date createTime, Date updateTime, Long version,
                       Set<AtlasClassification> propagatedClassifications, Set<AtlasClassification> blockedPropagatedClassifications) {
         setGuid(guid);
         setHomeId(homeId);
+        setProvenanceType(provenanceType);
         setEnd1(end1);
         setEnd2(end2);
         setLabel(label);
@@ -373,6 +387,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
         super.toString(sb);
         sb.append("guid='").append(guid).append('\'');
         sb.append(", homeId='").append(homeId).append('\'');
+        sb.append(", provenanceType=").append(provenanceType);
         sb.append(", end1=").append(end1);
         sb.append(", end2=").append(end2);
         sb.append(", label='").append(label).append('\'');
@@ -403,6 +418,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
         AtlasRelationship that = (AtlasRelationship) o;
         return Objects.equals(guid, that.guid) &&
                 Objects.equals(homeId, that.homeId) &&
+                Objects.equals(provenanceType, that.provenanceType) &&
                 Objects.equals(end1, that.end1) &&
                 Objects.equals(end2, that.end2) &&
                 Objects.equals(label, that.label) &&
@@ -419,7 +435,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), guid, homeId, end1, end2, label, propagateTags, status, createdBy, updatedBy,
+        return Objects.hash(super.hashCode(), guid, homeId, provenanceType, end1, end2, label, propagateTags, status, createdBy, updatedBy,
                             createTime, updateTime, version, propagatedClassifications, blockedPropagatedClassifications);
     }
 

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -1086,6 +1086,10 @@ public final class GraphHelper {
         return element.getProperty(Constants.IS_PROXY_KEY, Boolean.class);
     }
 
+    public static Integer getProvenanceType(AtlasElement element) {
+        return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class);
+    }
+
     public static String getTypeName(AtlasElement element) {
         return element.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class);
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java

@@ -74,6 +74,9 @@ import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
+
+
 import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
@@ -748,6 +751,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
             AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
             AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, guid);
             AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
+            AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY, relationship.getProvenanceType());
             AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
 
             // blocked propagated classifications

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java

@@ -165,6 +165,10 @@ public class EntityGraphMapper {
         if (entity.isProxy() != null) {
             AtlasGraphUtilsV2.setEncodedProperty(vertex, IS_PROXY_KEY, entity.isProxy());
         }
+
+        if (entity.getProvenanceType() != null) {
+            AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType());
+        }
     }
 
     public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException {

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java

@@ -525,6 +525,8 @@ public final class EntityGraphRetriever {
 
         entity.setIsProxy(GraphHelper.isProxy(entityVertex));
 
+        entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex));
+
         return entity;
     }
 
@@ -1166,12 +1168,17 @@ public final class EntityGraphRetriever {
         relationship.setUpdateTime(new Date(GraphHelper.getModifiedTime(edge)));
 
         Long version = GraphHelper.getVersion(edge);
-
         if (version == null) {
             version = Long.valueOf(1L);
         }
-
         relationship.setVersion(version);
+
+        Integer provenanceType = GraphHelper.getProvenanceType(edge);
+        if (provenanceType == null) {
+            provenanceType = Integer.valueOf(0);
+        }
+        relationship.setProvenanceType(provenanceType);
+
         relationship.setStatus(GraphHelper.getEdgeStatus(edge));
 
         AtlasVertex end1Vertex = edge.getOutVertex();