ATLAS-2936: Ability to store provenance type

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -103,6 +103,14 @@ public final class Constants {
      */
     public static final String IS_PROXY_KEY           = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isProxy");
 
+    /**
+     * The provenanceType field is used to record the provenance of an instance of an entity or relationship - this
+     * indicates how the instance was created. This corresponds to the InstanceProvenanceType enum defined in ODPi.
+     * To avoid creating a hard dependency on the ODPi class, the value is stored as an int corresponding to the
+     * ordinal in the ODPi enum.
+     */
+    public static final String PROVENANCE_TYPE_KEY    = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "provenanceType");
+
     public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp");
 
     public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp");

intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java

@@ -59,30 +59,32 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
 public class AtlasEntity extends AtlasStruct implements Serializable {
     private static final long serialVersionUID = 1L;
 
-    public static final String KEY_GUID        = "guid";
-    public static final String KEY_HOME_ID     = "homeId";
-    public static final String KEY_IS_PROXY    = "isProxy";
-    public static final String KEY_STATUS      = "status";
-    public static final String KEY_CREATED_BY  = "createdBy";
-    public static final String KEY_UPDATED_BY  = "updatedBy";
-    public static final String KEY_CREATE_TIME = "createTime";
-    public static final String KEY_UPDATE_TIME = "updateTime";
-    public static final String KEY_VERSION     = "version";
+    public static final String KEY_GUID               = "guid";
+    public static final String KEY_HOME_ID            = "homeId";
+    public static final String KEY_IS_PROXY           = "isProxy";
+    public static final String KEY_PROVENANCE_TYPE    = "provenanceType";
+    public static final String KEY_STATUS             = "status";
+    public static final String KEY_CREATED_BY         = "createdBy";
+    public static final String KEY_UPDATED_BY         = "updatedBy";
+    public static final String KEY_CREATE_TIME        = "createTime";
+    public static final String KEY_UPDATE_TIME        = "updateTime";
+    public static final String KEY_VERSION            = "version";
 
     /**
      * Status of the entity - can be active or deleted. Deleted entities are not removed from Atlas store.
      */
     public enum Status { ACTIVE, DELETED }
 
-    private String  guid       = null;
-    private String  homeId     = null;
-    private Boolean isProxy    = Boolean.FALSE;
-    private Status  status     = Status.ACTIVE;
-    private String  createdBy  = null;
-    private String  updatedBy  = null;
-    private Date    createTime = null;
-    private Date    updateTime = null;
-    private Long    version    = 0L;
+    private String  guid           = null;
+    private String  homeId         = null;
+    private Boolean isProxy        = Boolean.FALSE;
+    private Integer provenanceType = 0;
+    private Status  status         = Status.ACTIVE;
+    private String  createdBy      = null;
+    private String  updatedBy      = null;
+    private Date    createTime     = null;
+    private Date    updateTime     = null;
+    private Long    version        = 0L;
 
     private Map<String, Object>             relationshipAttributes;
     private List<AtlasClassification>       classifications;
@@ -119,15 +121,16 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         super(map);
 
         if (map != null) {
-            Object oGuid      = map.get(KEY_GUID);
-            Object homeId     = map.get(KEY_HOME_ID);
-            Object isProxy    = map.get(KEY_IS_PROXY);
-            Object status     = map.get(KEY_STATUS);
-            Object createdBy  = map.get(KEY_CREATED_BY);
-            Object updatedBy  = map.get(KEY_UPDATED_BY);
-            Object createTime = map.get(KEY_CREATE_TIME);
-            Object updateTime = map.get(KEY_UPDATE_TIME);
-            Object version    = map.get(KEY_VERSION);
+            Object oGuid             = map.get(KEY_GUID);
+            Object homeId            = map.get(KEY_HOME_ID);
+            Object isProxy           = map.get(KEY_IS_PROXY);
+            Object provenanceType    = map.get(KEY_PROVENANCE_TYPE);
+            Object status            = map.get(KEY_STATUS);
+            Object createdBy         = map.get(KEY_CREATED_BY);
+            Object updatedBy         = map.get(KEY_UPDATED_BY);
+            Object createTime        = map.get(KEY_CREATE_TIME);
+            Object updateTime        = map.get(KEY_UPDATE_TIME);
+            Object version           = map.get(KEY_VERSION);
 
             if (oGuid != null) {
                 setGuid(oGuid.toString());
@@ -144,6 +147,10 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
                 setIsProxy(Boolean.FALSE);
             }
 
+            if (provenanceType instanceof Number) {
+                setProvenanceType(((Number) version).intValue());
+            }
+
             if (status != null) {
                 setStatus(Status.valueOf(status.toString()));
             }
@@ -177,6 +184,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
             setGuid(other.getGuid());
             setHomeId(other.getHomeId());
             setIsProxy(other.isProxy());
+            setProvenanceType(other.getProvenanceType());
             setStatus(other.getStatus());
             setCreatedBy(other.getCreatedBy());
             setUpdatedBy(other.getUpdatedBy());
@@ -211,6 +219,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         this.isProxy = isProxy;
     }
 
+    public Integer getProvenanceType() {
+        return provenanceType;
+    }
+
+    public void setProvenanceType(Integer provenanceType) {
+        this.provenanceType = provenanceType;
+    }
+
     public Status getStatus() {
         return status;
     }
@@ -328,6 +344,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         setGuid(nextInternalId());
         setHomeId(null);
         setIsProxy(Boolean.FALSE);
+        setProvenanceType(0);
         setStatus(null);
         setCreatedBy(null);
         setUpdatedBy(null);
@@ -352,6 +369,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         sb.append("guid='").append(guid).append('\'');
         sb.append(", homeId='").append(homeId).append('\'');
         sb.append(", isProxy='").append(isProxy).append('\'');
+        sb.append(", provenanceType=").append(provenanceType);
         sb.append(", status=").append(status);
         sb.append(", createdBy='").append(createdBy).append('\'');
         sb.append(", updatedBy='").append(updatedBy).append('\'');
@@ -382,6 +400,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
         return Objects.equals(guid, that.guid) &&
                 Objects.equals(homeId, that.homeId) &&
                 Objects.equals(isProxy, that.isProxy) &&
+                Objects.equals(provenanceType, that.provenanceType) &&
                 status == that.status &&
                 Objects.equals(createdBy, that.createdBy) &&
                 Objects.equals(updatedBy, that.updatedBy) &&
@@ -394,7 +413,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), guid, homeId, isProxy, status, createdBy, updatedBy, createTime, updateTime, version,
+        return Objects.hash(super.hashCode(), guid, homeId, isProxy, provenanceType, status, createdBy, updatedBy, createTime, updateTime, version,
                             relationshipAttributes, classifications);
     }
 

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -1086,6 +1086,10 @@ public final class GraphHelper {
         return element.getProperty(Constants.IS_PROXY_KEY, Boolean.class);
     }
 
+    public static Integer getProvenanceType(AtlasElement element) {
+        return element.getProperty(Constants.PROVENANCE_TYPE_KEY, Integer.class);
+    }
+
     public static String getTypeName(AtlasElement element) {
         return element.getProperty(ENTITY_TYPE_PROPERTY_KEY, String.class);
     }

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java

@@ -74,6 +74,9 @@ import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
 import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
 import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
+
+
 import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
 import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
@@ -748,6 +751,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
             AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
             AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, guid);
             AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
+            AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY, relationship.getProvenanceType());
             AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
 
             // blocked propagated classifications

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java

@@ -165,6 +165,10 @@ public class EntityGraphMapper {
         if (entity.isProxy() != null) {
             AtlasGraphUtilsV2.setEncodedProperty(vertex, IS_PROXY_KEY, entity.isProxy());
         }
+
+        if (entity.getProvenanceType() != null) {
+            AtlasGraphUtilsV2.setEncodedProperty(vertex, PROVENANCE_TYPE_KEY, entity.getProvenanceType());
+        }
     }
 
     public EntityMutationResponse mapAttributesAndClassifications(EntityMutationContext context, final boolean isPartialUpdate, final boolean replaceClassifications) throws AtlasBaseException {

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java

@@ -525,6 +525,8 @@ public final class EntityGraphRetriever {
 
         entity.setIsProxy(GraphHelper.isProxy(entityVertex));
 
+        entity.setProvenanceType(GraphHelper.getProvenanceType(entityVertex));
+
         return entity;
     }
 
@@ -1166,12 +1168,17 @@ public final class EntityGraphRetriever {
         relationship.setUpdateTime(new Date(GraphHelper.getModifiedTime(edge)));
 
         Long version = GraphHelper.getVersion(edge);
-
         if (version == null) {
             version = Long.valueOf(1L);
         }
-
         relationship.setVersion(version);
+
+        Integer provenanceType = GraphHelper.getProvenanceType(edge);
+        if (provenanceType == null) {
+            provenanceType = Integer.valueOf(0);
+        }
+        relationship.setProvenanceType(provenanceType);
+
         relationship.setStatus(GraphHelper.getEdgeStatus(edge));
 
         AtlasVertex end1Vertex = edge.getOutVertex();