Skip to content

A
atlas
Commit 8fe110c3 by Sarath Subramanian Committed by Madhan Neethiraj
Options

ATLAS-1907: created RelationshipDefs for all base models and use it when…

ATLAS-1907: created RelationshipDefs for all base models and use it when creating edges for entities Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent b1350182
Showing with 757 additions and 59 deletions
addons/models/0010-base_model.json
...@@ -93,5 +93,45 @@ ...@@ -93,5 +93,45 @@
} }
] ]
} }
],
"relationshipDefs": [
{
"name": "dataset_process_inputs",
"typeVersion": "1.0",
"relationshipCategory": "AGGREGATION",
"endDef1": {
"type": "DataSet",
"name": "sourceToProcesses",
"isContainer": "false",
"cardinality": "SET"
},
"endDef2": {
"type": "Process",
"name": "inputs",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__Process.inputs"
},
"propagateTags": "NONE"
},
{
"name": "process_dataset_outputs",
"typeVersion": "1.0",
"relationshipCategory": "AGGREGATION",
"endDef1": {
"type": "Process",
"name": "outputs",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__Process.outputs"
},
"endDef2": {
"type": "DataSet",
"name": "sinkFromProcesses",
"isContainer": "false",
"cardinality": "SET"
},
"propagateTags": "NONE"
}
] ]
} }
\ No newline at end of file
addons/models/0030-hive_model.json
...@@ -521,5 +521,105 @@ ...@@ -521,5 +521,105 @@
} }
] ]
} }
],
"relationshipDefs": [
{
"name": "hive_db_tables",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_db",
"name": "tables",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "hive_table",
"name": "db",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_table.db"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_table_columns",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_table",
"name": "columns",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hive_table.columns"
},
"endDef2": {
"type": "hive_column",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_column.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_table_partitionkeys",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_table",
"name": "partitionKeys",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hive_table.partitionKeys"
},
"endDef2": {
"type": "hive_column",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_column.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_table_storagedesc",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_table",
"name": "sd",
"isContainer": "true",
"cardinality": "SINGLE",
"legacyLabel": "__hive_table.sd"
},
"endDef2": {
"type": "hive_storagedesc",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_storagedesc.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_process_column_lineage",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_process",
"name": "columnLineages",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "hive_column_lineage",
"name": "query",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_column_lineage.query"
},
"propagateTags": "NONE"
}
] ]
} }
\ No newline at end of file
addons/models/0050-falcon_model.json
...@@ -143,5 +143,64 @@ ...@@ -143,5 +143,64 @@
} }
] ]
} }
],
"relationshipDefs": [
{
"name": "falcon_feed_cluster",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "falcon_feed",
"name": "stored-in",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__falcon_feed.stored-in"
},
"endDef2": {
"type": "falcon_cluster",
"name": "feeds",
"isContainer": "true",
"cardinality": "SET"
},
"propagateTags": "NONE"
},
{
"name": "falcon_cluster_process",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "falcon_cluster",
"name": "processes",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "falcon_process",
"name": "runs-on",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__falcon_process.runs-on"
},
"propagateTags": "NONE"
},
{
"name": "falcon_cluster_feed_creation",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "falcon_cluster",
"name": "feedCreations",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "falcon_feed_creation",
"name": "stored-in",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__falcon_feed_creation.stored-in"
},
"propagateTags": "NONE"
}
] ]
} }
\ No newline at end of file
addons/models/0060-hbase_model.json
...@@ -96,5 +96,47 @@ ...@@ -96,5 +96,47 @@
], ],
"typeVersion": "1.0" "typeVersion": "1.0"
} }
],
"relationshipDefs": [
{
"name": "hbase_table_column_families",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hbase_table",
"name": "column_families",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hbase_table.column_families"
},
"endDef2": {
"type": "hbase_column_family",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hbase_column_family.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hbase_column_family_columns",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hbase_column_family",
"name": "columns",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hbase_column_family.columns"
},
"endDef2": {
"type": "hbase_column",
"name": "column_family",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hbase_column.column_family"
},
"propagateTags": "ONE_TO_TWO"
}
] ]
} }
\ No newline at end of file
addons/models/0080-storm_model.json
...@@ -142,5 +142,26 @@ ...@@ -142,5 +142,26 @@
} }
] ]
} }
],
"relationshipDefs": [
{
"name": "storm_topology_nodes",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"endDef1": {
"type": "storm_topology",
"name": "nodes",
"isContainer": "false",
"cardinality": "SET",
"legacyLabel": "__storm_topology.nodes"
},
"endDef2": {
"type": "storm_node",
"name": "topolgies",
"isContainer": "false",
"cardinality": "SET"
},
"propagateTags": "NONE"
}
] ]
} }
\ No newline at end of file
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
...@@ -80,13 +80,14 @@ public enum AtlasErrorCode { ...@@ -80,13 +80,14 @@ public enum AtlasErrorCode {
RELATIONSHIPDEF_COMPOSITION_MULTIPLE_PARENTS(400, "ATLAS-400-00-033", "COMPOSITION relationshipDef {0} can only have one parent; so cannot have SET cardinality on children"), RELATIONSHIPDEF_COMPOSITION_MULTIPLE_PARENTS(400, "ATLAS-400-00-033", "COMPOSITION relationshipDef {0} can only have one parent; so cannot have SET cardinality on children"),
RELATIONSHIPDEF_LIST_ON_END(400, "ATLAS-400-00-034", "relationshipDef {0} cannot have a LIST cardinality on an end"), RELATIONSHIPDEF_LIST_ON_END(400, "ATLAS-400-00-034", "relationshipDef {0} cannot have a LIST cardinality on an end"),
RELATIONSHIPDEF_INVALID_END_TYPE(400, "ATLAS-400-00-035", "relationshipDef {0} has invalid end type {1}"), RELATIONSHIPDEF_INVALID_END_TYPE(400, "ATLAS-400-00-035", "relationshipDef {0} has invalid end type {1}"),
INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid update for relationshipDef {0}: new end type {1}, existing end type {2}"), INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid relationshipDef: {0}: end type 1: {1}, end type 2: {2}"),
RELATIONSHIPDEF_INVALID_END1_UPDATE(400, "ATLAS-400-00-037", "invalid update for relationshipDef {0}: new end1 {1}, existing end1 {2}"), RELATIONSHIPDEF_INVALID_END1_UPDATE(400, "ATLAS-400-00-037", "invalid update for relationshipDef {0}: new end1 {1}, existing end1 {2}"),
RELATIONSHIPDEF_INVALID_END2_UPDATE(400, "ATLAS-400-00-038", "invalid update for relationshipDef {0}: new end2 {1}, existing end2 {2}"), RELATIONSHIPDEF_INVALID_END2_UPDATE(400, "ATLAS-400-00-038", "invalid update for relationshipDef {0}: new end2 {1}, existing end2 {2}"),
RELATIONSHIPDEF_INVALID_CATEGORY_UPDATE(400, "ATLAS-400-00-039", "invalid update for relationship {0}: new relationshipDef category {1}, existing relationshipDef category {2}"), RELATIONSHIPDEF_INVALID_CATEGORY_UPDATE(400, "ATLAS-400-00-039", "invalid update for relationship {0}: new relationshipDef category {1}, existing relationshipDef category {2}"),
RELATIONSHIPDEF_INVALID_NAME_UPDATE(400, "ATLAS-400-00-040", "invalid relationshipDef rename for relationship guid {0}: new name {1}, existing name {2}"), RELATIONSHIPDEF_INVALID_NAME_UPDATE(400, "ATLAS-400-00-040", "invalid relationshipDef rename for relationship guid {0}: new name {1}, existing name {2}"),
RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end1 name. Name must not contain query keywords"), RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-041", "{0}: invalid end1 name. Name must not contain query keywords"),
RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end2 name. Name must not contain query keywords"), RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-042", "{0}: invalid end2 name. Name must not contain query keywords"),
RELATIONSHIPDEF_NOT_DEFINED(400, "ATLAS-400-00-043", "No relationshipDef defined between {0} and {1} on attribute: {2}"),
// All Not found enums go here // All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"), TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS-404-00-002", "Given type guid {0} was invalid"), TYPE_GUID_NOT_FOUND(404, "ATLAS-404-00-002", "Given type guid {0} was invalid"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
...@@ -70,6 +70,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -70,6 +70,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private Date updateTime = null; private Date updateTime = null;
private Long version = 0L; private Long version = 0L;
private Map<String, Object> relationshipAttributes;
private List<AtlasClassification> classifications; private List<AtlasClassification> classifications;
@JsonIgnore @JsonIgnore
...@@ -170,6 +171,25 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -170,6 +171,25 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this.version = version; this.version = version;
} }
public Map<String, Object> getRelationshipAttributes() { return relationshipAttributes; }
public void setRelationshipAttributes(Map<String, Object> relationshipAttributes) {
this.relationshipAttributes = relationshipAttributes;
}
public void addRelationshipAttribute(String name, Object value) {
Map<String, Object> r = this.relationshipAttributes;
if (r != null) {
r.put(name, value);
} else {
r = new HashMap<>();
r.put(name, value);
this.relationshipAttributes = r;
}
}
public List<AtlasClassification> getClassifications() { return classifications; } public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; } public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
...@@ -204,6 +224,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -204,6 +224,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
dumpDateField(", createTime=", createTime, sb); dumpDateField(", createTime=", createTime, sb);
dumpDateField(", updateTime=", updateTime, sb); dumpDateField(", updateTime=", updateTime, sb);
sb.append(", version=").append(version); sb.append(", version=").append(version);
sb.append(", relationshipAttributes=[");
dumpObjects(relationshipAttributes, sb);
sb.append("]");
sb.append(", classifications=["); sb.append(", classifications=[");
AtlasBaseTypeDef.dumpObjects(classifications, sb); AtlasBaseTypeDef.dumpObjects(classifications, sb);
sb.append(']'); sb.append(']');
...@@ -227,13 +250,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable { ...@@ -227,13 +250,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
Objects.equals(createTime, that.createTime) && Objects.equals(createTime, that.createTime) &&
Objects.equals(updateTime, that.updateTime) && Objects.equals(updateTime, that.updateTime) &&
Objects.equals(version, that.version) && Objects.equals(version, that.version) &&
Objects.equals(relationshipAttributes, that.relationshipAttributes) &&
Objects.equals(classifications, that.classifications); Objects.equals(classifications, that.classifications);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(super.hashCode(), guid, status, createdBy, updatedBy, createTime, updateTime, version, return Objects.hash(super.hashCode(), guid, status, createdBy, updatedBy, createTime, updateTime, version,
classifications); relationshipAttributes, classifications);
} }
@Override @Override
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
...@@ -50,6 +50,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -50,6 +50,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
private String guid = null; private String guid = null;
private AtlasObjectId end1 = null; private AtlasObjectId end1 = null;
private AtlasObjectId end2 = null; private AtlasObjectId end2 = null;
private String label = null;
private Status status = Status.ACTIVE; private Status status = Status.ACTIVE;
private String createdBy = null; private String createdBy = null;
private String updatedBy = null; private String updatedBy = null;
...@@ -81,7 +82,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -81,7 +82,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) { public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) {
super(typeName); super(typeName);
init(nextInternalId(), end1, end2, null, null, null, null, null, 0L); init(nextInternalId(), end1, end2, null, null, null, null, null, null, 0L);
} }
public AtlasRelationship(String typeName, String attrName, Object attrValue) { public AtlasRelationship(String typeName, String attrName, Object attrValue) {
...@@ -98,7 +99,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -98,7 +99,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
super(other); super(other);
if (other != null) { if (other != null) {
init(other.guid, other.end1, other.end2, other.status, other.createdBy, other.updatedBy, init(other.guid, other.end1, other.end2, other.label, other.status, other.createdBy, other.updatedBy,
other.createTime, other.updateTime, other.version); other.createTime, other.updateTime, other.version);
} }
} }
...@@ -167,6 +168,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -167,6 +168,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public void setEnd2(AtlasObjectId end2) { this.end2 = end2; } public void setEnd2(AtlasObjectId end2) { this.end2 = end2; }
public String getLabel() { return label; }
public void setLabel(String label) { this.label = label; }
private static String nextInternalId() { private static String nextInternalId() {
return "-" + Long.toString(s_nextId.getAndIncrement()); return "-" + Long.toString(s_nextId.getAndIncrement());
} }
...@@ -174,15 +179,16 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -174,15 +179,16 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public String getRelationshipLabel() { return "r:" + super.getTypeName(); } public String getRelationshipLabel() { return "r:" + super.getTypeName(); }
private void init() { private void init() {
init(nextInternalId(), null, null, null, null, null, null, null, 0L); init(nextInternalId(), null, null, null, null, null, null, null, null, 0L);
} }
private void init(String guid, AtlasObjectId end1, AtlasObjectId end2, private void init(String guid, AtlasObjectId end1, AtlasObjectId end2, String label,
Status status, String createdBy, String updatedBy, Status status, String createdBy, String updatedBy,
Date createTime, Date updateTime, Long version) { Date createTime, Date updateTime, Long version) {
setGuid(guid); setGuid(guid);
setEnd1(end1); setEnd1(end1);
setEnd2(end2); setEnd2(end2);
setLabel(label);
setStatus(status); setStatus(status);
setCreatedBy(createdBy); setCreatedBy(createdBy);
setUpdatedBy(updatedBy); setUpdatedBy(updatedBy);
...@@ -202,6 +208,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -202,6 +208,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
sb.append("guid='").append(guid).append('\''); sb.append("guid='").append(guid).append('\'');
sb.append(", end1=").append(end1); sb.append(", end1=").append(end1);
sb.append(", end2=").append(end2); sb.append(", end2=").append(end2);
sb.append(", label='").append(label).append('\'');
sb.append(", status=").append(status); sb.append(", status=").append(status);
sb.append(", createdBy='").append(createdBy).append('\''); sb.append(", createdBy='").append(createdBy).append('\'');
sb.append(", updatedBy='").append(updatedBy).append('\''); sb.append(", updatedBy='").append(updatedBy).append('\'');
...@@ -223,6 +230,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -223,6 +230,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
return Objects.equals(guid, that.guid) && return Objects.equals(guid, that.guid) &&
Objects.equals(end1, that.end1) && Objects.equals(end1, that.end1) &&
Objects.equals(end2, that.end2) && Objects.equals(end2, that.end2) &&
Objects.equals(label, that.label) &&
status == that.status && status == that.status &&
Objects.equals(createdBy, that.createdBy) && Objects.equals(createdBy, that.createdBy) &&
Objects.equals(updatedBy, that.updatedBy) && Objects.equals(updatedBy, that.updatedBy) &&
...@@ -233,7 +241,8 @@ public class AtlasRelationship extends AtlasStruct implements Serializable { ...@@ -233,7 +241,8 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(super.hashCode(), guid, end1, end2, status, createdBy, updatedBy, createTime, updateTime, version); return Objects.hash(super.hashCode(), guid, end1, end2, label, status, createdBy,
updatedBy, createTime, updateTime, version);
} }
@Override @Override
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
...@@ -148,7 +148,7 @@ public class AtlasStruct implements Serializable { ...@@ -148,7 +148,7 @@ public class AtlasStruct implements Serializable {
AtlasStruct that = (AtlasStruct) o; AtlasStruct that = (AtlasStruct) o;
return Objects.equals(typeName, that.typeName) && return Objects.equals(typeName, that.typeName) &&
Objects.equals(attributes, that.attributes); Objects.equals(attributes, that.attributes);
} }
@Override @Override
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
...@@ -26,6 +26,7 @@ import org.apache.atlas.model.typedef.AtlasEntityDef; ...@@ -26,6 +26,7 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -47,11 +48,13 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -47,11 +48,13 @@ public class AtlasEntityType extends AtlasStructType {
private final AtlasEntityDef entityDef; private final AtlasEntityDef entityDef;
private List<AtlasEntityType> superTypes = Collections.emptyList(); private List<AtlasEntityType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet(); private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet(); private Set<String> allSubTypes = Collections.emptySet();
private Set<String> typeAndAllSubTypes = Collections.emptySet(); private Set<String> typeAndAllSubTypes = Collections.emptySet();
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap(); private Set<String> typeAndAllSuperTypes = Collections.emptySet();
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
public AtlasEntityType(AtlasEntityDef entityDef) { public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef); super(entityDef);
...@@ -89,15 +92,20 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -89,15 +92,20 @@ public class AtlasEntityType extends AtlasStructType {
} }
} }
this.superTypes = Collections.unmodifiableList(s); this.superTypes = Collections.unmodifiableList(s);
this.allSuperTypes = Collections.unmodifiableSet(allS); this.allSuperTypes = Collections.unmodifiableSet(allS);
this.allAttributes = Collections.unmodifiableMap(allA); this.allAttributes = Collections.unmodifiableMap(allA);
this.uniqAttributes = getUniqueAttributes(this.allAttributes); this.uniqAttributes = getUniqueAttributes(this.allAttributes);
this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2() this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase2() this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase3()
this.relationshipAttributesType = new HashMap<>(); // this will be populated in resolveReferencesPhase3()
this.typeAndAllSubTypes.add(this.getTypeName()); this.typeAndAllSubTypes.add(this.getTypeName());
this.typeAndAllSuperTypes = new HashSet<>(this.allSuperTypes);
this.typeAndAllSuperTypes.add(this.getTypeName());
this.typeAndAllSuperTypes = Collections.unmodifiableSet(this.typeAndAllSuperTypes);
} }
@Override @Override
...@@ -110,6 +118,43 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -110,6 +118,43 @@ public class AtlasEntityType extends AtlasStructType {
} }
} }
@Override
public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
for (AtlasAttributeDef attributeDef : getStructDef().getAttributeDefs()) {
String attributeName = attributeDef.getName();
AtlasType attributeType = typeRegistry.getType(attributeDef.getTypeName());
AtlasEntityType attributeEntityType = getReferencedEntityType(attributeType);
// validate if RelationshipDefs is defined for all entityDefs
if (attributeEntityType != null && !hasRelationshipAttribute(attributeName)) {
LOG.warn("No RelationshipDef defined between {} and {} on attribute: {}.{}", getTypeName(),
attributeEntityType.getTypeName(), getTypeName(), attributeName);
}
}
for (String superTypeName : allSuperTypes) {
AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName);
Map<String, AtlasAttribute> superTypeRelationshipAttributes = superType.getRelationshipAttributes();
if (MapUtils.isNotEmpty(superTypeRelationshipAttributes)) {
relationshipAttributes.putAll(superTypeRelationshipAttributes);
}
Map<String, List<AtlasRelationshipType>> superTypeRelationshipAttributesType = superType.getRelationshipAttributesType();
if (MapUtils.isNotEmpty(superTypeRelationshipAttributesType)) {
relationshipAttributesType.putAll(superTypeRelationshipAttributesType);
}
}
allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
typeAndAllSuperTypes = Collections.unmodifiableSet(typeAndAllSuperTypes);
relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes);
relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType);
}
public Set<String> getSuperTypes() { public Set<String> getSuperTypes() {
return entityDef.getSuperTypes(); return entityDef.getSuperTypes();
} }
...@@ -118,9 +163,11 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -118,9 +163,11 @@ public class AtlasEntityType extends AtlasStructType {
return allSuperTypes; return allSuperTypes;
} }
public Set<String> getAllSubTypes() { return Collections.unmodifiableSet(allSubTypes); } public Set<String> getAllSubTypes() { return allSubTypes; }
public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); } public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
public Set<String> getTypeAndAllSuperTypes() { return typeAndAllSuperTypes; }
public boolean isSuperTypeOf(AtlasEntityType entityType) { public boolean isSuperTypeOf(AtlasEntityType entityType) {
return entityType != null && allSubTypes.contains(entityType.getTypeName()); return entityType != null && allSubTypes.contains(entityType.getTypeName());
...@@ -142,12 +189,37 @@ public class AtlasEntityType extends AtlasStructType { ...@@ -142,12 +189,37 @@ public class AtlasEntityType extends AtlasStructType {
return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName); return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
} }
public Map<String, AtlasAttribute> getRelationshipAttributes() { return Collections.unmodifiableMap(relationshipAttributes); } public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
public void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) { // this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
relationshipAttributes.put(attributeName, attribute); relationshipAttributes.put(attributeName, attribute);
} }
// this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
void addRelationshipAttributeType(String attributeName, AtlasRelationshipType relationshipType) {
List<AtlasRelationshipType> relationshipTypes = relationshipAttributesType.get(attributeName);
if (relationshipTypes == null) {
relationshipTypes = new ArrayList<>();
relationshipAttributesType.put(attributeName, relationshipTypes);
}
relationshipTypes.add(relationshipType);
}
public List<AtlasRelationshipType> getRelationshipAttributeType(String attributeName) {
return relationshipAttributesType.get(attributeName);
}
public Map<String, List<AtlasRelationshipType>> getRelationshipAttributesType() {
return relationshipAttributesType;
}
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
}
@Override @Override
public AtlasEntity createDefaultValue() { public AtlasEntity createDefaultValue() {
AtlasEntity ret = new AtlasEntity(entityDef.getName()); AtlasEntity ret = new AtlasEntity(entityDef.getName());
......
intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
...@@ -90,9 +90,26 @@ public class AtlasRelationshipType extends AtlasStructType { ...@@ -90,9 +90,26 @@ public class AtlasRelationshipType extends AtlasStructType {
public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferencesPhase2(typeRegistry); super.resolveReferencesPhase2(typeRegistry);
addRelationshipAttributeToEndType(relationshipDef.getEndDef1(), end1Type, end2Type.getTypeName(), typeRegistry); AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
String relationshipLabel = null;
addRelationshipAttributeToEndType(relationshipDef.getEndDef2(), end2Type, end1Type.getTypeName(), typeRegistry); // if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
// if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
// if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
if (!endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
relationshipLabel = relationshipDef.getName();
} else if (endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
relationshipLabel = endDef1.getLegacyLabel();
} else if (!endDef1.hasLegacyRelation() && endDef2.hasLegacyRelation()) {
relationshipLabel = endDef2.getLegacyLabel();
}
addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel);
addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel);
} }
@Override @Override
...@@ -198,10 +215,8 @@ public class AtlasRelationshipType extends AtlasStructType { ...@@ -198,10 +215,8 @@ public class AtlasRelationshipType extends AtlasStructType {
} }
} }
private void addRelationshipAttributeToEndType(AtlasRelationshipEndDef endDef, private void addRelationshipAttributeToEndType(AtlasRelationshipEndDef endDef, AtlasEntityType entityType, String attrTypeName,
AtlasEntityType entityType, AtlasTypeRegistry typeRegistry, String relationshipLabel) throws AtlasBaseException {
String attrTypeName,
AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
String attrName = (endDef != null) ? endDef.getName() : null; String attrName = (endDef != null) ? endDef.getName() : null;
...@@ -211,15 +226,29 @@ public class AtlasRelationshipType extends AtlasStructType { ...@@ -211,15 +226,29 @@ public class AtlasRelationshipType extends AtlasStructType {
AtlasAttribute attribute = entityType.getAttribute(attrName); AtlasAttribute attribute = entityType.getAttribute(attrName);
// if relationshipLabel is null, then legacyLabel is mentioned at both ends,
// use the respective end's legacyLabel as relationshipLabel
if (relationshipLabel == null) {
relationshipLabel = endDef.getLegacyLabel();
}
if (attribute == null) { //attr doesn't exist in type - is a new relationship attribute if (attribute == null) { //attr doesn't exist in type - is a new relationship attribute
if (endDef.getCardinality() == Cardinality.SET) { if (endDef.getCardinality() == Cardinality.SET) {
attrTypeName = AtlasBaseTypeDef.getArrayTypeName(attrTypeName); attrTypeName = AtlasBaseTypeDef.getArrayTypeName(attrTypeName);
} }
attribute = new AtlasAttribute(entityType, new AtlasAttributeDef(attrName, attrTypeName), typeRegistry.getType(attrTypeName)); attribute = new AtlasAttribute(entityType, new AtlasAttributeDef(attrName, attrTypeName),
typeRegistry.getType(attrTypeName), relationshipLabel);
} else {
// attribute already exists (legacy attribute which is also a relationship attribute)
// add relationshipLabel information to existing attribute
attribute.setRelationshipEdgeLabel(relationshipLabel);
} }
entityType.addRelationshipAttribute(attrName, attribute); entityType.addRelationshipAttribute(attrName, attribute);
entityType.addRelationshipAttributeType(attrName, this);
} }
} }
\ No newline at end of file
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
...@@ -28,7 +28,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef; ...@@ -28,7 +28,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
...@@ -170,9 +169,10 @@ public class AtlasStructType extends AtlasType { ...@@ -170,9 +169,10 @@ public class AtlasStructType extends AtlasType {
continue; continue;
} }
// Set the inverse reference attribute. // Set the inverse reference attribute.
AtlasType referencedType = typeRegistry.getType(attribute.getAttributeDef().getTypeName()); AtlasType referencedType = typeRegistry.getType(attribute.getAttributeDef().getTypeName());
AtlasEntityType referencedEntityType = getReferencedEntityType(referencedType); AtlasEntityType referencedEntityType = getReferencedEntityType(referencedType);
AtlasAttribute inverseReference = referencedEntityType.getAttribute(attribute.getInverseRefAttributeName()); AtlasAttribute inverseReference = referencedEntityType.getAttribute(attribute.getInverseRefAttributeName());
attribute.setInverseRefAttribute(inverseReference); attribute.setInverseRefAttribute(inverseReference);
} }
} }
...@@ -574,7 +574,7 @@ public class AtlasStructType extends AtlasType { ...@@ -574,7 +574,7 @@ public class AtlasStructType extends AtlasType {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, structDef.getName()); throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, structDef.getName());
} }
private AtlasEntityType getReferencedEntityType(AtlasType type) { AtlasEntityType getReferencedEntityType(AtlasType type) {
if (type instanceof AtlasArrayType) { if (type instanceof AtlasArrayType) {
type = ((AtlasArrayType)type).getElementType(); type = ((AtlasArrayType)type).getElementType();
} }
...@@ -609,16 +609,17 @@ public class AtlasStructType extends AtlasType { ...@@ -609,16 +609,17 @@ public class AtlasStructType extends AtlasType {
private final boolean isOwnedRef; private final boolean isOwnedRef;
private final String inverseRefAttributeName; private final String inverseRefAttributeName;
private AtlasAttribute inverseRefAttribute; private AtlasAttribute inverseRefAttribute;
private String relationshipEdgeLabel;
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
this.definedInType = definedInType; public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipLabel) {
this.attributeDef = attrDef; this.definedInType = definedInType;
this.attributeType = attributeType.getTypeForAttribute(); this.attributeDef = attrDef;
this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName()); this.attributeType = attributeType.getTypeForAttribute();
this.vertexPropertyName = encodePropertyKey(this.qualifiedName); this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
this.vertexPropertyName = encodePropertyKey(this.qualifiedName);
boolean isOwnedRef = false; this.relationshipEdgeLabel = getRelationshipEdgeLabel(relationshipLabel);
String inverseRefAttribute = null; boolean isOwnedRef = false;
String inverseRefAttribute = null;
if (CollectionUtils.isNotEmpty(attributeDef.getConstraints())) { if (CollectionUtils.isNotEmpty(attributeDef.getConstraints())) {
for (AtlasConstraintDef constraint : attributeDef.getConstraints()) { for (AtlasConstraintDef constraint : attributeDef.getConstraints()) {
...@@ -636,10 +637,14 @@ public class AtlasStructType extends AtlasType { ...@@ -636,10 +637,14 @@ public class AtlasStructType extends AtlasType {
} }
} }
this.isOwnedRef = isOwnedRef; this.isOwnedRef = isOwnedRef;
this.inverseRefAttributeName = inverseRefAttribute; this.inverseRefAttributeName = inverseRefAttribute;
} }
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
this(definedInType, attrDef, attributeType, null);
}
public AtlasStructType getDefinedInType() { return definedInType; } public AtlasStructType getDefinedInType() { return definedInType; }
public AtlasStructDef getDefinedInDef() { return definedInType.getStructDef(); } public AtlasStructDef getDefinedInDef() { return definedInType.getStructDef(); }
...@@ -666,7 +671,15 @@ public class AtlasStructType extends AtlasType { ...@@ -666,7 +671,15 @@ public class AtlasStructType extends AtlasType {
public AtlasAttribute getInverseRefAttribute() { return inverseRefAttribute; } public AtlasAttribute getInverseRefAttribute() { return inverseRefAttribute; }
public void setInverseRefAttribute(AtlasAttribute inverseAttr) { inverseRefAttribute = inverseAttr; }; public void setInverseRefAttribute(AtlasAttribute inverseAttr) { inverseRefAttribute = inverseAttr; }
public String getRelationshipEdgeLabel() { return relationshipEdgeLabel; }
public void setRelationshipEdgeLabel(String relationshipEdgeLabel) { this.relationshipEdgeLabel = relationshipEdgeLabel; }
public static String getEdgeLabel(String property) {
return "__" + property;
}
public static String encodePropertyKey(String key) { public static String encodePropertyKey(String key) {
if (StringUtils.isBlank(key)) { if (StringUtils.isBlank(key)) {
...@@ -692,6 +705,10 @@ public class AtlasStructType extends AtlasType { ...@@ -692,6 +705,10 @@ public class AtlasStructType extends AtlasType {
return key; return key;
} }
private String getRelationshipEdgeLabel(String relationshipLabel) {
return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel;
}
private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) { private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
final String typeName = structDef.getName(); final String typeName = structDef.getName();
return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName); return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName);
......
intg/src/main/java/org/apache/atlas/type/AtlasType.java
...@@ -57,6 +57,9 @@ public abstract class AtlasType { ...@@ -57,6 +57,9 @@ public abstract class AtlasType {
public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
} }
public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
}
public String getTypeName() { return typeName; } public String getTypeName() { return typeName; }
public TypeCategory getTypeCategory() { return typeCategory; } public TypeCategory getTypeCategory() { return typeCategory; }
......
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
...@@ -364,6 +364,10 @@ public class AtlasTypeRegistry { ...@@ -364,6 +364,10 @@ public class AtlasTypeRegistry {
for (AtlasType type : registryData.allTypes.getAllTypes()) { for (AtlasType type : registryData.allTypes.getAllTypes()) {
type.resolveReferencesPhase2(this); type.resolveReferencesPhase2(this);
} }
for (AtlasEntityType entityType : registryData.entityDefs.getAllTypes()) {
entityType.resolveReferencesPhase3(this);
}
} }
public void clear() { public void clear() {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
...@@ -27,6 +27,7 @@ import org.apache.atlas.RequestContext; ...@@ -27,6 +27,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity.Status; import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException; import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdge;
...@@ -36,6 +37,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; ...@@ -36,6 +37,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1; import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance; import org.apache.atlas.typesystem.ITypedInstance;
...@@ -345,6 +348,10 @@ public final class GraphHelper { ...@@ -345,6 +348,10 @@ public final class GraphHelper {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel); return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
} }
public Iterator<AtlasEdge> getBothEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.BOTH, edgeLabel);
}
/** /**
* Returns the active edge for the given edge label. * Returns the active edge for the given edge label.
* If the vertex is deleted and there is no active edge, it returns the latest deleted edge * If the vertex is deleted and there is no active edge, it returns the latest deleted edge
...@@ -1223,4 +1230,50 @@ public final class GraphHelper { ...@@ -1223,4 +1230,50 @@ public final class GraphHelper {
return condition.toString(); return condition.toString();
} }
}
/**
* Get relationshipDef name from entityType using relationship attribute.
* if more than one relationDefs are returned for an attribute.
* e.g. hive_column.table
*
* hive_table.columns -> hive_column.table
* hive_table.partitionKeys -> hive_column.table
*
* resolve by comparing all incoming edges typename with relationDefs name returned for an attribute
* to pick the right relationshipDef name
*/
public String getRelationshipDefName(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
AtlasRelationshipDef relationshipDef = getRelationshipDef(entityVertex, entityType, attributeName);
return (relationshipDef != null) ? relationshipDef.getName() : null;
}
public AtlasRelationshipDef getRelationshipDef(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
List<AtlasRelationshipType> relationshipTypes = entityType.getRelationshipAttributeType(attributeName);
AtlasRelationshipDef ret = null;
if (relationshipTypes.size() > 1) {
Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
while (iter.hasNext() && ret == null) {
String edgeTypeName = AtlasGraphUtilsV1.getTypeName(iter.next());
for (AtlasRelationshipType relationType : relationshipTypes) {
AtlasRelationshipDef relationshipDef = relationType.getRelationshipDef();
if (StringUtils.equals(edgeTypeName, relationshipDef.getName())) {
ret = relationshipDef;
break;
}
}
}
} else {
//relationshipTypes will have at least one relationshipDef
ret = relationshipTypes.get(0).getRelationshipDef();
}
return ret;
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
...@@ -24,6 +24,7 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef; ...@@ -24,6 +24,7 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef; import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef; import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef; import org.apache.atlas.model.typedef.AtlasTypesDef;
...@@ -164,6 +165,14 @@ public class AtlasTypeDefStoreInitializer { ...@@ -164,6 +165,14 @@ public class AtlasTypeDefStoreInitializer {
} }
} }
if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
if (!typeRegistry.isRegisteredType(relationshipDef.getName())) {
typesToCreate.getRelationshipDefs().add(relationshipDef);
}
}
}
return typesToCreate; return typesToCreate;
} }
...@@ -234,6 +243,20 @@ public class AtlasTypeDefStoreInitializer { ...@@ -234,6 +243,20 @@ public class AtlasTypeDefStoreInitializer {
} }
} }
if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
AtlasRelationshipDef oldRelationshipDef = typeRegistry.getRelationshipDefByName(relationshipDef.getName());
if (oldRelationshipDef == null) {
continue;
}
if (updateTypeAttributes(oldRelationshipDef, relationshipDef)) {
typesToUpdate.getRelationshipDefs().add(relationshipDef);
}
}
}
return typesToUpdate; return typesToUpdate;
} }
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
...@@ -76,8 +76,8 @@ public class AtlasGraphUtilsV1 { ...@@ -76,8 +76,8 @@ public class AtlasGraphUtilsV1 {
return vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); return vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
} }
public static String getTypeName(AtlasVertex instanceVertex) { public static String getTypeName(AtlasElement element) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class); return element.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
} }
public static String getEdgeLabel(String fromNode, String toNode) { public static String getEdgeLabel(String fromNode, String toNode) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
...@@ -29,6 +29,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; ...@@ -29,6 +29,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graph.GraphHelper;
...@@ -49,6 +51,7 @@ import java.util.ArrayList; ...@@ -49,6 +51,7 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
...@@ -65,6 +68,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG; ...@@ -65,6 +68,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING; import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX; import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
public final class EntityGraphRetriever { public final class EntityGraphRetriever {
...@@ -180,6 +184,8 @@ public final class EntityGraphRetriever { ...@@ -180,6 +184,8 @@ public final class EntityGraphRetriever {
mapAttributes(entityVertex, entity, entityExtInfo); mapAttributes(entityVertex, entity, entityExtInfo);
mapRelationshipAttributes(entityVertex, entity, entityExtInfo);
mapClassifications(entityVertex, entity, entityExtInfo); mapClassifications(entityVertex, entity, entityExtInfo);
} }
...@@ -278,6 +284,23 @@ public final class EntityGraphRetriever { ...@@ -278,6 +284,23 @@ public final class EntityGraphRetriever {
} }
} }
private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(entity.getTypeName());
if (!(objType instanceof AtlasEntityType)) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, entity.getTypeName());
}
AtlasEntityType entityType = (AtlasEntityType) objType;
for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attribute, entityExtInfo);
entity.addRelationshipAttribute(attribute.getName(), attrValue);
}
}
public List<AtlasClassification> getClassifications(String guid) throws AtlasBaseException { public List<AtlasClassification> getClassifications(String guid) throws AtlasBaseException {
AtlasVertex instanceVertex = AtlasGraphUtilsV1.findByGuid(guid); AtlasVertex instanceVertex = AtlasGraphUtilsV1.findByGuid(guid);
...@@ -394,6 +417,40 @@ public final class EntityGraphRetriever { ...@@ -394,6 +417,40 @@ public final class EntityGraphRetriever {
return ret; return ret;
} }
private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, AtlasAttribute attribute,
AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
Object ret = null;
AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
AtlasRelationshipEndDef attributeEndDef = null;
if (endDef1Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef1.getName(), attribute.getName())) {
attributeEndDef = endDef1;
} else if (endDef2Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef2.getName(), attribute.getName())) {
attributeEndDef = endDef2;
}
String relationshipLabel = attribute.getRelationshipEdgeLabel();
switch (attributeEndDef.getCardinality()) {
case SINGLE:
ret = mapVertexToObjectId(entityVertex, relationshipLabel, null, entityExtInfo, attributeEndDef.getIsContainer());
break;
case LIST:
case SET:
ret = mapVertexToRelationshipArrayAttribute(entityVertex, (AtlasArrayType) attribute.getAttributeType(), relationshipLabel,
entityExtInfo, attributeEndDef.getIsContainer());
break;
}
return ret;
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName, private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException { AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName); List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName);
...@@ -451,6 +508,40 @@ public final class EntityGraphRetriever { ...@@ -451,6 +508,40 @@ public final class EntityGraphRetriever {
return arrValues; return arrValues;
} }
private List<Object> mapVertexToRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasArrayType arrayType,
String relationshipName, AtlasEntityExtInfo entityExtInfo,
boolean isContainer) throws AtlasBaseException {
Iterator<AtlasEdge> relationshipEdges = graphHelper.getBothEdgesByLabel(entityVertex, relationshipName);
AtlasType arrayElementType = arrayType.getElementType();
List<AtlasEdge> arrayElements = new ArrayList<>();
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping array attribute {} for vertex {}", arrayElementType.getTypeName(), entityVertex);
}
while (relationshipEdges.hasNext()) {
arrayElements.add(relationshipEdges.next());
}
if (CollectionUtils.isEmpty(arrayElements)) {
return null;
}
List arrValues = new ArrayList(arrayElements.size());
for (Object element : arrayElements) {
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, relationshipName,
entityExtInfo, isContainer);
if (arrValue != null) {
arrValues.add(arrValue);
}
}
return arrValues;
}
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value, String edgeLabel, private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value, String edgeLabel,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException { AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
Object ret = null; Object ret = null;
...@@ -538,7 +629,11 @@ public final class EntityGraphRetriever { ...@@ -538,7 +629,11 @@ public final class EntityGraphRetriever {
} }
if (GraphHelper.elementExists(edge)) { if (GraphHelper.elementExists(edge)) {
final AtlasVertex referenceVertex = edge.getInVertex(); AtlasVertex referenceVertex = edge.getInVertex();
if (StringUtils.equals(getIdFromVertex(referenceVertex), getIdFromVertex(entityVertex))) {
referenceVertex = edge.getOutVertex();
}
if (referenceVertex != null) { if (referenceVertex != null) {
if (entityExtInfo != null && isOwnedAttribute) { if (entityExtInfo != null && isOwnedAttribute) {
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import static org.mockito.Mockito.mock;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasRelationshipStoreV1Test {
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
AtlasTypeDefStore typeDefStore;
@Inject
DeleteHandlerV1 deleteHandler;
@Inject
EntityGraphMapper graphMapper;
AtlasEntityStore entityStore;
AtlasRelationshipStore relationshipStore;
AtlasEntityWithExtInfo dbEntity;
AtlasEntityWithExtInfo tblEntity;
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@BeforeClass
public void setUp() throws Exception {
new GraphBackedSearchIndexer(typeRegistry);
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
TestUtilsV2.defineHiveTypes() };
for (AtlasTypesDef typesDef : testTypesDefs) {
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
}
}
dbEntity = TestUtilsV2.createDBEntityV2();
tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
}
@AfterClass
public void clear() {
AtlasGraphProvider.cleanup();
}
@BeforeTest
public void init() throws Exception {
entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
relationshipStore = new AtlasRelationshipStoreV1(typeRegistry);
RequestContextV1.clear();
}
@Test
public void testDbTableRelationship() throws Exception {
// Add tests - in progress
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment