Skip to content

A
atlas
Commit 8fe110c3 by Sarath Subramanian Committed by Madhan Neethiraj
Options

ATLAS-1907: created RelationshipDefs for all base models and use it when…

ATLAS-1907: created RelationshipDefs for all base models and use it when creating edges for entities Signed-off-by: 's avatarMadhan Neethiraj <madhan@apache.org>
parent b1350182
Showing with 727 additions and 29 deletions
addons/models/0010-base_model.json
......@@ -93,5 +93,45 @@
}
]
}
],
"relationshipDefs": [
{
"name": "dataset_process_inputs",
"typeVersion": "1.0",
"relationshipCategory": "AGGREGATION",
"endDef1": {
"type": "DataSet",
"name": "sourceToProcesses",
"isContainer": "false",
"cardinality": "SET"
},
"endDef2": {
"type": "Process",
"name": "inputs",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__Process.inputs"
},
"propagateTags": "NONE"
},
{
"name": "process_dataset_outputs",
"typeVersion": "1.0",
"relationshipCategory": "AGGREGATION",
"endDef1": {
"type": "Process",
"name": "outputs",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__Process.outputs"
},
"endDef2": {
"type": "DataSet",
"name": "sinkFromProcesses",
"isContainer": "false",
"cardinality": "SET"
},
"propagateTags": "NONE"
}
]
}
\ No newline at end of file
addons/models/0030-hive_model.json
......@@ -521,5 +521,105 @@
}
]
}
],
"relationshipDefs": [
{
"name": "hive_db_tables",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_db",
"name": "tables",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "hive_table",
"name": "db",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_table.db"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_table_columns",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_table",
"name": "columns",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hive_table.columns"
},
"endDef2": {
"type": "hive_column",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_column.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_table_partitionkeys",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_table",
"name": "partitionKeys",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hive_table.partitionKeys"
},
"endDef2": {
"type": "hive_column",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_column.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_table_storagedesc",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_table",
"name": "sd",
"isContainer": "true",
"cardinality": "SINGLE",
"legacyLabel": "__hive_table.sd"
},
"endDef2": {
"type": "hive_storagedesc",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_storagedesc.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hive_process_column_lineage",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hive_process",
"name": "columnLineages",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "hive_column_lineage",
"name": "query",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hive_column_lineage.query"
},
"propagateTags": "NONE"
}
]
}
\ No newline at end of file
addons/models/0050-falcon_model.json
......@@ -143,5 +143,64 @@
}
]
}
],
"relationshipDefs": [
{
"name": "falcon_feed_cluster",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "falcon_feed",
"name": "stored-in",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__falcon_feed.stored-in"
},
"endDef2": {
"type": "falcon_cluster",
"name": "feeds",
"isContainer": "true",
"cardinality": "SET"
},
"propagateTags": "NONE"
},
{
"name": "falcon_cluster_process",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "falcon_cluster",
"name": "processes",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "falcon_process",
"name": "runs-on",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__falcon_process.runs-on"
},
"propagateTags": "NONE"
},
{
"name": "falcon_cluster_feed_creation",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "falcon_cluster",
"name": "feedCreations",
"isContainer": "true",
"cardinality": "SET"
},
"endDef2": {
"type": "falcon_feed_creation",
"name": "stored-in",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__falcon_feed_creation.stored-in"
},
"propagateTags": "NONE"
}
]
}
\ No newline at end of file
addons/models/0060-hbase_model.json
......@@ -96,5 +96,47 @@
],
"typeVersion": "1.0"
}
],
"relationshipDefs": [
{
"name": "hbase_table_column_families",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hbase_table",
"name": "column_families",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hbase_table.column_families"
},
"endDef2": {
"type": "hbase_column_family",
"name": "table",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hbase_column_family.table"
},
"propagateTags": "ONE_TO_TWO"
},
{
"name": "hbase_column_family_columns",
"typeVersion": "1.0",
"relationshipCategory": "COMPOSITION",
"endDef1": {
"type": "hbase_column_family",
"name": "columns",
"isContainer": "true",
"cardinality": "SET",
"legacyLabel": "__hbase_column_family.columns"
},
"endDef2": {
"type": "hbase_column",
"name": "column_family",
"isContainer": "false",
"cardinality": "SINGLE",
"legacyLabel": "__hbase_column.column_family"
},
"propagateTags": "ONE_TO_TWO"
}
]
}
\ No newline at end of file
addons/models/0080-storm_model.json
......@@ -142,5 +142,26 @@
}
]
}
],
"relationshipDefs": [
{
"name": "storm_topology_nodes",
"typeVersion": "1.0",
"relationshipCategory": "ASSOCIATION",
"endDef1": {
"type": "storm_topology",
"name": "nodes",
"isContainer": "false",
"cardinality": "SET",
"legacyLabel": "__storm_topology.nodes"
},
"endDef2": {
"type": "storm_node",
"name": "topolgies",
"isContainer": "false",
"cardinality": "SET"
},
"propagateTags": "NONE"
}
]
}
\ No newline at end of file
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
......@@ -80,13 +80,14 @@ public enum AtlasErrorCode {
RELATIONSHIPDEF_COMPOSITION_MULTIPLE_PARENTS(400, "ATLAS-400-00-033", "COMPOSITION relationshipDef {0} can only have one parent; so cannot have SET cardinality on children"),
RELATIONSHIPDEF_LIST_ON_END(400, "ATLAS-400-00-034", "relationshipDef {0} cannot have a LIST cardinality on an end"),
RELATIONSHIPDEF_INVALID_END_TYPE(400, "ATLAS-400-00-035", "relationshipDef {0} has invalid end type {1}"),
INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid update for relationshipDef {0}: new end type {1}, existing end type {2}"),
INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid relationshipDef: {0}: end type 1: {1}, end type 2: {2}"),
RELATIONSHIPDEF_INVALID_END1_UPDATE(400, "ATLAS-400-00-037", "invalid update for relationshipDef {0}: new end1 {1}, existing end1 {2}"),
RELATIONSHIPDEF_INVALID_END2_UPDATE(400, "ATLAS-400-00-038", "invalid update for relationshipDef {0}: new end2 {1}, existing end2 {2}"),
RELATIONSHIPDEF_INVALID_CATEGORY_UPDATE(400, "ATLAS-400-00-039", "invalid update for relationship {0}: new relationshipDef category {1}, existing relationshipDef category {2}"),
RELATIONSHIPDEF_INVALID_NAME_UPDATE(400, "ATLAS-400-00-040", "invalid relationshipDef rename for relationship guid {0}: new name {1}, existing name {2}"),
RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end1 name. Name must not contain query keywords"),
RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end2 name. Name must not contain query keywords"),
RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-041", "{0}: invalid end1 name. Name must not contain query keywords"),
RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-042", "{0}: invalid end2 name. Name must not contain query keywords"),
RELATIONSHIPDEF_NOT_DEFINED(400, "ATLAS-400-00-043", "No relationshipDef defined between {0} and {1} on attribute: {2}"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS-404-00-002", "Given type guid {0} was invalid"),
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
......@@ -70,6 +70,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private Date updateTime = null;
private Long version = 0L;
private Map<String, Object> relationshipAttributes;
private List<AtlasClassification> classifications;
@JsonIgnore
......@@ -170,6 +171,25 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this.version = version;
}
public Map<String, Object> getRelationshipAttributes() { return relationshipAttributes; }
public void setRelationshipAttributes(Map<String, Object> relationshipAttributes) {
this.relationshipAttributes = relationshipAttributes;
}
public void addRelationshipAttribute(String name, Object value) {
Map<String, Object> r = this.relationshipAttributes;
if (r != null) {
r.put(name, value);
} else {
r = new HashMap<>();
r.put(name, value);
this.relationshipAttributes = r;
}
}
public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
......@@ -204,6 +224,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
dumpDateField(", createTime=", createTime, sb);
dumpDateField(", updateTime=", updateTime, sb);
sb.append(", version=").append(version);
sb.append(", relationshipAttributes=[");
dumpObjects(relationshipAttributes, sb);
sb.append("]");
sb.append(", classifications=[");
AtlasBaseTypeDef.dumpObjects(classifications, sb);
sb.append(']');
......@@ -227,13 +250,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
Objects.equals(createTime, that.createTime) &&
Objects.equals(updateTime, that.updateTime) &&
Objects.equals(version, that.version) &&
Objects.equals(relationshipAttributes, that.relationshipAttributes) &&
Objects.equals(classifications, that.classifications);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, status, createdBy, updatedBy, createTime, updateTime, version,
classifications);
relationshipAttributes, classifications);
}
@Override
......
intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
......@@ -50,6 +50,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
private String guid = null;
private AtlasObjectId end1 = null;
private AtlasObjectId end2 = null;
private String label = null;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
......@@ -81,7 +82,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) {
super(typeName);
init(nextInternalId(), end1, end2, null, null, null, null, null, 0L);
init(nextInternalId(), end1, end2, null, null, null, null, null, null, 0L);
}
public AtlasRelationship(String typeName, String attrName, Object attrValue) {
......@@ -98,7 +99,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
super(other);
if (other != null) {
init(other.guid, other.end1, other.end2, other.status, other.createdBy, other.updatedBy,
init(other.guid, other.end1, other.end2, other.label, other.status, other.createdBy, other.updatedBy,
other.createTime, other.updateTime, other.version);
}
}
......@@ -167,6 +168,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public void setEnd2(AtlasObjectId end2) { this.end2 = end2; }
public String getLabel() { return label; }
public void setLabel(String label) { this.label = label; }
private static String nextInternalId() {
return "-" + Long.toString(s_nextId.getAndIncrement());
}
......@@ -174,15 +179,16 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public String getRelationshipLabel() { return "r:" + super.getTypeName(); }
private void init() {
init(nextInternalId(), null, null, null, null, null, null, null, 0L);
init(nextInternalId(), null, null, null, null, null, null, null, null, 0L);
}
private void init(String guid, AtlasObjectId end1, AtlasObjectId end2,
private void init(String guid, AtlasObjectId end1, AtlasObjectId end2, String label,
Status status, String createdBy, String updatedBy,
Date createTime, Date updateTime, Long version) {
setGuid(guid);
setEnd1(end1);
setEnd2(end2);
setLabel(label);
setStatus(status);
setCreatedBy(createdBy);
setUpdatedBy(updatedBy);
......@@ -202,6 +208,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
sb.append("guid='").append(guid).append('\'');
sb.append(", end1=").append(end1);
sb.append(", end2=").append(end2);
sb.append(", label='").append(label).append('\'');
sb.append(", status=").append(status);
sb.append(", createdBy='").append(createdBy).append('\'');
sb.append(", updatedBy='").append(updatedBy).append('\'');
......@@ -223,6 +230,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
return Objects.equals(guid, that.guid) &&
Objects.equals(end1, that.end1) &&
Objects.equals(end2, that.end2) &&
Objects.equals(label, that.label) &&
status == that.status &&
Objects.equals(createdBy, that.createdBy) &&
Objects.equals(updatedBy, that.updatedBy) &&
......@@ -233,7 +241,8 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, end1, end2, status, createdBy, updatedBy, createTime, updateTime, version);
return Objects.hash(super.hashCode(), guid, end1, end2, label, status, createdBy,
updatedBy, createTime, updateTime, version);
}
@Override
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -26,6 +26,7 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -51,7 +52,9 @@ public class AtlasEntityType extends AtlasStructType {
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> allSubTypes = Collections.emptySet();
private Set<String> typeAndAllSubTypes = Collections.emptySet();
private Set<String> typeAndAllSuperTypes = Collections.emptySet();
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
......@@ -95,9 +98,14 @@ public class AtlasEntityType extends AtlasStructType {
this.uniqAttributes = getUniqueAttributes(this.allAttributes);
this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase2()
this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase3()
this.relationshipAttributesType = new HashMap<>(); // this will be populated in resolveReferencesPhase3()
this.typeAndAllSubTypes.add(this.getTypeName());
this.typeAndAllSuperTypes = new HashSet<>(this.allSuperTypes);
this.typeAndAllSuperTypes.add(this.getTypeName());
this.typeAndAllSuperTypes = Collections.unmodifiableSet(this.typeAndAllSuperTypes);
}
@Override
......@@ -110,6 +118,43 @@ public class AtlasEntityType extends AtlasStructType {
}
}
@Override
public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
for (AtlasAttributeDef attributeDef : getStructDef().getAttributeDefs()) {
String attributeName = attributeDef.getName();
AtlasType attributeType = typeRegistry.getType(attributeDef.getTypeName());
AtlasEntityType attributeEntityType = getReferencedEntityType(attributeType);
// validate if RelationshipDefs is defined for all entityDefs
if (attributeEntityType != null && !hasRelationshipAttribute(attributeName)) {
LOG.warn("No RelationshipDef defined between {} and {} on attribute: {}.{}", getTypeName(),
attributeEntityType.getTypeName(), getTypeName(), attributeName);
}
}
for (String superTypeName : allSuperTypes) {
AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName);
Map<String, AtlasAttribute> superTypeRelationshipAttributes = superType.getRelationshipAttributes();
if (MapUtils.isNotEmpty(superTypeRelationshipAttributes)) {
relationshipAttributes.putAll(superTypeRelationshipAttributes);
}
Map<String, List<AtlasRelationshipType>> superTypeRelationshipAttributesType = superType.getRelationshipAttributesType();
if (MapUtils.isNotEmpty(superTypeRelationshipAttributesType)) {
relationshipAttributesType.putAll(superTypeRelationshipAttributesType);
}
}
allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
typeAndAllSuperTypes = Collections.unmodifiableSet(typeAndAllSuperTypes);
relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes);
relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType);
}
public Set<String> getSuperTypes() {
return entityDef.getSuperTypes();
}
......@@ -118,9 +163,11 @@ public class AtlasEntityType extends AtlasStructType {
return allSuperTypes;
}
public Set<String> getAllSubTypes() { return Collections.unmodifiableSet(allSubTypes); }
public Set<String> getAllSubTypes() { return allSubTypes; }
public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); }
public Set<String> getTypeAndAllSuperTypes() { return typeAndAllSuperTypes; }
public boolean isSuperTypeOf(AtlasEntityType entityType) {
return entityType != null && allSubTypes.contains(entityType.getTypeName());
......@@ -142,12 +189,37 @@ public class AtlasEntityType extends AtlasStructType {
return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
}
public Map<String, AtlasAttribute> getRelationshipAttributes() { return Collections.unmodifiableMap(relationshipAttributes); }
public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
public void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
// this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
relationshipAttributes.put(attributeName, attribute);
}
// this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
void addRelationshipAttributeType(String attributeName, AtlasRelationshipType relationshipType) {
List<AtlasRelationshipType> relationshipTypes = relationshipAttributesType.get(attributeName);
if (relationshipTypes == null) {
relationshipTypes = new ArrayList<>();
relationshipAttributesType.put(attributeName, relationshipTypes);
}
relationshipTypes.add(relationshipType);
}
public List<AtlasRelationshipType> getRelationshipAttributeType(String attributeName) {
return relationshipAttributesType.get(attributeName);
}
public Map<String, List<AtlasRelationshipType>> getRelationshipAttributesType() {
return relationshipAttributesType;
}
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
}
@Override
public AtlasEntity createDefaultValue() {
AtlasEntity ret = new AtlasEntity(entityDef.getName());
......
intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
......@@ -90,9 +90,26 @@ public class AtlasRelationshipType extends AtlasStructType {
public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferencesPhase2(typeRegistry);
addRelationshipAttributeToEndType(relationshipDef.getEndDef1(), end1Type, end2Type.getTypeName(), typeRegistry);
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
String relationshipLabel = null;
// if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
// if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
// if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
if (!endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
relationshipLabel = relationshipDef.getName();
} else if (endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
relationshipLabel = endDef1.getLegacyLabel();
} else if (!endDef1.hasLegacyRelation() && endDef2.hasLegacyRelation()) {
relationshipLabel = endDef2.getLegacyLabel();
}
addRelationshipAttributeToEndType(relationshipDef.getEndDef2(), end2Type, end1Type.getTypeName(), typeRegistry);
addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel);
addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel);
}
@Override
......@@ -198,10 +215,8 @@ public class AtlasRelationshipType extends AtlasStructType {
}
}
private void addRelationshipAttributeToEndType(AtlasRelationshipEndDef endDef,
AtlasEntityType entityType,
String attrTypeName,
AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
private void addRelationshipAttributeToEndType(AtlasRelationshipEndDef endDef, AtlasEntityType entityType, String attrTypeName,
AtlasTypeRegistry typeRegistry, String relationshipLabel) throws AtlasBaseException {
String attrName = (endDef != null) ? endDef.getName() : null;
......@@ -211,15 +226,29 @@ public class AtlasRelationshipType extends AtlasStructType {
AtlasAttribute attribute = entityType.getAttribute(attrName);
// if relationshipLabel is null, then legacyLabel is mentioned at both ends,
// use the respective end's legacyLabel as relationshipLabel
if (relationshipLabel == null) {
relationshipLabel = endDef.getLegacyLabel();
}
if (attribute == null) { //attr doesn't exist in type - is a new relationship attribute
if (endDef.getCardinality() == Cardinality.SET) {
attrTypeName = AtlasBaseTypeDef.getArrayTypeName(attrTypeName);
}
attribute = new AtlasAttribute(entityType, new AtlasAttributeDef(attrName, attrTypeName), typeRegistry.getType(attrTypeName));
attribute = new AtlasAttribute(entityType, new AtlasAttributeDef(attrName, attrTypeName),
typeRegistry.getType(attrTypeName), relationshipLabel);
} else {
// attribute already exists (legacy attribute which is also a relationship attribute)
// add relationshipLabel information to existing attribute
attribute.setRelationshipEdgeLabel(relationshipLabel);
}
entityType.addRelationshipAttribute(attrName, attribute);
entityType.addRelationshipAttributeType(attrName, this);
}
}
\ No newline at end of file
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -28,7 +28,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
......@@ -173,6 +172,7 @@ public class AtlasStructType extends AtlasType {
AtlasType referencedType = typeRegistry.getType(attribute.getAttributeDef().getTypeName());
AtlasEntityType referencedEntityType = getReferencedEntityType(referencedType);
AtlasAttribute inverseReference = referencedEntityType.getAttribute(attribute.getInverseRefAttributeName());
attribute.setInverseRefAttribute(inverseReference);
}
}
......@@ -574,7 +574,7 @@ public class AtlasStructType extends AtlasType {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, structDef.getName());
}
private AtlasEntityType getReferencedEntityType(AtlasType type) {
AtlasEntityType getReferencedEntityType(AtlasType type) {
if (type instanceof AtlasArrayType) {
type = ((AtlasArrayType)type).getElementType();
}
......@@ -609,14 +609,15 @@ public class AtlasStructType extends AtlasType {
private final boolean isOwnedRef;
private final String inverseRefAttributeName;
private AtlasAttribute inverseRefAttribute;
private String relationshipEdgeLabel;
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipLabel) {
this.definedInType = definedInType;
this.attributeDef = attrDef;
this.attributeType = attributeType.getTypeForAttribute();
this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
this.vertexPropertyName = encodePropertyKey(this.qualifiedName);
this.relationshipEdgeLabel = getRelationshipEdgeLabel(relationshipLabel);
boolean isOwnedRef = false;
String inverseRefAttribute = null;
......@@ -640,6 +641,10 @@ public class AtlasStructType extends AtlasType {
this.inverseRefAttributeName = inverseRefAttribute;
}
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
this(definedInType, attrDef, attributeType, null);
}
public AtlasStructType getDefinedInType() { return definedInType; }
public AtlasStructDef getDefinedInDef() { return definedInType.getStructDef(); }
......@@ -666,7 +671,15 @@ public class AtlasStructType extends AtlasType {
public AtlasAttribute getInverseRefAttribute() { return inverseRefAttribute; }
public void setInverseRefAttribute(AtlasAttribute inverseAttr) { inverseRefAttribute = inverseAttr; };
public void setInverseRefAttribute(AtlasAttribute inverseAttr) { inverseRefAttribute = inverseAttr; }
public String getRelationshipEdgeLabel() { return relationshipEdgeLabel; }
public void setRelationshipEdgeLabel(String relationshipEdgeLabel) { this.relationshipEdgeLabel = relationshipEdgeLabel; }
public static String getEdgeLabel(String property) {
return "__" + property;
}
public static String encodePropertyKey(String key) {
if (StringUtils.isBlank(key)) {
......@@ -692,6 +705,10 @@ public class AtlasStructType extends AtlasType {
return key;
}
private String getRelationshipEdgeLabel(String relationshipLabel) {
return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel;
}
private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
final String typeName = structDef.getName();
return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName);
......
intg/src/main/java/org/apache/atlas/type/AtlasType.java
......@@ -57,6 +57,9 @@ public abstract class AtlasType {
public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
}
public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
}
public String getTypeName() { return typeName; }
public TypeCategory getTypeCategory() { return typeCategory; }
......
intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
......@@ -364,6 +364,10 @@ public class AtlasTypeRegistry {
for (AtlasType type : registryData.allTypes.getAllTypes()) {
type.resolveReferencesPhase2(this);
}
for (AtlasEntityType entityType : registryData.entityDefs.getAllTypes()) {
entityType.resolveReferencesPhase3(this);
}
}
public void clear() {
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -27,6 +27,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
......@@ -36,6 +37,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
......@@ -345,6 +348,10 @@ public final class GraphHelper {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
}
public Iterator<AtlasEdge> getBothEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.BOTH, edgeLabel);
}
/**
* Returns the active edge for the given edge label.
* If the vertex is deleted and there is no active edge, it returns the latest deleted edge
......@@ -1223,4 +1230,50 @@ public final class GraphHelper {
return condition.toString();
}
/**
* Get relationshipDef name from entityType using relationship attribute.
* if more than one relationDefs are returned for an attribute.
* e.g. hive_column.table
*
* hive_table.columns -> hive_column.table
* hive_table.partitionKeys -> hive_column.table
*
* resolve by comparing all incoming edges typename with relationDefs name returned for an attribute
* to pick the right relationshipDef name
*/
public String getRelationshipDefName(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
AtlasRelationshipDef relationshipDef = getRelationshipDef(entityVertex, entityType, attributeName);
return (relationshipDef != null) ? relationshipDef.getName() : null;
}
public AtlasRelationshipDef getRelationshipDef(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
List<AtlasRelationshipType> relationshipTypes = entityType.getRelationshipAttributeType(attributeName);
AtlasRelationshipDef ret = null;
if (relationshipTypes.size() > 1) {
Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
while (iter.hasNext() && ret == null) {
String edgeTypeName = AtlasGraphUtilsV1.getTypeName(iter.next());
for (AtlasRelationshipType relationType : relationshipTypes) {
AtlasRelationshipDef relationshipDef = relationType.getRelationshipDef();
if (StringUtils.equals(edgeTypeName, relationshipDef.getName())) {
ret = relationshipDef;
break;
}
}
}
} else {
//relationshipTypes will have at least one relationshipDef
ret = relationshipTypes.get(0).getRelationshipDef();
}
return ret;
}
}
\ No newline at end of file
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -24,6 +24,7 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
......@@ -164,6 +165,14 @@ public class AtlasTypeDefStoreInitializer {
}
}
if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
if (!typeRegistry.isRegisteredType(relationshipDef.getName())) {
typesToCreate.getRelationshipDefs().add(relationshipDef);
}
}
}
return typesToCreate;
}
......@@ -234,6 +243,20 @@ public class AtlasTypeDefStoreInitializer {
}
}
if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
AtlasRelationshipDef oldRelationshipDef = typeRegistry.getRelationshipDefByName(relationshipDef.getName());
if (oldRelationshipDef == null) {
continue;
}
if (updateTypeAttributes(oldRelationshipDef, relationshipDef)) {
typesToUpdate.getRelationshipDefs().add(relationshipDef);
}
}
}
return typesToUpdate;
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
......@@ -76,8 +76,8 @@ public class AtlasGraphUtilsV1 {
return vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
}
public static String getTypeName(AtlasVertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
public static String getTypeName(AtlasElement element) {
return element.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
}
public static String getEdgeLabel(String fromNode, String toNode) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
......@@ -29,6 +29,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
......@@ -49,6 +51,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
......@@ -65,6 +68,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
public final class EntityGraphRetriever {
......@@ -180,6 +184,8 @@ public final class EntityGraphRetriever {
mapAttributes(entityVertex, entity, entityExtInfo);
mapRelationshipAttributes(entityVertex, entity, entityExtInfo);
mapClassifications(entityVertex, entity, entityExtInfo);
}
......@@ -278,6 +284,23 @@ public final class EntityGraphRetriever {
}
}
private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(entity.getTypeName());
if (!(objType instanceof AtlasEntityType)) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, entity.getTypeName());
}
AtlasEntityType entityType = (AtlasEntityType) objType;
for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attribute, entityExtInfo);
entity.addRelationshipAttribute(attribute.getName(), attrValue);
}
}
public List<AtlasClassification> getClassifications(String guid) throws AtlasBaseException {
AtlasVertex instanceVertex = AtlasGraphUtilsV1.findByGuid(guid);
......@@ -394,6 +417,40 @@ public final class EntityGraphRetriever {
return ret;
}
private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, AtlasAttribute attribute,
AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
Object ret = null;
AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
AtlasRelationshipEndDef attributeEndDef = null;
if (endDef1Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef1.getName(), attribute.getName())) {
attributeEndDef = endDef1;
} else if (endDef2Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef2.getName(), attribute.getName())) {
attributeEndDef = endDef2;
}
String relationshipLabel = attribute.getRelationshipEdgeLabel();
switch (attributeEndDef.getCardinality()) {
case SINGLE:
ret = mapVertexToObjectId(entityVertex, relationshipLabel, null, entityExtInfo, attributeEndDef.getIsContainer());
break;
case LIST:
case SET:
ret = mapVertexToRelationshipArrayAttribute(entityVertex, (AtlasArrayType) attribute.getAttributeType(), relationshipLabel,
entityExtInfo, attributeEndDef.getIsContainer());
break;
}
return ret;
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName);
......@@ -451,6 +508,40 @@ public final class EntityGraphRetriever {
return arrValues;
}
private List<Object> mapVertexToRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasArrayType arrayType,
String relationshipName, AtlasEntityExtInfo entityExtInfo,
boolean isContainer) throws AtlasBaseException {
Iterator<AtlasEdge> relationshipEdges = graphHelper.getBothEdgesByLabel(entityVertex, relationshipName);
AtlasType arrayElementType = arrayType.getElementType();
List<AtlasEdge> arrayElements = new ArrayList<>();
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping array attribute {} for vertex {}", arrayElementType.getTypeName(), entityVertex);
}
while (relationshipEdges.hasNext()) {
arrayElements.add(relationshipEdges.next());
}
if (CollectionUtils.isEmpty(arrayElements)) {
return null;
}
List arrValues = new ArrayList(arrayElements.size());
for (Object element : arrayElements) {
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, relationshipName,
entityExtInfo, isContainer);
if (arrValue != null) {
arrValues.add(arrValue);
}
}
return arrValues;
}
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value, String edgeLabel,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
Object ret = null;
......@@ -538,7 +629,11 @@ public final class EntityGraphRetriever {
}
if (GraphHelper.elementExists(edge)) {
final AtlasVertex referenceVertex = edge.getInVertex();
AtlasVertex referenceVertex = edge.getInVertex();
if (StringUtils.equals(getIdFromVertex(referenceVertex), getIdFromVertex(entityVertex))) {
referenceVertex = edge.getOutVertex();
}
if (referenceVertex != null) {
if (entityExtInfo != null && isOwnedAttribute) {
......
repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java 0 → 100644
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import static org.mockito.Mockito.mock;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasRelationshipStoreV1Test {
@Inject
AtlasTypeRegistry typeRegistry;
@Inject
AtlasTypeDefStore typeDefStore;
@Inject
DeleteHandlerV1 deleteHandler;
@Inject
EntityGraphMapper graphMapper;
AtlasEntityStore entityStore;
AtlasRelationshipStore relationshipStore;
AtlasEntityWithExtInfo dbEntity;
AtlasEntityWithExtInfo tblEntity;
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@BeforeClass
public void setUp() throws Exception {
new GraphBackedSearchIndexer(typeRegistry);
AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
TestUtilsV2.defineHiveTypes() };
for (AtlasTypesDef typesDef : testTypesDefs) {
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry);
if (!typesToCreate.isEmpty()) {
typeDefStore.createTypesDef(typesToCreate);
}
}
dbEntity = TestUtilsV2.createDBEntityV2();
tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
}
@AfterClass
public void clear() {
AtlasGraphProvider.cleanup();
}
@BeforeTest
public void init() throws Exception {
entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
relationshipStore = new AtlasRelationshipStoreV1(typeRegistry);
RequestContextV1.clear();
}
@Test
public void testDbTableRelationship() throws Exception {
// Add tests - in progress
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment