Skip to content

A
atlas
Commit 10f34ed8 by Madhan Neethiraj
Options

ATLAS-3065: include removed legacy attributes in entity.attributes for backward…

ATLAS-3065: include removed legacy attributes in entity.attributes for backward compatibility (in addition to entity.relationshipAttributes) - #3
parent 24715256
Showing with 358 additions and 218 deletions
addons/models/1000-Hadoop/1030-hive_model.json
......@@ -175,17 +175,12 @@
"DataSet"
],
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.1",
"typeDefOptions": {
"schemaElementsAttribute": "columns"
},
"attributeDefs": [
{
"name": "db",
"typeName": "hive_db",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "createTime",
"typeName": "date",
"cardinality": "SINGLE",
......@@ -218,32 +213,6 @@
"isUnique": false
},
{
"name": "sd",
"typeName": "hive_storagedesc",
"cardinality": "SINGLE",
"constraints": [
{
"type": "ownedRef"
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "partitionKeys",
"typeName": "array<hive_column>",
"cardinality": "SET",
"constraints": [
{
"type": "ownedRef"
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "aliases",
"typeName": "array<string>",
"cardinality": "SET",
......@@ -252,19 +221,6 @@
"isUnique": false
},
{
"name": "columns",
"typeName": "array<hive_column>",
"cardinality": "SET",
"constraints": [
{
"type": "ownedRef"
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "parameters",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
......@@ -315,22 +271,6 @@
"typeVersion": "1.0",
"attributeDefs": [
{
"name": "table",
"typeName": "hive_table",
"cardinality": "SINGLE",
"constraints": [
{
"type": "inverseRef",
"params": {
"attribute": "sd"
}
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
"name": "location",
"typeName": "string",
"cardinality": "SINGLE",
......@@ -461,7 +401,10 @@
"DataSet"
],
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.3",
"typeDefOptions": {
"schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
},
"attributeDefs": [
{
"name": "type",
......@@ -480,20 +423,12 @@
"isUnique": false
},
{
"name": "table",
"typeName": "hive_table",
"cardinality": "SINGLE",
"constraints": [
{
"type": "inverseRef",
"params": {
"attribute": "columns"
}
}
],
"isIndexable": false,
"isOptional": true,
"isUnique": false
"name": "position",
"typeName": "int",
"cardinality": "SINGLE",
"isIndexable": false,
"isOptional": true,
"isUnique": false
}
]
},
......@@ -506,14 +441,6 @@
"typeVersion" : "1.0",
"attributeDefs" : [
{
"name": "query",
"typeName": "hive_process",
"cardinality" : "SINGLE",
"isIndexable": false,
"isOptional": false,
"isUnique": false
},
{
"name": "depenendencyType",
"typeName": "string",
"cardinality" : "SINGLE",
......@@ -536,8 +463,9 @@
{
"name": "hive_table_db",
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hive_table.db",
"endDef1": {
"type": "hive_table",
"name": "db",
......@@ -556,8 +484,9 @@
{
"name": "hive_table_columns",
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hive_table.columns",
"endDef1": {
"type": "hive_table",
"name": "columns",
......@@ -577,8 +506,9 @@
{
"name": "hive_table_partitionkeys",
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hive_table.partitionkeys",
"endDef1": {
"type": "hive_table",
"name": "partitionKeys",
......@@ -598,8 +528,9 @@
{
"name": "hive_table_storagedesc",
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hive_table.sd",
"endDef1": {
"type": "hive_table",
"name": "sd",
......@@ -619,21 +550,22 @@
{
"name": "hive_process_column_lineage",
"serviceType": "hive",
"typeVersion": "1.0",
"typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
"relationshipLabel": "__hive_column_lineage.query",
"endDef1": {
"type": "hive_process",
"name": "columnLineages",
"isContainer": true,
"cardinality": "SET"
},
"endDef2": {
"type": "hive_column_lineage",
"name": "query",
"isContainer": false,
"cardinality": "SINGLE",
"isLegacyAttribute": true
},
"endDef2": {
"type": "hive_process",
"name": "columnLineages",
"isContainer": true,
"cardinality": "SET"
},
"propagateTags": "NONE"
}
]
......
addons/models/2000-RDBMS/2010-rdbms_model.json
......@@ -148,7 +148,10 @@
"description": "a table in an rdbms database (schema)",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
"typeVersion": "1.1",
"typeVersion": "1.2",
"typeDefOptions": {
"schemaElementsAttribute": "columns"
},
"attributeDefs": [
{
"name": "db",
......@@ -254,7 +257,10 @@
"description": "a column in an rdbms table",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
"typeVersion": "1.1",
"typeVersion": "1.2",
"typeDefOptions": {
"schemaAttributes": "[\"name\", \"description\", \"owner\", \"data_type\", \"comment\", \" isPrimaryKey\", \" isNullable\"]"
},
"attributeDefs": [
{
"name": "data_type",
......
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
......@@ -19,6 +19,7 @@
package org.apache.atlas.repository.graphdb.janus.migration;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
......@@ -72,6 +73,15 @@ public class TypesWithCollectionsFinder {
addIfCollectionAttribute(attr, collectionProperties);
}
if (type instanceof AtlasEntityType) {
AtlasEntityType entityType = (AtlasEntityType) type;
for (Map<String, AtlasAttribute> attrs : entityType.getRelationshipAttributes().values()) {
for (AtlasAttribute attr : attrs.values()) {
addIfCollectionAttribute(attr, collectionProperties);
}
}
}
return collectionProperties;
} catch (Exception e) {
LOG.error("addVertexPropertiesForCollectionAttributes", e);
......
intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
......@@ -195,7 +195,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
super(TypeCategory.RELATIONSHIP, name, description, typeVersion, attributeDefs, null);
setRelationshipCategory(relationshipCategory);
setRelationshipLabel(getDefaultRelationshipLabel());
setRelationshipLabel(null);
setPropagateTags(propagatetags);
setEndDef1(endDef1);
setEndDef2(endDef2);
......@@ -226,7 +226,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
}
public String getRelationshipLabel() {
return relationshipLabel != null ? relationshipLabel : ("r:" + super.getName());
return relationshipLabel;
}
public void setPropagateTags(PropagateTags propagateTags) {
......@@ -253,13 +253,6 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
return this.endDef2;
}
@JsonIgnore
private String getDefaultRelationshipLabel() {
String name = super.getName();
return name != null ? ("r:" + name) : null;
}
@Override
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
......
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
......@@ -34,7 +34,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
......@@ -335,7 +334,7 @@ public class AtlasEntityType extends AtlasStructType {
attributes.put(relationshipType.getTypeName(), attribute);
}
public Collection<String> getAttributeRelationshipTypes(String attributeName) {
public Set<String> getAttributeRelationshipTypes(String attributeName) {
Map<String, AtlasAttribute> attributes = relationshipAttributes.get(attributeName);
return attributes != null ? attributes.keySet() : null;
......@@ -353,6 +352,10 @@ public class AtlasEntityType extends AtlasStructType {
return typeQryStr;
}
public boolean hasAttribute(String attributeName) {
return allAttributes.containsKey(attributeName);
}
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
}
......
intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
......@@ -50,6 +50,7 @@ public class AtlasRelationshipType extends AtlasStructType {
private final AtlasRelationshipDef relationshipDef;
private final boolean hasLegacyAttributeEnd;
private String relationshipLabel;
private AtlasEntityType end1Type;
private AtlasEntityType end2Type;
......@@ -75,6 +76,10 @@ public class AtlasRelationshipType extends AtlasStructType {
return this.hasLegacyAttributeEnd;
}
public String getRelationshipLabel() {
return this.relationshipLabel;
}
@Override
void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferences(typeRegistry);
......@@ -110,19 +115,23 @@ public class AtlasRelationshipType extends AtlasStructType {
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
String relationshipLabel = null;
// if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
// if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
// if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
if (!endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
relationshipLabel = relationshipDef.getRelationshipLabel();
} else if (endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
relationshipLabel = getLegacyEdgeLabel(end1Type, endDef1.getName());
} else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
relationshipLabel = getLegacyEdgeLabel(end2Type, endDef2.getName());
String relationshipLabel = relationshipDef.getRelationshipLabel();
if (relationshipLabel == null) {
// if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
// if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
// if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
if (!endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
relationshipLabel = "r:" + getTypeName();
} else if (endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
relationshipLabel = getLegacyEdgeLabel(end1Type, endDef1.getName());
} else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
relationshipLabel = getLegacyEdgeLabel(end2Type, endDef2.getName());
}
}
this.relationshipLabel = relationshipLabel;
addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel);
addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel);
......
intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
......@@ -141,6 +141,24 @@ public class AtlasTypeUtil {
}
public static AtlasRelationshipType findRelationshipWithLegacyRelationshipEnd(String entityTypeName, String attributeName, AtlasTypeRegistry typeRegistry) {
AtlasRelationshipType ret = null;
for (AtlasRelationshipDef relationshipDef : typeRegistry.getAllRelationshipDefs()) {
AtlasRelationshipEndDef end1Def = relationshipDef.getEndDef1();
AtlasRelationshipEndDef end2Def = relationshipDef.getEndDef2();
if ((end1Def.getIsLegacyAttribute() && StringUtils.equals(end1Def.getType(), entityTypeName) && StringUtils.equals(end1Def.getName(), attributeName)) ||
(end2Def.getIsLegacyAttribute() && StringUtils.equals(end2Def.getType(), entityTypeName) && StringUtils.equals(end2Def.getName(), attributeName))) {
ret = typeRegistry.getRelationshipTypeByName(relationshipDef.getName());
break;
}
}
return ret;
}
public static AtlasAttributeDef createOptionalAttrDef(String name, AtlasType dataType) {
return new AtlasAttributeDef(name, dataType.getTypeName(), true,
Cardinality.SINGLE, 0, 1,
......
repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
......@@ -103,7 +103,7 @@ class RegistryBasedLookup implements Lookup {
return true;
}
AtlasType at = et.getAttributeType(attributeName);
AtlasType at = getAttributeType(et, attributeName);
if(at == null) {
return false;
}
......@@ -135,14 +135,17 @@ class RegistryBasedLookup implements Lookup {
return "";
}
AtlasStructType.AtlasAttribute attr = et.getAttribute(attributeName);
AtlasStructType.AtlasAttribute attr = getAttribute(et, attributeName);
return (attr != null) ? attr.getRelationshipEdgeLabel() : "";
}
@Override
public boolean hasAttribute(GremlinQueryComposer.Context context, String typeName) {
return (context.getActiveEntityType() != null) &&
(isSystemAttribute(typeName) || context.getActiveEntityType().getAttribute(typeName) != null);
AtlasEntityType entityType = context.getActiveEntityType();
return (entityType != null) &&
(isSystemAttribute(typeName) || entityType.hasAttribute(typeName) || entityType.hasRelationshipAttribute(typeName));
}
@Override
......@@ -197,7 +200,7 @@ class RegistryBasedLookup implements Lookup {
return "";
}
AtlasStructType.AtlasAttribute attr = et.getAttribute(item);
AtlasStructType.AtlasAttribute attr = getAttribute(et, item);
if(attr == null) {
return null;
}
......@@ -213,7 +216,7 @@ class RegistryBasedLookup implements Lookup {
return getCollectionElementType(mapType.getValueType());
}
return context.getActiveEntityType().getAttribute(item).getTypeName();
return getAttribute(context.getActiveEntityType(), item).getTypeName();
}
private String getCollectionElementType(AtlasType elemType) {
......@@ -231,7 +234,7 @@ class RegistryBasedLookup implements Lookup {
return false;
}
AtlasType attr = et.getAttributeType(attributeName);
AtlasType attr = getAttributeType(et, attributeName);
return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE);
}
......@@ -242,7 +245,7 @@ class RegistryBasedLookup implements Lookup {
return false;
}
AtlasType attr = et.getAttributeType(attrName);
AtlasType attr = getAttributeType(et, attrName);
boolean ret = attr != null && NUMERIC_ATTRIBUTES.containsKey(attr.getTypeName());
if(ret) {
context.setNumericTypeFormatter(NUMERIC_ATTRIBUTES.get(attr.getTypeName()));
......@@ -250,4 +253,24 @@ class RegistryBasedLookup implements Lookup {
return ret;
}
private AtlasStructType.AtlasAttribute getAttribute(AtlasEntityType entityType, String attrName) {
AtlasStructType.AtlasAttribute ret = null;
if (entityType != null) {
ret = entityType.getAttribute(attrName);
if (ret == null) {
ret = entityType.getRelationshipAttribute(attrName, null);
}
}
return ret;
}
private AtlasType getAttributeType(AtlasEntityType entityType, String attrName) {
AtlasStructType.AtlasAttribute attribute = getAttribute(entityType, attrName);
return attribute != null ? attribute.getAttributeType() : null;
}
}
repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
......@@ -39,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -127,12 +128,30 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND);
}
final Map v2Attribs = (Map) v2Map.get(ATTRIBUTES_PROPERTY_KEY);
final Map v2Attribs = (Map) v2Map.get(ATTRIBUTES_PROPERTY_KEY);
final Map v2RelationshipAttribs = (Map) v2Map.get(RELATIONSHIP_ATTRIBUTES_PROPERTY_KEY);
final Map attributes;
if (MapUtils.isNotEmpty(v2RelationshipAttribs)) {
if (MapUtils.isNotEmpty(v2Attribs)) {
attributes = new HashMap(v2RelationshipAttribs);
for (Object key : v2Attribs.keySet()) {
if (!attributes.containsKey(key)) {
attributes.put(key, v2Attribs.get(key));
}
}
} else {
attributes = v2RelationshipAttribs;
}
} else {
attributes = v2Attribs;
}
if (MapUtils.isEmpty(v2Attribs)) {
ret = new Id(idStr, 0, typeName);
} else {
ret = new Referenceable(idStr, typeName, super.fromV2ToV1(entityType, v2Attribs, context));
ret = new Referenceable(idStr, typeName, super.fromV2ToV1(entityType, attributes, context));
}
} else if (v2Obj instanceof AtlasEntity) {
AtlasEntity entity = (AtlasEntity) v2Obj;
......@@ -142,8 +161,28 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
status = Status.ACTIVE;
}
final Map<String, Object> v2Attribs = entity.getAttributes();
final Map<String, Object> v2RelationshipAttribs = entity.getRelationshipAttributes();
final Map<String, Object> attributes;
if (MapUtils.isNotEmpty(v2RelationshipAttribs)) {
if (MapUtils.isNotEmpty(v2Attribs)) {
attributes = new HashMap(v2RelationshipAttribs);
for (String key : v2Attribs.keySet()) {
if (!attributes.containsKey(key)) {
attributes.put(key, v2Attribs.get(key));
}
}
} else {
attributes = v2RelationshipAttribs;
}
} else {
attributes = v2Attribs;
}
Referenceable referenceable = new Referenceable(entity.getGuid(), entity.getTypeName(), status.name(),
fromV2ToV1(entityType, entity.getAttributes(), context),
fromV2ToV1(entityType, attributes, context),
new AtlasSystemAttributes(entity.getCreatedBy(), entity.getUpdatedBy(), entity.getCreateTime(), entity.getUpdateTime()));
if (CollectionUtils.isNotEmpty(entity.getClassifications())) {
......
repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
......@@ -41,7 +41,8 @@ import java.util.Map;
public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasStructFormatConverter.class);
public static final String ATTRIBUTES_PROPERTY_KEY = "attributes";
public static final String ATTRIBUTES_PROPERTY_KEY = "attributes";
public static final String RELATIONSHIP_ATTRIBUTES_PROPERTY_KEY = "relationshipAttributes";
public AtlasStructFormatConverter(AtlasFormatConverters registry, AtlasTypeRegistry typeRegistry) {
this(registry, typeRegistry, TypeCategory.STRUCT);
......@@ -136,8 +137,14 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
AtlasAttribute attr = structType.getAttribute(attrName);
if (attr == null) {
LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
continue;
if (isEntityType) {
attr = ((AtlasEntityType) structType).getRelationshipAttribute(attrName, null);
}
if (attr == null) {
LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
continue;
}
}
AtlasType attrType = attr.getAttributeType();
......@@ -240,7 +247,8 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
}
protected Map<String, Object> fromV1ToV2(AtlasStructType structType, Map attributes, ConverterContext context) throws AtlasBaseException {
Map<String, Object> ret = null;
Map<String, Object> ret = null;
AtlasEntityType entityType = (structType instanceof AtlasEntityType) ? ((AtlasEntityType) structType) : null;
if (MapUtils.isNotEmpty(attributes)) {
ret = new HashMap<>();
......@@ -251,8 +259,14 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
AtlasAttribute attr = structType.getAttribute(attrName);
if (attr == null) {
LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
continue;
if (entityType != null) {
attr = entityType.getRelationshipAttribute(attrName, null);
}
if (attr == null) {
LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
continue;
}
}
AtlasType attrType = attr.getAttributeType();
......
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
......@@ -1773,25 +1773,29 @@ public final class GraphHelper {
* to pick the right relationshipDef name
*/
public String getRelationshipTypeName(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
String ret = null;
Collection<String> relationshipTypes = entityType.getAttributeRelationshipTypes(attributeName);
String ret = null;
Set<String> relationshipTypes = entityType.getAttributeRelationshipTypes(attributeName);
if (CollectionUtils.isNotEmpty(relationshipTypes)) {
Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
if (relationshipTypes.size() == 1) {
ret = relationshipTypes.iterator().next();
} else {
Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
while (iter.hasNext() && ret == null) {
String edgeTypeName = AtlasGraphUtilsV2.getTypeName(iter.next());
while (iter.hasNext() && ret == null) {
String edgeTypeName = AtlasGraphUtilsV2.getTypeName(iter.next());
if (relationshipTypes.contains(edgeTypeName)) {
ret = edgeTypeName;
if (relationshipTypes.contains(edgeTypeName)) {
ret = edgeTypeName;
break;
break;
}
}
}
if (ret == null) {
//relationshipTypes will have at least one relationshipDef
ret = relationshipTypes.iterator().next();
if (ret == null) {
//relationshipTypes will have at least one relationshipDef
ret = relationshipTypes.iterator().next();
}
}
}
......
repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
......@@ -27,7 +27,9 @@ import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -117,7 +119,11 @@ public class TypeAttributeDifference {
AtlasStructDef.AtlasAttributeDef incoming) throws AtlasBaseException {
AtlasStructDef.AtlasAttributeDef existingAttribute = existing.getAttribute(incoming.getName());
if (existingAttribute == null) {
difference.add(incoming);
AtlasRelationshipType relationshipType = AtlasTypeUtil.findRelationshipWithLegacyRelationshipEnd(existing.getName(), incoming.getName(), typeRegistry);
if (relationshipType == null) {
difference.add(incoming);
}
} else {
if (!existingAttribute.getTypeName().equals(incoming.getTypeName())) {
LOG.error("Attribute definition difference found: {}, {}", existingAttribute, incoming);
......
repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
......@@ -793,8 +793,6 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
updatedDef.setRelationshipCategory(newRelationshipCategory);
}
end1Def.setIsLegacyAttribute(false);
end2Def.setIsLegacyAttribute(false);
updatedDef.setTypeVersion(patch.getUpdateToVersion());
AtlasEntityDef updatedEntityDef1 = new AtlasEntityDef(end1Type.getEntityDef());
......
repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
......@@ -77,6 +77,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
protected abstract AtlasDefStore<AtlasRelationshipDef> getRelationshipDefStore(AtlasTypeRegistry typeRegistry);
public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; }
@Override
public void init() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.init()");
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
......@@ -690,7 +690,11 @@ public abstract class DeleteHandlerV1 {
AtlasEntityType parentEntityType = (AtlasEntityType) parentType;
AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName());
if (attribute.getInverseRefAttribute() != null) {
if (attribute == null) {
attribute = parentEntityType.getRelationshipAttribute(atlasEdgeLabel.getAttributeName(), AtlasGraphUtilsV2.getTypeName(edge));
}
if (attribute != null && attribute.getInverseRefAttribute() != null) {
deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute());
}
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
......@@ -26,6 +26,7 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
......@@ -322,7 +323,11 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
AtlasAttribute attr = entityType.getAttribute(attrName);
if (attr == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entity.getTypeName());
attr = entityType.getRelationshipAttribute(attrName, null);
if (attr == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entity.getTypeName());
}
}
AtlasType attrType = attr.getAttributeType();
......@@ -856,6 +861,14 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
AtlasEntity entity = entityStream.getByGuid(guid);
if (entity != null) { // entity would be null if guid is not in the stream but referenced by an entity in the stream
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
compactAttributes(entity, entityType);
AtlasVertex vertex = getResolvedEntityVertex(discoveryContext, entity);
if (vertex != null) {
......@@ -865,8 +878,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
graphDiscoverer.validateAndNormalizeForUpdate(entity);
}
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
String guidVertex = AtlasGraphUtilsV2.getIdFromVertex(vertex);
String guidVertex = AtlasGraphUtilsV2.getIdFromVertex(vertex);
if (!StringUtils.equals(guidVertex, guid)) { // if entity was found by unique attribute
entity.setGuid(guidVertex);
......@@ -878,8 +890,6 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
} else {
graphDiscoverer.validateAndNormalize(entity);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
//Create vertices which do not exist in the repository
if (RequestContext.get().isImportInProgress() && AtlasTypeUtil.isAssignedGuid(entity.getGuid())) {
vertex = entityGraphMapper.createVertexWithGuid(entity, entity.getGuid());
......@@ -1039,4 +1049,32 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
return ret;
}
// move/remove relationship-attributes present in 'attributes'
private void compactAttributes(AtlasEntity entity, AtlasEntityType entityType) {
if (entity != null) {
for (String attrName : entityType.getRelationshipAttributes().keySet()) {
if (entity.hasAttribute(attrName)) { // relationship attribute is present in 'attributes'
Object attrValue = entity.removeAttribute(attrName);
if (attrValue != null) {
// if the attribute doesn't exist in relationshipAttributes, add it
Object relationshipAttrValue = entity.getRelationshipAttribute(attrName);
if (relationshipAttrValue == null) {
entity.setRelationshipAttribute(attrName, attrValue);
if (LOG.isDebugEnabled()) {
LOG.debug("moved attribute {}.{} from attributes to relationshipAttributes", entityType.getTypeName(), attrName);
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("attribute {}.{} is present in attributes and relationshipAttributes. Removed from attributes", entityType.getTypeName(), attrName);
}
}
}
}
}
}
}
}
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
......@@ -470,9 +470,11 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
// Update RelationshipCategory
vertex.setProperty(Constants.RELATIONSHIPTYPE_CATEGORY_KEY, relationshipCategory);
vertex.setProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY, relationshipDef.getRelationshipLabel());
vertex.setProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY, relationshipDef.getRelationshipLabel());
if (relationshipDef.getRelationshipLabel() == null) {
vertex.removeProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY);
} else {
vertex.setProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY, relationshipDef.getRelationshipLabel());
}
if (relationshipDef.getPropagateTags() == null) {
vertex.setProperty(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, AtlasRelationshipDef.PropagateTags.NONE.name());
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
......@@ -798,7 +798,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
}
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationshipTypeName);
String ret = relationshipType.getRelationshipDef().getRelationshipLabel();
String ret = relationshipType.getRelationshipLabel();
AtlasRelationshipEndDef endDef1 = relationshipType.getRelationshipDef().getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipType.getRelationshipDef().getEndDef2();
Set<String> fromVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV2.getTypeName(fromVertex));
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
......@@ -29,15 +29,14 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.v1.model.typedef.AttributeDefinition;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.v1.model.typedef.Multiplicity;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
......@@ -425,6 +424,19 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) {
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) {
// this could have been an attribute removed by a REMOVE_LEGACY_REF_ATTRIBUTE patch
// in such case, don't add this attribute; ignore and continue
AtlasRelationshipType relationship = AtlasTypeUtil.findRelationshipWithLegacyRelationshipEnd(structDef.getName(), attributeDef.getName(), typeDefStore.getTypeRegistry());
if (relationship != null) {
attrNames.remove(attributeDef.getName());
LOG.warn("Ignoring attempt to add legacy attribute {}.{}, which is already present in relationship {}", structDef.getName(), attributeDef.getName(), relationship.getTypeName());
continue;
}
// new attribute - only allow if optional
if (!attributeDef.getIsOptional()) {
throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName());
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
......@@ -189,8 +189,6 @@ public class EntityGraphMapper {
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = context.getType(guid);
compactAttributes(createdEntity, entityType);
mapRelationshipAttributes(createdEntity, entityType, vertex, CREATE, context);
mapAttributes(createdEntity, entityType, vertex, CREATE, context);
......@@ -206,8 +204,6 @@ public class EntityGraphMapper {
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = context.getType(guid);
compactAttributes(updatedEntity, entityType);
mapRelationshipAttributes(updatedEntity, entityType, vertex, UPDATE, context);
mapAttributes(updatedEntity, entityType, vertex, UPDATE, context);
......@@ -1901,33 +1897,6 @@ public class EntityGraphMapper {
}
}
// move/remove relationship-attributes present in 'attributes'
private static void compactAttributes(AtlasEntity entity, AtlasEntityType entityType) {
if (entity != null) {
for (String attrName : entityType.getRelationshipAttributes().keySet()) {
if (entity.hasAttribute(attrName)) {
Object attrValue = entity.getAttribute(attrName);
if (LOG.isDebugEnabled()) {
LOG.debug("relationship attribute {}.{} is present in entity, removing it", entityType.getTypeName(), attrName);
}
entity.removeAttribute(attrName);
if (attrValue != null) { // relationship attribute is present in 'attributes'
// if the attribute doesn't exist in relationshipAttributes, add it
Object relationshipAttrValue = entity.getRelationshipAttribute(attrName);
if (relationshipAttrValue == null) {
entity.setRelationshipAttribute(attrName, attrValue);
}
}
}
}
}
}
private String getIdFromInVertex(AtlasEdge edge) {
return getIdFromVertex(edge.getInVertex());
}
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
......@@ -386,7 +386,7 @@ public class EntityGraphRetriever {
mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
if (!ignoreRelationshipAttr) { // only map when really needed
mapRelationshipAttributes(entityVertex, entity);
mapRelationshipAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
}
mapClassifications(entityVertex, entity);
......@@ -990,7 +990,7 @@ public class EntityGraphRetriever {
return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null;
}
private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException {
private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType == null) {
......@@ -998,13 +998,11 @@ public class EntityGraphRetriever {
}
for (String attributeName : entityType.getRelationshipAttributes().keySet()) {
Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attributeName);
entity.setRelationshipAttribute(attributeName, attrValue);
mapVertexToRelationshipAttribute(entityVertex, entityType, attributeName, entity, entityExtInfo, isMinExtInfo);
}
}
private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) throws AtlasBaseException {
private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
Object ret = null;
String relationshipTypeName = graphHelper.getRelationshipTypeName(entityVertex, entityType, attributeName);
AtlasRelationshipType relationshipType = relationshipTypeName != null ? typeRegistry.getRelationshipTypeByName(relationshipTypeName) : null;
......@@ -1013,6 +1011,7 @@ public class EntityGraphRetriever {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, "relationshipDef is null");
}
AtlasAttribute attribute = entityType.getRelationshipAttribute(attributeName, relationshipTypeName);
AtlasRelationshipDef relationshipDef = relationshipType.getRelationshipDef();
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
......@@ -1032,25 +1031,33 @@ public class EntityGraphRetriever {
switch (attributeEndDef.getCardinality()) {
case SINGLE:
ret = mapRelatedVertexToObjectId(entityVertex, entityType.getRelationshipAttribute(attributeName, relationshipTypeName));
ret = mapRelatedVertexToObjectId(entityVertex, attribute, entityExtInfo, isMinExtInfo);
break;
case LIST:
case SET:
ret = mapRelationshipArrayAttribute(entityVertex, entityType.getRelationshipAttribute(attributeName, relationshipTypeName));
ret = mapRelationshipArrayAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo);
break;
}
if (ret != null) {
entity.setRelationshipAttribute(attributeName, ret);
if (attributeEndDef.getIsLegacyAttribute() && !entity.hasAttribute(attributeName)) {
entity.setAttribute(attributeName, ret);
}
}
return ret;
}
private AtlasObjectId mapRelatedVertexToObjectId(AtlasVertex entityVertex, AtlasAttribute attribute) throws AtlasBaseException {
private AtlasObjectId mapRelatedVertexToObjectId(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEdge edge = graphHelper.getEdgeForLabel(entityVertex, attribute.getRelationshipEdgeLabel(), attribute.getRelationshipEdgeDirection());
return mapVertexToRelatedObjectId(entityVertex, edge);
return mapVertexToRelatedObjectId(entityVertex, edge, attribute.isOwnedRef(), entityExtInfo, isMinExtInfo);
}
private List<AtlasRelatedObjectId> mapRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasAttribute attribute) throws AtlasBaseException {
private List<AtlasRelatedObjectId> mapRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
List<AtlasRelatedObjectId> ret = new ArrayList<>();
Iterator<AtlasEdge> edges = null;
......@@ -1066,7 +1073,7 @@ public class EntityGraphRetriever {
while (edges.hasNext()) {
AtlasEdge relationshipEdge = edges.next();
AtlasRelatedObjectId relatedObjectId = mapVertexToRelatedObjectId(entityVertex, relationshipEdge);
AtlasRelatedObjectId relatedObjectId = mapVertexToRelatedObjectId(entityVertex, relationshipEdge, attribute.isOwnedRef(), entityExtInfo, isMinExtInfo);
ret.add(relatedObjectId);
}
......@@ -1075,7 +1082,7 @@ public class EntityGraphRetriever {
return ret;
}
private AtlasRelatedObjectId mapVertexToRelatedObjectId(AtlasVertex entityVertex, AtlasEdge edge) throws AtlasBaseException {
private AtlasRelatedObjectId mapVertexToRelatedObjectId(AtlasVertex entityVertex, AtlasEdge edge, boolean isOwnedRef, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasRelatedObjectId ret = null;
if (GraphHelper.elementExists(edge)) {
......@@ -1100,6 +1107,15 @@ public class EntityGraphRetriever {
if (displayText != null) {
ret.setDisplayText(displayText.toString());
}
if (isOwnedRef && entityExtInfo != null) {
if (isMinExtInfo) {
mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo);
} else {
mapVertexToAtlasEntity(referenceVertex, entityExtInfo);
}
}
}
}
......
repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
......@@ -162,6 +162,9 @@ public abstract class BasicTestSetup {
column("weekDay", "int", "week Day"));
entities.addAll(timeDimColumns);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
"Dimension");
entities.add(timeDim);
......@@ -170,14 +173,23 @@ public abstract class BasicTestSetup {
database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
entities.add(reportingDB);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity salesFactDaily =
table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed",
salesFactColumns, "Metric");
entities.add(salesFactDaily);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
entities.add(circularLineageTable1);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal 2", "Managed", salesFactColumns, "Metric");
entities.add(circularLineageTable2);
......@@ -196,6 +208,9 @@ public abstract class BasicTestSetup {
AtlasEntity logDB = database("Logging", "logging database", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging");
entities.add(logDB);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity loggingFactDaily =
table("log_fact_daily_mv", "log fact daily materialized view", logDB, sd, "Tim ETL", "Managed",
logFactColumns, "Log Data");
......@@ -207,6 +222,9 @@ public abstract class BasicTestSetup {
column("brand_name", "int", "brand name"));
entities.addAll(productDimColumns);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity productDim =
table("product_dim", "product dimension table", salesDB, sd, "John Doe 2", "Managed", productDimColumns,
"Dimension");
......@@ -221,6 +239,9 @@ public abstract class BasicTestSetup {
column("address", "string", "customer address", "PII"));
entities.addAll(customerDimColumns);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity customerDim =
table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns,
"Dimension");
......@@ -229,6 +250,9 @@ public abstract class BasicTestSetup {
AtlasEntity customerDimView = view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
entities.add(customerDimView);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity salesFactMonthly =
table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI",
"Managed", salesFactColumns, "Metric");
......@@ -238,6 +262,9 @@ public abstract class BasicTestSetup {
ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
entities.add(loadSalesMonthly);
sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
entities.add(sd);
AtlasEntity loggingFactMonthly =
table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL 2",
"Managed", logFactColumns, "Log Data");
......@@ -339,8 +366,13 @@ public abstract class BasicTestSetup {
sd.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "@" + clusterName + "_storage");
AtlasObjectId tableId = getAtlasObjectId(table);
sd.setAttribute("table", tableId);
for (AtlasEntity column : columns) {
column.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "." + column.getAttribute(AtlasClient.NAME).toString() + "@" + clusterName);
column.setAttribute("table", tableId);
}
return table;
......
repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
......@@ -21,6 +21,11 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
......@@ -31,9 +36,18 @@ import java.util.List;
public class TypeAttributeDifferenceTest {
private TypeAttributeDifference typeAttributeDifference;
@Mock
AtlasTypeDefStore typeDefStore;
@Mock
AtlasTypeRegistry typeRegistry;
@BeforeClass
public void setup() {
typeAttributeDifference = new TypeAttributeDifference(null, null);
MockitoAnnotations.initMocks(this);
typeAttributeDifference = new TypeAttributeDifference(typeDefStore, typeRegistry);
}
private List<AtlasEnumDef.AtlasEnumElementDef> getEnumElementDefs(int startIndex, String... names) {
......
repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
......@@ -591,10 +591,6 @@ public class ClassificationPropagationTest {
loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/patches/001-hive_column_add_position.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/patches/002-hive_column_table_add_options.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/patches/003-hive_column_update_table_remove_constraint.json", typeDefStore, typeRegistry);
loadSampleClassificationDefs();
runImportWithNoParameters(importService, getZipSource(IMPORT_FILE));
......
webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java
......@@ -998,8 +998,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
List<Referenceable> refs = (List<Referenceable>) hiveTableInstance.get("columns");
Assert.assertEquals(refs.size(), 2);
Assert.assertTrue(refs.get(0).getValuesMap().equals(values1));
Assert.assertTrue(refs.get(1).getValuesMap().equals(values2));
Assert.assertEquals(refs.get(0).getValuesMap(), values1);
Assert.assertEquals(refs.get(1).getValuesMap(), values2);
}
@Test
......
webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
......@@ -81,7 +81,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
ZipSource zs = new ZipSource(exportedStream);
assertNotNull(zs.getExportResult());
assertTrue(zs.getCreationOrder().size() > EXPECTED_CREATION_ORDER_SIZE);
assertTrue(zs.getCreationOrder().size() > EXPECTED_CREATION_ORDER_SIZE, "expected creationOrderSize > " + EXPECTED_CREATION_ORDER_SIZE + ", but found " + zs.getCreationOrder().size());
}
private void performImport(String fileToImport, int expectedProcessedEntitiesCount) throws AtlasServiceException {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment