Skip to content

A
atlas
Commit d185522b by Madhan Neethiraj
Options

ATLAS-1227: Added support for attribute constraints in the API

parent 92dc6faa
Showing with 584 additions and 65 deletions
intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
......@@ -20,9 +20,11 @@ package org.apache.atlas.model.typedef;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
......@@ -256,23 +258,25 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
public static final int COUNT_NOT_SET = -1;
private String name;
private String typeName;
private boolean isOptional;
private Cardinality cardinality;
private int valuesMinCount;
private int valuesMaxCount;
private boolean isUnique;
private boolean isIndexable;
private String name;
private String typeName;
private boolean isOptional;
private Cardinality cardinality;
private int valuesMinCount;
private int valuesMaxCount;
private boolean isUnique;
private boolean isIndexable;
private List<AtlasConstraintDef> constraintDefs;
public AtlasAttributeDef() { this(null, null); }
public AtlasAttributeDef(String name, String typeName) {
this(name, typeName, false, Cardinality.SINGLE, 1, 1, false, false);
this(name, typeName, false, Cardinality.SINGLE, 1, 1, false, false, null);
}
public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality,
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable) {
int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable,
List<AtlasConstraintDef> constraintDefs) {
setName(name);
setTypeName(typeName);
setOptional(isOptional);
......@@ -281,6 +285,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setValuesMaxCount(valuesMaxCount);
setUnique(isUnique);
setIndexable(isIndexable);
setConstraintDefs(constraintDefs);
}
public AtlasAttributeDef(AtlasAttributeDef other) {
......@@ -293,6 +298,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
setValuesMaxCount(other.getValuesMaxCount());
setUnique(other.isUnique());
setIndexable(other.isIndexable());
setConstraintDefs(other.getConstraintDefs());
}
}
......@@ -358,6 +364,33 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
isIndexable = idexable;
}
public List<AtlasConstraintDef> getConstraintDefs() { return constraintDefs; }
public void setConstraintDefs(List<AtlasConstraintDef> constraintDefs) {
if (this.constraintDefs != null && this.constraintDefs == constraintDefs) {
return;
}
if (CollectionUtils.isEmpty(constraintDefs)) {
this.constraintDefs = null;
} else {
this.constraintDefs = new ArrayList<AtlasConstraintDef>(constraintDefs);
}
}
public void addConstraint(AtlasConstraintDef constraintDef) {
List<AtlasConstraintDef> cDefs = constraintDefs;
if (cDefs == null) {
cDefs = new ArrayList<>();
} else {
cDefs = new ArrayList<>(cDefs);
}
cDefs.add(constraintDef);
this.constraintDefs = cDefs;
}
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
......@@ -372,6 +405,18 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append(", valuesMaxCount=").append(valuesMaxCount);
sb.append(", isUnique=").append(isUnique);
sb.append(", isIndexable=").append(isIndexable);
sb.append(", constraintDefs=[");
if (CollectionUtils.isNotEmpty(constraintDefs)) {
int i = 0;
for (AtlasConstraintDef constraintDef : constraintDefs) {
constraintDef.toString(sb);
if (i > 0) {
sb.append(", ");
}
i++;
}
}
sb.append("]");
sb.append('}');
return sb;
......@@ -394,6 +439,9 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
if (valuesMaxCount != that.valuesMaxCount) { return false; }
if (isUnique != that.isUnique) { return false; }
if (isIndexable != that.isIndexable) { return false; }
if (constraintDefs != null ? !constraintDefs.equals(that.constraintDefs) : that.constraintDefs != null) {
return false;
}
return true;
}
......@@ -408,6 +456,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
result = 31 * result + valuesMaxCount;
result = 31 * result + (isUnique ? 1 : 0);
result = 31 * result + (isIndexable ? 1 : 0);
result = 31 * result + (constraintDefs != null ? constraintDefs.hashCode() : 0);
return result;
}
......@@ -418,6 +467,104 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
}
/**
* class that captures details of a constraint.
*/
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public static class AtlasConstraintDef implements Serializable {
private static final long serialVersionUID = 1L;
public static final String CONSTRAINT_TYPE_FOREIGN_KEY = "foreignKey";
public static final String CONSTRAINT_TYPE_MAPPED_FROM_REF = "mappedFromRef";
public static final String CONSTRAINT_PARAM_REF_ATTRIBUTE = "refAttribute";
public static final String CONSTRAINT_PARAM_ON_DELETE = "onDelete";
public static final String CONSTRAINT_PARAM_VAL_CASCADE = "cascade";
private String type; // foreignKey/mappedFromRef/valueInRange
private Map<String, Object> params; // onDelete=cascade/refAttribute=attr2/min=0,max=23
public AtlasConstraintDef() { }
public AtlasConstraintDef(String type) {
this(type, null);
}
public AtlasConstraintDef(String type, Map<String, Object> params) {
this.type = type;
if (params != null) {
this.params = new HashMap<String, Object>(params);
}
}
public AtlasConstraintDef(AtlasConstraintDef that) {
if (that != null) {
this.type = that.type;
if (that.params != null) {
this.params = new HashMap<String, Object>(that.params);
}
}
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Map<String, Object> getParams() {
return params;
}
public void setParams(Map<String, Object> params) {
this.params = params;
}
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
}
sb.append("AtlasConstraintDef{");
sb.append("type='").append(type).append('\'');
sb.append(", params='").append(params).append('\'');
sb.append('}');
return sb;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AtlasConstraintDef that = (AtlasConstraintDef) o;
if (type != null ? !type.equals(that.type) : that.type != null) return false;
if (params != null ? !params.equals(that.params) : that.params != null) return false;
return true;
}
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
result = 31 * result + (params != null ? params.hashCode() : 0);
return result;
}
@Override
public String toString() { return toString(new StringBuilder()).toString(); }
}
/**
* REST serialization friendly list.
*/
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
......
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
......@@ -22,8 +22,14 @@ import java.util.*;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -36,7 +42,9 @@ public class AtlasStructType extends AtlasType {
private final AtlasStructDef structDef;
private Map<String, AtlasType> attrTypes = Collections.emptyMap();
private Map<String, AtlasType> attrTypes = Collections.emptyMap();
private Set<String> foreignKeyAttributes = new HashSet<>();
private Map<String, TypeAttributePair> mappedFromRefAttributes = new HashMap<String, TypeAttributePair>();
public AtlasStructType(AtlasStructDef structDef) {
......@@ -57,6 +65,31 @@ public class AtlasStructType extends AtlasType {
public AtlasAttributeDef getAttributeDef(String attributeName) { return structDef.getAttribute(attributeName); }
public boolean isForeignKeyAttribute(String attributeName) {
return foreignKeyAttributes.contains(attributeName);
}
public boolean isMappedFromRefAttribute(String attributeName) {
return mappedFromRefAttributes.containsKey(attributeName);
}
public String getMappedFromRefAttribute(String typeName, String attribName) {
String ret = null;
for (Map.Entry<String, TypeAttributePair> e : mappedFromRefAttributes.entrySet()) {
String refTypeName = e.getValue().typeName;
String refAttribName = e.getValue().attributeName;
if(StringUtils.equals(refTypeName, typeName) && StringUtils.equals(refAttribName, attribName)) {
ret = e.getKey();
break;
}
}
return ret;
}
@Override
public void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
Map<String, AtlasType> a = new HashMap<String, AtlasType>();
......@@ -69,6 +102,8 @@ public class AtlasStructType extends AtlasType {
+ structDef.getName() + "." + attributeDef.getName());
}
resolveConstraints(attributeDef, attrType);
Cardinality cardinality = attributeDef.getCardinality();
if (cardinality == Cardinality.LIST || cardinality == Cardinality.SET) {
......@@ -290,4 +325,111 @@ public class AtlasStructType extends AtlasType {
return null;
}
private void resolveConstraints(AtlasAttributeDef attribDef, AtlasType attribType) throws AtlasBaseException {
if (attribDef == null || CollectionUtils.isEmpty(attribDef.getConstraintDefs()) || attribType == null) {
return;
}
for (AtlasStructDef.AtlasConstraintDef constraintDef : attribDef.getConstraintDefs()) {
String constraintType = constraintDef != null ? constraintDef.getType() : null;
if (StringUtils.isBlank(constraintType)) {
continue;
}
if (constraintType.equals(CONSTRAINT_TYPE_FOREIGN_KEY)) {
resolveForeignKeyConstraint(attribDef, constraintDef, attribType);
} else if (constraintType.equals(CONSTRAINT_TYPE_MAPPED_FROM_REF)) {
resolveMappedFromRefConstraint(attribDef, constraintDef, attribType);
} else {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName()
+ ": unknown constraint " + constraintType);
}
}
}
/*
* valid conditions for foreign-key constraint:
* - supported only in entity-type
* - attribute should be an entity-type or an array of entity-type
*/
private void resolveForeignKeyConstraint(AtlasAttributeDef attribDef, AtlasConstraintDef constraintDef,
AtlasType attribType) throws AtlasBaseException {
if (!(this instanceof AtlasEntityType)) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
+ AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY + " constraint not supported");
}
if (attribType instanceof AtlasArrayType) {
attribType = ((AtlasArrayType)attribType).getElementType();
}
if (!(attribType instanceof AtlasEntityType)) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
+ AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY + " incompatible attribute type "
+ attribType.getTypeName());
}
foreignKeyAttributes.add(attribDef.getName());
}
/*
* valid conditions for mapped-from-ref constraint:
* - supported only in entity-type
* - attribute should be an entity-type or an array of entity-type
* - attribute's entity-type should have a foreign-key constraint to this type
*/
private void resolveMappedFromRefConstraint(AtlasAttributeDef attribDef, AtlasConstraintDef constraintDef,
AtlasType attribType) throws AtlasBaseException {
if (!(this instanceof AtlasEntityType)) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
+ CONSTRAINT_TYPE_MAPPED_FROM_REF + " constraint not supported");
}
if (attribType instanceof AtlasArrayType) {
attribType = ((AtlasArrayType)attribType).getElementType();
}
if (!(attribType instanceof AtlasEntityType)) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
+ CONSTRAINT_TYPE_MAPPED_FROM_REF + " incompatible attribute type "
+ attribType.getTypeName());
}
String refAttribName = AtlasTypeUtil.getStringValue(constraintDef.getParams(), CONSTRAINT_PARAM_REF_ATTRIBUTE);
if (StringUtils.isBlank(refAttribName)) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": "
+ CONSTRAINT_TYPE_MAPPED_FROM_REF + " invalid constraint. missing parameter "
+ CONSTRAINT_PARAM_REF_ATTRIBUTE);
}
AtlasStructType structType = (AtlasStructType)attribType;
AtlasAttributeDef refAttrib = structType.getAttributeDef(refAttribName);
if (refAttrib == null) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. "
+ CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName
+ " does not exist");
}
if (!StringUtils.equals(getTypeName(), refAttrib.getTypeName())) {
throw new AtlasBaseException(getTypeName() + "." + attribDef.getName() + ": invalid constraint. Datatype of"
+ CONSTRAINT_PARAM_REF_ATTRIBUTE + " " + structType.getTypeName() + "." + refAttribName
+ " should be " + getTypeName() + ", but found " + refAttrib.getTypeName());
}
mappedFromRefAttributes.put(attribDef.getName(), new TypeAttributePair(refAttrib.getTypeName(), refAttribName));
}
private class TypeAttributePair {
public final String typeName;
public final String attributeName;
public TypeAttributePair(String typeName, String attributeName) {
this.typeName = typeName;
this.attributeName = attributeName;
}
}
}
intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
......@@ -26,6 +26,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_MAP_KEY
import org.apache.commons.lang.StringUtils;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
......@@ -52,6 +53,22 @@ public class AtlasTypeUtil {
return ATLAS_BUILTIN_TYPENAMES.contains(typeName);
}
public static boolean isArrayType(String typeName) {
return StringUtils.startsWith(typeName, ATLAS_TYPE_ARRAY_PREFIX)
&& StringUtils.endsWith(typeName, ATLAS_TYPE_ARRAY_SUFFIX);
}
public static boolean isMapType(String typeName) {
return StringUtils.startsWith(typeName, ATLAS_TYPE_MAP_PREFIX)
&& StringUtils.endsWith(typeName, ATLAS_TYPE_MAP_SUFFIX);
}
public static String getStringValue(Map map, Object key) {
Object ret = map != null ? map.get(key) : null;
return ret != null ? ret.toString() : null;
}
private static void getReferencedTypeNames(String typeName, Set<String> referencedTypeNames) {
if (StringUtils.isNotBlank(typeName) && !referencedTypeNames.contains(typeName)) {
if (typeName.startsWith(ATLAS_TYPE_ARRAY_PREFIX) && typeName.endsWith(ATLAS_TYPE_ARRAY_SUFFIX)) {
......
intg/src/test/java/org/apache/atlas/type/TestAtlasEntityType.java
......@@ -24,6 +24,8 @@ import org.apache.atlas.model.ModelTestUtil;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.testng.annotations.Test;
import static org.testng.Assert.*;
......@@ -115,6 +117,74 @@ public class TestAtlasEntityType {
}
}
@Test
public void testForeignKeyConstraintValid() {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
List<AtlasEntityDef> entityDefs = new ArrayList<>();
String failureMsg = null;
entityDefs.add(createTableEntityDef());
entityDefs.add(createColumnEntityDef());
try {
typeRegistry.addTypesWithNoRefResolve(entityDefs);
typeRegistry.resolveReferences();
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
assertNull(failureMsg, "failed to create types my_table and my_column");
}
@Test
public void testForeignKeyConstraintInValidMappedFromRef() {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
List<AtlasEntityDef> entityDefs = new ArrayList<>();
String failureMsg = null;
entityDefs.add(createTableEntityDef());
try {
typeRegistry.addTypes(entityDefs);
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
assertNotNull(failureMsg, "expected invalid constraint failure - unknown attribute in mappedFromRef");
}
@Test
public void testForeignKeyConstraintInValidMappedFromRef2() {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
List<AtlasEntityDef> entityDefs = new ArrayList<>();
String failureMsg = null;
entityDefs.add(createTableEntityDefWithMissingRefAttribute());
entityDefs.add(createColumnEntityDef());
try {
typeRegistry.addTypesWithNoRefResolve(entityDefs);
typeRegistry.resolveReferences();
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
assertNotNull(failureMsg, "expected invalid constraint failure - missing refAttribute in mappedFromRef");
}
@Test
public void testForeignKeyConstraintInValidForeignKey() {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
List<AtlasEntityDef> entityDefs = new ArrayList<>();
String failureMsg = null;
entityDefs.add(createColumnEntityDef());
try {
typeRegistry.addTypes(entityDefs);
} catch (AtlasBaseException excp) {
failureMsg = excp.getMessage();
}
assertNotNull(failureMsg, "expected invalid constraint failure - unknown attribute in foreignKey");
}
private static AtlasEntityType getEntityType(AtlasEntityDef entityDef) {
try {
return new AtlasEntityType(entityDef, ModelTestUtil.getTypesRegistry());
......@@ -122,4 +192,39 @@ public class TestAtlasEntityType {
return null;
}
}
private AtlasEntityDef createTableEntityDef() {
AtlasEntityDef table = new AtlasEntityDef("my_table");
AtlasAttributeDef attrColumns = new AtlasAttributeDef("columns",
AtlasBaseTypeDef.getArrayTypeName("my_column"));
Map<String, Object> params = new HashMap<>();
params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, "table");
attrColumns.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, params));
table.addAttribute(attrColumns);
return table;
}
private AtlasEntityDef createTableEntityDefWithMissingRefAttribute() {
AtlasEntityDef table = new AtlasEntityDef("my_table");
AtlasAttributeDef attrColumns = new AtlasAttributeDef("columns",
AtlasBaseTypeDef.getArrayTypeName("my_column"));
attrColumns.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF, null));
table.addAttribute(attrColumns);
return table;
}
private AtlasEntityDef createColumnEntityDef() {
AtlasEntityDef column = new AtlasEntityDef("my_column");
AtlasAttributeDef attrTable = new AtlasAttributeDef("table", "my_table");
attrTable.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY));
column.addAttribute(attrTable);
return column;
}
}
release-log.txt
......@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1227 Added support for attribute constraints in the API
ATLAS-1225 Optimize AtlasTypeDefGraphStore to use typeRegistry (mneethiraj via sumasai)
ATLAS-1221 build failure in windows SyntaxError: invalid syntax (zhangqiang2 via shwethags)
ATLAS-1226 Servlet init-params in web.xml are unused (mneethiraj via shwethags)
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasClassificationDefStoreV1.java
......@@ -345,7 +345,7 @@ public class AtlasClassificationDefStoreV1 implements AtlasClassificationDefStor
typeDefStore.createSuperTypeEdges(vertex, classificationDef.getSuperTypes());
}
private AtlasClassificationDef toClassificationDef(AtlasVertex vertex) {
private AtlasClassificationDef toClassificationDef(AtlasVertex vertex) throws AtlasBaseException {
AtlasClassificationDef ret = null;
if (vertex != null && typeDefStore.isTypeVertex(vertex, TypeCategory.TRAIT)) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityDefStoreV1.java
......@@ -344,7 +344,7 @@ public class AtlasEntityDefStoreV1 implements AtlasEntityDefStore {
typeDefStore.createSuperTypeEdges(vertex, entityDef.getSuperTypes());
}
private AtlasEntityDef toEntityDef(AtlasVertex vertex) {
private AtlasEntityDef toEntityDef(AtlasVertex vertex) throws AtlasBaseException {
AtlasEntityDef ret = null;
if (vertex != null && typeDefStore.isTypeVertex(vertex, TypeCategory.CLASS)) {
......
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
......@@ -21,16 +21,27 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasStructDefStore;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -62,17 +73,17 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.create({})", structDef);
}
AtlasVertex AtlasVertex = typeDefStore.findTypeVertexByName(structDef.getName());
AtlasVertex vertex = typeDefStore.findTypeVertexByName(structDef.getName());
if (AtlasVertex != null) {
if (vertex != null) {
throw new AtlasBaseException(structDef.getName() + ": type already exists");
}
AtlasVertex = typeDefStore.createTypeVertex(structDef);
vertex = typeDefStore.createTypeVertex(structDef);
toVertex(structDef, AtlasVertex, typeDefStore);
toVertex(structDef, vertex, typeDefStore);
AtlasStructDef ret = toStructDef(AtlasVertex);
AtlasStructDef ret = toStructDef(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.create({}): {}", structDef, ret);
......@@ -127,15 +138,15 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.getByName({})", name);
}
AtlasVertex atlasVertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
if (atlasVertex == null) {
if (vertex == null) {
throw new AtlasBaseException("no structDef exists with name " + name);
}
atlasVertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, String.class);
vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, String.class);
AtlasStructDef ret = toStructDef(atlasVertex);
AtlasStructDef ret = toStructDef(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.getByName({}): {}", name, ret);
......@@ -150,13 +161,13 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.getByGuid({})", guid);
}
AtlasVertex AtlasVertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT);
AtlasVertex vertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT);
if (AtlasVertex == null) {
if (vertex == null) {
throw new AtlasBaseException("no structDef exists with guid " + guid);
}
AtlasStructDef ret = toStructDef(AtlasVertex);
AtlasStructDef ret = toStructDef(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.getByGuid({}): {}", guid, ret);
......@@ -171,15 +182,15 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.updateByName({}, {})", name, structDef);
}
AtlasVertex atlasVertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
if (atlasVertex == null) {
if (vertex == null) {
throw new AtlasBaseException("no structDef exists with name " + name);
}
toVertex(structDef, atlasVertex);
toVertex(structDef, vertex);
AtlasStructDef ret = toStructDef(atlasVertex);
AtlasStructDef ret = toStructDef(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.updateByName({}, {}): {}", name, structDef, ret);
......@@ -194,15 +205,15 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.updateByGuid({})", guid);
}
AtlasVertex atlasVertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT);
AtlasVertex vertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT);
if (atlasVertex == null) {
if (vertex == null) {
throw new AtlasBaseException("no structDef exists with guid " + guid);
}
toVertex(structDef, atlasVertex);
toVertex(structDef, vertex);
AtlasStructDef ret = toStructDef(atlasVertex);
AtlasStructDef ret = toStructDef(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.updateByGuid({}): {}", guid, ret);
......@@ -239,13 +250,13 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.deleteByName({})", name);
}
AtlasVertex AtlasVertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
if (AtlasVertex == null) {
if (vertex == null) {
throw new AtlasBaseException("no structDef exists with name " + name);
}
typeDefStore.deleteTypeVertex(AtlasVertex);
typeDefStore.deleteTypeVertex(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.deleteByName({})", name);
......@@ -277,13 +288,13 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
LOG.debug("==> AtlasStructDefStoreV1.deleteByGuid({})", guid);
}
AtlasVertex AtlasVertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT);
AtlasVertex vertex = typeDefStore.findTypeVertexByGuidAndCategory(guid, TypeCategory.STRUCT);
if (AtlasVertex == null) {
if (vertex == null) {
throw new AtlasBaseException("no structDef exists with guid " + guid);
}
typeDefStore.deleteTypeVertex(AtlasVertex);
typeDefStore.deleteTypeVertex(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.deleteByGuid({})", guid);
......@@ -341,48 +352,53 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
return ret;
}
private void toVertex(AtlasStructDef structDef, AtlasVertex AtlasVertex) {
toVertex(structDef, AtlasVertex, typeDefStore);
private void toVertex(AtlasStructDef structDef, AtlasVertex vertex) {
toVertex(structDef, vertex, typeDefStore);
}
private AtlasStructDef toStructDef(AtlasVertex AtlasVertex) {
private AtlasStructDef toStructDef(AtlasVertex vertex) throws AtlasBaseException {
AtlasStructDef ret = null;
if (AtlasVertex != null && typeDefStore.isTypeVertex(AtlasVertex, TypeCategory.STRUCT)) {
ret = toStructDef(AtlasVertex, new AtlasStructDef(), typeDefStore);
if (vertex != null && typeDefStore.isTypeVertex(vertex, TypeCategory.STRUCT)) {
ret = toStructDef(vertex, new AtlasStructDef(), typeDefStore);
}
return ret;
}
public static void toVertex(AtlasStructDef structDef, AtlasVertex atlasVertex, AtlasTypeDefGraphStoreV1 typeDefStore) {
List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size());
public static void toVertex(AtlasStructDef structDef, AtlasVertex vertex, AtlasTypeDefGraphStoreV1 typeDefStore) {
AtlasTypeRegistry typeRegistry = typeDefStore.getTypeRegistry();
AtlasStructType structType = (AtlasStructType)typeRegistry.getType(structDef.getName());
List<String> attrNames = new ArrayList<>(structDef.getAttributeDefs().size());
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName());
AtlasGraphUtilsV1.setProperty(atlasVertex, propertyKey, toJsonFromAttributeDef(attributeDef));
AtlasGraphUtilsV1.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType, typeRegistry));
attrNames.add(attributeDef.getName());
addReferencesForAttribute(atlasVertex, attributeDef, typeDefStore);
addReferencesForAttribute(vertex, attributeDef, typeDefStore);
}
AtlasGraphUtilsV1.setProperty(atlasVertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
}
public static AtlasStructDef toStructDef(AtlasVertex atlasVertex, AtlasStructDef structDef, AtlasTypeDefGraphStoreV1 typeDefStore) {
public static AtlasStructDef toStructDef(AtlasVertex vertex,
AtlasStructDef structDef,
AtlasTypeDefGraphStoreV1 typeDefStore) throws AtlasBaseException {
AtlasStructDef ret = (structDef != null) ? structDef :new AtlasStructDef();
typeDefStore.vertexToTypeDef(atlasVertex, ret);
typeDefStore.vertexToTypeDef(vertex, ret);
List<AtlasAttributeDef> attributeDefs = new ArrayList<>();
List<String> attrNames = atlasVertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(ret), List.class);
List<String> attrNames = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(ret), List.class);
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(ret, attrName);
String attribJson = atlasVertex.getProperty(propertyKey, String.class);
String attribJson = vertex.getProperty(propertyKey, String.class);
attributeDefs.add(toAttributeDefFromJson(AtlasType.fromJson(attribJson, Map.class)));
attributeDefs.add(toAttributeDefFromJson(structDef, AtlasType.fromJson(attribJson, Map.class),
typeDefStore));
}
}
ret.setAttributeDefs(attributeDefs);
......@@ -390,10 +406,12 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
return ret;
}
private static void addReferencesForAttribute(AtlasVertex atlasVertex, AtlasAttributeDef attributeDef, AtlasTypeDefGraphStoreV1 typeDefStore) {
private static void addReferencesForAttribute(AtlasVertex vertex,
AtlasAttributeDef attributeDef,
AtlasTypeDefGraphStoreV1 typeDefStore) {
Set<String> referencedTypeNames = AtlasTypeUtil.getReferencedTypeNames(attributeDef.getTypeName());
String AtlasVertexTypeName = atlasVertex.getProperty(Constants.TYPENAME_PROPERTY_KEY, String.class);
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY, String.class);
for (String referencedTypeName : referencedTypeNames) {
if (!AtlasTypeUtil.isBuiltInType(referencedTypeName)) {
......@@ -404,18 +422,36 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
}
if (referencedTypeAtlasVertex != null) {
String label = AtlasGraphUtilsV1.getEdgeLabel(AtlasVertexTypeName, attributeDef.getName());
String label = AtlasGraphUtilsV1.getEdgeLabel(typeName, attributeDef.getName());
typeDefStore.getOrCreateEdge(atlasVertex, referencedTypeAtlasVertex, label);
typeDefStore.getOrCreateEdge(vertex, referencedTypeAtlasVertex, label);
}
}
}
}
private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef) {
Boolean isComposite = Boolean.FALSE;
private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef,
AtlasStructType structType,
AtlasTypeRegistry typeRegistry) {
boolean isForeignKey = structType.isForeignKeyAttribute(attributeDef.getName());
boolean isMappedFromRef = structType.isMappedFromRefAttribute(attributeDef.getName());
String reverseAttribName = null;
if (isForeignKey) { // check if the referenced entity has foreignKeyRef to this attribute
AtlasType attribType = typeRegistry.getType(attributeDef.getTypeName());
if (attribType instanceof AtlasArrayType) {
attribType = ((AtlasArrayType)attribType).getElementType();
}
if (attribType instanceof AtlasEntityType) {
reverseAttribName = ((AtlasStructType)attribType).getMappedFromRefAttribute(structType.getTypeName(),
attributeDef.getName());
}
}
boolean isComposite = isMappedFromRef || (isForeignKey && StringUtils.isBlank(reverseAttribName));
Map<String, Object> attribInfo = new HashMap<String, Object>();
attribInfo.put("name", attributeDef.getName());
......@@ -434,7 +470,10 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
return AtlasType.toJson(attribInfo);
}
private static AtlasAttributeDef toAttributeDefFromJson(Map attribInfo) {
private static AtlasAttributeDef toAttributeDefFromJson(AtlasStructDef structDef,
Map attribInfo,
AtlasTypeDefGraphStoreV1 typeDefStore)
throws AtlasBaseException {
AtlasAttributeDef ret = new AtlasAttributeDef();
ret.setName((String) attribInfo.get("name"));
......@@ -442,10 +481,78 @@ public class AtlasStructDefStoreV1 implements AtlasStructDefStore {
ret.setUnique((Boolean) attribInfo.get("isUnique"));
ret.setIndexable((Boolean) attribInfo.get("isIndexable"));
/*
String reverseAttribName = (String)attribInfo.get("reverseAttributeName");
Boolean isComposite = (Boolean) attribInfo.get("isComposite");
*/
String attrTypeName = ret.getTypeName();
if (AtlasTypeUtil.isArrayType(attrTypeName)) {
Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(ret.getTypeName());
if (typeNames.size() > 0) {
attrTypeName = typeNames.iterator().next();
}
}
if (! AtlasTypeUtil.isBuiltInType(attrTypeName)) {
AtlasVertex attributeType = typeDefStore.findTypeVertexByName(attrTypeName);
// check for isComposite/reverseAttributeName for entity types
if (attributeType != null && typeDefStore.isTypeVertex(attributeType, TypeCategory.CLASS)) {
String reverseAttribName = (String) attribInfo.get("reverseAttributeName");
Boolean isComposite = (Boolean) attribInfo.get("isComposite");
// find the attribute that refers to this attribute
if (StringUtils.isNotBlank(reverseAttribName) || isComposite) {
if (AtlasTypeUtil.isMapType(attrTypeName)) {
throw new AtlasBaseException(structDef.getName() + "." + ret.getName()
+ ": constraints not supported on map type " + attrTypeName);
}
if (attributeType != null) {
String refAttributeName = null;
List<String> attrNames = attributeType.getProperty(
AtlasGraphUtilsV1.getPropertyKey(attrTypeName), List.class);
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String attribJson = attributeType.getProperty(
AtlasGraphUtilsV1.getPropertyKey(attrTypeName, attrName), String.class);
Map refAttrInfo = AtlasType.fromJson(attribJson, Map.class);
String refAttribType = (String) refAttrInfo.get("dataType");
String refAttribRevAttribName = (String) refAttrInfo.get("reverseAttributeName");
if (StringUtils.equals(refAttribType, structDef.getName()) &&
StringUtils.equals(refAttribRevAttribName, ret.getName())) {
refAttributeName = (String) refAttrInfo.get("name");
break;
}
}
}
if (isComposite) {
if (StringUtils.isNotBlank(refAttributeName)) { // ex: hive_table.columns, hive_column.table
Map<String, Object> params = new HashMap<String, Object>();
params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, refAttributeName);
ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params));
} else { // ex: hive_table.partitionKeys, with no reverseAttribute-reference
ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY));
}
}
if (StringUtils.isNotBlank(reverseAttribName)) { // ex: hive_column.table
Map<String, Object> params = new HashMap<>();
params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE);
ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params));
}
}
}
}
}
Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class);
Number minCount = (Number) multiplicity.get("lower");
Number maxCount = (Number) multiplicity.get("upper");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment